diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index aa84e125e0..d67bdf9bd8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -5,7 +5,7 @@ ARG USER_UID=1000 ARG USER_GID=$USER_UID ARG HELM_VERSION=3.3.1 -ARG KUBECTL_VERSION=1.18.8 +ARG KUBECTL_VERSION=1.19.15 ARG ISTIOCTL_VERSION=1.8.1 RUN : INSTALL APT REQUIREMENTS \ diff --git a/Dockerfile b/Dockerfile index f202d9ad15..de1fff615a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG USER_UID=1000 ARG USER_GID=$USER_UID ARG HELM_VERSION=3.3.1 -ARG KUBECTL_VERSION=1.18.8 +ARG KUBECTL_VERSION=1.19.15 ARG ISTIOCTL_VERSION=1.8.1 ENV EPICLI_DOCKER_SHARED_DIR=/shared diff --git a/ansible/playbooks/roles/helm_charts/files/system/epi-dummy/templates/ingress.yaml b/ansible/playbooks/roles/helm_charts/files/system/epi-dummy/templates/ingress.yaml index 3db10019a2..4345f939e0 100644 --- a/ansible/playbooks/roles/helm_charts/files/system/epi-dummy/templates/ingress.yaml +++ b/ansible/playbooks/roles/helm_charts/files/system/epi-dummy/templates/ingress.yaml @@ -1,7 +1,7 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "epi-dummy.fullname" . -}} {{- $svcPort := .Values.service.port -}} -apiVersion: networking.k8s.io/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ $fullName }} diff --git a/ansible/playbooks/roles/kubernetes_common/tasks/update-in-cluster-config.yml b/ansible/playbooks/roles/kubernetes_common/tasks/update-in-cluster-config.yml index d6579f8f89..6a1a54f2a9 100644 --- a/ansible/playbooks/roles/kubernetes_common/tasks/update-in-cluster-config.yml +++ b/ansible/playbooks/roles/kubernetes_common/tasks/update-in-cluster-config.yml @@ -1,10 +1,8 @@ --- - name: Update in-cluster configuration - shell: | + command: | kubeadm init phase upload-config kubeadm \ --config /etc/kubeadm/kubeadm-config.yml - args: - executable: /bin/bash register: upload_config until: upload_config is succeeded retries: 30 diff --git a/ansible/playbooks/roles/kubernetes_master/files/coredns-configmap.yml b/ansible/playbooks/roles/kubernetes_master/files/coredns-configmap.yml index fa25cfd30d..9fa9891d55 100644 --- a/ansible/playbooks/roles/kubernetes_master/files/coredns-configmap.yml +++ b/ansible/playbooks/roles/kubernetes_master/files/coredns-configmap.yml @@ -1,4 +1,4 @@ -# Based on https://github.com/kubernetes/kubernetes/blob/v1.18.6/cluster/addons/dns/coredns/coredns.yaml.in +# Based on https://github.com/kubernetes/kubernetes/blob/v1.19.15/cluster/addons/dns/coredns/coredns.yaml.in # Hosts plugin added # Note: /etc/hosts in pods is managed by kubelet thus we mount it under different path, @@ -28,7 +28,9 @@ data: ttl 30 } prometheus :9153 - forward . /etc/resolv.conf + forward . /etc/resolv.conf { + max_concurrent 1000 + } cache 30 loop reload diff --git a/ansible/playbooks/roles/kubernetes_master/templates/calico.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/calico.yml.j2 index 9661a5a63c..e8cbbeebc5 100644 --- a/ansible/playbooks/roles/kubernetes_master/templates/calico.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_master/templates/calico.yml.j2 @@ -12,12 +12,9 @@ data: # Configure the backend to use. calico_backend: "bird" # Configure the MTU to use for workload interfaces and tunnels. - # - If Wireguard is enabled, set to your network MTU - 60 - # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50 - # - Otherwise, if IPIP is enabled, set to your network MTU - 20 - # - Otherwise, if not using any encapsulation, set to your network MTU. - veth_mtu: "1440" - + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- @@ -28,6 +25,7 @@ data: { "type": "calico", "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, @@ -52,18 +50,11 @@ data: } ] } - --- # Source: calico/templates/kdd-crds.yaml - - ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: bgpconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -99,6 +90,32 @@ spec: 64512]' format: int32 type: integer + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' @@ -107,13 +124,36 @@ spec: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array serviceClusterIPs: description: ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. If specified, Calico will advertise these blocks, as well as any cluster IPs within them. items: - description: ServiceClusterIPBlock represents a single whitelisted - CIDR block for ClusterIPs. + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. properties: cidr: type: string @@ -124,8 +164,20 @@ spec: Service External IPs. Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks. items: - description: ServiceExternalIPBlock represents a single whitelisted - CIDR External IP block. + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. properties: cidr: type: string @@ -141,16 +193,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: bgppeers.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -184,17 +230,55 @@ spec: description: The AS Number of the peer. format: int32 type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, this + is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string node: description: The node name identifying the Calico node instance that - is peering with this peer. If this is not set, this represents a - global peer, i.e. a peer that peers with every node in the deployment. + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. type: string nodeSelector: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object peerIP: - description: The IP address of the peer. + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. type: string peerSelector: description: Selector for the remote nodes to peer with. When this @@ -202,12 +286,15 @@ spec: peering between the local node and selected remote nodes, we configure an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node’s NodeBGPSpec.ASNumber, + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, or the global default if that is not set. type: string - required: - - asNumber - - peerIP + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string type: object type: object served: true @@ -218,16 +305,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: blockaffinities.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -284,16 +365,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: clusterinformations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -353,16 +428,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: felixconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -393,6 +462,25 @@ spec: spec: description: FelixConfigurationSpec contains the values of the Felix configuration. properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enabled" or "Disabled". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string bpfConnectTimeLoadBalancingEnabled: description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The @@ -402,13 +490,13 @@ spec: true]' type: boolean bpfDataIfacePattern: - description: 'BPFDataIfacePattern is a regular expression that controls + description: BPFDataIfacePattern is a regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually - named cali...). [Default: ^(en.*|eth.*|tunl0$)]' + named cali...). type: string bpfDisableUnprivileged: description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled @@ -430,6 +518,14 @@ spec: node appears to use the IP of the ingress node; this requires a permissive L2 network. [Default: Tunnel]' type: string + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, controls a + 32bit mark that is set on connections from an external client to + a local service. This mark allows us to control how packets of + that connection are routed within the host and how is routing + intepreted by RPF check. [Default: 0]' + type: integer + bpfKubeProxyEndpointSlicesEnabled: description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls whether Felix's embedded kube-proxy accepts EndpointSlices or not. @@ -453,10 +549,10 @@ spec: `tc exec bpf debug`. [Default: Off].' type: string chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel’s + description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico’s rules from being bypassed. If you switch + since it prevents Calico''s rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' @@ -476,11 +572,11 @@ spec: traffic that goes from a workload endpoint to the host itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload endpoints to the host itself with an - iptables “DROP” action. If you want to allow some or all traffic + iptables "DROP" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables “INPUT” chain; - Calico will insert its rules at the top of that chain, then “RETURN” - packets to the “INPUT” chain once it has completed processing workload + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]' @@ -508,19 +604,21 @@ spec: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a comma-delimited list of - UDP/TCP ports that Felix will allow incoming traffic to host endpoints + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. Each - port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all inbound host ports, use the value none. - The default value allows ssh access and DHCP. [Default: tcp:22, + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' items: - description: ProtoPort is combination of protocol and port, both - must be specified. + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. properties: + net: + type: string port: type: integer protocol: @@ -531,21 +629,23 @@ spec: type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a comma-delimited list - of UDP/TCP ports that Felix will allow outgoing traffic from host - endpoints to irrespective of the security policy. This is useful - to avoid accidentally cutting off a host with incorrect configuration. - Each port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all outbound host ports, use the value none. - The default value opens etcd’s standard ports to ensure that Felix - does not get cut off from etcd as well as allowing DHCP and DNS. - [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, - udp:53, udp:67]' + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' items: - description: ProtoPort is combination of protocol and port, both - must be specified. + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. properties: + net: + type: string port: type: integer protocol: @@ -555,6 +655,13 @@ spec: - protocol type: object type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -583,8 +690,13 @@ spec: workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker - integrations set the ‘cali’ value, and our OpenStack integration - sets the ‘tap’ value. [Default: cali]' + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. type: string ipipEnabled: type: boolean @@ -595,7 +707,7 @@ spec: ipsetsRefreshInterval: description: 'IpsetsRefreshInterval is the period at which Felix re-checks all iptables state to ensure that no other process has accidentally - broken Calico’s rules. Set to 0 to disable iptables refresh. [Default: + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: 90s]' type: string iptablesBackend: @@ -607,7 +719,7 @@ spec: iptablesLockFilePath: description: 'IptablesLockFilePath is the location of the iptables lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix’s + its standard location (for example if you have mapped it into Felix''s container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: @@ -639,16 +751,16 @@ spec: description: 'IptablesPostWriteCheckInterval is the period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn’t + This should only occur if another application on the system doesn''t respect the iptables lock. [Default: 1s]' type: string iptablesRefreshInterval: description: 'IptablesRefreshInterval is the period at which Felix re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico’s rules. Set to 0 to disable IP sets - refresh. Note: the default for this value is lower than the other - refresh intervals as a workaround for a Linux kernel bug that was - fixed in kernel version 4.11. If you are using v4.11 or greater + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage. [Default: 10s]' type: string @@ -699,10 +811,15 @@ spec: type: string metadataPort: description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ‘None’), is used to set - up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed [Default: 8775].' type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string natOutgoingAddress: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the @@ -773,9 +890,9 @@ spec: status reports. [Default: 90s]' type: string routeRefreshInterval: - description: 'RouterefreshInterval is the period at which Felix re-checks + description: 'RouteRefreshInterval is the period at which Felix re-checks the routes in the dataplane to ensure that no other process has - accidentally broken Calico’s rules. Set to 0 to disable route refresh. + accidentally broken Calico''s rules. Set to 0 to disable route refresh. [Default: 90s]' type: string routeSource: @@ -796,6 +913,13 @@ spec: - max - min type: object + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string sidecarAccelerationEnabled: description: 'SidecarAccelerationEnabled enables experimental sidecar acceleration [Default: false]' @@ -857,8 +981,6 @@ spec: Calico''s BPF maps or attached programs. Set to 0 to disable XDP refresh. [Default: 90s]' type: string - required: - - bpfLogLevel type: object type: object served: true @@ -869,16 +991,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: globalnetworkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -926,7 +1042,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -942,16 +1058,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1009,9 +1126,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1039,6 +1156,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -1081,7 +1218,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1110,7 +1247,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1147,16 +1284,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1214,9 +1352,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1244,6 +1382,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -1257,7 +1415,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -1273,16 +1431,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1340,9 +1499,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1370,6 +1529,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -1412,7 +1591,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1441,7 +1620,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1478,16 +1657,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1545,9 +1725,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1575,6 +1755,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -1645,16 +1845,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: globalnetworksets.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1703,16 +1897,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: hostendpoints.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1768,7 +1956,7 @@ spec: is empty - through the specific interface that has one of the IPs in ExpectedIPs. Therefore, when InterfaceName is empty, at least one expected IP must be specified. Only external interfaces (such - as “eth0”) are supported here; it isn't possible for a HostEndpoint + as \"eth0\") are supported here; it isn't possible for a HostEndpoint to protect traffic through a specific local workload interface. \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; initially just pre-DNAT policy. Please check Calico documentation @@ -1816,16 +2004,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamblocks.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1890,7 +2072,6 @@ spec: - allocations - attributes - cidr - - deleted - strictAffinity - unallocated type: object @@ -1903,16 +2084,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamconfigs.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1945,6 +2120,10 @@ spec: properties: autoAllocateBlocks: type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + type: integer strictAffinity: type: boolean required: @@ -1960,16 +2139,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamhandles.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2004,6 +2177,8 @@ spec: additionalProperties: type: integer type: object + deleted: + type: boolean handleID: type: string required: @@ -2019,16 +2194,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ippools.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2089,7 +2258,7 @@ spec: type: object ipipMode: description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunelling + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling is disabled). type: string nat-outgoing: @@ -2109,7 +2278,7 @@ spec: vxlanMode: description: Contains configuration for VXLAN tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunelling is disabled). + tunneling is disabled). type: string required: - cidr @@ -2123,16 +2292,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: kubecontrollersconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2189,6 +2352,11 @@ spec: host endpoints for every node. [Default: Disabled]' type: string type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' @@ -2238,6 +2406,10 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer required: - controllers type: object @@ -2285,6 +2457,12 @@ spec: of host endpoints for every node. [Default: Disabled]' type: string type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: @@ -2338,6 +2516,11 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer required: - controllers type: object @@ -2351,16 +2534,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: networkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2397,7 +2574,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -2413,16 +2590,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2480,9 +2658,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2510,6 +2688,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -2552,7 +2750,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2581,7 +2779,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2618,16 +2816,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2685,9 +2884,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2715,6 +2914,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -2728,7 +2947,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -2744,16 +2963,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2811,9 +3031,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2841,6 +3061,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -2883,7 +3123,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2912,7 +3152,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2949,16 +3189,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -3016,9 +3257,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -3046,6 +3287,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -3108,16 +3369,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: networksets.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3164,8 +3419,6 @@ status: plural: "" conditions: [] storedVersions: [] - ---- --- # Source: calico/templates/calico-kube-controllers-rbac.yaml @@ -3184,12 +3437,14 @@ rules: - watch - list - get - # Pods are queried to check for existence. + # Pods are watched to check for existence as part of IPAM controller. - apiGroups: [""] resources: - pods verbs: - get + - list + - watch # IPAM resources are manipulated when nodes are deleted. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3207,6 +3462,7 @@ rules: - create - update - delete + - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3252,8 +3508,6 @@ subjects: name: calico-kube-controllers namespace: kube-system --- - ---- # Source: calico/templates/calico-node-rbac.yaml # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. @@ -3270,6 +3524,14 @@ rules: - namespaces verbs: - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list - apiGroups: [""] resources: - endpoints @@ -3393,7 +3655,6 @@ rules: - daemonsets verbs: - get - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -3407,7 +3668,6 @@ subjects: - kind: ServiceAccount name: calico-node namespace: kube-system - --- # Source: calico/templates/calico-node.yaml # This manifest installs the calico-node container, as well @@ -3455,8 +3715,13 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: {{ image_registry_address }}/calico/cni:v3.15.0 + image: {{ image_registry_address }}/calico/cni:v3.20.2 command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: - name: KUBERNETES_NODE_NAME valueFrom: @@ -3477,8 +3742,13 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: {{ image_registry_address }}/calico/cni:v3.15.0 - command: ["/install-cni.sh"] + image: {{ image_registry_address }}/calico/cni:v3.20.2 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # Name of the CNI config file to create. - name: CNI_CONF_NAME @@ -3513,7 +3783,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: {{ image_registry_address }}/calico/pod2daemon-flexvol:v3.15.0 + image: {{ image_registry_address }}/calico/pod2daemon-flexvol:v3.20.2 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -3524,7 +3794,12 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: {{ image_registry_address }}/calico/node:v3.15.0 + image: {{ image_registry_address }}/calico/node:v3.20.2 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -3587,9 +3862,6 @@ spec: # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - name: FELIX_HEALTHENABLED value: "true" securityContext: @@ -3606,6 +3878,7 @@ spec: periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 + timeoutSeconds: 10 readinessProbe: exec: command: @@ -3613,7 +3886,12 @@ spec: - -felix-ready - -bird-ready periodSeconds: 10 + timeoutSeconds: 10 volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false - mountPath: /lib/modules name: lib-modules readOnly: true @@ -3628,6 +3906,16 @@ spec: readOnly: false - name: policysync mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true volumes: # Used by calico-node. - name: lib-modules @@ -3643,6 +3931,10 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate # Used to install CNI. - name: cni-bin-dir hostPath: @@ -3650,6 +3942,10 @@ spec: - name: cni-net-dir hostPath: path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni # Mount in the directory for host-local IPAM allocations. This is # used when upgrading from host-local to calico-ipam, and can be removed # if not using the upgrade-ipam init container. @@ -3667,13 +3963,11 @@ spec: type: DirectoryOrCreate path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds --- - apiVersion: v1 kind: ServiceAccount metadata: name: calico-node namespace: kube-system - --- # Source: calico/templates/calico-kube-controllers.yaml # See https://github.com/projectcalico/kube-controllers @@ -3711,27 +4005,48 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: {{ image_registry_address }}/calico/kube-controllers:v3.15.0 + image: {{ image_registry_address }}/calico/kube-controllers:v3.20.2 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS value: node - name: DATASTORE_TYPE value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 readinessProbe: exec: command: - /usr/bin/check-status - -r - + periodSeconds: 10 --- - apiVersion: v1 kind: ServiceAccount metadata: name: calico-kube-controllers namespace: kube-system - +--- +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers --- # Source: calico/templates/calico-etcd-secrets.yaml @@ -3740,4 +4055,3 @@ metadata: --- # Source: calico/templates/configure-canal.yaml - diff --git a/ansible/playbooks/roles/kubernetes_master/templates/canal.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/canal.yml.j2 index e504ffb801..71f5716a89 100644 --- a/ansible/playbooks/roles/kubernetes_master/templates/canal.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_master/templates/canal.yml.j2 @@ -13,14 +13,13 @@ data: # If left blank, then the interface is chosen using the node's # default route. canal_iface: "" - # Whether or not to masquerade traffic to destinations not within # the pod network. masquerade: "true" - - # Configure the MTU to use - veth_mtu: "1450" - + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- @@ -31,6 +30,7 @@ data: { "type": "calico", "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, @@ -56,7 +56,6 @@ data: } ] } - # Flannel network configuration. Mounted into the flannel container. net-conf.json: | { @@ -65,18 +64,11 @@ data: "Type": "vxlan" } } - --- # Source: calico/templates/kdd-crds.yaml - - ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: bgpconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -112,6 +104,32 @@ spec: 64512]' format: int32 type: integer + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' @@ -120,13 +138,36 @@ spec: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array serviceClusterIPs: description: ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. If specified, Calico will advertise these blocks, as well as any cluster IPs within them. items: - description: ServiceClusterIPBlock represents a single whitelisted - CIDR block for ClusterIPs. + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. properties: cidr: type: string @@ -137,8 +178,20 @@ spec: Service External IPs. Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks. items: - description: ServiceExternalIPBlock represents a single whitelisted - CIDR External IP block. + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. properties: cidr: type: string @@ -154,16 +207,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: bgppeers.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -197,17 +244,55 @@ spec: description: The AS Number of the peer. format: int32 type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, this + is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string node: description: The node name identifying the Calico node instance that - is peering with this peer. If this is not set, this represents a - global peer, i.e. a peer that peers with every node in the deployment. + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. type: string nodeSelector: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object peerIP: - description: The IP address of the peer. + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. type: string peerSelector: description: Selector for the remote nodes to peer with. When this @@ -215,12 +300,15 @@ spec: peering between the local node and selected remote nodes, we configure an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node’s NodeBGPSpec.ASNumber, + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, or the global default if that is not set. type: string - required: - - asNumber - - peerIP + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string type: object type: object served: true @@ -231,16 +319,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: blockaffinities.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -297,16 +379,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: clusterinformations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -366,16 +442,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: felixconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -406,6 +476,25 @@ spec: spec: description: FelixConfigurationSpec contains the values of the Felix configuration. properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enabled" or "Disabled". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string bpfConnectTimeLoadBalancingEnabled: description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The @@ -415,13 +504,13 @@ spec: true]' type: boolean bpfDataIfacePattern: - description: 'BPFDataIfacePattern is a regular expression that controls + description: BPFDataIfacePattern is a regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually - named cali...). [Default: ^(en.*|eth.*|tunl0$)]' + named cali...). type: string bpfDisableUnprivileged: description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled @@ -443,6 +532,14 @@ spec: node appears to use the IP of the ingress node; this requires a permissive L2 network. [Default: Tunnel]' type: string + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, controls a + 32bit mark that is set on connections from an external client to + a local service. This mark allows us to control how packets of + that connection are routed within the host and how is routing + intepreted by RPF check. [Default: 0]' + type: integer + bpfKubeProxyEndpointSlicesEnabled: description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls whether Felix's embedded kube-proxy accepts EndpointSlices or not. @@ -466,10 +563,10 @@ spec: `tc exec bpf debug`. [Default: Off].' type: string chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel’s + description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico’s rules from being bypassed. If you switch + since it prevents Calico''s rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' @@ -489,11 +586,11 @@ spec: traffic that goes from a workload endpoint to the host itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload endpoints to the host itself with an - iptables “DROP” action. If you want to allow some or all traffic + iptables "DROP" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables “INPUT” chain; - Calico will insert its rules at the top of that chain, then “RETURN” - packets to the “INPUT” chain once it has completed processing workload + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]' @@ -521,19 +618,21 @@ spec: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a comma-delimited list of - UDP/TCP ports that Felix will allow incoming traffic to host endpoints + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. Each - port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all inbound host ports, use the value none. - The default value allows ssh access and DHCP. [Default: tcp:22, + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' items: - description: ProtoPort is combination of protocol and port, both - must be specified. + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. properties: + net: + type: string port: type: integer protocol: @@ -544,21 +643,23 @@ spec: type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a comma-delimited list - of UDP/TCP ports that Felix will allow outgoing traffic from host - endpoints to irrespective of the security policy. This is useful - to avoid accidentally cutting off a host with incorrect configuration. - Each port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all outbound host ports, use the value none. - The default value opens etcd’s standard ports to ensure that Felix - does not get cut off from etcd as well as allowing DHCP and DNS. - [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, - udp:53, udp:67]' + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' items: - description: ProtoPort is combination of protocol and port, both - must be specified. + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. properties: + net: + type: string port: type: integer protocol: @@ -568,6 +669,13 @@ spec: - protocol type: object type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -596,8 +704,13 @@ spec: workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker - integrations set the ‘cali’ value, and our OpenStack integration - sets the ‘tap’ value. [Default: cali]' + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. type: string ipipEnabled: type: boolean @@ -608,7 +721,7 @@ spec: ipsetsRefreshInterval: description: 'IpsetsRefreshInterval is the period at which Felix re-checks all iptables state to ensure that no other process has accidentally - broken Calico’s rules. Set to 0 to disable iptables refresh. [Default: + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: 90s]' type: string iptablesBackend: @@ -620,7 +733,7 @@ spec: iptablesLockFilePath: description: 'IptablesLockFilePath is the location of the iptables lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix’s + its standard location (for example if you have mapped it into Felix''s container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: @@ -652,16 +765,16 @@ spec: description: 'IptablesPostWriteCheckInterval is the period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn’t + This should only occur if another application on the system doesn''t respect the iptables lock. [Default: 1s]' type: string iptablesRefreshInterval: description: 'IptablesRefreshInterval is the period at which Felix re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico’s rules. Set to 0 to disable IP sets - refresh. Note: the default for this value is lower than the other - refresh intervals as a workaround for a Linux kernel bug that was - fixed in kernel version 4.11. If you are using v4.11 or greater + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage. [Default: 10s]' type: string @@ -712,10 +825,15 @@ spec: type: string metadataPort: description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ‘None’), is used to set - up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed [Default: 8775].' type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string natOutgoingAddress: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the @@ -786,9 +904,9 @@ spec: status reports. [Default: 90s]' type: string routeRefreshInterval: - description: 'RouterefreshInterval is the period at which Felix re-checks + description: 'RouteRefreshInterval is the period at which Felix re-checks the routes in the dataplane to ensure that no other process has - accidentally broken Calico’s rules. Set to 0 to disable route refresh. + accidentally broken Calico''s rules. Set to 0 to disable route refresh. [Default: 90s]' type: string routeSource: @@ -809,6 +927,13 @@ spec: - max - min type: object + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string sidecarAccelerationEnabled: description: 'SidecarAccelerationEnabled enables experimental sidecar acceleration [Default: false]' @@ -870,8 +995,6 @@ spec: Calico''s BPF maps or attached programs. Set to 0 to disable XDP refresh. [Default: 90s]' type: string - required: - - bpfLogLevel type: object type: object served: true @@ -882,16 +1005,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: globalnetworkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -939,7 +1056,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -955,16 +1072,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1022,9 +1140,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1052,6 +1170,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -1094,7 +1232,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1123,7 +1261,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1160,16 +1298,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1227,9 +1366,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1257,6 +1396,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -1270,7 +1429,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -1286,16 +1445,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1353,9 +1513,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1383,6 +1543,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -1425,7 +1605,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1454,7 +1634,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1491,16 +1671,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1558,9 +1739,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1588,6 +1769,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -1658,16 +1859,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: globalnetworksets.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1716,16 +1911,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: hostendpoints.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1781,7 +1970,7 @@ spec: is empty - through the specific interface that has one of the IPs in ExpectedIPs. Therefore, when InterfaceName is empty, at least one expected IP must be specified. Only external interfaces (such - as “eth0”) are supported here; it isn't possible for a HostEndpoint + as \"eth0\") are supported here; it isn't possible for a HostEndpoint to protect traffic through a specific local workload interface. \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; initially just pre-DNAT policy. Please check Calico documentation @@ -1829,16 +2018,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamblocks.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1903,7 +2086,6 @@ spec: - allocations - attributes - cidr - - deleted - strictAffinity - unallocated type: object @@ -1916,16 +2098,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamconfigs.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1958,6 +2134,10 @@ spec: properties: autoAllocateBlocks: type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + type: integer strictAffinity: type: boolean required: @@ -1973,16 +2153,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamhandles.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2017,6 +2191,8 @@ spec: additionalProperties: type: integer type: object + deleted: + type: boolean handleID: type: string required: @@ -2032,16 +2208,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ippools.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2102,7 +2272,7 @@ spec: type: object ipipMode: description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunelling + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling is disabled). type: string nat-outgoing: @@ -2122,7 +2292,7 @@ spec: vxlanMode: description: Contains configuration for VXLAN tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunelling is disabled). + tunneling is disabled). type: string required: - cidr @@ -2136,16 +2306,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: kubecontrollersconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2202,6 +2366,11 @@ spec: host endpoints for every node. [Default: Disabled]' type: string type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' @@ -2251,6 +2420,10 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer required: - controllers type: object @@ -2298,6 +2471,12 @@ spec: of host endpoints for every node. [Default: Disabled]' type: string type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: @@ -2351,6 +2530,11 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer required: - controllers type: object @@ -2364,16 +2548,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: networkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2410,7 +2588,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -2426,16 +2604,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2493,9 +2672,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2523,6 +2702,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -2565,7 +2764,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2594,7 +2793,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2631,16 +2830,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2698,9 +2898,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2728,6 +2928,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -2741,7 +2961,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -2757,16 +2977,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2824,9 +3045,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2854,6 +3075,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -2896,7 +3137,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2925,7 +3166,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2962,16 +3203,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -3029,9 +3271,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -3059,6 +3301,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -3121,16 +3383,10 @@ status: plural: "" conditions: [] storedVersions: [] - ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: networksets.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3177,8 +3433,6 @@ status: plural: "" conditions: [] storedVersions: [] - ---- --- # Source: calico/templates/calico-kube-controllers-rbac.yaml @@ -3197,12 +3451,14 @@ rules: - watch - list - get - # Pods are queried to check for existence. + # Pods are watched to check for existence as part of IPAM controller. - apiGroups: [""] resources: - pods verbs: - get + - list + - watch # IPAM resources are manipulated when nodes are deleted. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3220,6 +3476,7 @@ rules: - create - update - delete + - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3265,8 +3522,6 @@ subjects: name: calico-kube-controllers namespace: kube-system --- - ---- # Source: calico/templates/calico-node-rbac.yaml # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. @@ -3283,6 +3538,14 @@ rules: - namespaces verbs: - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list - apiGroups: [""] resources: - endpoints @@ -3339,6 +3602,7 @@ rules: - bgpconfigurations - ippools - ipamblocks + - ipamhandles - globalnetworkpolicies - globalnetworksets - networkpolicies @@ -3376,7 +3640,6 @@ rules: verbs: - create - update - --- # Flannel ClusterRole # Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml @@ -3428,7 +3691,6 @@ subjects: - kind: ServiceAccount name: canal namespace: kube-system - --- # Source: calico/templates/calico-node.yaml # This manifest installs the canal container, as well @@ -3486,8 +3748,13 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: {{ image_registry_address }}/calico/cni:v3.15.0 - command: ["/install-cni.sh"] + image: {{ image_registry_address }}/calico/cni:v3.20.2 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # Name of the CNI config file to create. - name: CNI_CONF_NAME @@ -3522,7 +3789,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: {{ image_registry_address }}/calico/pod2daemon-flexvol:v3.15.0 + image: {{ image_registry_address }}/calico/pod2daemon-flexvol:v3.20.2 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -3533,14 +3800,16 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: {{ image_registry_address }}/calico/node:v3.15.0 + image: {{ image_registry_address }}/calico/node:v3.20.2 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" - # Configure route aggregation based on pod CIDR. - - name: USE_POD_CIDR - value: "true" # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" @@ -3566,12 +3835,6 @@ spec: # no effect. This should fall within `--cluster-cidr`. - name: CALICO_IPV4POOL_CIDR value: "10.244.0.0/16" - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: canal-config - key: veth_mtu # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -3581,9 +3844,6 @@ spec: # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - name: FELIX_HEALTHENABLED value: "true" securityContext: @@ -3599,13 +3859,19 @@ spec: periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 + timeoutSeconds: 10 readinessProbe: httpGet: path: /readiness port: 9099 host: localhost periodSeconds: 10 + timeoutSeconds: 10 volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false - mountPath: /lib/modules name: lib-modules readOnly: true @@ -3620,10 +3886,20 @@ spec: readOnly: false - name: policysync mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true # This container runs flannel using the kube-subnet-mgr backend # for allocating subnets. - name: kube-flannel - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-{{ canal_arch }} + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-{{ canal_arch }} command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] securityContext: privileged: true @@ -3667,6 +3943,10 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate # Used by flannel. - name: flannel-cfg configMap: @@ -3678,6 +3958,10 @@ spec: - name: cni-net-dir hostPath: path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni # Used to create per-pod Unix Domain Sockets - name: policysync hostPath: @@ -3688,14 +3972,12 @@ spec: hostPath: type: DirectoryOrCreate path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - --- apiVersion: v1 kind: ServiceAccount metadata: name: canal namespace: kube-system - --- # Source: calico/templates/calico-kube-controllers.yaml # See https://github.com/projectcalico/kube-controllers @@ -3733,27 +4015,48 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: {{ image_registry_address }}/calico/kube-controllers:v3.15.0 + image: {{ image_registry_address }}/calico/kube-controllers:v3.20.2 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS value: node - name: DATASTORE_TYPE value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 readinessProbe: exec: command: - /usr/bin/check-status - -r - + periodSeconds: 10 --- - apiVersion: v1 kind: ServiceAccount metadata: name: calico-kube-controllers namespace: kube-system - +--- +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers --- # Source: calico/templates/calico-etcd-secrets.yaml @@ -3762,4 +4065,3 @@ metadata: --- # Source: calico/templates/configure-canal.yaml - diff --git a/ansible/playbooks/roles/kubernetes_master/templates/kube-flannel.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/kube-flannel.yml.j2 index db32c6668c..d4cbf21d2d 100644 --- a/ansible/playbooks/roles/kubernetes_master/templates/kube-flannel.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_master/templates/kube-flannel.yml.j2 @@ -1,3 +1,6 @@ +# Modified according to: +# * https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#-strong-api-groups-strong- +# * https://raw.githubusercontent.com/flannel-io/flannel/v0.14.0/Documentation/kube-flannel.yml --- apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -31,7 +34,7 @@ spec: allowPrivilegeEscalation: false defaultAllowPrivilegeEscalation: false # Capabilities - allowedCapabilities: ['NET_ADMIN'] + allowedCapabilities: ['NET_ADMIN', 'NET_RAW'] defaultAddCapabilities: [] requiredDropCapabilities: [] # Host namespaces @@ -47,7 +50,7 @@ spec: rule: 'RunAsAny' --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel rules: @@ -76,7 +79,7 @@ rules: - patch --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel roleRef: @@ -165,13 +168,14 @@ spec: values: - amd64 hostNetwork: true + priorityClassName: system-node-critical tolerations: - operator: Exists effect: NoSchedule serviceAccountName: flannel initContainers: - name: install-cni - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-amd64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-amd64 command: - cp args: @@ -185,7 +189,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-amd64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-amd64 command: - /opt/bin/flanneld args: @@ -201,7 +205,7 @@ spec: securityContext: privileged: false capabilities: - add: ["NET_ADMIN"] + add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: @@ -259,13 +263,14 @@ spec: values: - arm64 hostNetwork: true + priorityClassName: system-node-critical tolerations: - operator: Exists effect: NoSchedule serviceAccountName: flannel initContainers: - name: install-cni - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-arm64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-arm64 command: - cp args: @@ -279,7 +284,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-arm64 + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-arm64 command: - /opt/bin/flanneld args: @@ -295,7 +300,7 @@ spec: securityContext: privileged: false capabilities: - add: ["NET_ADMIN"] + add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: @@ -353,13 +358,14 @@ spec: values: - arm hostNetwork: true + priorityClassName: system-node-critical tolerations: - operator: Exists effect: NoSchedule serviceAccountName: flannel initContainers: - name: install-cni - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-arm + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-arm command: - cp args: @@ -373,7 +379,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-arm + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-arm command: - /opt/bin/flanneld args: @@ -389,7 +395,7 @@ spec: securityContext: privileged: false capabilities: - add: ["NET_ADMIN"] + add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: @@ -447,13 +453,14 @@ spec: values: - ppc64le hostNetwork: true + priorityClassName: system-node-critical tolerations: - operator: Exists effect: NoSchedule serviceAccountName: flannel initContainers: - name: install-cni - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-ppc64le + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-ppc64le command: - cp args: @@ -467,7 +474,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-ppc64le + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-ppc64le command: - /opt/bin/flanneld args: @@ -483,7 +490,7 @@ spec: securityContext: privileged: false capabilities: - add: ["NET_ADMIN"] + add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: @@ -541,13 +548,14 @@ spec: values: - s390x hostNetwork: true + priorityClassName: system-node-critical tolerations: - operator: Exists effect: NoSchedule serviceAccountName: flannel initContainers: - name: install-cni - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-s390x + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-s390x command: - cp args: @@ -561,7 +569,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.12.0-s390x + image: {{ image_registry_address }}/quay.io/coreos/flannel:v0.14.0-s390x command: - /opt/bin/flanneld args: @@ -577,7 +585,7 @@ spec: securityContext: privileged: false capabilities: - add: ["NET_ADMIN"] + add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: diff --git a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 index 606325f7e5..2962cb6163 100644 --- a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 @@ -45,3 +45,8 @@ networking: imageRepository: {{ image_registry_address }}/{{ specification.advanced.imageRepository }} certificatesDir: {{ specification.advanced.certificates.location }} +--- +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +cgroupDriver: systemd +rotateCertificates: true diff --git a/ansible/playbooks/roles/kubernetes_master/templates/kubernetes-dashboard.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/kubernetes-dashboard.yml.j2 index 3b34d17661..1f1568f93e 100644 --- a/ansible/playbooks/roles/kubernetes_master/templates/kubernetes-dashboard.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_master/templates/kubernetes-dashboard.yml.j2 @@ -187,7 +187,7 @@ spec: spec: containers: - name: kubernetes-dashboard - image: {{ image_registry_address }}/kubernetesui/dashboard:v2.0.3 + image: {{ image_registry_address }}/kubernetesui/dashboard:v2.3.1 imagePullPolicy: IfNotPresent ports: - containerPort: 8443 @@ -271,7 +271,7 @@ spec: spec: containers: - name: dashboard-metrics-scraper - image: {{ image_registry_address }}/kubernetesui/metrics-scraper:v1.0.4 + image: {{ image_registry_address }}/kubernetesui/metrics-scraper:v1.0.7 ports: - containerPort: 8000 protocol: TCP diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index 140ce7e299..8f1255454f 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -120,15 +120,20 @@ yum-utils # to make remote-to-remote "synchronize" work in ansible rsync -# K8s upgrade v1.18.6 (Epiphany >= v0.7.1) +# K8s v1.18.6 (Epiphany >= v0.7.1) kubeadm-1.18.6 kubectl-1.18.6 kubelet-1.18.6 +# K8s v1.19.15 (Epiphany >= v1.3, transitional version) +kubeadm-1.19.15 +kubectl-1.19.15 +kubelet-1.19.15 + # Kubernetes Generic -kubernetes-cni-0.7.5-0 -# kubernetes-cni-0.8.6-0 since K8s v1.18.6 -kubernetes-cni-0.8.6-0 +kubernetes-cni-0.8.6-0 # since K8s v1.18.6 +# https://github.com/kubernetes/kubernetes/blob/v1.19.15/build/dependencies.yaml +kubernetes-cni-0.8.7-0 # since K8s v1.19.15 [files] # --- Packages --- @@ -184,8 +189,8 @@ https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard [images] haproxy:2.2.2-alpine -kubernetesui/dashboard:v2.0.3 -kubernetesui/metrics-scraper:v1.0.4 +kubernetesui/dashboard:v2.3.1 +kubernetesui/metrics-scraper:v1.0.7 registry:2 hashicorp/vault-k8s:0.10.0 vault:1.7.0 @@ -208,3 +213,17 @@ calico/cni:v3.15.0 calico/kube-controllers:v3.15.0 calico/node:v3.15.0 calico/pod2daemon-flexvol:v3.15.0 +## v1.19.15 +k8s.gcr.io/kube-apiserver:v1.19.15 +k8s.gcr.io/kube-controller-manager:v1.19.15 +k8s.gcr.io/kube-scheduler:v1.19.15 +k8s.gcr.io/kube-proxy:v1.19.15 +k8s.gcr.io/coredns:1.7.0 +k8s.gcr.io/etcd:3.4.13-0 +k8s.gcr.io/pause:3.2 +quay.io/coreos/flannel:v0.14.0-arm64 +quay.io/coreos/flannel:v0.14.0 +calico/cni:v3.20.2 +calico/kube-controllers:v3.20.2 +calico/node:v3.20.2 +calico/pod2daemon-flexvol:v3.20.2 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 0601295053..340ab81e6e 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -123,14 +123,20 @@ yum-utils # to make remote-to-remote "synchronize" work in ansible rsync -# K8s upgrade v1.18.6 (Epiphany >= v0.7.1) +# K8s v1.18.6 (Epiphany >= v0.7.1) kubeadm-1.18.6 kubectl-1.18.6 kubelet-1.18.6 +# K8s v1.19.15 (Epiphany >= v1.3, transitional version) +kubeadm-1.19.15 +kubectl-1.19.15 +kubelet-1.19.15 + # Kubernetes Generic -# kubernetes-cni-0.8.6-0 since K8s v1.18.6 -kubernetes-cni-0.8.6-0 +kubernetes-cni-0.8.6-0 # since K8s v1.18.6 +# https://github.com/kubernetes/kubernetes/blob/v1.19.15/build/dependencies.yaml +kubernetes-cni-0.8.7-0 # since K8s v1.19.15 [files] # --- Packages --- @@ -186,8 +192,8 @@ https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard [images] haproxy:2.2.2-alpine -kubernetesui/dashboard:v2.0.3 -kubernetesui/metrics-scraper:v1.0.4 +kubernetesui/dashboard:v2.3.1 +kubernetesui/metrics-scraper:v1.0.7 registry:2 hashicorp/vault-k8s:0.10.0 vault:1.7.0 @@ -201,7 +207,7 @@ istio/proxyv2:1.8.1 istio/operator:1.8.1 epiphanyplatform/keycloak:14.0.0 rabbitmq:3.8.9 -# K8s upgrade +# K8s ## v1.18.6 k8s.gcr.io/kube-apiserver:v1.18.6 k8s.gcr.io/kube-controller-manager:v1.18.6 @@ -216,3 +222,17 @@ calico/cni:v3.15.0 calico/kube-controllers:v3.15.0 calico/node:v3.15.0 calico/pod2daemon-flexvol:v3.15.0 +## v1.19.15 +k8s.gcr.io/kube-apiserver:v1.19.15 +k8s.gcr.io/kube-controller-manager:v1.19.15 +k8s.gcr.io/kube-scheduler:v1.19.15 +k8s.gcr.io/kube-proxy:v1.19.15 +k8s.gcr.io/coredns:1.7.0 +k8s.gcr.io/etcd:3.4.13-0 +k8s.gcr.io/pause:3.2 +quay.io/coreos/flannel:v0.14.0-amd64 +quay.io/coreos/flannel:v0.14.0 +calico/cni:v3.20.2 +calico/kube-controllers:v3.20.2 +calico/node:v3.20.2 +calico/pod2daemon-flexvol:v3.20.2 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index c0e5bc850d..3eb4a41821 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -119,15 +119,20 @@ yum-utils # to make remote-to-remote "synchronize" work in ansible rsync -# K8s upgrade v1.18.6 (Epiphany >= v0.7.1) +# K8s v1.18.6 (Epiphany >= v0.7.1) kubeadm-1.18.6 kubectl-1.18.6 kubelet-1.18.6 +# K8s v1.19.15 (Epiphany >= v1.3, transitional version) +kubeadm-1.19.15 +kubectl-1.19.15 +kubelet-1.19.15 + # Kubernetes Generic -kubernetes-cni-0.7.5-0 -# kubernetes-cni-0.8.6-0 since K8s v1.18.6 -kubernetes-cni-0.8.6-0 +kubernetes-cni-0.8.6-0 # since K8s v1.18.6 +# https://github.com/kubernetes/kubernetes/blob/v1.19.15/build/dependencies.yaml +kubernetes-cni-0.8.7-0 # since K8s v1.19.15 [files] # --- Packages --- @@ -183,8 +188,8 @@ https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard [images] haproxy:2.2.2-alpine -kubernetesui/dashboard:v2.0.3 -kubernetesui/metrics-scraper:v1.0.4 +kubernetesui/dashboard:v2.3.1 +kubernetesui/metrics-scraper:v1.0.7 registry:2 hashicorp/vault-k8s:0.10.0 vault:1.7.0 @@ -198,7 +203,7 @@ istio/proxyv2:1.8.1 istio/operator:1.8.1 epiphanyplatform/keycloak:14.0.0 rabbitmq:3.8.9 -# K8s upgrade +# K8s ## v1.18.6 k8s.gcr.io/kube-apiserver:v1.18.6 k8s.gcr.io/kube-controller-manager:v1.18.6 @@ -213,3 +218,17 @@ calico/cni:v3.15.0 calico/kube-controllers:v3.15.0 calico/node:v3.15.0 calico/pod2daemon-flexvol:v3.15.0 +## v1.19.15 +k8s.gcr.io/kube-apiserver:v1.19.15 +k8s.gcr.io/kube-controller-manager:v1.19.15 +k8s.gcr.io/kube-scheduler:v1.19.15 +k8s.gcr.io/kube-proxy:v1.19.15 +k8s.gcr.io/coredns:1.7.0 +k8s.gcr.io/etcd:3.4.13-0 +k8s.gcr.io/pause:3.2 +quay.io/coreos/flannel:v0.14.0-amd64 +quay.io/coreos/flannel:v0.14.0 +calico/cni:v3.20.2 +calico/kube-controllers:v3.20.2 +calico/node:v3.20.2 +calico/pod2daemon-flexvol:v3.20.2 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt index 71d3c1b84d..9ff9bc1318 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt @@ -179,15 +179,21 @@ python-idna python-ipaddress python-six -# K8s upgrade v1.18.6 (Epiphany >= v0.7.1) +# K8s v1.18.6 (Epiphany >= v0.7.1) kubeadm 1.18.6 kubectl 1.18.6 kubelet 1.18.6 +# K8s v1.19.15 (Epiphany >= v1.3, transitional version) +kubeadm 1.19.15 +kubectl 1.19.15 +kubelet 1.19.15 + # Kubernetes Generic -kubernetes-cni 0.7.5-00 -# kubernetes-cni-0.8.6-0 since K8s v1.18.6 +# kubernetes-cni-0.8.6 since K8s v1.18.6 kubernetes-cni 0.8.6-00 +# kubernetes-cni-0.8.7 since K8s v1.19.15 +kubernetes-cni 0.8.7-00 [files] # --- Packages --- @@ -243,8 +249,8 @@ https://grafana.com/api/dashboards/10991/revisions/11/download grafana_dashboard [images] haproxy:2.2.2-alpine -kubernetesui/dashboard:v2.0.3 -kubernetesui/metrics-scraper:v1.0.4 +kubernetesui/dashboard:v2.3.1 +kubernetesui/metrics-scraper:v1.0.7 registry:2 hashicorp/vault-k8s:0.10.0 vault:1.7.0 @@ -258,7 +264,7 @@ istio/proxyv2:1.8.1 istio/operator:1.8.1 epiphanyplatform/keycloak:14.0.0 rabbitmq:3.8.9 -# K8s upgrade +# K8s ## v1.18.6 k8s.gcr.io/kube-apiserver:v1.18.6 k8s.gcr.io/kube-controller-manager:v1.18.6 @@ -273,3 +279,17 @@ calico/cni:v3.15.0 calico/kube-controllers:v3.15.0 calico/node:v3.15.0 calico/pod2daemon-flexvol:v3.15.0 +## v1.19.15 +k8s.gcr.io/kube-apiserver:v1.19.15 +k8s.gcr.io/kube-controller-manager:v1.19.15 +k8s.gcr.io/kube-scheduler:v1.19.15 +k8s.gcr.io/kube-proxy:v1.19.15 +k8s.gcr.io/coredns:1.7.0 +k8s.gcr.io/etcd:3.4.13-0 +k8s.gcr.io/pause:3.2 +quay.io/coreos/flannel:v0.14.0-amd64 +quay.io/coreos/flannel:v0.14.0 +calico/cni:v3.20.2 +calico/kube-controllers:v3.20.2 +calico/node:v3.20.2 +calico/pod2daemon-flexvol:v3.20.2 diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-kubeadm.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-kubeadm.yml index cbe6f7e7ff..8678c66406 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-kubeadm.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-kubeadm.yml @@ -6,25 +6,19 @@ - name: Install kubeadm package vars: - install_cni_as_new_package: >- - {{ (ansible_facts.packages['kubernetes-cni'] is undefined - and (cni_in_kubelet is undefined or not cni_in_kubelet)) }} - # unhold kubelet to avoid errors: # The following packages have unmet dependencies: kubelet : Conflicts: kubernetes-cni # The following packages have unmet dependencies: kubeadm : Depends: kubernetes-cni (>= 0.8.6) - packages_to_unhold: >- - {{ ['kubeadm', 'kubelet'] if (install_cni_as_new_package) else - ['kubeadm'] }} + packages_to_unhold: + - kubeadm + - kubelet _packages: kubeadm: "{{ version }}-00" kubelet: "{{ version }}-00" kubernetes-cni: "{{ cni_version }}-00" - packages_to_install: >- - {{ _packages if (install_cni_as_new_package) - else { 'kubeadm': _packages.kubeadm } }} + packages_to_install: "{{ _packages }}" packages_to_hold: "{{ packages_to_unhold | union(packages_to_install.keys() | list) }}" diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages-cni-in-kubelet.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages-cni-in-kubelet.yml deleted file mode 100644 index c5b072beeb..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/Debian/install-packages-cni-in-kubelet.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- name: k8s/install | Get information about installed packages as facts - package_facts: - manager: auto - changed_when: false - -# Unhold before changing to avoid error -- name: k8s/install | Include unhold packages task - include_tasks: unhold-packages.yml - vars: - packages: >- - {%- if ansible_facts.packages['kubernetes-cni'] is defined -%} - ['kubelet', 'kubectl', 'kubernetes-cni'] - {%- else -%} - ['kubelet', 'kubectl'] - {%- endif -%} - -- name: k8s/install | Remove newer Debian packages installed as dependencies if they exist # as there is no allow_downgrade parameter in ansible apt module - apt: - name: - - kubelet - - kubectl - state: absent - when: ansible_facts.packages['kubelet'][0].version is version(version + '-00', '>') - or ansible_facts.packages['kubectl'][0].version is version(version + '-00', '>') - -- name: k8s/install | Install kubelet {{ version }} and kubectl {{ version }} packages for Debian family - apt: - name: - - kubelet={{ version }}-00 - - kubectl={{ version }}-00 - update_cache: yes - state: present - -- name: k8s/install | Include hold packages task - include_tasks: hold-packages.yml - vars: - packages: [kubelet, kubectl] diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/RedHat/install-packages-cni-in-kubelet.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/RedHat/install-packages-cni-in-kubelet.yml deleted file mode 100644 index 436a62fe9d..0000000000 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/RedHat/install-packages-cni-in-kubelet.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: k8s/install | Install kubelet-{{ version }} and kubectl-{{ version }} for RedHat family - yum: - name: - - kubelet-{{ version }}-0 - - kubectl-{{ version }}-0 - update_cache: true - allow_downgrade: true - disable_excludes: kubernetes - state: present - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/backup-kubeadm-config.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/backup-kubeadm-config.yml index 5ef9bf6d1b..ad88bcd302 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/backup-kubeadm-config.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/backup-kubeadm-config.yml @@ -1,6 +1,15 @@ --- # The kubeadm-config.yml file is no longer used during upgrade process, # but we keep it for backup and reference purposes. -- name: k8s/master | Save kubeadm-config ConfigMap to file +- name: k8s/master | Collect kubeadm-config ConfigMap command: >- - kubeadm config view > /etc/kubeadm/kubeadm-config.yml + kubectl get cm kubeadm-config -o yaml -n kube-system + register: kubeadm_config + +- name: k8s/master | Save kubeadm-config ConfigMap to file + copy: + content: "{{ kubeadm_config.stdout | to_nice_yaml(indent=2) }}" + dest: /etc/kubeadm/kubeadm-config.yml + owner: root + group: root + mode: u=rw diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml index 646d915771..0a6de7b1de 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/get-cluster-version.yml @@ -1,8 +1,8 @@ --- - name: Get cluster version command: kubectl version --output yaml - register: cluster_version - until: cluster_version is success + register: kubectl_cluster_version + until: kubectl_cluster_version is success retries: 60 delay: 5 changed_when: false @@ -10,4 +10,8 @@ - name: Set cluster version as fact set_fact: cluster_version: >- - {{ (cluster_version.stdout | from_yaml).serverVersion.gitVersion }} + {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.gitVersion }} + cluster_version_major: >- + {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.major }} + cluster_version_minor: >- + {{ (kubectl_cluster_version.stdout | from_yaml).serverVersion.minor }} diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-cgroup-driver.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-cgroup-driver.yml index f43b7ba6e8..1a86d895d3 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-cgroup-driver.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-cgroup-driver.yml @@ -1,28 +1,29 @@ --- -# A standalone version of this procedure can be found in tools/development/k8s/memory/patch_cgroup_driver/. -# It has been reported that Epiphany behaves unstable with high resource utilization, this patch seems to be fixing these problems. - # K8s documentation (https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-drivers) states: # > A single cgroup manager simplifies the view of what resources are being allocated and will by default have a more consistent view of the available and in-use resources. # > When there are two cgroup managers on a system, you end up with two views of those resources. # > In the field, people have reported cases where nodes that are configured to use cgroupfs for the kubelet and Docker, # > but systemd for the rest of the processes, become unstable under resource pressure. -- name: k8s/cgroups | Read /var/lib/kubelet/kubeadm-flags.env +# This procedure is based on https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/#migrating-to-the-systemd-driver + +- name: k8s/cgroups | Read /var/lib/kubelet/config.yaml slurp: - path: /var/lib/kubelet/kubeadm-flags.env - register: slurp_var_lib_kubelet_kubeadm_flags_env + path: /var/lib/kubelet/config.yaml + register: slurp_var_lib_kubelet_config_yaml -- name: k8s/cgroups | Process /var/lib/kubelet/kubeadm-flags.env +- name: k8s/cgroups | Process /var/lib/kubelet/config.yaml set_fact: - var_lib_kubelet_kubeadm_flags_env: - output: "{{ _output }}" - changed: "{{ _output != _input }}" + var_lib_kubelet_config_yaml: + output: "{{ _output_str }}" + changed: "{{ _output_str != (_input | to_nice_yaml(indent=2)) }}" vars: _input: >- - {{ slurp_var_lib_kubelet_kubeadm_flags_env.content | b64decode }} - _output: >- - {{ _input.replace('--cgroup-driver=cgroupfs', '--cgroup-driver=systemd') }} + {{ slurp_var_lib_kubelet_config_yaml.content | b64decode | from_yaml }} + _cgroup_driver: + cgroupDriver: systemd + _output_str: >- + {{ _input | combine(_cgroup_driver) | to_nice_yaml(indent=2) }} - name: k8s/cgroups | Read /etc/docker/daemon.json slurp: @@ -49,15 +50,15 @@ {{ _input | combine(_update, recursive=true) }} - name: k8s/cgroups | Perform cgroup driver patching (switch to systemd) - when: var_lib_kubelet_kubeadm_flags_env.changed or etc_docker_daemon_json.changed + when: var_lib_kubelet_config_yaml.changed or etc_docker_daemon_json.changed block: # At this point we assume that currently processed node has been drained already. - - name: k8s/cgroups | Write /var/lib/kubelet/kubeadm-flags.env + - name: k8s/cgroups | Write /var/lib/kubelet/config.yaml copy: - dest: /var/lib/kubelet/kubeadm-flags.env + dest: /var/lib/kubelet/config.yaml content: | - {{ var_lib_kubelet_kubeadm_flags_env.output }} + {{ var_lib_kubelet_config_yaml.output }} owner: root group: root mode: preserve diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml new file mode 100644 index 0000000000..bfa3184c11 --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubelet-cm.yml @@ -0,0 +1,31 @@ +--- +- name: k8s/kubelet-cm | Include get-cluster-version.yml + include_tasks: get-cluster-version.yml # sets cluster_version + +- name: k8s/kubelet-cm | Get kubelet config from ConfigMap + command: |- + kubectl get cm kubelet-config-{{ cluster_version_major }}.{{ cluster_version_minor }} \ + --namespace kube-system \ + --output=jsonpath={{ jsonpath }} + vars: + jsonpath: >- + '{.data.kubelet}' + changed_when: false + register: kubelet_config + +- name: 'k8s/kubelet-cm | Update kubelet ConfigMap with cgroupDriver: systemd' + vars: + _cgroup_driver: + cgroupDriver: systemd + _kubelet_desired_config: >- + {{ kubelet_config.stdout | from_yaml | combine(_cgroup_driver) | to_nice_yaml(indent=2) }} + _patch: + data: + kubelet: | + {{ _kubelet_desired_config }} + command: |- + kubectl patch cm kubelet-config-{{ cluster_version_major }}.{{ cluster_version_minor }} \ + --namespace kube-system + --patch '{{ _patch | to_yaml }}' + register: patch_result + changed_when: not 'no change' in patch_result.stdout diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-k8s-dashboard.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-k8s-dashboard.yml index e37ee66014..305d9ae991 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-k8s-dashboard.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-k8s-dashboard.yml @@ -27,8 +27,9 @@ when: upgrade_k8s_dashboard block: - name: k8s/master | Delete kubernetes-dashboard namespace + # --ignore-not-found is used to be able re-run upgrade if it fails command: >- - kubectl delete ns kubernetes-dashboard + kubectl delete ns kubernetes-dashboard --ignore-not-found=true # Deploy new version of kubernetes-dashboard - name: k8s/master | Apply Kubernetes Dashboard diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml index 99c9da76d6..a2369cea05 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-master0.yml @@ -1,11 +1,20 @@ --- +# During HA control plane upgrade server address in kubeconfig is switched to local for +# * compatibility between client and server versions +# * identifying correct server version + +- name: k8s/master0 | Switch apiserver address to local + include_tasks: utils/set-local-apiserver.yml # sets kubectl_context_cluster + when: + - groups.kubernetes_master | length > 1 + - name: k8s/master0 | Wait for cluster's readiness include_tasks: utils/wait.yml # This resolves issues (related to the etcd encryption) causing upgrades to hang. # Legacy clusters may have incomplete configs, thus it is corrected here, before any `kubeadm upgrade` command is executed. # If config is incomplete, kubeadm rewrites the kube-apiserver.yaml manifest file without the etcd feature enabled. -# In turn, this causes Kuberentes components such as the controller-manager to lose ability to read internal (kube-system) secrets, then +# In turn, this causes Kubernetes components such as the controller-manager to lose ability to read internal (kube-system) secrets, then # any upgrade attempt freezes and the cluster at hand becomes unusable. - name: k8s/master0 | Make sure the etcd encryption feature is properly configured (if enabled) include_tasks: patch-kubeadm-etcd-encryption.yml @@ -30,7 +39,7 @@ # Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16 - name: k8s/master0 | Validate whether cluster is upgradeable command: >- - kubeadm upgrade plan v{{ version }} + kubeadm upgrade plan v{{ version }} register: result until: - result is succeeded @@ -41,7 +50,7 @@ # Note: Usage of the --config flag for reconfiguring the cluster during upgrade is not recommended since v1.16 - name: k8s/master0 | Upgrade K8s cluster to v{{ version }} command: >- - kubeadm upgrade apply -y v{{ version }} + kubeadm upgrade apply -y v{{ version }} register: result until: - result is succeeded @@ -53,17 +62,19 @@ - name: Install kubelet and kubectl for {{ version }} include_tasks: >- - {%- if cni_in_kubelet is undefined or not cni_in_kubelet -%} - {{ ansible_os_family }}/install-packages.yml - {%- else -%} - {{ ansible_os_family }}/install-packages-cni-in-kubelet.yml - {%- endif -%} + {{ ansible_os_family }}/install-packages.yml when: - result is succeeded - name: k8s/master0 | Wait for cluster's readiness include_tasks: utils/wait.yml +# Even though cluster is seen as ready here, the next task may occasionally fail with error like +# "Error from server (NotFound): the server could not find the requested resource". +# The following task prevents this issue. +- name: k8s/master0 | Refresh api-resources cache + shell: kubectl api-resources --cached=false > /dev/null + # 'kubeadm upgrade apply' overwrites Epiphany's customized CoreDNS so we patch it again. # This task restores 'hosts' plugin and should be run each time K8s was upgraded in order to support "--wait-for-pods" epicli feature (issue #1218). - name: k8s/master0 | Customize CoreDNS @@ -79,18 +90,15 @@ - name: k8s/master0 | Upgrade CNI plugin pod include_tasks: upgrade-cni-plugin-pod.yml -- name: k8s/master0 | Upgrade Kubernetes Dashboard - include_tasks: upgrade-k8s-dashboard.yml - when: - - upgrade_to_final_version - - name: k8s/master0 | Backup kubeadm-config.yml include_tasks: backup-kubeadm-config.yml - name: k8s/master0 | Upgrade Docker # this may restart Docker daemon include_tasks: docker.yml -# This is considered a bugfix for existing clusters created prior Epiphany v0.10.x and after v0.5.x. +- name: k8s/master0 | Patch kubelet ConfigMap with systemd cgroup driver + include_tasks: patch-kubelet-cm.yml + - name: k8s/master0 | Replace cgroupfs driver with systemd driver include_tasks: patch-cgroup-driver.yml vars: { _requires_restart: false } # it will be properly restarted anyways @@ -105,3 +113,10 @@ - name: k8s/master0 | Verify component versions and node status include_tasks: kubernetes/verify-upgrade.yml + +- name: k8s/master0 | Switch apiserver address to HAProxy + command: |- + kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446 + when: + - groups.kubernetes_master | length > 1 + changed_when: true diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml index ca6d8f23ff..0ff766d3bc 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-masterN.yml @@ -1,4 +1,7 @@ --- +- name: k8s/masterN | Switch apiserver address to local + include_tasks: utils/set-local-apiserver.yml # sets kubectl_context_cluster + - name: k8s/masterN | Drain master in preparation for maintenance include_tasks: utils/drain.yml @@ -19,11 +22,7 @@ - name: k8s/masterN | Install kubelet and kubectl for {{ version }} include_tasks: >- - {%- if cni_in_kubelet is undefined or not cni_in_kubelet -%} - {{ ansible_os_family }}/install-packages.yml - {%- else -%} - {{ ansible_os_family }}/install-packages-cni-in-kubelet.yml - {%- endif -%} + {{ ansible_os_family }}/install-packages.yml when: - result is succeeded @@ -39,7 +38,6 @@ - name: k8s/masterN | Upgrade Docker # this may restart Docker daemon include_tasks: docker.yml -# This is considered a bugfix for existing clusters created prior Epiphany v0.10.x and after v0.5.x. - name: k8s/masterN | Replace cgroupfs driver with systemd driver include_tasks: patch-cgroup-driver.yml vars: { _requires_restart: false } # it will be properly restarted anyways @@ -57,3 +55,8 @@ - name: k8s/masterN | Verify component versions and node status include_tasks: kubernetes/verify-upgrade.yml + +- name: k8s/masterN | Switch apiserver address to HAProxy + command: |- + kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:3446 + changed_when: true diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml index 1125caf802..b7fdb06cb1 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-node.yml @@ -5,7 +5,6 @@ - name: k8s/node | Upgrade Docker # this may restart Docker daemon include_tasks: docker.yml -# This is considered a bugfix for existing clusters created prior Epiphany v0.10.x and after v0.5.x. - name: k8s/node | Replace cgroupfs driver with systemd driver include_tasks: patch-cgroup-driver.yml vars: { _requires_restart: true } @@ -28,11 +27,7 @@ - name: k8s/node | Install packages include_tasks: >- - {%- if cni_in_kubelet is undefined or not cni_in_kubelet -%} - {{ ansible_os_family }}/install-packages.yml - {%- else -%} - {{ ansible_os_family }}/install-packages-cni-in-kubelet.yml - {%- endif -%} + {{ ansible_os_family }}/install-packages.yml - name: k8s/node | Upgrade CNI plugin pod import_tasks: upgrade-cni-plugin-pod.yml @@ -53,4 +48,8 @@ include_tasks: utils/uncordon.yml - name: k8s/node | Verify component versions and node status - include_tasks: verify-upgrade.yml + include_tasks: + file: verify-upgrade.yml + apply: + environment: + KUBECONFIG: /etc/kubernetes/kubelet.conf diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/utils/set-local-apiserver.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/utils/set-local-apiserver.yml new file mode 100644 index 0000000000..9c71a4075e --- /dev/null +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/utils/set-local-apiserver.yml @@ -0,0 +1,16 @@ +--- +- name: k8s/utils | Get cluster name for the current kubectl context + shell: |- + set -o pipefail && \ + kubectl config get-contexts \ + | awk 'NR==1 { for (col=1; col<=NF; col++) { columns[$col] = toupper(col) } }; \ + NR>1 { if ($1 ~ /^*/) print $columns["CLUSTER"]}' + args: + executable: /bin/bash + register: kubectl_context_cluster + changed_when: false + +- name: k8s/utils | Set apiserver address to local + command: |- + kubectl config set-cluster {{ kubectl_context_cluster.stdout }} --server=https://localhost:6443 + changed_when: true # it's assumed to be used only in HA mode as for single control plane node it's useless diff --git a/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml b/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml index e30cd2ef95..bf886df408 100644 --- a/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml +++ b/ansible/playbooks/roles/upgrade/tasks/kubernetes/verify-upgrade.yml @@ -1,62 +1,66 @@ --- -- name: k8s/verify | Verify cluster versions - environment: - KUBECONFIG: /etc/kubernetes/kubelet.conf +- name: k8s/verify | Verify cluster version + when: + - inventory_hostname in groups.kubernetes_master block: - - name: k8s/verify | Verify cluster version - when: - - inventory_hostname in groups.kubernetes_master - block: - - name: k8s/verify | Include wait-for-kube-apiserver.yml - include_tasks: utils/wait-for-kube-apiserver.yml + - name: k8s/verify | Include wait-for-kube-apiserver.yml + include_tasks: utils/wait-for-kube-apiserver.yml - - name: k8s/verify | Include get-cluster-version.yml - include_tasks: get-cluster-version.yml # sets cluster_version + - name: k8s/verify | Include get-cluster-version.yml + include_tasks: get-cluster-version.yml # sets cluster_version - - name: k8s/verify | Verify cluster version - assert: - that: version in cluster_version + - name: k8s/verify | Verify cluster version + assert: + that: version in cluster_version - - name: k8s/verify | Verify kubectl version - block: - - name: k8s/verify | Get kubectl version - command: kubectl version --client --short - register: kubectl_version - changed_when: false +- name: k8s/verify | Verify kubectl version + block: + - name: k8s/verify | Get kubectl version + command: kubectl version --client --short + register: kubectl_version + changed_when: false - - name: k8s/verify | Verify kubectl version - assert: - that: version in kubectl_version.stdout + - name: k8s/verify | Verify kubectl version + assert: + that: version in kubectl_version.stdout - - name: k8s/verify | Verify kubeadm version - block: - - name: k8s/verify | Get kubeadm version - command: >- - kubeadm version -o short - register: kubeadm_version - changed_when: false - - - name: k8s/verify | Verify kubeadm version - assert: - that: version in kubeadm_version.stdout - - - name: k8s/verify | Verify kubelet version from API server and get node status - run_once: true - shell: |- - set -o pipefail && - kubectl get nodes {{ inventory_hostname }} | - # get values only for STATUS and VERSION columns, example output: 'Ready v1.14.6' - awk 'NR==1 { for (col=1; col<=NF; col++) { columns[$col] = col } }; - NR>1 { print $columns["STATUS"], $columns["VERSION"] }' - register: node_status_and_version - until: - - version in node_status_and_version.stdout - retries: 30 # 1min - delay: 2 - args: - executable: /bin/bash +- name: k8s/verify | Verify kubeadm version + block: + - name: k8s/verify | Get kubeadm version + command: >- + kubeadm version -o short + register: kubeadm_version changed_when: false - - name: k8s/verify | Verify node status + - name: k8s/verify | Verify kubeadm version assert: - that: "'Ready' in node_status_and_version.stdout" + that: version in kubeadm_version.stdout + +- name: k8s/verify | Verify kubelet version from API server and get node status + run_once: true + shell: |- + set -o pipefail && + kubectl get nodes {{ inventory_hostname }} | + # get values only for STATUS and VERSION columns, example output: 'Ready v1.14.6' + awk 'NR==1 { for (col=1; col<=NF; col++) { columns[$col] = col } }; + NR>1 { print $columns["STATUS"], $columns["VERSION"] }' + register: node_status_and_version + until: + - version in node_status_and_version.stdout + retries: 30 # 1min + delay: 2 + args: + executable: /bin/bash + changed_when: false + +- name: k8s/verify | Verify node status + assert: + that: "'Ready' in node_status_and_version.stdout" + +- name: k8s/verify | Verify cgroup driver is set to systemd in kubelet config + lineinfile: + dest: /var/lib/kubelet/config.yaml + line: "cgroupDriver: systemd" + check_mode: true + register: cgroup_driver + failed_when: cgroup_driver.changed diff --git a/ansible/playbooks/upgrade.yml b/ansible/playbooks/upgrade.yml index 527442647b..aaf9c0ad96 100644 --- a/ansible/playbooks/upgrade.yml +++ b/ansible/playbooks/upgrade.yml @@ -55,7 +55,22 @@ - import_role: name: upgrade tasks_from: kubernetes - vars: { ver: "1.18.6", cni_ver: "0.8.6", upgrade_to_final_version: true } + vars: { ver: "1.19.15", cni_ver: "0.8.7", upgrade_to_final_version: true } + when: "'kubernetes' in upgrade_components or upgrade_components|length == 0" + environment: + KUBECONFIG: "{{ kubeconfig.remote }}" + +# K8s dashboard upgraded only after control plane upgrade +# Otherwise there can be issues, such as hitting not upgraded apiserver by controller manager +# which results to forever terminating namespace after deletion +- hosts: kubernetes_master + become: true + become_method: sudo + tasks: + - include_role: + name: upgrade + tasks_from: kubernetes/upgrade-k8s-dashboard + run_once: true when: "'kubernetes' in upgrade_components or upgrade_components|length == 0" environment: KUBECONFIG: "{{ kubeconfig.remote }}" diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index 9d43de27c5..5fce0e92d5 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -18,9 +18,16 @@ - [#2497](https://github.com/epiphany-platform/epiphany/issues/2497) - Fix epicli apply --full region values - [#1743](https://github.com/epiphany-platform/epiphany/issues/1743) - Virtual machine "kind" mismatch - [#2656](https://github.com/epiphany-platform/epiphany/issues/2656) - WAL files are not removed from $PGDATA/pg_wal directory +- [#1587](https://github.com/epiphany-platform/epiphany/issues/1587) - Duplicated SANs for K8s apiserver certificate ### Updated +- Upgrade Flannel to v0.14.0 +- Upgrade Calico and Canal to v3.20.2 +- Upgrade Coredns to v1.7.0 +- Upgrade Kubernetes dashboard to v2.3.1 +- Upgrade Kubernetes metrics-scraper to v1.0.7 +- [#2093](https://github.com/epiphany-platform/epiphany/issues/2093) - Upgrade K8s to v1.19.15 - [#2494](https://github.com/epiphany-platform/epiphany/issues/2494) - Duplicated MOTD after ssh to servers - [#1974](https://github.com/epiphany-platform/epiphany/issues/1974) - [documentation] Azure Files Persistent Volume Support - [#2454](https://github.com/epiphany-platform/epiphany/issues/2454) - Remove dependencies for K8s v1.17 diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index 4be19e3cc8..910f2d9ed3 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -6,11 +6,13 @@ Note that versions are default versions and can be changed in certain cases thro | Component | Version | Repo/Website | License | | -------------------------- | -------- | ----------------------------------------------------- | ----------------------------------------------------------------- | -| Kubernetes | 1.18.6 | https://github.com/kubernetes/kubernetes | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Kubernetes Dashboard | 2.0.3 | https://github.com/kubernetes/dashboard | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Calico | 3.15.0 | https://github.com/projectcalico/calico | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | -| Flannel | 0.12.0 | https://github.com/coreos/flannel/ | [Apache License](https://www.apache.org/licenses/LICENSE-1.0) | -| Canal | 3.15.0 | https://github.com/projectcalico/calico | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Kubernetes | 1.19.15 | https://github.com/kubernetes/kubernetes | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Kubernetes Dashboard | 2.3.1 | https://github.com/kubernetes/dashboard | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Kubernetes metrics-scraper | 1.0.7 | https://github.com/kubernetes-sigs/dashboard-metrics-scraper | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Calico | 3.20.2 | https://github.com/projectcalico/calico | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Flannel | 0.14.0 | https://github.com/coreos/flannel/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Canal | 3.20.2 | https://github.com/projectcalico/calico | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| Coredns | 1.7.0 | https://github.com/coredns/coredns | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Kafka | 2.6.0 | https://github.com/apache/kafka | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Zookeeper | 3.5.8 | https://github.com/apache/zookeeper | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | RabbitMQ | 3.8.9 | https://github.com/rabbitmq/rabbitmq-server | [Mozilla Public License](https://www.mozilla.org/en-US/MPL/) | diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index fe3fe5e0d5..106d54f58b 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -8,12 +8,11 @@ it currently can upgrade and will add are: --- **NOTE** -Starting from v0.7.1 to the latest, Epiphany provides an ability to install K8s v1.18.6 only. There is an assertion to -check whether K8s version is supported before running upgrade. +There is an assertion to check whether K8s version is supported before running upgrade. --- -- Kubernetes (master and nodes). At the moment only v1.18.6 is supported +- Kubernetes (master and nodes). Supported versions: v1.18.6 (Epiphany 0.7.1+), v1.19.15 (Epiphany 1.3.0+) - common: Upgrades all common configurations to match them to current Epiphany version - repository: Adds the repository role needed for component installation in current Epiphany version - image_registry: Adds the image_registry role needed for offline installation in current Epiphany version @@ -26,7 +25,7 @@ see [Run apply after upgrade](./UPGRADE.md#run-apply-after-upgrade) chapter for Note about upgrade from pre-0.8 Epiphany: - If you need to upgrade a cluster deployed with `epicli` in version earlier than 0.8, you should make sure that you've got enough disk space on master (which - is used as repository) host. If you didn't extend OS disk on master during deployment process, you probably have only + is used as repository). If you didn't extend OS disk on master during deployment process, you probably have only 32 GB disk which is not enough to properly upgrade cluster (we recommend at least 64 GB). Before you run upgrade, please extend OS disk on master machine according to cloud provider documentation: [AWS](https://aws.amazon.com/premiumsupport/knowledge-center/expand-root-ebs-linux/) @@ -317,7 +316,7 @@ advised not to use this approach when Erlang needs to be upgraded. Before K8s version upgrade make sure that deprecated API versions are not used: -1. [v1.18](https://v1-18.docs.kubernetes.io/docs/setup/release/notes/#deprecation) +1. [v1.19](https://v1-19.docs.kubernetes.io/docs/setup/release/notes/#deprecation) ### Upgrade diff --git a/schema/common/defaults/configuration/image-registry.yml b/schema/common/defaults/configuration/image-registry.yml index 4a433c3665..76efacbfc8 100644 --- a/schema/common/defaults/configuration/image-registry.yml +++ b/schema/common/defaults/configuration/image-registry.yml @@ -15,10 +15,10 @@ specification: file_name: rabbitmq-3.8.9.tar - name: "epiphanyplatform/ignite:2.9.1" file_name: ignite-2.9.1.tar - - name: "kubernetesui/dashboard:v2.0.3" - file_name: dashboard-v2.0.3.tar - - name: "kubernetesui/metrics-scraper:v1.0.4" - file_name: metrics-scraper-v1.0.4.tar + - name: "kubernetesui/dashboard:v2.3.1" + file_name: dashboard-v2.3.1.tar + - name: "kubernetesui/metrics-scraper:v1.0.7" + file_name: metrics-scraper-v1.0.7.tar - name: "vault:1.7.0" file_name: vault-1.7.0.tar - name: "hashicorp/vault-k8s:0.10.0" @@ -37,6 +37,38 @@ specification: current: - name: "haproxy:2.2.2-alpine" file_name: haproxy-2.2.2-alpine.tar + # K8s v1.19.15 - Epiphany 1.3 (transitional version) + # https://github.com/kubernetes/kubernetes/blob/v1.19.15/build/dependencies.yaml + - name: "k8s.gcr.io/kube-apiserver:v1.19.15" + file_name: kube-apiserver-v1.19.15.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.19.15" + file_name: kube-controller-manager-v1.19.15.tar + - name: "k8s.gcr.io/kube-proxy:v1.19.15" + file_name: kube-proxy-v1.19.15.tar + - name: "k8s.gcr.io/kube-scheduler:v1.19.15" + file_name: kube-scheduler-v1.19.15.tar + - name: "k8s.gcr.io/coredns:1.7.0" + file_name: coredns-1.7.0.tar + - name: "k8s.gcr.io/etcd:3.4.13-0" + file_name: etcd-3.4.13-0.tar + - name: "k8s.gcr.io/pause:3.2" + file_name: pause-3.2.tar + # flannel + - name: "quay.io/coreos/flannel:v0.14.0-amd64" + file_name: flannel-v0.14.0-amd64.tar + - name: "quay.io/coreos/flannel:v0.14.0" + file_name: flannel-v0.14.0.tar + # canal & calico + - name: "calico/cni:v3.20.2" + file_name: cni-v3.20.2.tar + - name: "calico/kube-controllers:v3.20.2" + file_name: kube-controllers-v3.20.2.tar + - name: "calico/node:v3.20.2" + file_name: node-v3.20.2.tar + - name: "calico/pod2daemon-flexvol:v3.20.2" + file_name: pod2daemon-flexvol-v3.20.2.tar + legacy: + # K8s v1.18.6 - Epiphany 0.7.1 - 1.2 - name: "k8s.gcr.io/kube-apiserver:v1.18.6" file_name: kube-apiserver-v1.18.6.tar - name: "k8s.gcr.io/kube-controller-manager:v1.18.6" @@ -49,8 +81,6 @@ specification: file_name: coredns-1.6.7.tar - name: "k8s.gcr.io/etcd:3.4.3-0" file_name: etcd-3.4.3-0.tar - - name: "k8s.gcr.io/pause:3.2" - file_name: pause-3.2.tar # flannel - name: "quay.io/coreos/flannel:v0.12.0-amd64" file_name: flannel-v0.12.0-amd64.tar @@ -65,7 +95,6 @@ specification: file_name: node-v3.15.0.tar - name: "calico/pod2daemon-flexvol:v3.15.0" file_name: pod2daemon-flexvol-v3.15.0.tar - legacy: [] aarch64: generic: - name: "epiphanyplatform/keycloak:14.0.0" @@ -74,10 +103,10 @@ specification: file_name: rabbitmq-3.8.9.tar - name: "epiphanyplatform/ignite:2.9.1" file_name: ignite-2.9.1.tar - - name: "kubernetesui/dashboard:v2.0.3" - file_name: dashboard-v2.0.3.tar - - name: "kubernetesui/metrics-scraper:v1.0.4" - file_name: metrics-scraper-v1.0.4.tar + - name: "kubernetesui/dashboard:v2.3.1" + file_name: dashboard-v2.3.1.tar + - name: "kubernetesui/metrics-scraper:v1.0.7" + file_name: metrics-scraper-v1.0.7.tar - name: "vault:1.7.0" file_name: vault-1.7.0.tar - name: "hashicorp/vault-k8s:0.10.0" @@ -85,6 +114,37 @@ specification: current: - name: "haproxy:2.2.2-alpine" file_name: haproxy-2.2.2-alpine.tar + # K8s v1.19.15 - Epiphany 1.3 (transition version) + - name: "k8s.gcr.io/kube-apiserver:v1.19.15" + file_name: kube-apiserver-v1.19.15.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.19.15" + file_name: kube-controller-manager-v1.19.15.tar + - name: "k8s.gcr.io/kube-proxy:v1.19.15" + file_name: kube-proxy-v1.19.15.tar + - name: "k8s.gcr.io/kube-scheduler:v1.19.15" + file_name: kube-scheduler-v1.19.15.tar + - name: "k8s.gcr.io/coredns:1.7.0" + file_name: coredns-1.7.0.tar + - name: "k8s.gcr.io/etcd:3.4.13-0" + file_name: etcd-3.4.13-0.tar + - name: "k8s.gcr.io/pause:3.2" + file_name: pause-3.2.tar + # flannel + - name: "quay.io/coreos/flannel:v0.14.0-arm64" + file_name: flannel-v0.14.0-arm64.tar + - name: "quay.io/coreos/flannel:v0.14.0" + file_name: flannel-v0.14.0.tar + # canal & calico + - name: "calico/cni:v3.20.2" + file_name: cni-v3.20.2.tar + - name: "calico/kube-controllers:v3.20.2" + file_name: kube-controllers-v3.20.2.tar + - name: "calico/node:v3.20.2" + file_name: node-v3.20.2.tar + - name: "calico/pod2daemon-flexvol:v3.20.2" + file_name: pod2daemon-flexvol-v3.20.2.tar + legacy: + # K8s v1.18.6 - Epiphany 0.7.1 - 1.2 - name: "k8s.gcr.io/kube-apiserver:v1.18.6" file_name: kube-apiserver-v1.18.6.tar - name: "k8s.gcr.io/kube-controller-manager:v1.18.6" @@ -97,8 +157,6 @@ specification: file_name: coredns-1.6.7.tar - name: "k8s.gcr.io/etcd:3.4.3-0" file_name: etcd-3.4.3-0.tar - - name: "k8s.gcr.io/pause:3.2" - file_name: pause-3.2.tar # flannel - name: "quay.io/coreos/flannel:v0.12.0-arm64" file_name: flannel-v0.12.0-arm64.tar @@ -113,5 +171,3 @@ specification: file_name: node-v3.15.0.tar - name: "calico/pod2daemon-flexvol:v3.15.0" file_name: pod2daemon-flexvol-v3.15.0.tar - # No legacy images for aarch64 jet as we dont support aarch64 < 1.1.0 - legacy: [] diff --git a/schema/common/defaults/configuration/kubernetes-master.yml b/schema/common/defaults/configuration/kubernetes-master.yml index 4b7d14f893..7fb948c4f7 100644 --- a/schema/common/defaults/configuration/kubernetes-master.yml +++ b/schema/common/defaults/configuration/kubernetes-master.yml @@ -2,8 +2,8 @@ kind: configuration/kubernetes-master title: Kubernetes Master Config name: default specification: - version: 1.18.6 - cni_version: 0.8.6 + version: 1.19.15 + cni_version: 0.8.7 cluster_name: "kubernetes-epiphany" allow_pods_on_master: False storage: diff --git a/schema/common/defaults/configuration/kubernetes-node.yml b/schema/common/defaults/configuration/kubernetes-node.yml index 6f4af852de..0a9488d5b5 100644 --- a/schema/common/defaults/configuration/kubernetes-node.yml +++ b/schema/common/defaults/configuration/kubernetes-node.yml @@ -2,6 +2,6 @@ kind: configuration/kubernetes-node title: Kubernetes Node Config name: default specification: - version: 1.18.6 - cni_version: 0.8.6 + version: 1.19.15 + cni_version: 0.8.7 node_labels: "node-type=epiphany" diff --git a/tests/spec/spec/kubernetes_master/kubernetes_master_spec.rb b/tests/spec/spec/kubernetes_master/kubernetes_master_spec.rb index a1a3edee77..6642b0e8e3 100644 --- a/tests/spec/spec/kubernetes_master/kubernetes_master_spec.rb +++ b/tests/spec/spec/kubernetes_master/kubernetes_master_spec.rb @@ -233,3 +233,21 @@ its(:exit_status) { should eq 0 } end end + +describe 'Check Kubernetes namespace creation and deletion' do + ns_name = 'ns-spectest' + describe command("kubectl create ns #{ns_name}") do + its(:stdout) { should match %r{namespace/#{ns_name} created} } + its(:exit_status) { should eq 0 } + end + describe command("kubectl get ns #{ns_name} -o json") do + its(:stdout_as_json) { should include('metadata' => include('name' => ns_name.to_s)) } + its(:stdout_as_json) { should include('status' => include('phase' => 'Active')) } + its(:exit_status) { should eq 0 } + end + describe command("kubectl delete ns #{ns_name} --timeout=1m") do + its(:stdout) { should match %r{namespace "#{ns_name}" deleted} } + its(:stderr) { should_not match %r{error}i } + its(:exit_status) { should eq 0 } + end +end diff --git a/tools/development/k8s/memory/patch_cgroup_driver/readme.md b/tools/development/k8s/memory/patch_cgroup_driver/README.md similarity index 72% rename from tools/development/k8s/memory/patch_cgroup_driver/readme.md rename to tools/development/k8s/memory/patch_cgroup_driver/README.md index 06da69a02c..6be5e2e741 100644 --- a/tools/development/k8s/memory/patch_cgroup_driver/readme.md +++ b/tools/development/k8s/memory/patch_cgroup_driver/README.md @@ -1,11 +1,21 @@ +--- +**NOTE** + +This tool is deprecated and needs to be adjusted according +to [deprecation](https://github.com/kubernetes/kubernetes/pull/90513) of `--cgroup-driver` flag in `kubeadm-flags.env` +since 1.19 version. Epicli ensures that cgroup driver is set to `systemd` during apply and upgrade. + +--- + [K8s documentation](https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-drivers) states: > A single cgroup manager simplifies the view of what resources are being allocated and will by default have a more consistent view of the available and in-use resources. > When there are two cgroup managers on a system, you end up with two views of those resources. > In the field, people have reported cases where nodes that are configured to use cgroupfs for the kubelet and Docker, but systemd for the rest of the processes, become unstable under resource pressure. -Unfortunately (before this workaround) Epiphany had never switched to the `systemd` cgroup driver for `docker` and `kubelet` services. -Our aim here is to take an existing Epiphany cluster, patch worker nodes and perform memory and cpu stress tests on it. +Unfortunately (before this workaround) Epiphany had never switched to the `systemd` cgroup driver for `docker` +and `kubelet` services. Our aim here is to take an existing Epiphany cluster, patch worker nodes and perform memory and +cpu stress tests on it. ## Requirements @@ -31,7 +41,8 @@ Ansible will replace back `systemd` driver with `cgroupfs` driver. ## Procedure -Ansible will sequentially (rolling update but **without waiting for pods to be `Ready`**) reconfigure `docker` and `kubelet` services on each worker node. +Ansible will sequentially (rolling update but **without waiting for pods to be `Ready`**) reconfigure `docker` +and `kubelet` services on each worker node. When there are any changes found in config files, ansible will (for each worker node):