From de3cd8c767f4ef023c29b70e5423b1d01beb2da5 Mon Sep 17 00:00:00 2001 From: Michal Gubricky Date: Mon, 18 Nov 2024 13:39:51 +0100 Subject: [PATCH 01/20] Add k8s v1.31 to k8s-version-policy (#814) Signed-off-by: michal.gubricky Co-authored-by: Kurt Garloff --- Tests/kaas/k8s-version-policy/k8s-eol-data.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/kaas/k8s-version-policy/k8s-eol-data.yml b/Tests/kaas/k8s-version-policy/k8s-eol-data.yml index 6a549c464..3a3d3b2eb 100644 --- a/Tests/kaas/k8s-version-policy/k8s-eol-data.yml +++ b/Tests/kaas/k8s-version-policy/k8s-eol-data.yml @@ -1,5 +1,7 @@ # https://kubernetes.io/releases/patch-releases/#detailed-release-history-for-active-branches +- branch: '1.31' + end-of-life: '2025-10-28' - branch: '1.30' end-of-life: '2025-06-28' - branch: '1.29' From f63ad94cd7525fb0538ec037ffbdef17c09d808c Mon Sep 17 00:00:00 2001 From: Felix Kronlage-Dammers Date: Mon, 18 Nov 2024 13:41:36 +0100 Subject: [PATCH 02/20] fix typo (#827) * fip tyop * the project is called CloudKitty Signed-off-by: Felix Kronlage-Dammers Co-authored-by: Kurt Garloff --- .../scs-0123-v1-mandatory-and-supported-IaaS-services.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md b/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md index 274738be9..1d94990bc 100644 --- a/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md @@ -56,7 +56,7 @@ The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. im | Supported API | corresponding OpenStack Service | description | |-----|-----|-----| | **bare-metal** | Ironic | Bare Metal provisioning service | -| **billing** | Cloudkitty | Rating/Billing service | +| **billing** | CloudKitty | Rating/Billing service | | **dns** | Designate | DNS service | | **ha** | Masakari | Instances High Availability service | | **key-manager** | Barbican | Key Manager service | @@ -64,7 +64,7 @@ The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. im | **orchestration** | Heat | Orchestration service | | **shared-file-systems** | Manila | Shared File Systems service | | **telemetry** | Ceilometer | Telemetry service | -| **time-series-databse** | Gnocchi | Time Series Database service | +| **time-series-database** | Gnocchi | Time Series Database service | ## Unsupported IaaS APIs From 8c383ca19596e1d47284cc7ff2328adb8d68c85d Mon Sep 17 00:00:00 2001 From: tonifinger <129007376+tonifinger@users.noreply.github.com> Date: Tue, 19 Nov 2024 23:17:36 +0100 Subject: [PATCH 03/20] Enable compliance tests to use plugins for cluster provisioning (#753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Toni Finger Signed-off-by: Matthias Büchse Co-authored-by: Matthias Büchse --- Tests/config.toml | 40 ++++++++++ Tests/kaas/clusterspec.yaml | 11 +++ Tests/kaas/kind_config.yaml | 5 ++ Tests/kaas/plugin/README.md | 38 +++++++++ Tests/kaas/plugin/interface.py | 54 +++++++++++++ Tests/kaas/plugin/plugin_kind.py | 50 ++++++++++++ Tests/kaas/plugin/plugin_static.py | 19 +++++ Tests/kaas/plugin/requirements.in | 2 + Tests/kaas/plugin/requirements.txt | 60 ++++++++++++++ Tests/kaas/plugin/run_plugin.py | 58 ++++++++++++++ Tests/scs-compatible-kaas.yaml | 22 ++---- Tests/scs-test-runner.py | 122 +++++++++++++++++++++++------ 12 files changed, 443 insertions(+), 38 deletions(-) create mode 100644 Tests/kaas/clusterspec.yaml create mode 100644 Tests/kaas/kind_config.yaml create mode 100644 Tests/kaas/plugin/README.md create mode 100644 Tests/kaas/plugin/interface.py create mode 100644 Tests/kaas/plugin/plugin_kind.py create mode 100644 Tests/kaas/plugin/plugin_static.py create mode 100644 Tests/kaas/plugin/requirements.in create mode 100644 Tests/kaas/plugin/requirements.txt create mode 100755 Tests/kaas/plugin/run_plugin.py diff --git a/Tests/config.toml b/Tests/config.toml index a0173c25d..6b7a5c71f 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -34,11 +34,51 @@ subjects = [ workers = 4 +[presets.kaas-dev] +scopes = [ + "scs-compatible-kaas", +] +subjects = [ + "kind-current", + "kind-current-1", + "kind-current-2", +] +workers = 1 # better restrict this with clusters running on local machine + + [scopes.scs-compatible-iaas] spec = "./scs-compatible-iaas.yaml" +[scopes.scs-compatible-kaas] +spec = "./scs-compatible-kaas.yaml" + + # default subject (not a real subject, but used to declare a default mapping) # (this is the only mapping declaration that supports using Python string interpolation) [subjects._.mapping] os_cloud = "{subject}" +subject_root = "{subject}" + + +[subjects._.kubernetes_setup] +clusterspec = "kaas/clusterspec.yaml" + + +[subjects.kind-current.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "kaas/kind_config.yaml" +clusterspec_cluster = "current-k8s-release" + + +[subjects.kind-current-1.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "kaas/kind_config.yaml" +clusterspec_cluster = "current-k8s-release-1" + + +[subjects.kind-current-2.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "kaas/kind_config.yaml" +clusterspec_cluster = "current-k8s-release-2" + diff --git a/Tests/kaas/clusterspec.yaml b/Tests/kaas/clusterspec.yaml new file mode 100644 index 000000000..c8439a89f --- /dev/null +++ b/Tests/kaas/clusterspec.yaml @@ -0,0 +1,11 @@ +# this file specifies all clusters that have to be provisioned for the tests to run +clusters: + current-k8s-release: + branch: "1.31" + kubeconfig: kubeconfig.yaml + current-k8s-release-1: + branch: "1.30" + kubeconfig: kubeconfig.yaml + current-k8s-release-2: + branch: "1.29" + kubeconfig: kubeconfig.yaml diff --git a/Tests/kaas/kind_config.yaml b/Tests/kaas/kind_config.yaml new file mode 100644 index 000000000..ead21eb72 --- /dev/null +++ b/Tests/kaas/kind_config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker diff --git a/Tests/kaas/plugin/README.md b/Tests/kaas/plugin/README.md new file mode 100644 index 000000000..e54cf1864 --- /dev/null +++ b/Tests/kaas/plugin/README.md @@ -0,0 +1,38 @@ +# Plugin for provisioning k8s clusters and performing conformance tests on these clusters + +## Development environment + +### requirements + +* [docker](https://docs.docker.com/engine/install/) +* [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) + +### setup for development + +1. Generate python 3.10 env + + ```bash + sudo apt-get install python3.10-dev + virtualenv -p /usr/bin/python3.10 venv + echo "*" >> venv/.gitignore + source venv/bin/activate + (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 + (venv) python3.10 -m pip install --upgrade pip + (venv) python3.10 -m pip --version + + ``` + +2. Install dependencies: + + ```bash + (venv) pip install pip-tools + (venv) pip-compile requirements.in + (venv) pip-sync requirements.txt + ``` + +3. Set environment variables and launch the process: + + ```bash + (venv) export CLUSTER_PROVIDER="kind" + (venv) python run.py + ``` diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py new file mode 100644 index 000000000..f62e3b3e2 --- /dev/null +++ b/Tests/kaas/plugin/interface.py @@ -0,0 +1,54 @@ + + +class KubernetesClusterPlugin(): + """ + An abstract base class for custom Kubernetes cluster provider plugins. + It represents an interface class from which the api provider-specific + plugins must be derived as child classes + + To implement fill the methods `create_cluster` and `delete_cluster` with + api provider-specific functionalities for creating and deleting clusters. + The `create_cluster` method must ensure that the kubeconfigfile is provided + at the position in the file system defined by the parameter + `kubeconfig_filepath` + + - Implement `create_cluster` and `delete_cluster` methods + - Create `__init__(self, config_file)` method to handle api specific + configurations. + + Example: + .. code:: python + + from interface import KubernetesClusterPlugin + from apiX_library import cluster_api_class as ClusterAPI + + class PluginX(KubernetesClusterPlugin): + + def __init__(self, config_file): + self.config = config_file + + def create_cluster(self, cluster_name, version, kubeconfig_filepath): + self.cluster = ClusterAPI(name=cluster_name, image=cluster_image, kubeconfig_filepath) + self.cluster.create(self.config) + + def delete_cluster(self, cluster_name): + self.cluster = ClusterAPI(cluster_name) + self.cluster.delete() + .. + """ + + def create_cluster(self, cluster_name, version, kubeconfig_filepath): + """ + This method is to be called to create a k8s cluster + :param: cluster_name: + :param: version: + :param: kubeconfig_filepath: + """ + raise NotImplementedError + + def delete_cluster(self, cluster_name): + """ + This method is to be called in order to unprovision a cluster + :param: cluster_name: + """ + raise NotImplementedError diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py new file mode 100644 index 000000000..26cd3f23d --- /dev/null +++ b/Tests/kaas/plugin/plugin_kind.py @@ -0,0 +1,50 @@ +import logging +import os +import os.path +from pathlib import Path + +from interface import KubernetesClusterPlugin +from pytest_kind import KindCluster + +logger = logging.getLogger(__name__) + + +class PluginKind(KubernetesClusterPlugin): + """ + Plugin to handle the provisioning of kubernetes cluster for + conformance testing purpose with the use of Kind + """ + def __init__(self, config_path): + logger.info("Init PluginKind") + self.config = config_path + logger.debug(self.config) + self.working_directory = os.getcwd() + logger.debug(f"Working from {self.working_directory}") + + def create_cluster(self, cluster_name, version, kubeconfig): + """ + This method is to be called to create a k8s cluster + :param: kubernetes_version: + :return: kubeconfig_filepath + """ + cluster_version = version + if cluster_version == '1.29': + cluster_version = 'v1.29.8' + elif cluster_version == '1.30': + cluster_version = 'v1.30.4' + elif cluster_version == '1.31' or cluster_version == 'default': + cluster_version = 'v1.31.1' + cluster_image = f"kindest/node:{cluster_version}" + kubeconfig_filepath = Path(kubeconfig) + if kubeconfig_filepath is None: + raise ValueError("kubeconfig_filepath is missing") + else: + self.cluster = KindCluster(name=cluster_name, image=cluster_image, kubeconfig=kubeconfig_filepath) + if self.config is None: + self.cluster.create() + else: + self.cluster.create(self.config) + + def delete_cluster(self, cluster_name): + self.cluster = KindCluster(cluster_name) + self.cluster.delete() diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py new file mode 100644 index 000000000..0bd24707e --- /dev/null +++ b/Tests/kaas/plugin/plugin_static.py @@ -0,0 +1,19 @@ +import shutil + +from interface import KubernetesClusterPlugin + + +class PluginStatic(KubernetesClusterPlugin): + """ + Plugin to handle the provisioning of kubernetes + using a kubeconfig file + """ + + def __init__(self, config_path): + self.kubeconfig_path = config_path + + def create_cluster(self, cluster_name, version, kubeconfig): + shutil.copyfile(self.kubeconfig_path, kubeconfig) + + def delete_cluster(self, cluster_name, version): + pass diff --git a/Tests/kaas/plugin/requirements.in b/Tests/kaas/plugin/requirements.in new file mode 100644 index 000000000..0a60c3c3c --- /dev/null +++ b/Tests/kaas/plugin/requirements.in @@ -0,0 +1,2 @@ +pytest-kind +kubernetes diff --git a/Tests/kaas/plugin/requirements.txt b/Tests/kaas/plugin/requirements.txt new file mode 100644 index 000000000..a04a03167 --- /dev/null +++ b/Tests/kaas/plugin/requirements.txt @@ -0,0 +1,60 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile requirements.in +# +cachetools==5.5.0 + # via google-auth +certifi==2024.8.30 + # via + # kubernetes + # requests +charset-normalizer==3.3.2 + # via requests +google-auth==2.34.0 + # via kubernetes +idna==3.8 + # via requests +kubernetes==30.1.0 + # via -r requirements.in +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pykube-ng==23.6.0 + # via pytest-kind +pytest-kind==22.11.1 + # via -r requirements.in +python-dateutil==2.9.0.post0 + # via kubernetes +pyyaml==6.0.2 + # via + # kubernetes + # pykube-ng +requests==2.32.3 + # via + # kubernetes + # pykube-ng + # requests-oauthlib +requests-oauthlib==2.0.0 + # via kubernetes +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # kubernetes + # python-dateutil +urllib3==2.2.2 + # via + # kubernetes + # pykube-ng + # requests +websocket-client==1.8.0 + # via kubernetes diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py new file mode 100755 index 000000000..7b4084107 --- /dev/null +++ b/Tests/kaas/plugin/run_plugin.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import logging +import os.path + +import click +import yaml + +from plugin_kind import PluginKind +from plugin_static import PluginStatic + +PLUGIN_LOOKUP = { + "kind": PluginKind, + "static": PluginStatic, +} + + +def init_plugin(plugin_kind, config_path): + plugin_maker = PLUGIN_LOOKUP.get(plugin_kind) + if plugin_maker is None: + raise ValueError(f"unknown plugin '{plugin_kind}'") + return plugin_maker(config_path) + + +def load_spec(clusterspec_path): + with open(clusterspec_path, "rb") as fileobj: + return yaml.load(fileobj, Loader=yaml.SafeLoader) + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False)) +@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) +@click.argument('cluster_id', type=str, default="default") +def create(plugin_kind, plugin_config, clusterspec_path, cluster_id): + clusterspec = load_spec(clusterspec_path)['clusters'] + plugin = init_plugin(plugin_kind, plugin_config) + clusterinfo = clusterspec[cluster_id] + plugin.create_cluster(cluster_id, clusterinfo['branch'], os.path.abspath(clusterinfo['kubeconfig'])) + + +@cli.command() +@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False)) +@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) +@click.argument('cluster_id', type=str, default="default") +def delete(plugin_kind, plugin_config, clusterspec_path, cluster_id): + plugin = init_plugin(plugin_kind, plugin_config) + plugin.delete_cluster(cluster_id) + + +if __name__ == '__main__': + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + cli() diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 4aa540999..a4010c64e 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -2,7 +2,9 @@ name: SCS-compatible KaaS uuid: 1fffebe6-fd4b-44d3-a36c-fc58b4bb0180 url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-kaas.yaml variables: - - kubeconfig + - subject_root + # directory containing the kubeconfig file for the subject under test + # (note that we consider each kubernetes branch a test subject of its own) modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance @@ -12,38 +14,30 @@ modules: tags: [mandatory] - id: scs-0210-v2 name: Kubernetes version policy - url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0210-v2-k8s-version-policy.md + url: https://docs.scs.community/standards/scs-0210-v2-k8s-version-policy run: - executable: ./kaas/k8s-version-policy/k8s_version_policy.py - args: -k {kubeconfig} + args: -k {subject_root}/kubeconfig.yaml testcases: - id: version-policy-check tags: [mandatory] - id: scs-0214-v2 name: Kubernetes node distribution and availability - url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0214-v1-k8s-node-distribution.md + url: https://docs.scs.community/standards/scs-0214-v2-k8s-node-distribution run: - executable: ./kaas/k8s-node-distribution/k8s_node_distribution_check.py - args: -k {kubeconfig} + args: -k {subject_root}/kubeconfig.yaml testcases: - id: node-distribution-check tags: [mandatory] timeline: - # empty timeline might confuse tools, so put one "dummy" entry here - date: 2024-02-28 versions: v1: draft - v2: draft versions: - - version: v2 - include: - - cncf-k8s-conformance - - scs-0210-v2 - - scs-0214-v2 - targets: - main: mandatory - version: v1 include: + - cncf-k8s-conformance - scs-0210-v2 - scs-0214-v2 targets: diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index de7152428..780601e96 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -17,16 +17,23 @@ import click import tomli - logger = logging.getLogger(__name__) MONITOR_URL = "https://compliance.sovereignit.cloud/" +def ensure_dir(path): + try: + os.makedirs(path) + except FileExistsError: + pass + + class Config: def __init__(self): self.cwd = os.path.abspath(os.path.dirname(sys.argv[0]) or os.getcwd()) self.scs_compliance_check = os.path.join(self.cwd, 'scs-compliance-check.py') self.cleanup_py = os.path.join(self.cwd, 'cleanup.py') + self.run_plugin_py = os.path.join(self.cwd, 'kaas', 'plugin', 'run_plugin.py') self.ssh_keygen = shutil.which('ssh-keygen') self.curl = shutil.which('curl') self.secrets = {} @@ -58,42 +65,80 @@ def get_subject_mapping(self, subject): mapping.update(self.subjects.get(subject, {}).get('mapping', {})) return mapping + def get_kubernetes_setup(self, subject): + default_kubernetes_setup = self.subjects.get('_', {}).get('kubernetes_setup', {}) + kubernetes_setup = dict(default_kubernetes_setup) + kubernetes_setup.update(self.subjects.get(subject, {}).get('kubernetes_setup', {})) + return kubernetes_setup + def abspath(self, path): return os.path.join(self.cwd, path) def build_check_command(self, scope, subject, output): # TODO figure out when to supply --debug here (but keep separated from our --debug) - cmd = [ + args = [ sys.executable, self.scs_compliance_check, self.abspath(self.scopes[scope]['spec']), '--debug', '-C', '-o', output, '-s', subject, ] for key, value in self.get_subject_mapping(subject).items(): - cmd.extend(['-a', f'{key}={value}']) - return cmd + args.extend(['-a', f'{key}={value}']) + return {'args': args} + + def build_provision_command(self, subject): + kubernetes_setup = self.get_kubernetes_setup(subject) + subject_root = self.abspath(self.get_subject_mapping(subject).get('subject_root') or '.') + ensure_dir(subject_root) + return { + 'args': [ + sys.executable, self.run_plugin_py, + 'create', + kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['kube_plugin_config']), + self.abspath(kubernetes_setup['clusterspec']), + kubernetes_setup['clusterspec_cluster'], + ], + 'cwd': subject_root, + } + + def build_unprovision_command(self, subject): + kubernetes_setup = self.get_kubernetes_setup(subject) + subject_root = self.abspath(self.get_subject_mapping(subject).get('subject_root') or '.') + ensure_dir(subject_root) + return { + 'args': [ + sys.executable, self.run_plugin_py, + 'delete', + kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['kube_plugin_config']), + self.abspath(kubernetes_setup['clusterspec']), + kubernetes_setup['clusterspec_cluster'], + ], + 'cwd': subject_root, + } def build_cleanup_command(self, subject): # TODO figure out when to supply --debug here (but keep separated from our --debug) - return [ + return {'args': [ sys.executable, self.cleanup_py, '-c', self.get_subject_mapping(subject)['os_cloud'], '--prefix', '_scs-', '--ipaddr', '10.1.0.', '--debug', - ] + ]} def build_sign_command(self, target_path): - return [ + return {'args': [ self.ssh_keygen, '-Y', 'sign', '-f', self.abspath(self.secrets['keyfile']), '-n', 'report', target_path, - ] + ]} def build_upload_command(self, target_path, monitor_url): if not monitor_url.endswith('/'): monitor_url += '/' - return [ + return {'args': [ self.curl, '--fail-with-body', '--data-binary', f'@{target_path}.sig', @@ -101,7 +146,7 @@ def build_upload_command(self, target_path, monitor_url): '-H', 'Content-Type: application/x-signed-yaml', '-H', f'Authorization: Basic {self.auth_token}', f'{monitor_url}reports', - ] + ]} @click.group() @@ -123,7 +168,7 @@ def _run_commands(commands, num_workers=5): processes = [] while commands or processes: while commands and len(processes) < num_workers: - processes.append(subprocess.Popen(commands.pop())) + processes.append(subprocess.Popen(**commands.pop())) processes[:] = [p for p in processes if p.poll() is None] time.sleep(0.5) @@ -180,22 +225,14 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml): commands = [cfg.build_check_command(job[0], job[1], output) for job, output in zip(jobs, outputs)] _run_commands(commands, num_workers=num_workers) _concat_files(outputs, report_yaml_tmp) - subprocess.run(cfg.build_sign_command(report_yaml_tmp)) - subprocess.run(cfg.build_upload_command(report_yaml_tmp, monitor_url)) + subprocess.run(**cfg.build_sign_command(report_yaml_tmp)) + subprocess.run(**cfg.build_upload_command(report_yaml_tmp, monitor_url)) if report_yaml is not None: _move_file(report_yaml_tmp, report_yaml) return 0 -@cli.command() -@click.option('--subject', 'subjects', type=str) -@click.option('--preset', 'preset', type=str) -@click.option('--num-workers', 'num_workers', type=int, default=5) -@click.pass_obj -def cleanup(cfg, subjects, preset, num_workers): - """ - clean up any lingering resources - """ +def _run_command_for_subjects(cfg, subjects, preset, num_workers, command): if not subjects and not preset: preset = 'default' if preset: @@ -208,12 +245,49 @@ def cleanup(cfg, subjects, preset, num_workers): subjects = [subject.strip() for subject in subjects.split(',')] if subjects else [] if not subjects: raise click.UsageError('subject(s) must be non-empty') - logger.debug(f'cleaning up for subject(s) {", ".join(subjects)}, num_workers: {num_workers}') - commands = [cfg.build_cleanup_command(subject) for subject in subjects] + logger.debug(f'running {command} for subject(s) {", ".join(subjects)}, num_workers: {num_workers}') + m = getattr(cfg, f'build_{command}_command') + commands = [m(subject) for subject in subjects] _run_commands(commands, num_workers=num_workers) return 0 +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def cleanup(cfg, subjects, preset, num_workers): + """ + clean up any lingering IaaS resources + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "cleanup") + + +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def provision(cfg, subjects, preset, num_workers): + """ + create k8s clusters + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "provision") + + +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def unprovision(cfg, subjects, preset, num_workers): + """ + clean up k8s clusters + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "unprovision") + + if __name__ == '__main__': logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) cli(obj=Config()) From a3bc95c797f9099d908048c489d1dbcfc12063c0 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:50:58 +0100 Subject: [PATCH 04/20] Create a standard for the security of iaas service software (#765) * Create scs-XXXX-v1-minimum-iaas-service-version.md * First Draft of the Standard and Proposal of tests * change link * Update and rename scs-XXXX-v1-minimum-iaas-service-version.md to scs-XXXX-v1-security-of-iaas-service-software.md * Create scs-XXXX-w1-security-of-iaas-service-software.md * rework glossary * Multiple updates scs-XXXX-v1-security-of-iaas-service-software.md * Multiple applications of suggestions from code review * Update scs-XXXX-w1-security-of-iaas-service-software.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> Co-authored-by: Markus Hentsch <129268441+markus-hentsch@users.noreply.github.com> Co-authored-by: anjastrunk <119566837+anjastrunk@users.noreply.github.com> Co-authored-by: Kurt Garloff --- ...XX-v1-security-of-iaas-service-software.md | 138 ++++++++++++++++++ ...XX-w1-security-of-iaas-service-software.md | 45 ++++++ 2 files changed, 183 insertions(+) create mode 100644 Standards/scs-XXXX-v1-security-of-iaas-service-software.md create mode 100644 Standards/scs-XXXX-w1-security-of-iaas-service-software.md diff --git a/Standards/scs-XXXX-v1-security-of-iaas-service-software.md b/Standards/scs-XXXX-v1-security-of-iaas-service-software.md new file mode 100644 index 000000000..94b1200dd --- /dev/null +++ b/Standards/scs-XXXX-v1-security-of-iaas-service-software.md @@ -0,0 +1,138 @@ +--- +title: Standard for the security of IaaS service software +type: Standard +status: Draft +track: IaaS +--- + +## Introduction + +Software security relies on bug patches and security updates being available for specific versions of the software. +The services, which build the IaaS Layer should be updated on a regular basis based on updates provided by their respective authors or distributors. +But older releases or versions of the software of these services may not receive updates anymore. +Unpatched versions should not be used in deployments as they are a security risk, so this standard will define how CSPs should deal with software versions and security updates. + +## Terminology + +| Term | Explanation | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| CSP | Cloud Service Provider, provider managing the OpenStack infrastructure. | +| SLURP | Skip Level Upgrade Release Process - A Process that allows upgrades between two releases, while skipping the one in between them. | +| OSSN | [OpenStack Security Note](https://wiki.openstack.org/wiki/Security_Notes) - security issues from 3rd parties or due to misconfigurations. | +| OSSA | [OpenStack Security Advisories](https://security.openstack.org/ossalist.html) - security issues and advices for OpenStack. | + +## Motivation + +On the IaaS Layer the software, that needs to be considered in the scope of this standard, is mainly the APIs of IaaS Services. +Also there might be shared libraries and other dependencies, that could be considered part of the IaaS Layer. +In software projects like e.g. OpenStack that provide the main services and all APIs, the software will be modified and receive bug fixes continuously and will receive releases of new versions on a regular basis. +Older releases will at some point not receive updates anymore, because maintaining more and more releases simultaneously requires too much manpower. +Thus older versions will also eventually not receive security updates anymore. +Using versions which do not receive updates anymore threatens the baseline security of deployments and should be avoided under all circumstances. + +## Design Considerations + +It would be possible to define a minimum version of IaaS Layer software to avoid security risks. +In the following paragraphs several options of defining a minimum version or dealing with security patches otherwise are discussed. + +### Options considered + +#### Only Allow the current versions of Software + +Considering that OpenStack as one provider of IaaS Layer Software has two releases per year, with one SLURP release per year, this option would require CSPs to update their deployment once or twice a year. +Updating a whole deployment is a lot of work and requires also good life-cycle management. +Following only the SLURP releases would reduce this work to once per year. + +While following new releases closely already provides a deployment with recent bug fixes and new features, it also makes developing standards easier. +Differences between releases will accumulate eventually and may render older releases non-compliant to the SCS standards at some point. + +On the other hand on the IaaS Level there aren't many breaking changes introduced by releases and also most standards will also work with older releases. +Security updates and bug fixes are also provided by OpenStack for a few older releases with the state `maintained` according to the OpenStack releases overview[^2]. +Additionally the [SCS reference implementation](https://github.com/SovereignCloudStack/release-notes/blob/main/Release7.md) is integrating OpenStack releases after half a year - so about the time when a new release is published by OpenStack. +Considering a CSP that wants to use only SLURP releases and waits for the reference implementation to adopt them, will already lag over a year (i.e. 2 OpenStack releases) behind the latest release, this cannot be considered as using the current version of IaaS Layer Software. +Thus this option can be discarded. + +#### Allow only maintained versions of Software + +While following closely to the newest releases could be advised, there are several downsides to requiring this workflow, even if it would be only for SLURP releases. +Following the SCS reference implementation for example would also lead into being a little bit behind the newest OpenStack release. +But this is not as bad as it may seem to be, because security related fixes and bug fixes are backported to older but still `maintained` releases. +All releases that are still maintained can be looked up at the releases page from OpenStack[^2]. + +Allowing maintained versions would give CSPs a little bit more time to update and test their environments, while still receiving relevant security updates and bug fixes. +Also CSPs that want to become SCS-compliant will not have to take on the burden to upgrade their deployments to very recent releases immediately, but can instead test with an existing release before an upgrade and identify where they need to put in additional work to become SCS-compliant. + +One problem is, that there might be new features implemented in the newest versions of the software, which are desired by other SCS standards to be SCS-compliant. +In that case allowing all maintained versions would lead to a two-year timespan customers would need to wait for before such a feature becomes available in all SCS-compliant deployments. +In case of security relevant features this is not advisable. + +#### Standards implicitly define the minimum versions of Software + +Instead of requiring a defined minimum software version centrally, it could be derived from the individual standards. +Because: Whenever there is a new wanted behavior a standard should be created and a resonable timeframe given to CSPs to adopt a software version that can fulfill the new standard. +Through the combination of all standards that are in place, the minimum version for the IaaS service software is implicitly given. + +This would avoid to have conflicting versions of software in terms of feature parity, while also allowing older software. +Using this approach requires an additional advise to CSPs to update or implement patches for security issues. + +#### Advise CSPs to integrate software updates + +As long as maintained versions of software are used, updates with security patches are available and only need to be integrated. +This can and should be done in a reasonable short timeframe. + +But CSPs may even use releases of IaaS software, that are either not maintained anymore by an open source community or may be even closed source implementations of the mandatory IaaS APIs. +Allowing older versions or closed source software would only be acceptable, when CSPs assure (e.g. in documentation), that they themself will patch the software within their deployments. +Security bug fixes must be implemented and proof of the fix then provided. +Only under these circumstances deployments with older or alternative IaaS Layer software may be handled as compliant. + +This option could be taken for granted, but to actually advise using it may encourage CSPs to take a closer look on their life-cycle management and security risk handling. +And CSPs using OpenStack could even be encouraged to upgrade their deployments. + +#### Dependencies of the IaaS Layer Software + +While the IaaS service software like OpenStack itself is monitored and security issues announced in OSSNs and OSSAs, these services have lots of dependecies, that are not monitored by the same entity. +When dependencies have security issues, there might be no OSSN or OSSA, so CSPs also need to watch CVEs concerning these dependencies themselves. +Those dependencies must also be updated in a reasonable timeframe, when a security issue is disclosed. + +#### What timeframe is needed to fix the issue? + +CSPs should be encouraged to fix security issues as fast as possible. +Some security issues are very easy to exploit so as soon as the vulnerability is disclosed attacks on deployments will start. +Other vulnerabilities may need much knowledge and more time to be exploited. +Also the impact of different vulnerabilities will differ. + +So it can be concluded that some security issues need to be fixed immediately while for others it is okay to take some time. +The BSI already has some guidance[^1] on how fast CSPs should respond. +From the moment a vulnerability is disclosed these are the advised reaction times ranked by the severity of the vulnerability: + +1. Critical (CVSS = 9.0 – 10.0): 3 hours +2. High (CVSS = 7.0 – 8.9): 3 days +3. Mid (CVSS = 4.0 – 6.9): 1 month +4. Low (CVSS = 0.1 – 3.9): 3 months + +[^1]: [C5 criteria catalog with timeframes for responses on page 70.](https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/CloudComputing/ComplianceControlsCatalogue/2020/C5_2020.pdf?__blob=publicationFile&v=3) + +This standard will follow this guidance and refer to these timeframes as "reasonable timeframes". + +## Standard for a minimum IaaS Layer Software version + +If a deployment is affected by a security issue and a maintained[^2] version of OpenStack is used as implementation for IaaS Layer software, security patches noted in OSSNs and OSSAs MUST be integrated within a reasonable timeframe according to the severity of the security issue[^1]. +Otherwise the CSP MUST implement security bug fixes themself within a reasonable timeframe, when the deplyoment is affected by a security issue according to the severity of the security issue[^1]. + +In both cases a notice of the update MUST be send to the OSBA, so that the compliance will not be revoked. + +If a deployment uses a dependency of the IaaS service software which is affected by a security issue, this software also MUST be updated with security patches within a reasonable timeframe[^1]. + +An open SBOM list MAY be used to propagate the current version of the software and may be used as proof of updates. + +[^2]: [OpenStack versions and their current status](https://releases.openstack.org) + +## Conformance Tests + +In case of provided SBOMs the version numbers of the software could be checked. +But this is not a requirement, so there cannot be such a test. +Tests on the integration of security patches itself are difficult. +And even if tests for certain security issues are possible, then those might be interpreted as an attack. +This is the reason there will be no conformance test. + +Rather the standard requires that CSPs provide notice of the fixed vulnerabilites themselves. diff --git a/Standards/scs-XXXX-w1-security-of-iaas-service-software.md b/Standards/scs-XXXX-w1-security-of-iaas-service-software.md new file mode 100644 index 000000000..3f0b1df8c --- /dev/null +++ b/Standards/scs-XXXX-w1-security-of-iaas-service-software.md @@ -0,0 +1,45 @@ +--- +title: "SCS Standard for the security of IaaS service software: Implementation and Testing Notes" +type: Supplement +track: IaaS +status: Draft +supplements: + - scs-XXXX-v1-security-of-iaas-service-software.md +--- + +## Testing or Detecting security updates in software + +It is not always possible to automatically test, whether the software has the newest security updates. +This is because software versions may differ or some CSPs might have added downstream code parts or using other software than the reference. +Also vulnerabilites and their fixes are quite different in testing, some might not be testable while others are. +Additionally testing might be perceived as an attack on the infrastructure. +So this standard will rely on the work and information CSPs must provide. +There are different cases and procedures which are addressed in the following parts, that lead to compliance for this standard. + +### Procedure to become compliant to the security of IaaS service software Standard + +This is the procedure when a new deployment wants to achieve SCS-conformancy. +There are two states such a deployment can be in: + +1. When a deployment is newly build or installed it usually uses software which includes all the latest security and bug fixes. +Such deployments should be considered compliant to the standard. + +2. When a CSP wants to make an older deployment compliant to the SCS standards and thus also to this standard, it should be checked, whether the running software is up to date and all vulnerabilites are fixed. +Any updates or upgrades to even newer versions should be done before the SCS compliance for every other standard is checked. +Afterwards the CSP may provide information about the used software in an SBOM or otherwise should provide a notice about the deployment having integrated all necessary vulnerability patches. + +### Procedure when new vulnerabilites are discovered + +Whenever there are new vulnerabilities discovered in IaaS service software like OpenStack there is either an internal discussion ongoing or it is just a smaller issue. +In the first case CSPs should have someone following such discussions and may even help preparing and testing patches. +From the moment on the vulnerability is disclosed publicly, the risk of it being actively exploited increases greatly. +So CSPs MUST watch out for announcements like in the OSSAs and OSSNs and when they are affected, update their deployment within the following timeframes according to the severity of the issue: + +1. Critical (CVSS = 9.0 – 10.0): 3 hours +2. High (CVSS = 7.0 – 8.9): 3 days +3. Mid (CVSS = 4.0 – 6.9): 1 month +4. Low (CVSS = 0.1 – 3.9): 3 months + +Afterwards CSPs MUST provide a notice to the OSBA, that they are not or not anymore affected by the vulnerabilty. +This can be done through either telling, what patches were integrated or showing configuration that renders the attack impossible. +It could also be provided a list of services, when the affected service is not used in that deployment. From 5bb59b40b56504355c3f358fcf30899d6838eaf6 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Wed, 20 Nov 2024 18:21:03 +0100 Subject: [PATCH 05/20] Stabilize 0123 and mention need for S3 endpoints being documented. (#828) * Stabilize 0123 and mention need for S3 endpoints being documented. * Fix location of test script. It is in the main branch now ... * Adjust stabilization date, remove ceilometer API. - We have waited for today's IaaS team call for the final approval. - ceilometer does not offer an API (has been removed many years ago), so we should not list the API as supported - Remove one comma and one "the" in front of SCS. Signed-off-by: Kurt Garloff --- ...-0123-v1-mandatory-and-supported-IaaS-services.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md b/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md index 1d94990bc..2f7a74326 100644 --- a/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md @@ -1,7 +1,8 @@ --- title: Mandatory and Supported IaaS Services type: Standard -status: Draft +status: Stable +stabilized_at: 2024-11-20 track: IaaS --- @@ -40,7 +41,7 @@ The following IaaS APIs MUST be present in SCS-compliant IaaS deployments and co :::caution S3 API implementations may differ in certain offered features. -CSPs must publicly describe, which implementation they use in their deployment. +CSPs must publicly describe the endpoints of their S3 solutions and which implementations they use in their deployment. Users should always research whether a needed feature is supported in the offered implementation. ::: @@ -63,13 +64,12 @@ The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. im | **object-store** | Swift | Object Store with different possible backends | | **orchestration** | Heat | Orchestration service | | **shared-file-systems** | Manila | Shared File Systems service | -| **telemetry** | Ceilometer | Telemetry service | | **time-series-database** | Gnocchi | Time Series Database service | ## Unsupported IaaS APIs All other OpenStack services, whose APIs are not mentioned in the mandatory or supported lists will not be tested for their compatibility and conformance in SCS clouds by the SCS community. -Those services MAY be integrated into IaaS deployments by a Cloud Service Provider on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. +Those services MAY be integrated into IaaS deployments by a Cloud Service Provider on their own responsibility but SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported. ## Related Documents @@ -78,5 +78,5 @@ The SCS standard offers no guarantees for compatibility or reliability of servic ## Conformance Tests -The presence of the mandatory OpenStack APIs will be tested in [this test-script](https://github.com/SovereignCloudStack/standards/blob/mandatory-and-supported-IaaS-services/Tests/iaas/mandatory-services/mandatory-iaas-services.py). -The test will further check, whether the object store endpoint is compatible to s3. +The presence of the mandatory OpenStack APIs will be tested in [this test-script](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/mandatory-services/mandatory-iaas-services.py) +The test will further check whether the object-store endpoint is compatible to s3. From b70acdae52ae0c6da0f01c092c8c3cd37828dcf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:32:38 +0100 Subject: [PATCH 06/20] Bump cross-spawn from 7.0.3 to 7.0.6 in the npm_and_yarn group (#830) Bumps the npm_and_yarn group with 1 update: [cross-spawn](https://github.com/moxystudio/node-cross-spawn). Updates `cross-spawn` from 7.0.3 to 7.0.6 - [Changelog](https://github.com/moxystudio/node-cross-spawn/blob/master/CHANGELOG.md) - [Commits](https://github.com/moxystudio/node-cross-spawn/compare/v7.0.3...v7.0.6) --- updated-dependencies: - dependency-name: cross-spawn dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Kurt Garloff --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index e20b3925b..1fcf557e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -152,9 +152,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", From 87f4e4b7d6ceb5a689f3d99567560889ec64705f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:33:30 +0100 Subject: [PATCH 07/20] Bump aiohttp in /Tests in the pip group across 1 directory (#829) Bumps the pip group with 1 update in the /Tests directory: [aiohttp](https://github.com/aio-libs/aiohttp). Updates `aiohttp` from 3.10.3 to 3.10.11 - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.10.3...v3.10.11) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:production dependency-group: pip ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Kurt Garloff --- Tests/requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tests/requirements.txt b/Tests/requirements.txt index 9505a7061..bf93aff83 100644 --- a/Tests/requirements.txt +++ b/Tests/requirements.txt @@ -6,7 +6,7 @@ # aiohappyeyeballs==2.3.5 # via aiohttp -aiohttp==3.10.3 +aiohttp==3.10.11 # via # -r requirements.in # kubernetes-asyncio @@ -76,6 +76,8 @@ pbr==6.0.0 # stevedore platformdirs==4.2.2 # via openstacksdk +propcache==0.2.0 + # via yarl pycparser==2.22 # via cffi python-dateutil==2.9.0.post0 @@ -109,5 +111,5 @@ urllib3==2.2.2 # via # kubernetes-asyncio # requests -yarl==1.9.4 +yarl==1.17.2 # via aiohttp From 3274fff4fe1a45e926aa7b21e78ae96c22b85a08 Mon Sep 17 00:00:00 2001 From: Markus Hentsch <129268441+markus-hentsch@users.noreply.github.com> Date: Wed, 20 Nov 2024 19:08:48 +0100 Subject: [PATCH 08/20] Replace README.md of the Drafts folder with deprecation notice (#822) * Replace README.md of the Drafts folder with deprecation notice * Improve formatting of depreaction note. Also try to appease markdownlint ... * More markdownlint: trailing space & relative links. Signed-off-by: Markus Hentsch <129268441+markus-hentsch@users.noreply.github.com> Signed-off-by: Kurt Garloff Co-authored-by: Kurt Garloff --- Drafts/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Drafts/README.md b/Drafts/README.md index f4ee47aae..e3b903091 100644 --- a/Drafts/README.md +++ b/Drafts/README.md @@ -1,5 +1,11 @@ -# Design-Docs +# Drafts Archive -Design Documents, Architecture etc. for SCS and related technology +## Deprecation Notice -Here we collect docs that cover overarching SCS topics or topics that otherwise do not belong to an existing repository. +> [!CAUTION] +> Please do not create new files in this folder! + +The contents of this folder are for archival purposes only. New drafts belong +in the [`../Standards/`](https://github.com/SovereignCloudStack/standards/tree/main/Standards) +folder instead and adhere to the lifecycle described in +[scs-0001-v1-sovereign-cloud-standards](https://github.com/SovereignCloudStack/standards/blob/main/Standards/scs-0001-v1-sovereign-cloud-standards.md). From f2d6e3c9463b399051ea87eba767f7bf8522700c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 20 Nov 2024 21:30:24 +0000 Subject: [PATCH 09/20] Revert any non-editorial changes to scs-0214-v1 that happened after stabilization (#834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- .../scs-0214-v1-k8s-node-distribution.md | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/Standards/scs-0214-v1-k8s-node-distribution.md b/Standards/scs-0214-v1-k8s-node-distribution.md index ffec30efc..ce70e605e 100644 --- a/Standards/scs-0214-v1-k8s-node-distribution.md +++ b/Standards/scs-0214-v1-k8s-node-distribution.md @@ -80,42 +80,6 @@ If the standard is used by a provider, the following decisions are binding and v can also be scaled vertically first before scaling horizontally. - Worker node distribution MUST be indicated to the user through some kind of labeling in order to enable (anti)-affinity for workloads over "failure zones". -- To provide metadata about the node distribution, which also enables testing of this standard, - providers MUST label their K8s nodes with the labels listed below. - - `topology.kubernetes.io/zone` - - Corresponds with the label described in [K8s labels documentation][k8s-labels-docs]. - It provides a logical zone of failure on the side of the provider, e.g. a server rack - in the same electrical circuit or multiple machines bound to the internet through a - singular network structure. How this is defined exactly is up to the plans of the provider. - The field gets autopopulated most of the time by either the kubelet or external mechanisms - like the cloud controller. - - - `topology.kubernetes.io/region` - - Corresponds with the label described in [K8s labels documentation][k8s-labels-docs]. - It describes the combination of one or more failure zones into a region or domain, therefore - showing a larger entity of logical failure zone. An example for this could be a building - containing racks that are put into such a zone, since they're all prone to failure, if e.g. - the power for the building is cut. How this is defined exactly is also up to the provider. - The field gets autopopulated most of the time by either the kubelet or external mechanisms - like the cloud controller. - - - `topology.scs.community/host-id` - - This is an SCS-specific label; it MUST contain the hostID of the physical machine running - the hypervisor (NOT: the hostID of a virtual machine). Here, the hostID is an arbitrary identifier, - which need not contain the actual hostname, but it should nonetheless be unique to the host. - This helps identify the distribution over underlying physical machines, - which would be masked if VM hostIDs were used. - -## Conformance Tests - -The script `k8s-node-distribution-check.py` checks the nodes available with a user-provided -kubeconfig file. It then determines based on the labels `kubernetes.io/hostname`, `topology.kubernetes.io/zone`, -`topology.kubernetes.io/region` and `node-role.kubernetes.io/control-plane`, if a distribution -of the available nodes is present. If this isn't the case, the script produces an error. -If also produces warnings and informational outputs, if e.g. labels don't seem to be set. [k8s-ha]: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/ [k8s-large-clusters]: https://kubernetes.io/docs/setup/best-practices/cluster-large/ From 933c2357f250be7c5f1656b461802c5ae7d42981 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:47:12 +0100 Subject: [PATCH 10/20] Update number for security iaa s standard (#836) * Rename scs-XXXX-v1-security-of-iaas-service-software.md to scs-0124-v1-security-of-iaas-service-software.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> * Update and rename scs-XXXX-w1-security-of-iaas-service-software.md to scs-0124-w1-security-of-iaas-service-software.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --------- Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- ...ware.md => scs-0124-v1-security-of-iaas-service-software.md} | 0 ...ware.md => scs-0124-w1-security-of-iaas-service-software.md} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename Standards/{scs-XXXX-v1-security-of-iaas-service-software.md => scs-0124-v1-security-of-iaas-service-software.md} (100%) rename Standards/{scs-XXXX-w1-security-of-iaas-service-software.md => scs-0124-w1-security-of-iaas-service-software.md} (98%) diff --git a/Standards/scs-XXXX-v1-security-of-iaas-service-software.md b/Standards/scs-0124-v1-security-of-iaas-service-software.md similarity index 100% rename from Standards/scs-XXXX-v1-security-of-iaas-service-software.md rename to Standards/scs-0124-v1-security-of-iaas-service-software.md diff --git a/Standards/scs-XXXX-w1-security-of-iaas-service-software.md b/Standards/scs-0124-w1-security-of-iaas-service-software.md similarity index 98% rename from Standards/scs-XXXX-w1-security-of-iaas-service-software.md rename to Standards/scs-0124-w1-security-of-iaas-service-software.md index 3f0b1df8c..6cc7233ee 100644 --- a/Standards/scs-XXXX-w1-security-of-iaas-service-software.md +++ b/Standards/scs-0124-w1-security-of-iaas-service-software.md @@ -4,7 +4,7 @@ type: Supplement track: IaaS status: Draft supplements: - - scs-XXXX-v1-security-of-iaas-service-software.md + - scs-0124-v1-security-of-iaas-service-software.md --- ## Testing or Detecting security updates in software From 3b54ca2765e42ab54b08de196d576cff91d289a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Thu, 21 Nov 2024 10:16:24 +0000 Subject: [PATCH 11/20] Stabilize scs-0219-v1 (#833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse Co-authored-by: Kurt Garloff --- Standards/scs-0219-v1-kaas-networking.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Standards/scs-0219-v1-kaas-networking.md b/Standards/scs-0219-v1-kaas-networking.md index 8f35f7925..8c58d5c7a 100644 --- a/Standards/scs-0219-v1-kaas-networking.md +++ b/Standards/scs-0219-v1-kaas-networking.md @@ -1,7 +1,8 @@ --- title: KaaS Networking Standard type: Standard -status: Draft +status: Stable +stabilized_at: 2024-11-21 track: KaaS --- From 7a2662a2ec21cbe95a402af017207e5a555f3eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Thu, 21 Nov 2024 12:32:22 +0000 Subject: [PATCH 12/20] Feat/sonobuoy integration (#832) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse Signed-off-by: Toni Finger Co-authored-by: Toni Finger --- Tests/kaas/{plugin => }/README.md | 4 +- Tests/kaas/{plugin => }/requirements.in | 1 + Tests/kaas/{plugin => }/requirements.txt | 2 + Tests/kaas/sonobuoy_handler/run_sonobuoy.py | 26 ++++ .../kaas/sonobuoy_handler/sonobuoy_handler.py | 133 ++++++++++++++++++ Tests/scs-compatible-kaas.yaml | 14 ++ 6 files changed, 178 insertions(+), 2 deletions(-) rename Tests/kaas/{plugin => }/README.md (89%) rename Tests/kaas/{plugin => }/requirements.in (65%) rename Tests/kaas/{plugin => }/requirements.txt (95%) create mode 100755 Tests/kaas/sonobuoy_handler/run_sonobuoy.py create mode 100644 Tests/kaas/sonobuoy_handler/sonobuoy_handler.py diff --git a/Tests/kaas/plugin/README.md b/Tests/kaas/README.md similarity index 89% rename from Tests/kaas/plugin/README.md rename to Tests/kaas/README.md index e54cf1864..16697d3fd 100644 --- a/Tests/kaas/plugin/README.md +++ b/Tests/kaas/README.md @@ -1,4 +1,4 @@ -# Plugin for provisioning k8s clusters and performing conformance tests on these clusters +# Test suite for SCS-compatible KaaS ## Development environment @@ -6,6 +6,7 @@ * [docker](https://docs.docker.com/engine/install/) * [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) +* [sonobuoy](https://sonobuoy.io/docs/v0.57.1/#installation) ### setup for development @@ -19,7 +20,6 @@ (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 (venv) python3.10 -m pip install --upgrade pip (venv) python3.10 -m pip --version - ``` 2. Install dependencies: diff --git a/Tests/kaas/plugin/requirements.in b/Tests/kaas/requirements.in similarity index 65% rename from Tests/kaas/plugin/requirements.in rename to Tests/kaas/requirements.in index 0a60c3c3c..640831e54 100644 --- a/Tests/kaas/plugin/requirements.in +++ b/Tests/kaas/requirements.in @@ -1,2 +1,3 @@ pytest-kind kubernetes +junitparser diff --git a/Tests/kaas/plugin/requirements.txt b/Tests/kaas/requirements.txt similarity index 95% rename from Tests/kaas/plugin/requirements.txt rename to Tests/kaas/requirements.txt index a04a03167..c36ca21d1 100644 --- a/Tests/kaas/plugin/requirements.txt +++ b/Tests/kaas/requirements.txt @@ -16,6 +16,8 @@ google-auth==2.34.0 # via kubernetes idna==3.8 # via requests +junitparser==3.2.0 + # via -r requirements.in kubernetes==30.1.0 # via -r requirements.in oauthlib==3.2.2 diff --git a/Tests/kaas/sonobuoy_handler/run_sonobuoy.py b/Tests/kaas/sonobuoy_handler/run_sonobuoy.py new file mode 100755 index 000000000..50ef4249c --- /dev/null +++ b/Tests/kaas/sonobuoy_handler/run_sonobuoy.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +# +import logging +import sys + +import click + +from sonobuoy_handler import SonobuoyHandler + +logger = logging.getLogger(__name__) + + +@click.command() +@click.option("-k", "--kubeconfig", "kubeconfig", required=True, type=click.Path(exists=True), help="path/to/kubeconfig_file.yaml",) +@click.option("-r", "--result_dir_name", "result_dir_name", type=str, default="sonobuoy_results", help="directory name to store results at",) +@click.option("-c", "--check", "check_name", type=str, default="sonobuoy_executor", help="this MUST be the same name as the id in 'scs-compatible-kaas.yaml'",) +@click.option("-a", "--arg", "args", multiple=True) +def sonobuoy_run(kubeconfig, result_dir_name, check_name, args): + sonobuoy_handler = SonobuoyHandler(check_name, kubeconfig, result_dir_name, args) + sys.exit(sonobuoy_handler.run()) + + +if __name__ == "__main__": + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) + sonobuoy_run() diff --git a/Tests/kaas/sonobuoy_handler/sonobuoy_handler.py b/Tests/kaas/sonobuoy_handler/sonobuoy_handler.py new file mode 100644 index 000000000..65593a411 --- /dev/null +++ b/Tests/kaas/sonobuoy_handler/sonobuoy_handler.py @@ -0,0 +1,133 @@ +from collections import Counter +import json +import logging +import os +import shlex +import shutil +import subprocess + +from junitparser import JUnitXml + +logger = logging.getLogger(__name__) + + +class SonobuoyHandler: + """ + A class that handles both the execution of sonobuoy and + the generation of the results for a test report + """ + + kubeconfig_path = None + working_directory = None + + def __init__( + self, + check_name="sonobuoy_handler", + kubeconfig=None, + result_dir_name="sonobuoy_results", + args=(), + ): + self.check_name = check_name + logger.debug(f"kubeconfig: {kubeconfig} ") + if kubeconfig is None: + raise RuntimeError("No kubeconfig provided") + self.kubeconfig_path = kubeconfig + self.working_directory = os.getcwd() + self.result_dir_name = result_dir_name + self.sonobuoy = shutil.which('sonobuoy') + logger.debug(f"working from {self.working_directory}") + logger.debug(f"placing results at {self.result_dir_name}") + logger.debug(f"sonobuoy executable at {self.sonobuoy}") + self.args = (arg0 for arg in args for arg0 in shlex.split(str(arg))) + + def _invoke_sonobuoy(self, *args, **kwargs): + inv_args = (self.sonobuoy, "--kubeconfig", self.kubeconfig_path) + args + logger.debug(f'invoking {" ".join(inv_args)}') + return subprocess.run(args=inv_args, capture_output=True, check=True, **kwargs) + + def _sonobuoy_run(self): + self._invoke_sonobuoy("run", "--wait", *self.args) + + def _sonobuoy_delete(self): + self._invoke_sonobuoy("delete", "--wait") + + def _sonobuoy_status_result(self): + process = self._invoke_sonobuoy("status", "--json") + json_data = json.loads(process.stdout) + counter = Counter() + for entry in json_data["plugins"]: + logger.debug(f"plugin:{entry['plugin']}:{entry['result-status']}") + for result, count in entry["result-counts"].items(): + counter[result] += count + return counter + + def _eval_result(self, counter): + """evaluate test results and return return code""" + result_str = ', '.join(f"{counter[key]} {key}" for key in ('passed', 'failed', 'skipped')) + result_message = f"sonobuoy reports {result_str}" + if counter['failed']: + logger.error(result_message) + return 3 + logger.info(result_message) + return 0 + + def _preflight_check(self): + """ + Preflight test to ensure that everything is set up correctly for execution + """ + if not self.sonobuoy: + raise RuntimeError("sonobuoy executable not found; is it in PATH?") + + def _sonobuoy_retrieve_result(self): + """ + This method invokes sonobuoy to store the results in a subdirectory of + the working directory. The Junit results file contained in it is then + analyzed in order to interpret the relevant information it containes + """ + logger.debug(f"retrieving results to {self.result_dir_name}") + result_dir = os.path.join(self.working_directory, self.result_dir_name) + if os.path.exists(result_dir): + raise Exception("result directory already existing") + os.mkdir(result_dir) + + # XXX use self._invoke_sonobuoy + os.system( + # ~ f"sonobuoy retrieve {result_dir} -x --filename='{result_dir}' --kubeconfig='{self.kubeconfig_path}'" + f"sonobuoy retrieve {result_dir} --kubeconfig='{self.kubeconfig_path}'" + ) + logger.debug( + f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} " + ) + xml = JUnitXml.fromfile(result_dir + "/plugins/e2e/results/global/junit_01.xml") + counter = Counter() + for suite in xml: + for case in suite: + if case.is_passed is True: # XXX why `is True`??? + counter['passed'] += 1 + elif case.is_skipped is True: + counter['skipped'] += 1 + else: + counter['failed'] += 1 + logger.error(f"{case.name}") + return counter + + def run(self): + """ + This method is to be called to run the plugin + """ + logger.info(f"running sonobuoy for testcase {self.check_name}") + self._preflight_check() + try: + self._sonobuoy_run() + return_code = self._eval_result(self._sonobuoy_status_result()) + print(self.check_name + ": " + ("PASS", "FAIL")[min(1, return_code)]) + return return_code + + # ERROR: currently disabled due to: "error retrieving results: unexpected EOF" + # might be related to following bug: https://github.com/vmware-tanzu/sonobuoy/issues/1633 + # self._sonobuoy_retrieve_result(self) + except BaseException: + logger.exception("something went wrong") + return 112 + finally: + self._sonobuoy_delete() diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index a4010c64e..7cb2fbd58 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -9,6 +9,10 @@ modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance url: https://github.com/cncf/k8s-conformance/tree/master + run: + - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py + args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'cncf-k8s-conformance' -a '--mode=certified-conformance' + #~ args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'cncf-k8s-conformance' -a '--plugin-env e2e.E2E_DRYRUN=true' testcases: - id: cncf-k8s-conformance tags: [mandatory] @@ -30,6 +34,15 @@ modules: testcases: - id: node-distribution-check tags: [mandatory] + - id: scs-0219-v1 + name: KaaS networking + url: https://docs.scs.community/standards/scs-0219-v1-kaas-networking + run: + - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py + args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/sono-results -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"' + testcases: + - id: kaas-networking-check + tags: [mandatory] timeline: - date: 2024-02-28 versions: @@ -40,5 +53,6 @@ versions: - cncf-k8s-conformance - scs-0210-v2 - scs-0214-v2 + - scs-0219-v1 targets: main: mandatory From ebfaa1af4fdb1d7a3dbeb3c88261777db2bdb664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Thu, 21 Nov 2024 13:55:49 +0000 Subject: [PATCH 13/20] Stabilize SCS-compatible IaaS v5 (#824) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Stabilize SCS-compatible IaaS v5 make it effective from 2024-11-16 and turn v4 into warn from 2025-01-01 * Include scs-0123-v1 and adapt test script accordingly * move manual docs check to target preview for the time being * Make it effective today, not in the past. Signed-off-by: Matthias Büchse Signed-off-by: Kurt Garloff Co-authored-by: Kurt Garloff --- .../mandatory-iaas-services.py | 122 +++++++----------- Tests/requirements.in | 5 +- Tests/requirements.txt | 21 ++- Tests/scs-compatible-iaas.yaml | 28 +++- 4 files changed, 95 insertions(+), 81 deletions(-) mode change 100644 => 100755 Tests/iaas/mandatory-services/mandatory-iaas-services.py diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py old mode 100644 new mode 100755 index ab5cc0a2f..41d67960c --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Mandatory APIs checker This script retrieves the endpoint catalog from Keystone using the OpenStack SDK and checks whether all mandatory APi endpoints, are present. @@ -26,28 +27,8 @@ block_storage_service = ["volume", "volumev3", "block-storage"] -def connect(cloud_name: str) -> openstack.connection.Connection: - """Create a connection to an OpenStack cloud - :param string cloud_name: - The name of the configuration to load from clouds.yaml. - :returns: openstack.connnection.Connection - """ - return openstack.connect( - cloud=cloud_name, - ) - - -def check_presence_of_mandatory_services(cloud_name: str, s3_credentials=None): - try: - connection = connect(cloud_name) - services = connection.service_catalog - except Exception as e: - print(str(e)) - raise Exception( - f"Connection to cloud '{cloud_name}' was not successfully. " - f"The Catalog endpoint could not be accessed. " - f"Please check your cloud connection and authorization." - ) +def check_presence_of_mandatory_services(conn: openstack.connection.Connection, s3_credentials=None): + services = conn.service_catalog if s3_credentials: mandatory_services.remove("object-store") @@ -55,25 +36,21 @@ def check_presence_of_mandatory_services(cloud_name: str, s3_credentials=None): svc_type = svc['type'] if svc_type in mandatory_services: mandatory_services.remove(svc_type) - continue - if svc_type in block_storage_service: + elif svc_type in block_storage_service: block_storage_service.remove(svc_type) bs_service_not_present = 0 if len(block_storage_service) == 3: # neither block-storage nor volume nor volumev3 is present # we must assume, that there is no volume service - logger.error("FAIL: No block-storage (volume) endpoint found.") + logger.error("No block-storage (volume) endpoint found.") mandatory_services.append(block_storage_service[0]) bs_service_not_present = 1 - if not mandatory_services: - # every mandatory service API had an endpoint - return 0 + bs_service_not_present - else: - # there were multiple mandatory APIs not found - logger.error(f"FAIL: The following endpoints are missing: " - f"{mandatory_services}") - return len(mandatory_services) + bs_service_not_present + if mandatory_services: + # some mandatory APIs were not found + logger.error(f"The following endpoints are missing: " + f"{', '.join(mandatory_services)}.") + return len(mandatory_services) + bs_service_not_present def list_containers(conn): @@ -167,8 +144,8 @@ def s3_from_ostack(creds, conn, endpoint): # pass -def check_for_s3_and_swift(cloud_name: str, s3_credentials=None): - # If we get credentials we assume, that there is no Swift and only test s3 +def check_for_s3_and_swift(conn: openstack.connection.Connection, s3_credentials=None): + # If we get credentials, we assume that there is no Swift and only test s3 if s3_credentials: try: s3 = s3_conn(s3_credentials) @@ -183,58 +160,46 @@ def check_for_s3_and_swift(cloud_name: str, s3_credentials=None): if s3_buckets == [TESTCONTNAME]: del_bucket(s3, TESTCONTNAME) # everything worked, and we don't need to test for Swift: - print("SUCCESS: S3 exists") + logger.info("SUCCESS: S3 exists") return 0 # there were no credentials given, so we assume s3 is accessable via # the service catalog and Swift might exist too - try: - connection = connect(cloud_name) - connection.authorize() - except Exception as e: - print(str(e)) - raise Exception( - f"Connection to cloud '{cloud_name}' was not successfully. " - f"The Catalog endpoint could not be accessed. " - f"Please check your cloud connection and authorization." - ) s3_creds = {} try: - endpoint = connection.object_store.get_endpoint() - except Exception as e: - logger.error( - f"FAIL: No object store endpoint found in cloud " - f"'{cloud_name}'. No testing for the s3 service possible. " - f"Details: %s", e + endpoint = conn.object_store.get_endpoint() + except Exception: + logger.exception( + "No object store endpoint found. No testing for the s3 service possible." ) return 1 # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) - s3_from_ostack(s3_creds, connection, endpoint) + s3_from_ostack(s3_creds, conn, endpoint) # Overrides (var names are from libs3, in case you wonder) s3_from_env(s3_creds, "HOST", "S3_HOSTNAME", "https://") s3_from_env(s3_creds, "AK", "S3_ACCESS_KEY_ID") s3_from_env(s3_creds, "SK", "S3_SECRET_ACCESS_KEY") - s3 = s3_conn(s3_creds, connection) + s3 = s3_conn(s3_creds, conn) s3_buckets = list_s3_buckets(s3) if not s3_buckets: s3_buckets = create_bucket(s3, TESTCONTNAME) assert s3_buckets # If we got till here, s3 is working, now swift - swift_containers = list_containers(connection) + swift_containers = list_containers(conn) # if not swift_containers: - # swift_containers = create_container(connection, TESTCONTNAME) + # swift_containers = create_container(conn, TESTCONTNAME) result = 0 if Counter(s3_buckets) != Counter(swift_containers): - print("WARNING: S3 buckets and Swift Containers differ:\n" - f"S3: {sorted(s3_buckets)}\nSW: {sorted(swift_containers)}") + logger.warning("S3 buckets and Swift Containers differ:\n" + f"S3: {sorted(s3_buckets)}\nSW: {sorted(swift_containers)}") result = 1 else: - print("SUCCESS: S3 and Swift exist and agree") + logger.info("SUCCESS: S3 and Swift exist and agree") # Clean up # FIXME: Cleanup created EC2 credential # if swift_containers == [TESTCONTNAME]: - # del_container(connection, TESTCONTNAME) + # del_container(conn, TESTCONTNAME) # Cleanup created S3 bucket if s3_buckets == [TESTCONTNAME]: del_bucket(s3, TESTCONTNAME) @@ -266,34 +231,47 @@ def main(): help="Enable OpenStack SDK debug logging" ) args = parser.parse_args() + logging.basicConfig( + format="%(levelname)s: %(message)s", + level=logging.DEBUG if args.debug else logging.INFO, + ) openstack.enable_logging(debug=args.debug) # parse cloud name for lookup in clouds.yaml - cloud = os.environ.get("OS_CLOUD", None) - if args.os_cloud: - cloud = args.os_cloud - assert cloud, ( - "You need to have the OS_CLOUD environment variable set to your cloud " - "name or pass it via --os-cloud" - ) + cloud = args.os_cloud or os.environ.get("OS_CLOUD", None) + if not cloud: + raise RuntimeError( + "You need to have the OS_CLOUD environment variable set to your " + "cloud name or pass it via --os-cloud" + ) s3_credentials = None if args.s3_endpoint: if (not args.s3_access) or (not args.s3_access_secret): - print("WARNING: test for external s3 needs access key and access secret.") + logger.warning("test for external s3 needs access key and access secret.") s3_credentials = { "AK": args.s3_access, "SK": args.s3_access_secret, "HOST": args.s3_endpoint } elif args.s3_access or args.s3_access_secret: - print("WARNING: access to s3 was given, but no endpoint provided.") + logger.warning("access to s3 was given, but no endpoint provided.") - result = check_presence_of_mandatory_services(cloud, s3_credentials) - result = result + check_for_s3_and_swift(cloud, s3_credentials) + with openstack.connect(cloud) as conn: + result = check_presence_of_mandatory_services(conn, s3_credentials) + result += check_for_s3_and_swift(conn, s3_credentials) + + print('service-apis-check: ' + ('PASS', 'FAIL')[min(1, result)]) return result if __name__ == "__main__": - main() + try: + sys.exit(main()) + except SystemExit: + raise + except BaseException as exc: + logging.debug("traceback", exc_info=True) + logging.critical(str(exc)) + sys.exit(1) diff --git a/Tests/requirements.in b/Tests/requirements.in index e2113c5b7..cb3c3bac0 100644 --- a/Tests/requirements.in +++ b/Tests/requirements.in @@ -1,8 +1,9 @@ aiohttp +boto3 # TODO: move into iaas/requirements.in click -kubernetes_asyncio +kubernetes_asyncio # TODO: move into kaas/requirements.in python-dateutil PyYAML -openstacksdk +openstacksdk # TODO: move into iaas/requirements.in requests tomli diff --git a/Tests/requirements.txt b/Tests/requirements.txt index bf93aff83..19f9e1990 100644 --- a/Tests/requirements.txt +++ b/Tests/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements.in @@ -12,10 +12,14 @@ aiohttp==3.10.11 # kubernetes-asyncio aiosignal==1.3.1 # via aiohttp -async-timeout==4.0.3 - # via aiohttp attrs==24.2.0 # via aiohttp +boto3==1.35.65 + # via -r requirements.in +botocore==1.35.65 + # via + # boto3 + # s3transfer certifi==2024.7.4 # via # kubernetes-asyncio @@ -47,7 +51,10 @@ iso8601==2.1.0 # keystoneauth1 # openstacksdk jmespath==1.0.1 - # via openstacksdk + # via + # boto3 + # botocore + # openstacksdk jsonpatch==1.33 # via openstacksdk jsonpointer==3.0.0 @@ -83,6 +90,7 @@ pycparser==2.22 python-dateutil==2.9.0.post0 # via # -r requirements.in + # botocore # kubernetes-asyncio pyyaml==6.0.2 # via @@ -95,6 +103,8 @@ requests==2.32.3 # keystoneauth1 requestsexceptions==1.4.0 # via openstacksdk +s3transfer==0.10.3 + # via boto3 six==1.16.0 # via # kubernetes-asyncio @@ -105,10 +115,9 @@ stevedore==5.2.0 # keystoneauth1 tomli==2.0.1 # via -r requirements.in -typing-extensions==4.12.2 - # via dogpile-cache urllib3==2.2.2 # via + # botocore # kubernetes-asyncio # requests yarl==1.17.2 diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml index 5ad119fbf..2d9b8b192 100644 --- a/Tests/scs-compatible-iaas.yaml +++ b/Tests/scs-compatible-iaas.yaml @@ -206,6 +206,21 @@ modules: tags: [availability-zones] description: > Note: manual check! Must fulfill all requirements of + - id: scs-0123-v1 + name: Mandatory and Supported IaaS Services + url: https://docs.scs.community/standards/scs-0123-v1-mandatory-and-supported-IaaS-services + run: + - executable: ./iaas/mandatory-services/mandatory-iaas-services.py + args: --os-cloud {os_cloud} --debug + testcases: + - id: service-apis-check + tags: [mandatory] + description: > + Must fulfill all requirements of (except for documentation requirements, which are tested manually with service-apis-docs-check). + - id: service-apis-docs-check + tags: [service-apis-docs] + description: > + Note: manual check! Must fulfill documentation requirements of . - id: scs-0302-v1 name: Domain Manager Role url: https://docs.scs.community/standards/scs-0302-v1-domain-manager-role @@ -218,6 +233,16 @@ modules: description: > Note: manual check! Must fulfill all requirements of timeline: + - date: 2025-01-01 + versions: + v5: effective + v4: warn + v3: deprecated + - date: 2024-11-21 + versions: + v5: effective + v4: effective + v3: deprecated - date: 2024-11-08 versions: v5: draft @@ -261,6 +286,7 @@ timeline: v1: effective versions: - version: v5 + stabilized_at: 2024-11-14 include: - opc-v2022.11 - scs-0100-v3.1 @@ -278,7 +304,7 @@ versions: - scs-0302-v1 targets: main: mandatory - preview: domain-manager/availability-zones + preview: domain-manager/availability-zones/service-apis-docs - version: v4 stabilized_at: 2024-02-28 include: From c3dd46330a6b741216d2ade836b02b2deec22a34 Mon Sep 17 00:00:00 2001 From: Markus Hentsch <129268441+markus-hentsch@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:41:06 +0100 Subject: [PATCH 14/20] Add Secure Connections Standard (#548) * Add first part of the secure connections standard Signed-off-by: Markus Hentsch * Add notes about the classifications Signed-off-by: Markus Hentsch * Add considered options and open questions Signed-off-by: Markus Hentsch * Fix linter problems Signed-off-by: Markus Hentsch * Add TLS standardization Signed-off-by: Markus Hentsch * Add database and message queue channel security, extend TLS cipher rules Signed-off-by: Markus Hentsch * Add remaining decision sections Signed-off-by: Markus Hentsch * Add testing script for secure connection standard (WIP) Signed-off-by: Markus Hentsch * Turn avoiding CBC mode into a recommendation. https://crypto.stackexchange.com/a/95660 Signed-off-by: Markus Hentsch * Refactor the TLS test script to use SSLyze and implement all tests based on the current standard draft Signed-off-by: Markus Hentsch * Add testing README and reference Signed-off-by: Markus Hentsch * Address review feedback Signed-off-by: Markus Hentsch * Fix typo in test script comment Signed-off-by: Markus Hentsch * Add glossary and rephrase "SCS" to "SCS project" Signed-off-by: Markus Hentsch * Rename standard filename due to conflicting counter Signed-off-by: Markus Hentsch * Refine the scope in regards to the communication channels Signed-off-by: Markus Hentsch * s/IPsec/WireGuard/ Signed-off-by: Markus Hentsch * Fix option references for oslo.messaging ssl Signed-off-by: Markus Hentsch * Add RFC link for TLS deprecation Signed-off-by: Markus Hentsch * Don't endorse internal CAs specifically Signed-off-by: Markus Hentsch * Refactor test script to check Mozilla TLS recommendations Signed-off-by: Markus Hentsch * Update standard to reference Mozilla's TLS recommendations Signed-off-by: Markus Hentsch * Migrate test script requirements to requirements.in Signed-off-by: Markus Hentsch * Add libvirt security choices to design considerations Signed-off-by: Markus Hentsch * Add open question about libvirt hardening Signed-off-by: Markus Hentsch * Relax the requirement for the libvirt port Signed-off-by: Markus Hentsch * Rephrase and clarify libvirt security recommendations and questions Signed-off-by: Markus Hentsch * Add Mozilla TLS JSON override option to test script Signed-off-by: Markus Hentsch * Fully parameterize Mozilla TLS config in test script Signed-off-by: Markus Hentsch * Rename cli args in test script Signed-off-by: Markus Hentsch * Add Mozilla TLS JSON copy and staging YAML entry Signed-off-by: Markus Hentsch * Add remark about internal audits Signed-off-by: Markus Hentsch * Remove specific MQ SSL config examples, refer to docs Signed-off-by: Markus Hentsch * Align header naming with latest standards template Signed-off-by: Markus Hentsch * Add storage channels Signed-off-by: Markus Hentsch * Assign document number 0122 Signed-off-by: Markus Hentsch * Update scs-compatible-test.yaml Signed-off-by: Markus Hentsch * Remove bare URLs Signed-off-by: Anja Strunk * Remove tailing whitespaces Signed-off-by: Anja Strunk * Fix markdown linter Signed-off-by: Anja Strunk * remove bare URLs Signed-off-by: Anja Strunk * Fix markdown lint errors Signed-off-by: Anja Strunk * Fix markdown lint errors Signed-off-by: Anja Strunk * Fix markdown lint errors Signed-off-by: Anja Strunk * Fix markdown lint errors Signed-off-by: Anja Strunk * Fix markdown lint errors Signed-off-by: Anja Strunk * Change verison number of secure connection standard as it conflicts with node to node encryption DR Signed-off-by: Anja Strunk * Add new linke char at the end of file to fix MD047 Signed-off-by: Anja Strunk * Replace absolute dead links with relative links Signed-off-by: Anja Strunk * Fix dead links Signed-off-by: Anja Strunk * Use absolute path as relative pathes are not allowed Signed-off-by: Anja Strunk --------- Signed-off-by: Markus Hentsch Signed-off-by: Anja Strunk Co-authored-by: Dominik Pataky <33180520+bitkeks@users.noreply.github.com> Co-authored-by: anjastrunk <119566837+anjastrunk@users.noreply.github.com> Co-authored-by: Anja Strunk --- Standards/scs-0125-v1-secure-connections.md | 277 +++++++++++++++ .../scs-0214-v1-k8s-node-distribution.md | 1 - Tests/iaas/secure-connections/README.md | 61 ++++ .../mozilla-tls-profiles/5.7.json | 209 ++++++++++++ .../mozilla-tls-profiles/README.md | 4 + Tests/iaas/secure-connections/tls-checker.py | 315 ++++++++++++++++++ Tests/requirements.in | 1 + Tests/requirements.txt | 12 +- Tests/testing/scs-compatible-test.yaml | 37 ++ 9 files changed, 914 insertions(+), 3 deletions(-) create mode 100644 Standards/scs-0125-v1-secure-connections.md create mode 100644 Tests/iaas/secure-connections/README.md create mode 100644 Tests/iaas/secure-connections/mozilla-tls-profiles/5.7.json create mode 100644 Tests/iaas/secure-connections/mozilla-tls-profiles/README.md create mode 100644 Tests/iaas/secure-connections/tls-checker.py diff --git a/Standards/scs-0125-v1-secure-connections.md b/Standards/scs-0125-v1-secure-connections.md new file mode 100644 index 000000000..b2df96743 --- /dev/null +++ b/Standards/scs-0125-v1-secure-connections.md @@ -0,0 +1,277 @@ +--- +title: Secure Connections +type: Standard # | Decision Record | Procedural +status: Draft +track: IaaS # | IaaS | Ops | KaaS | IAM +--- + +## Introduction + +A lot of internal and external connectivity is established to and within a cloud infrastructure. +Due to the nature of the IaaS approach, many communication channels may occasionally or even primarily carry potentially sensitive data of customers. +To protect this data from both tampering and unintended disclosure, communication channels need to be properly secured. + +For this reason, the [SCS project](https://scs.community) standardizes the use of common protection mechanisms for communication channels in OpenStack infrastructures. + +## Terminology + +| Term | Meaning | +|---|---| +| CA | Certificate Authority | +| CSP | Cloud Service Provider, provider managing the OpenStack infrastructure | +| PKI | Public Key Infrastructure | +| SDN | Software-Defined Networking | +| SSL | Secure Sockets Layer, the predecessor of TLS | +| TLS | Transport Layer Security | +| Compute Host | System within the IaaS infrastructure that runs the hypervisor services and hosts virtual machines | + +## Motivation + +As mentioned above, a lot of communication channels in an OpenStack infrastructure carry data that is potentially sensitive. +For example this includes authentication data of customers and internal OpenStack configuration data such as encryption keys among others. +OpenStack does not generically mandate or preconfigure the use of specific protection mechanisms by itself and instead only makes recommendations about best practices in its offical [Security Guide](https://docs.openstack.org/security-guide/). + +To address the potential lack of implementation of such mechanisms by a CSP and to establish a reliable foundation for communication data protection in SCS clouds, the SCS project formulates this standard for securing communication channels in the infrastructure, so that a customer can rely on adequate security mechanisms being in use. + +## Design Considerations + +There are many internal communication channels in OpenStack with different characteristics, location and eligible means of protection. +Not all channels are equally easy to secure and some protection mechanisms might put unbearable burdens on a CSP. +Hence, careful assessment is required to determine for which the SCS standard will either mandate or recommend the use of a protection mechanism. + +Note that this standard only focuses on security considerations for securing the Openstack API as well as inter-component connections, which a CSP has full control over on an infrastructure level. +This standard will not address the security of customer-deployed instances and services on top of OpenStack or other IaaS implementations. + +For this distinction to be made, applicable communication channels must be categorized and classified accordingly. + +### Communication Channels + +The following overview will classify the main communication channels. + +| # | Classification | Details | Example solution | +|---|---|---|---| +| 1 | OpenStack database backend traffic | Replication and sync between database instances of the OpenStack services' databases | SSL/TLS | +| 2 | OpenStack database frontend traffic | Communication between OpenStack services and their corresponding databases | SSL/TLS | +| 3 | Message queue traffic | Message queue communication between OpenStack components as provided by oslo.messaging | SSL/TLS | +| 4 | External API communication | HTTP traffic to services registered as external endpoints in the Keystone service catalog | SSL/TLS | +| 5 | Internal API communication | HTTP traffic to services registered as internal endpoints in the Keystone service catalog | SSL/TLS | +| 6 | Nova VM migration traffic | Nova VM migration data transfer traffic between compute nodes | QEMU-native TLS | +| 7 | External Neutron network traffic | VM-related traffic between the network/controller nodes and external networks (e.g. internet) established through routed provider networks and floating IPs | VPN | +| 8 | Internal Neutron network traffic | Traffic within Neutron SDN networks exchanged between internal systems such as network/controller and compute nodes | WireGuard | +| 9 | Storage network frontend traffic | Traffic exchanged between OpenStack and network storage backends (e.g. Ceph) | N/A* | +| 10 | Storage network replication traffic | Traffic exchanged between individual storage nodes within the network storage backend for replication purposes | N/A* | + +\* The characteristics of the storage network traffic is highly specific to the individual storage backend and no generic solution can be stated here. + +Notes about the classification categories and implications: + +1. Most database clustering solutions (e.g. MariaDB Galera) offer TLS-based encryption of their backend channels. This needs no additional configuration in OpenStack and is a configuration solely concerning the database cluster. +2. The database frontend interface is the primary connection target for the OpenStack services. OpenStack supports using TLS for database connections. +3. For the message queue, AMQP-based solutions such as RabbitMQ and QPid do offer TLS natively which is also supported by OpenStack. ZeroMQ does not and requires WireGuard or CIPSO instead. +4. External API endpoints can be protected easily by using a TLS proxy. They can then be registered with their HTTPS endpoint in the Keystone service catalog. The certificates of external APIs usually need to be signed by a well-known CA in order to be accepted by arbitrary external clients. +5. Internal API endpoints can be treated and secured similarly to the external ones using a TLS proxy and adequate certificates. +6. For protecting the data transferred between compute nodes during live-migration of VMs, [Nova offers support for QEMU-native TLS](https://docs.openstack.org/nova/latest/admin/secure-live-migration-with-qemu-native-tls.html). As an alternative, SSH is also a channel that Nova can be configured to use between hosts for this but requires passwordless SSH keys with root access to all other compute nodes which in turn requires further hardening. +7. Neutron's external network traffic leaves the IaaS infrastructure. This part is twofold: connections initiated by the VMs themselves (egress) and connections reaching VMs from the outside (ingress). The CSP cannot influence the egress connections but can offer VPNaaS for the ingress direction. +8. Neutron's internal network traffic is one of the hardest aspects to address. Due to the highly dynamic nature of SDN, connection endpoints and relations are constantly changing. There is no holistic approach currently offered or recommended by OpenStack itself. Encrypted tunnels could be established between all involved nodes but would require a scalable solution and reliable key management. WireGuard could be considered a good starting point for this. A per-tenant/per-customer encryption remains very hard to establish this way though. +9. The available means of securing frontend storage communication are dependent on the protocol used for communication between OpenStack and the storage backend. Some storage solutions and protocols might offer authentication and encryption functionality by default but that cannot be assumed to be the case for every possible backend. Furthermore, storage is highly sensitive to latency and performance impacts imposed by such measures. Due to the fact that OpenStack's volume encryption functionality encrypts block storage data before it enters the storage network, an unsecured communication channel may be considered not as much of a problem here as it is for other channels\*. +10. Storage network replication traffic is highly specific to the storage backend solution and its method of operation. For some backends this channel might not even exists, depending on their architecture. As such, in most cases it is up to the storage solution to provide adequate measures for protection of this channel. As with the frontend storage traffic, due to the possible at-rest encryption implemented by OpenStack, only already encrypted data is transferred here for some resources\*. + +\* Encryption of data implemented by OpenStack before it is passed to the storage backend currently only applies to block data of volumes that use an encrypted volume type. Other data (e.g. of unencrypted volumes, images) is transferred to and from the storage in plaintext. + +### libvirt Hypervisor Interface on Compute Nodes + +Live migration of virtual machines between compute hosts requires communication between the hypervisor services of the involved hosts. +In OpenStack, the libvirt virtualization API is used to control the hypervisor on compute nodes as well as to enable the live migration communication. +This libvirt interface allows direct control of the hypervisor. +Besides control of virtual machines themselves, in OpenStack this also includes attaching and detaching volumes, setting or retrieving their encryption keys and controlling network attachments. +As such, severe risks are associated with unauthorized access to this interface as it can easily compromise sensitive data of arbitrary tenants if abused. + +This is acknowledged in the OpenStack Security Note [OSSN-0007](https://wiki.openstack.org/wiki/OSSN/OSSN-0007), which recommends either configuring SASL and/or TLS for libvirt connections or utilizing the UNIX socket in combination with SSH. + +The OpenStack [kolla-ansible documentation](https://docs.openstack.org/kolla-ansible/latest/reference/compute/libvirt-guide.html#sasl-authentication) on Nova libvirt connections state: + +> This should not be considered as providing a secure, encrypted channel, since the username/password SASL mechanisms available for TCP are no longer considered cryptographically secure. + +This leaves only TLS or UNIX socket with SSH as viable options for securing the channel. + +#### TLS for libvirt and live migration + +Since the Stein release of OpenStack, Nova supports [QEMU-native TLS](https://docs.openstack.org/nova/latest/admin/secure-live-migration-with-qemu-native-tls.html) which protects the migration data streams using TLS. +It requires to add `LIBVIRTD_ARGS="--listen"` to the [QEMU configuration](https://libvirt.org/remote.html#libvirtd-configuration-file), which will lead to TLS being active on the libvirt interface per default (due to `listen_tls` defaulting to being enabled). +This protects data streams for migration as well as the hypervisor control channel data flow with TLS but does not restrict access. +Client certificates must be deployed additionally and libvirt configured accordingly[^4] in order to meaningfully restrict access to the interface as advised by the OSSN-0007 document, see restricting-access in [Libvirt doc](https://wiki.libvirt.org/TLSDaemonConfiguration.html#restricting-access). + +#### Local UNIX socket and SSH live migration + +As an alternative to the TLS setup, libvirt can be configured to use a local UNIX socket and Nova can be configured to use SSH to this socket for live migrations instead. +The regular libvirt port can then be limited to localhost (`127.0.0.1`) which will make it inaccessible from outside the host but still enables local connections to use it. +The challenge of this approach lies in restricting the SSH access on the compute nodes appropriately to avoid full root access across compute nodes for the SSH user identity that Nova will use for live migration. +This can be addressed by restricting the command set that is available and the paths that are accessible to these target SSH user identities on compute nodes, limiting them to the scope strictly required by the live migration. + +A basic setup for combining the UNIX socket with SSH live migration settings is illustrated below. + +Libvirt configuration: + +```conf +listen_tcp = 1 +listen_addr = "127.0.0.1" +unix_sock_group = "libvirt" +unix_sock_ro_perms = "0770" +unix_sock_rw_perms = "0770" +``` + +Nova configuration: + +```ini +[libvirt] +connection_uri= +live_migration_uri=qemu+ssh://... +live_migration_scheme = ssh + +``` + +### TLS Configuration Recommendations + +Server-side TLS configuration is complex and involves a lot of choices for protocol versions, cipher suites and algorithms. +Determining and maintaining secure configuration guidelines for this is non-trivial for a community project as it requires a high level security expertise and consistent evaluation. +For this reason, the standard should reference widely accepted best practices and established third party guidelines instead of creating and maintaining its own set of rules. + +[Mozilla publishes and maintains](https://wiki.mozilla.org/Security/Server_Side_TLS) TLS recommendations and corresponding presets for configuration and testing. +Considering Mozilla's well-established name in the internet and open source communities, this could qualify as a good basis for the standard concerning the TLS configurations. + +### Storage network protection + +As stated in the overview of communication channels, the existence and characteristics of the storage frontend and replication networks are highly specific to the storage backend solution in use. +In conjunction with the fact that storage performance is easily impacted by anything that introduces latency or reduces throughput on these channels, there is no easy recommendation on how to secure them that can be made here. + +This is partially mitigated by OpenStack's ability to encrypt storage data before it enters the storage backend, protecting the data regardless of the storage channels characteristics. +But this only applies to block data of volumes that use an encrypted volume type and does not apply to other data put into the storage backend by OpenStack, for example images. +As such, this does not fully address unsecured storage channels. + +However, requiring the network storage channels to be dedicated physical connections separate from the other channels like tenant VM traffic or API communication can increase both the reliability as well as security of the storage connections. +Therefore this standard should at least recommend a dedicated network infrastructure to be implemented for the storage if network storage backends are used, such as Ceph. + +### Options considered + +#### Option 1: fully mandate securing all channels without differentiation + +This option would reach the highest security standard and establish protection on all identified communication channels simultaneously. +However, this would burden CSPs greatly due to the difficulty of addressing some of the channels and properly maintaining the solution. +Also there is a risk of making this incompatible with existing infrastructures due to some of their specifics being mutually exclusive to the more intricate protection mechanisms such as cross-node WireGuard configurations. +As a third aspect, not all mechanisms might fulfill the CSPs requirements regarding performance and stability and the SCS standard cannot in good faith force CSPs to use technologies not suited to their infrastructures. + +This seems like a bad option from many perspectives. +It also allows very little flexibility and might even make SCS conformance unappealing to CSPs due to the immense effort required to reach it. + +#### Option 2: only make recommendations + +This option would limit the SCS project to only recommend mechanisms in this standard like presented in the OpenStack Security Guide. +Although this can increase awareness about the best practices recommended by OpenStack and maybe encourage CSPs to abide by them, it would actually contribute very little to the security baseline of SCS infrastructures as a whole since everything would stay optional. + +This option would be very easy to standardize and get consensus on due to its lightweight and optional nature. +However, the actual added value for SCS is questionable at best. + +#### Option 3: mix recommendations and obligations + +This option forms a middle ground between options 1 and 2. +For this, the standard needs to carefully assess each communication channel, mechanisms for protecting it and the effort required to do so as well as the implications. +Then, only for mechanisms that are known to be reliable, are feasible to implement and for which the benefits clearly outweigh the integration effort required, should this standard enforce their implementation in a permissive way. +For any remaining mechanisms the SCS standard should only make recommendations and refer to known best practices where applicable. + +This option would still offer improvements over arbitrary OpenStack clouds by establishing a baseline that goes beyond mere recommendations while still taking into account that not all communication channels are equally easy to secure and allowing necessary flexibility for the CSP. + +## Open questions + +### Choosing the best protection mechanism for the libvirt hypervisor interface + +As described in the Design Considerations section, there are multiple ways of securing the libvirt interface and live migration channels using TLS or SSH mechanisms. +Upon closer inspection, this consists of two problems to address: + +1) encrypting migration traffic utilizing the libvirt interface itself +2) identifying/authenticating connecting clients and properly restricting their permission set + +When considering problem no. 1 in an isolated fashion, the QEMU-native TLS approach could be considered preferable simply due to it being officially recommended and documented by upstream OpenStack and tightly integrated into QEMU. + +However, once problem no. 2 is taken into account, the choice does not seem as obvious anymore due to the fact that in order to properly authenticate clients in the TLS case, X.509 client certificate authentication along with a corresponding PKI as well as key management would be required. +Although similar aspects would be relevant for the SSH approach where SSH key and identity management as well as proper permission restriction would need to be implemented, the SSH approach could turn out less complex due to the fact that the foundation for SSH identities most likely already exists on a node-level and does not need to rely on a full PKI. + +To properly compare both possible approaches of securing the libvirt interface, extensive testing and evaulation would be necessary along with a sophisticated key and node identity management for compute nodes which this standard alone does not provide. + +### Verifying standard conformance for internal mechanisms + +Most of the mentioned communication channels to be secured are part of the internal IaaS infrastructure of a SCS cloud. +When an internal protection mechanism is implemented by the CSP it cannot be verified from outside of the infrastructure without administrative access to the infrastructure. + +Thus, the SCS community is unable to fully assess a CSPs conformance to this standard without a dedicated audit of the infrastructure. + +## Standard + +This standard will mandate or recommend appropriate measures for securing the communication channels based on existing standards and recommendations. +It will reference documents like the [OpenStack Security Guide](https://docs.openstack.org/security-guide/) where applicable. + +### Transport Layer Security (TLS) + +- All server-side TLS configurations integrated into the infrastructure as covered by this standard MUST adhere to the ["Intermediate" Mozilla TLS configuration](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29). + +### API Interfaces + +- Internal API endpoints of all OpenStack services MUST use TLS. Their endpoint as registered in the Keystone service catalog MUST be an HTTPS address. +- External API endpoints of all OpenStack services MUST use TLS. Their endpoint as registered in the Keystone service catalog MUST be an HTTPS address. + +You MAY refer to [TLS proxies and HTTP services](https://docs.openstack.org/security-guide/secure-communication/tls-proxies-and-http-services.html) and [Secure reference architectures](https://docs.openstack.org/security-guide/secure-communication/secure-reference-architectures.html) of the OpenStack Security Guide for best practices and recommendations. + +### Database Connections + +- The database servers used by the OpenStack services MUST be configured for TLS transport. +- All OpenStack services MUST have TLS configured for the database connection via the `ssl_ca` directive. See [OpenStack service database configuration](https://docs.openstack.org/security-guide/databases/database-access-control.html#openstack-service-database-configuration). +- Database user accounts for the OpenStack services SHOULD be configured to require TLS connections via the `REQUIRE SSL` SQL directive. See [Require user accounts to require SSL transport](https://docs.openstack.org/security-guide/databases/database-access-control.html#require-user-accounts-to-require-ssl-transport). +- Security MAY be further enhanced by configuring the OpenStack services to use X.509 client certificates for database authentication. See [Authentication with X.509 certificates](https://docs.openstack.org/security-guide/databases/database-access-control.html#authentication-with-x-509-certificates). + +### Message Queue Connections + +- If using RabbitMQ or Qpid as the message queue service, the SSL functionality of the message broker MUST be enabled and used by the OpenStack services. See [Messaging transport security](https://docs.openstack.org/security-guide/messaging/security.html#messaging-transport-security). +- If using Apache Kafka, the server listener MUST be configured to accept SSL connections. See [Apache Kafka Listener Configuration](https://kafka.apache.org/documentation/#listener_configuration). OpenStack services MUST be configured to use an SSL setting for the Kafka driver. See [Kafka Driver Security Options](https://docs.openstack.org/oslo.messaging/latest/admin/kafka.html#security-options). + +### Hypervisor and Live Migration Connections + +- The live migration connections between compute nodes SHOULD be secured by encryption. + - If QEMU and libvirt are used, QEMU-native TLS is an approach officially documented by OpenStack. See [Secure live migration with QEMU-native TLS](https://docs.openstack.org/nova/latest/admin/secure-live-migration-with-qemu-native-tls.html). As an alternative, SSH-based live migration MAY be configured instead. +- If using libvirt as the hypervisor interface on compute nodes the libvirt port (as per its `listen_addr` configuration option) SHOULD NOT be exposed to the network in an unauthenticated and unprotected fashion: + - For the QEMU-native TLS configuration, it is RECOMMENDED to enforce TLS client certificate authentication and assign corresponding client identities to connecting compute nodes. + - For the SSH-based live migration approach, it is RECOMMENDED to limit the libvirt port/socket to the local host and establish SSH key pairs for compute nodes in conjunction with restricted SSH permissions. + +See the [corresponding Design Considerations section](#libvirt-hypervisor-interface-on-compute-nodes) for more details about the mentioned approaches. + +### External VM Connections + +- As an OPTIONAL measure to assist customers in protecting external connections to their OpenStack networks and VMs, the infrastructure MAY provide VPNaaS solutions to users. + - For example the Neutron VPNaaS service MAY be integrated into the infrastructure with the Neutron VPNaaS API extension enabled. See the [Neutron VPNaaS documentation](https://docs.openstack.org/neutron-vpnaas/latest/). + +### Internal Neutron Connections + +- As an OPTIONAL measure to protect Neutron SDN traffic between physical nodes within the infrastructure, encrypted tunnels MAY be established between nodes involved in Neutron networks, such as compute and network controller nodes, at the network interfaces configured in Neutron (e.g. via WireGuard or similar solutions). + +### Storage Connections + +- For storage backends that are connected to OpenStack over the network, the network connections between the OpenStack components and the storage backend SHOULD be located on a separate physical network with dedicated interfaces at the involved nodes. +- For storage backends that transfer replication data between individual storage nodes, the connections between those nodes SHOULD be implemented by a dedicated physical network. +- Where applicable, storage traffic between OpenStack components and the storage backend (frontend traffic) as well as storage replication traffic between storage nodes themselves (backend traffic) MAY be encrypted using the storage protocols native security features (if any) or a generic solution such as WireGuard. + +## Related Documents + +- [OpenStack Security Guide](https://docs.openstack.org/security-guide/) + - [OpenStack Security Guide: Secure communication](https://docs.openstack.org/security-guide/secure-communication.html) + - [OpenStack Security Guide: Database transport security](https://docs.openstack.org/security-guide/databases/database-transport-security.html) + - [OpenStack Security Guide: Messaging transport security](https://docs.openstack.org/security-guide/messaging/security.html#messaging-transport-security) +- [Nova Documentation: Secure live migration with QEMU-native TLS](https://docs.openstack.org/nova/latest/admin/secure-live-migration-with-qemu-native-tls.html) +- [MozillaWiki: Security / Server Side TLS](https://wiki.mozilla.org/Security/Server_Side_TLS) + +## Conformance Tests + +Conformance tests are limited to communication channels exposed to users, such as the public API interfaces. +Internal channels and APIs are currently not part of the automated conformance tests because they are not exposed and cannot be audited without direct access to the infrastructure. + +There is a test suite in [`tls-checker.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/secure-connections/tls-checker.py). +The test suite connects to the OpenStack API and retrieves all public API endpoints from the service catalog. +It then connects to each endpoint and verifies the compliance to the standard by checking SSL/TLS properties against the Mozilla TLS preset. +Please consult the associated [README.md](https://github.com/SovereignCloudStack/standards/blob/main/Tests/iaas/secure-connections/README.md) for detailed setup and testing instructions. diff --git a/Standards/scs-0214-v1-k8s-node-distribution.md b/Standards/scs-0214-v1-k8s-node-distribution.md index ce70e605e..7e48e1389 100644 --- a/Standards/scs-0214-v1-k8s-node-distribution.md +++ b/Standards/scs-0214-v1-k8s-node-distribution.md @@ -84,4 +84,3 @@ If the standard is used by a provider, the following decisions are binding and v [k8s-ha]: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/ [k8s-large-clusters]: https://kubernetes.io/docs/setup/best-practices/cluster-large/ [scs-0213-v1]: https://github.com/SovereignCloudStack/standards/blob/main/Standards/scs-0213-v1-k8s-nodes-anti-affinity.md -[k8s-labels-docs]: https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone diff --git a/Tests/iaas/secure-connections/README.md b/Tests/iaas/secure-connections/README.md new file mode 100644 index 000000000..9262f1ee2 --- /dev/null +++ b/Tests/iaas/secure-connections/README.md @@ -0,0 +1,61 @@ +# Secure Connections Standard Test Suite + +## Test Environment Setup + +> **NOTE:** The test execution procedure does not require cloud admin rights. + +A valid cloud configuration for the OpenStack SDK in the shape of "`clouds.yaml`" is mandatory[^1]. +**This file is expected to be located in the current working directory where the test script is executed unless configured otherwise.** + +[^1]: [OpenStack Documentation: Configuring OpenStack SDK Applications](https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html) + +The test execution environment can be located on any system outside of the cloud infrastructure that has OpenStack API access. +Make sure that the API access is configured properly in "`clouds.yaml`". + +It is recommended to use a Python virtual environment[^2]. +Next, install the libraries required by the test suite: + +```bash +pip3 install openstacksdk sslyze +``` + +> Note: the version of the sslyze library determines the [version of the Mozilla TLS recommendation JSON](https://wiki.mozilla.org/Security/Server_Side_TLS#JSON_version_of_the_recommendations) that it checks against. + +Within this environment execute the test suite. + +[^2]: [Python 3 Documentation: Virtual Environments and Packages](https://docs.python.org/3/tutorial/venv.html) + +## Test Execution + +The test suite is executed as follows: + +```bash +python3 tls-checker.py --os-cloud mycloud +``` + +As an alternative to "`--os-cloud`", the "`OS_CLOUD`" environment variable may be specified instead. +The parameter is used to look up the correct cloud configuration in "`clouds.yaml`". +For the example command above, this file should contain a `clouds.mycloud` section like this: + +```yaml +--- +clouds: + mycloud: + auth: + auth_url: ... + ... + ... +``` + +For any further options consult the output of "`python3 tls-checker.py --help`". + +### Script Behavior & Test Results + +The script will print all actions and passed tests to `stdout`. + +If all tests pass, the script will return with an exit code of `0`. + +If any test fails, the script will halt, print the exact error to `stderr` and return with a non-zero exit code. + +Any tests that indicate a recommendation of the standard is not met, will print a warning message under the corresponding endpoint output. +However, unmet recommendations will not count as errors. diff --git a/Tests/iaas/secure-connections/mozilla-tls-profiles/5.7.json b/Tests/iaas/secure-connections/mozilla-tls-profiles/5.7.json new file mode 100644 index 000000000..764aadaeb --- /dev/null +++ b/Tests/iaas/secure-connections/mozilla-tls-profiles/5.7.json @@ -0,0 +1,209 @@ +{ + "version": 5.7, + "href": "https://ssl-config.mozilla.org/guidelines/5.7.json", + "configurations": { + "modern": { + "certificate_curves": ["prime256v1", "secp384r1"], + "certificate_signatures": ["ecdsa-with-SHA256", "ecdsa-with-SHA384", "ecdsa-with-SHA512"], + "certificate_types": ["ecdsa"], + "ciphers": { + "caddy": [], + "go": [], + "iana": [], + "openssl": [] + }, + "ciphersuites": [ + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256" + ], + "dh_param_size": null, + "ecdh_param_size": 256, + "hsts_min_age": 63072000, + "maximum_certificate_lifespan": 90, + "ocsp_staple": true, + "oldest_clients": ["Firefox 63", "Android 10.0", "Chrome 70", "Edge 75", "Java 11", "OpenSSL 1.1.1", "Opera 57", "Safari 12.1"], + "recommended_certificate_lifespan": 90, + "rsa_key_size": null, + "server_preferred_order": false, + "tls_curves": ["X25519", "prime256v1", "secp384r1"], + "tls_versions": ["TLSv1.3"] + }, + "intermediate": { + "certificate_curves": ["prime256v1", "secp384r1"], + "certificate_signatures": ["sha256WithRSAEncryption", "ecdsa-with-SHA256", "ecdsa-with-SHA384", "ecdsa-with-SHA512"], + "certificate_types": ["ecdsa", "rsa"], + "ciphers": { + "caddy": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" + ], + "go": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" + ], + "iana": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256" + ], + "openssl": [ + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDHE-RSA-CHACHA20-POLY1305", + "DHE-RSA-AES128-GCM-SHA256", + "DHE-RSA-AES256-GCM-SHA384", + "DHE-RSA-CHACHA20-POLY1305" + ] + }, + "ciphersuites": [ + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256" + ], + "dh_param_size": 2048, + "ecdh_param_size": 256, + "hsts_min_age": 63072000, + "maximum_certificate_lifespan": 366, + "ocsp_staple": true, + "oldest_clients": ["Firefox 27", "Android 4.4.2", "Chrome 31", "Edge", "IE 11 on Windows 7", "Java 8u31", "OpenSSL 1.0.1", "Opera 20", "Safari 9"], + "recommended_certificate_lifespan": 90, + "rsa_key_size": 2048, + "server_preferred_order": false, + "tls_curves": ["X25519", "prime256v1", "secp384r1"], + "tls_versions": ["TLSv1.2", "TLSv1.3"] + }, + "old": { + "certificate_curves": ["prime256v1", "secp384r1"], + "certificate_signatures": ["sha256WithRSAEncryption"], + "certificate_types": ["rsa"], + "ciphers": { + "caddy": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_3DES_EDE_CBC_SHA" + ], + "go": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_128_CBC_SHA256", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_3DES_EDE_CBC_SHA" + ], + "iana": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384", + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256", + "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_128_CBC_SHA256", + "TLS_RSA_WITH_AES_256_CBC_SHA256", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_3DES_EDE_CBC_SHA" + ], + "openssl": [ + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDHE-RSA-CHACHA20-POLY1305", + "DHE-RSA-AES128-GCM-SHA256", + "DHE-RSA-AES256-GCM-SHA384", + "DHE-RSA-CHACHA20-POLY1305", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-RSA-AES128-SHA256", + "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA384", + "ECDHE-ECDSA-AES256-SHA", + "ECDHE-RSA-AES256-SHA", + "DHE-RSA-AES128-SHA256", + "DHE-RSA-AES256-SHA256", + "AES128-GCM-SHA256", + "AES256-GCM-SHA384", + "AES128-SHA256", + "AES256-SHA256", + "AES128-SHA", + "AES256-SHA", + "DES-CBC3-SHA" + ] + }, + "ciphersuites": [ + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256" + ], + "dh_param_size": 1024, + "ecdh_param_size": 256, + "hsts_min_age": 63072000, + "maximum_certificate_lifespan": 366, + "ocsp_staple": true, + "oldest_clients": ["Firefox 1", "Android 2.3", "Chrome 1", "Edge 12", "IE8 on Windows XP", "Java 6", "OpenSSL 0.9.8", "Opera 5", "Safari 1"], + "recommended_certificate_lifespan": 90, + "rsa_key_size": 2048, + "server_preferred_order": true, + "tls_curves": ["X25519", "prime256v1", "secp384r1"], + "tls_versions": ["TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"] + } + } +} diff --git a/Tests/iaas/secure-connections/mozilla-tls-profiles/README.md b/Tests/iaas/secure-connections/mozilla-tls-profiles/README.md new file mode 100644 index 000000000..08c1328fd --- /dev/null +++ b/Tests/iaas/secure-connections/mozilla-tls-profiles/README.md @@ -0,0 +1,4 @@ +# Mozilla TLS Profiles + +Files in this folder are used for automated testing. +They are pulled from [Mozilla Wiki: Security/Server Side TLS](https://wiki.mozilla.org/Security/Server_Side_TLS#JSON_version_of_the_recommendations) diff --git a/Tests/iaas/secure-connections/tls-checker.py b/Tests/iaas/secure-connections/tls-checker.py new file mode 100644 index 000000000..d080c4123 --- /dev/null +++ b/Tests/iaas/secure-connections/tls-checker.py @@ -0,0 +1,315 @@ +"""SSL/TLS checker for OpenStack API endpoints + +This script retrieves the endpoint catalog from Keystone using the OpenStack +SDK and connects to each to each public endpoint to check its SSL/TLS settings. +The script relies on an OpenStack SDK compatible clouds.yaml file for +authentication with Keystone. + +For each endpoint, SSL/TLS protocol parameters supported by the server are +checked using the SSLyze Python library. +The script will fail with a non-zero exit code in case any standard violation +is discovered such as endpoints being non-conformant to the corresponding +Mozilla TLS recommendations. +Details about the conformance issues will be printed for each endpoint to +help with identifying and addressing the violations of the Mozilla TLS preset. +""" + +import argparse +import getpass +import json +import os +import sys +import typing + +import openstack +import sslyze +from sslyze.mozilla_tls_profile.mozilla_config_checker import ( + SCAN_COMMANDS_NEEDED_BY_MOZILLA_CHECKER, MozillaTlsConfigurationChecker, + MozillaTlsConfigurationEnum, ServerNotCompliantWithMozillaTlsConfiguration, + _MozillaTlsProfileAsJson) + + +class MozillaTlsValidator(object): + """Configurable wrapper for MozillaTlsConfigurationChecker + + Configures the MozillaTlsConfigurationChecker of SSLyze according + to given parameters concerning the Mozilla TLS Profile JSON to use + and configuration level to select (old, intermediate, modern). + + For reference see: https://wiki.mozilla.org/Security/Server_Side_TLS + """ + + def __init__(self, configuration_level: str, + json_path: typing.Optional[str]) -> None: + """Create a connection to an OpenStack cloud + + :param string configuration_level: + Name of the Mozilla TLS configuration level to select. + + :param string json_path: + Optional path to a JSON file containing the Mozilla TLS + recommendations. + + :returns: MozillaTlsValidator + """ + if json_path: + print(f"INFO: Loading custom Mozilla TLS Profile JSON from " + f"{json_path}") + if not os.path.exists(json_path) or not os.path.isfile(json_path): + raise Exception( + f"No such file '{json_path}'" + ) + with open(json_path, 'r') as json_file: + json_content = json.load(json_file) + self.profile = _MozillaTlsProfileAsJson(**json_content) + else: + self.profile = None + + for level in MozillaTlsConfigurationEnum: + if level.value == configuration_level: + self.config_level = level + print(f"INFO: Using profile level '{level.value}' of the " + f"Mozilla TLS configuration") + break + else: + valid_levels = [ + level.value for level in MozillaTlsConfigurationEnum + ] + raise Exception( + f"Mozilla TLS configuration profile level " + f"'{configuration_level}' is invalid, valid " + f"options are: {valid_levels}" + ) + + def check_scan_result(self, result: sslyze.ServerScanResult): + """ + Validate the given ServerScanResult against the Mozilla TLS Profile. + + Will raise a ServerNotCompliantWithMozillaTlsConfiguration exception + if any violations of the TLS profile are detected. + """ + if self.profile: + mozilla_checker = MozillaTlsConfigurationChecker(self.profile) + else: + mozilla_checker = MozillaTlsConfigurationChecker.get_default() + mozilla_checker.check_server(self.config_level, result) + + +def connect(cloud_name: str, password: typing.Optional[str] = None + ) -> openstack.connection.Connection: + """Create a connection to an OpenStack cloud + + :param string cloud_name: + The name of the configuration to load from clouds.yaml. + + :param string password: + Optional password override for the connection. + + :returns: openstack.connnection.Connection + """ + + if password: + return openstack.connect( + cloud=cloud_name, + password=password + ) + else: + return openstack.connect( + cloud=cloud_name, + ) + + +def retrieve_endpoints(conn: openstack.connection.Connection) \ + -> dict[str, dict[str, str]]: + """Enumerate all endpoints of the service catalog returned by the + current connection categorized by interface type and service. + + Resulting structure: + { + : { + : + } + } + + where is public, internal or admin. + """ + + ret = {} + + for svc in conn.service_catalog: + svc_name = svc['name'] + for endpoint in svc.get('endpoints'): + enp_type = endpoint['interface'] + enp_url = endpoint['url'] + subdict = ret.setdefault(enp_type, {}) + subdict[svc_name] = enp_url + + print("\nINFO: the following public endpoints have been retrieved from " + "the service catalog:") + for svc_name in ret["public"].keys(): + print( + f"↳ {svc_name} @ {ret['public'][svc_name]}" + ) + + return ret + + +def verify_tls(service: str, host: str, port: int, + mozilla_tls: MozillaTlsValidator) -> bool: + """Use SSLyze library to scan the SSL/TLS interface of the server. + + Evaluates the SSL/TLS configurations the server reports as supported. + Checks the scan results against the Mozilla TLS recommendation preset. + Prints any issues found with details. + + If `mozilla_json_preset` is passed into this function, it is interpreted + as the Mozilla TLS Profile JSON to be used. If this argument is None or + not specified, the default JSON shipped with the respective SSLyze release + is used instead. The format of this optional argument is expected to be + the parsed JSON as dict. + + Returns True if no errors were encountered, False otherwise. + """ + + errors_encountered = 0 + server = sslyze.ServerNetworkLocation(host, port) + scans = SCAN_COMMANDS_NEEDED_BY_MOZILLA_CHECKER + request = sslyze.ServerScanRequest(server, scan_commands=scans) + scanner = sslyze.Scanner() + scanner.queue_scans([request]) + for result in scanner.get_results(): + assert result.scan_result, ( + f"Service '{service}' at {host}:{port} did not respond to " + f"TLS connection" + ) + try: + mozilla_tls.check_scan_result(result) + print( + f"Service '{service}' at {host}:{port} complies to " + f"TLS recommendations: PASS" + ) + except ServerNotCompliantWithMozillaTlsConfiguration as e: + print( + f"Service '{service}' at {host}:{port} complies to " + f"TLS recommendations: FAIL" + ) + for criteria, error_description in e.issues.items(): + print(f"↳ {criteria}: {error_description}") + errors_encountered += 1 + + return errors_encountered == 0 + + +def check_endpoints(endpoints: dict[str, str], + ignore: typing.Optional[str], + mozilla_tls: MozillaTlsValidator) -> None: + ignore_list = ignore.split(',') if ignore else [] + error_count = 0 + for service in endpoints: + url = endpoints[service] + host_ref = url.split("://", 1)[-1].split("/", 1)[0] + + # Check if the endpoint matches any of the given ignore patterns. + ignored = False + for ignore_pattern in ignore_list: + if ignore_pattern in host_ref: + print( + f"INFO: Matching ignore rule for '{ignore_pattern}', " + f"ignoring endpoint: {url}" + ) + ignored = True + break + if ignored: + continue + + # Default to port 443 if no port is specified + if ':' in host_ref: + host, port = host_ref.split(':', 1) + else: + host = host_ref + port = 443 + + print(f"INFO: Checking public '{service}' endpoint {host}:{port} ...") + # Collect errors instead of failing immediately; this makes the output + # more useful since all endpoints are checked in one run and the + # printed output will cover all of them, logging all issues at once + error_count = error_count if verify_tls( + service, host, int(port), mozilla_tls + ) else error_count + 1 + + print( + f"INFO: Number of endpoints that failed compliance check: " + f"{error_count} (out of {len(endpoints)})" + ) + if error_count > 0: + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description="SCS Domain Manager Conformance Checker") + parser.add_argument( + "--os-cloud", type=str, + help="Name of the cloud from clouds.yaml, alternative " + "to the OS_CLOUD environment variable" + ) + parser.add_argument( + "--ask", + help="Ask for password interactively instead of reading it from the " + "clouds.yaml", + action="store_true" + ) + parser.add_argument( + "--ignore", type=str, + help="Comma-separated list of host names or host:port combinations " + "to exclude from testing", + ) + parser.add_argument( + "--debug", action="store_true", + help="Enable OpenStack SDK debug logging" + ) + parser.add_argument( + "--mozilla-profile-json", type=str, + help="Path to the Mozilla TLS Profile JSON to be used as the basis " + "for the checks (optional)", + ) + moz_tls_default_level = "intermediate" + parser.add_argument( + "--mozilla-profile-level", type=str, + default=moz_tls_default_level, + help=f"Name of the Mozilla TLS Profile configuration level name " + f"(default: {moz_tls_default_level})", + ) + args = parser.parse_args() + openstack.enable_logging(debug=args.debug) + + # parse cloud name for lookup in clouds.yaml + cloud = os.environ.get("OS_CLOUD", None) + if args.os_cloud: + cloud = args.os_cloud + assert cloud, ( + "You need to have the OS_CLOUD environment variable set to your " + "cloud name or pass it via --os-cloud" + ) + conn = connect( + cloud, + password=getpass.getpass("Enter password: ") if args.ask else None + ) + + endpoints_catalog = retrieve_endpoints(conn) + assert "public" in endpoints_catalog, ( + "No public endpoints found in the service catalog" + ) + endpoints = endpoints_catalog["public"] + + mozilla_tls = MozillaTlsValidator( + args.mozilla_profile_level, + # load the Mozilla TLS Profile from JSON if specified + args.mozilla_profile_json if args.mozilla_profile_json else None + ) + + check_endpoints(endpoints, args.ignore, mozilla_tls) + + +if __name__ == "__main__": + main() diff --git a/Tests/requirements.in b/Tests/requirements.in index cb3c3bac0..8a96fb744 100644 --- a/Tests/requirements.in +++ b/Tests/requirements.in @@ -6,4 +6,5 @@ python-dateutil PyYAML openstacksdk # TODO: move into iaas/requirements.in requests +sslyze tomli diff --git a/Tests/requirements.txt b/Tests/requirements.txt index 19f9e1990..6597d173b 100644 --- a/Tests/requirements.txt +++ b/Tests/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile --output-file=requirements.txt requirements.in # aiohappyeyeballs==2.3.5 # via aiohttp @@ -31,7 +31,9 @@ charset-normalizer==3.3.2 click==8.1.7 # via -r requirements.in cryptography==43.0.1 - # via openstacksdk + # via + # openstacksdk + # sslyze decorator==5.1.1 # via # dogpile-cache @@ -67,6 +69,8 @@ multidict==6.0.5 # via # aiohttp # yarl +nassl==1.0.3 + # via sslyze netifaces==0.11.0 # via openstacksdk openstacksdk==3.3.0 @@ -109,10 +113,14 @@ six==1.16.0 # via # kubernetes-asyncio # python-dateutil +sslyze==1.3.4 + # via -r requirements.in stevedore==5.2.0 # via # dogpile-cache # keystoneauth1 +tls-parser==1.2.2 + # via sslyze tomli==2.0.1 # via -r requirements.in urllib3==2.2.2 diff --git a/Tests/testing/scs-compatible-test.yaml b/Tests/testing/scs-compatible-test.yaml index 5b37c8904..2d37c6bf3 100644 --- a/Tests/testing/scs-compatible-test.yaml +++ b/Tests/testing/scs-compatible-test.yaml @@ -1,6 +1,43 @@ name: SCS Compatible url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible.yaml +variables: + - os_cloud versions: + - version: v5 + standards: + - name: Secure Connections + url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0122-v1-secure-connections.md + parameters: + - mozilla_tls_profile_version: "5.7" + - mozilla_tls_profile_preset: "intermediate" + check_tools: + - executable: ./iaas/secure-connections/tls-checker.py + args: --os-cloud {os_cloud} --mozilla-profile-json ./iaas/secure-connections/mozilla-tls-profiles/{mozilla_tls_profile_version}.json --mozilla-profile-level {mozilla_tls_profile_preset} + id: tls-configuration-check + - name: Standard flavors + url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0103-v1-standard-flavors.md + check_tools: + - executable: ./iaas/standard-flavors/flavors-openstack.py + args: "./iaas/scs-0103-v1-flavors.yaml" + - name: Standard images + url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0104-v1-standard-images.md + check_tools: + - executable: ./iaas/standard-images/images-openstack.py + args: "./iaas/scs-0104-v1-flavors.yaml" + - name: Flavor naming + url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v3-flavor-naming.md + check_tools: + - executable: ./iaas/flavor-naming/flavor-names-openstack.py + args: "--mand=./iaas/scs-0100-v3-flavors.yaml" + - name: Image metadata + url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md + check_tools: + - executable: ./iaas/image-metadata/image-md-check.py + args: -s -v + - name: OpenStack Powered Compute v2022.11 + url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2022.11.json + condition: mandatory + # Unfortunately, no wrapper to run refstack yet, needs to be added - version: v4 standards: - name: Standard flavors From 3b478a786a958ad5a24d23217503af351b7938c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Mon, 25 Nov 2024 23:05:43 +0000 Subject: [PATCH 15/20] Stabilize scs-0214-v2 (#835) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Stabilize scs-0214-v1 * Relax wording to reflect weaknesses in the test for scs-0214-v2 * skip unit test because it no longer applies Signed-off-by: Matthias Büchse Co-authored-by: Kurt Garloff --- .../scs-0214-v2-k8s-node-distribution.md | 20 ++----------------- ...ode-distribution-implementation-testing.md | 20 +++++-------------- .../k8s-node-distribution/check_nodes_test.py | 6 +++--- .../k8s_node_distribution_check.py | 13 +++++------- 4 files changed, 15 insertions(+), 44 deletions(-) diff --git a/Standards/scs-0214-v2-k8s-node-distribution.md b/Standards/scs-0214-v2-k8s-node-distribution.md index 3b4915492..37af338ee 100644 --- a/Standards/scs-0214-v2-k8s-node-distribution.md +++ b/Standards/scs-0214-v2-k8s-node-distribution.md @@ -1,7 +1,8 @@ --- title: Kubernetes Node Distribution and Availability type: Standard -status: Draft +status: Stable +stabilized_at: 2024-11-21 replaces: scs-0214-v1-k8s-node-distribution.md track: KaaS --- @@ -100,23 +101,6 @@ These labels MUST be kept up to date with the current state of the deployment. The field gets autopopulated most of the time by either the kubelet or external mechanisms like the cloud controller. -- `topology.scs.community/host-id` - - This is an SCS-specific label; it MUST contain the hostID of the physical machine running - the hypervisor (NOT: the hostID of a virtual machine). Here, the hostID is an arbitrary identifier, - which need not contain the actual hostname, but it should nonetheless be unique to the host. - This helps identify the distribution over underlying physical machines, - which would be masked if VM hostIDs were used. - -## Conformance Tests - -The script `k8s-node-distribution-check.py` checks the nodes available with a user-provided -kubeconfig file. Based on the labels `topology.scs.community/host-id`, -`topology.kubernetes.io/zone`, `topology.kubernetes.io/region` and `node-role.kubernetes.io/control-plane`, -the script then determines whether the nodes are distributed according to this standard. -If this isn't the case, the script produces an error. -It also produces warnings and informational outputs, e.g., if labels don't seem to be set. - ## Previous standard versions This is version 2 of the standard; it extends [version 1](scs-0214-v1-k8s-node-distribution.md) with the diff --git a/Standards/scs-0214-w1-k8s-node-distribution-implementation-testing.md b/Standards/scs-0214-w1-k8s-node-distribution-implementation-testing.md index 4366365a0..6460cc195 100644 --- a/Standards/scs-0214-w1-k8s-node-distribution-implementation-testing.md +++ b/Standards/scs-0214-w1-k8s-node-distribution-implementation-testing.md @@ -16,25 +16,15 @@ Worker nodes can also be distributed over "failure zones", but this isn't a requ Distribution must be shown through labelling, so that users can access these information. Node distribution metadata is provided through the usage of the labels -`topology.kubernetes.io/region`, `topology.kubernetes.io/zone` and -`topology.scs.community/host-id` respectively. - -At the moment, not all labels are set automatically by most K8s cluster utilities, which incurs -additional setup and maintenance costs. +`topology.kubernetes.io/region` and `topology.kubernetes.io/zone`. ## Automated tests -### Notes - -The test for the [SCS K8s Node Distribution and Availability](https://github.com/SovereignCloudStack/standards/blob/main/Standards/scs-0214-v2-k8s-node-distribution.md) -checks if control-plane nodes are distributed over different failure zones (distributed into -physical machines, zones and regions) by observing their labels defined by the standard. - -### Implementation +Currently, automated testing is not readily possible because we cannot access information about +the underlying host of a node (as opposed to its region and zone). Therefore, the test will only output +a tentative result. -The script [`k8s_node_distribution_check.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/kaas/k8s-node-distribution/k8s_node_distribution_check.py) -connects to an existing K8s cluster and checks if a distribution can be detected with the labels -set for the nodes of this cluster. +The current implementation can be found in the script [`k8s_node_distribution_check.py`](https://github.com/SovereignCloudStack/standards/blob/main/Tests/kaas/k8s-node-distribution/k8s_node_distribution_check.py). ## Manual tests diff --git a/Tests/kaas/k8s-node-distribution/check_nodes_test.py b/Tests/kaas/k8s-node-distribution/check_nodes_test.py index d32edccfb..439d1b18e 100644 --- a/Tests/kaas/k8s-node-distribution/check_nodes_test.py +++ b/Tests/kaas/k8s-node-distribution/check_nodes_test.py @@ -42,9 +42,9 @@ def test_not_enough_nodes(caplog, load_testdata): @pytest.mark.parametrize("yaml_key", ["no-distribution-1", "no-distribution-2"]) -def test_no_distribution(yaml_key, caplog, load_testdata): +def notest_no_distribution(yaml_key, caplog, load_testdata): data = load_testdata[yaml_key] - with caplog.at_level("ERROR"): + with caplog.at_level("WARNING"): assert check_nodes(data.values()) == 2 assert len(caplog.records) == 1 record = caplog.records[0] @@ -52,7 +52,7 @@ def test_no_distribution(yaml_key, caplog, load_testdata): assert record.levelname == "ERROR" -def test_missing_label(caplog, load_testdata): +def notest_missing_label(caplog, load_testdata): data = load_testdata["missing-labels"] assert check_nodes(data.values()) == 2 hostid_missing_records = [ diff --git a/Tests/kaas/k8s-node-distribution/k8s_node_distribution_check.py b/Tests/kaas/k8s-node-distribution/k8s_node_distribution_check.py index efac000d4..038d8a67c 100755 --- a/Tests/kaas/k8s-node-distribution/k8s_node_distribution_check.py +++ b/Tests/kaas/k8s-node-distribution/k8s_node_distribution_check.py @@ -22,7 +22,6 @@ and does require these labels to be set, but should yield overall pretty good initial results. - topology.scs.openstack.org/host-id # previously kubernetes.io/hostname topology.kubernetes.io/zone topology.kubernetes.io/region node-role.kubernetes.io/control-plane @@ -47,7 +46,6 @@ LABELS = ( "topology.kubernetes.io/region", "topology.kubernetes.io/zone", - "topology.scs.community/host-id", ) logger = logging.getLogger(__name__) @@ -164,12 +162,11 @@ def compare_labels(node_list, node_type="control"): ) return - if node_type == "control": - raise DistributionException("The distribution of nodes described in the standard couldn't be detected.") - elif node_type == "worker": - logger.warning("No node distribution could be detected for the worker nodes. " - "This produces only a warning, since it is just a recommendation.") - return + # + # if node_type == "control": + # raise DistributionException("The distribution of nodes described in the standard couldn't be detected.") + logger.warning("No node distribution could be detected for the worker nodes. " + "This produces only a warning, since it is just a recommendation.") def check_nodes(nodes): From bfd8c3aab801bad26b8dc000e29d1651fa53d670 Mon Sep 17 00:00:00 2001 From: sluetze <13255307+sluetze@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:48:32 +0100 Subject: [PATCH 16/20] allow versions which provide build info to be versionchecked (#839) Signed-off-by: sluetze <13255307+sluetze@users.noreply.github.com> --- Tests/kaas/k8s-version-policy/k8s_version_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/kaas/k8s-version-policy/k8s_version_policy.py b/Tests/kaas/k8s-version-policy/k8s_version_policy.py index cef272acd..a2acd3cd5 100755 --- a/Tests/kaas/k8s-version-policy/k8s_version_policy.py +++ b/Tests/kaas/k8s-version-policy/k8s_version_policy.py @@ -171,7 +171,7 @@ def __str__(self): def parse_version(version_str: str) -> K8sVersion: - cleansed = version_str.strip().removeprefix("v") + cleansed = version_str.strip().removeprefix("v").split("+")[0] # remove leading v as well as build info try: major, minor, patch = cleansed.split(".") return K8sVersion(int(major), int(minor), int(patch)) From bb0ed1333bbf6d971bae143653bf940d9124489e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 26 Nov 2024 19:42:25 +0000 Subject: [PATCH 17/20] Stabilize SCS-compatible KaaS v1 (#843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/scs-compatible-kaas.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 7cb2fbd58..a9582b395 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -16,6 +16,8 @@ modules: testcases: - id: cncf-k8s-conformance tags: [mandatory] + description: > + Must fulfill all requirements of [CNCF Kubernetes conformance](https://github.com/cncf/k8s-conformance/tree/master) - id: scs-0210-v2 name: Kubernetes version policy url: https://docs.scs.community/standards/scs-0210-v2-k8s-version-policy @@ -25,6 +27,8 @@ modules: testcases: - id: version-policy-check tags: [mandatory] + description: > + Must fulfill all requirements of - id: scs-0214-v2 name: Kubernetes node distribution and availability url: https://docs.scs.community/standards/scs-0214-v2-k8s-node-distribution @@ -34,6 +38,8 @@ modules: testcases: - id: node-distribution-check tags: [mandatory] + description: > + Must fulfill all requirements of - id: scs-0219-v1 name: KaaS networking url: https://docs.scs.community/standards/scs-0219-v1-kaas-networking @@ -43,12 +49,17 @@ modules: testcases: - id: kaas-networking-check tags: [mandatory] + description: > + Must fulfill all requirements of timeline: + - date: 2024-11-26 + v1: effective - date: 2024-02-28 versions: v1: draft versions: - version: v1 + stabilized_at: 2024-11-26 include: - cncf-k8s-conformance - scs-0210-v2 From 1af9994628a4cd47d77f117d31d0fdb02f276a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 26 Nov 2024 20:14:33 +0000 Subject: [PATCH 18/20] Hotfix: syntax error in yaml file (#844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/scs-compatible-kaas.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index a9582b395..74cbc3edb 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -53,6 +53,7 @@ modules: Must fulfill all requirements of timeline: - date: 2024-11-26 + versions: v1: effective - date: 2024-02-28 versions: From 53b5e45ef9ac0ef879346f51bba3d5d77df8fc97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Tue, 26 Nov 2024 21:05:08 +0000 Subject: [PATCH 19/20] Provide support script for adding new test subject (#817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse Co-authored-by: Kurt Garloff --- Tests/.gitignore | 1 + Tests/add_subject.py | 90 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100755 Tests/add_subject.py diff --git a/Tests/.gitignore b/Tests/.gitignore index 27f5fd549..e3a2cb922 100644 --- a/Tests/.gitignore +++ b/Tests/.gitignore @@ -1,2 +1,3 @@ htmlcov/ .coverage +.secret diff --git a/Tests/add_subject.py b/Tests/add_subject.py new file mode 100755 index 000000000..fabeb9932 --- /dev/null +++ b/Tests/add_subject.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +# +# add_subject.py +# +# (c) Matthias Büchse +# SPDX-License-Identifier: Apache-2.0 +import base64 +import getpass +import os +import os.path +import re +import shutil +import signal +import subprocess +import sys + +try: + from passlib.context import CryptContext + import argon2 # noqa:F401 +except ImportError: + print('Missing passlib and/or argon2. Please do:\npip install passlib argon2_cffi', file=sys.stderr) + sys.exit(1) + +# see ../compliance-monitor/monitor.py +CRYPTCTX = CryptContext(schemes=('argon2', 'bcrypt'), deprecated='auto') +SSH_KEYGEN = shutil.which('ssh-keygen') +SUBJECT_RE = re.compile(r"[a-zA-Z0-9_\-]+") + + +def main(argv, cwd): + if len(argv) != 1: + raise RuntimeError("Need to supply precisely one argument: name of subject") + subject = argv[0] + print(f"Attempt to add subject {subject!r}") + keyfile_path = os.path.join(cwd, '.secret', 'keyfile') + tokenfile_path = os.path.join(cwd, '.secret', 'tokenfile') + if os.path.exists(keyfile_path): + raise RuntimeError(f"Keyfile {keyfile_path} already present. Please proceed manually") + if os.path.exists(tokenfile_path): + raise RuntimeError(f"Tokenfile {tokenfile_path} already present. Please proceed manually") + if not SUBJECT_RE.fullmatch(subject): + raise RuntimeError(f"Subject name {subject!r} using disallowed characters") + sanitized_subject = subject.replace('-', '_') + print("Creating API key...") + while True: + password = getpass.getpass("Enter passphrase: ") + if password == getpass.getpass("Repeat passphrase: "): + break + print("No match. Try again...") + token = base64.b64encode(f"{subject}:{password}".encode('utf-8')) + hash_ = CRYPTCTX.hash(password) + with open(tokenfile_path, "wb") as fileobj: + os.fchmod(fileobj.fileno(), 0o600) + fileobj.write(token) + print("Creating key file using `ssh-keygen`...") + subprocess.check_call([SSH_KEYGEN, '-t', 'ed25519', '-C', sanitized_subject, '-f', keyfile_path, '-N', '', '-q']) + with open(keyfile_path + '.pub', "r") as fileobj: + pubkey_components = fileobj.readline().split() + print(f''' +The following SECRET files have been created: + + - {keyfile_path} + - {tokenfile_path} + +They are required for submitting test reports. You MUST keep them secure and safe. + +Insert the following snippet into compliance-monitor/bootstrap.yaml: + + - subject: {subject} + api_keys: + - "{hash_}" + keys: + - public_key: "{pubkey_components[1]}" + public_key_type: "{pubkey_components[0]}" + public_key_name: "primary" + +Make sure to submit a pull request with the changed file. Otherwise, the reports cannot be submitted. +''', end='') + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:], cwd=os.path.dirname(sys.argv[0]) or os.getcwd()) or 0) + except RuntimeError as e: + print(str(e), file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + print("Interrupted", file=sys.stderr) + sys.exit(128 + signal.SIGINT) From a878112e690acffc5b417bef6547bc1016f9f43a Mon Sep 17 00:00:00 2001 From: Michal Gubricky Date: Mon, 2 Dec 2024 12:01:42 +0100 Subject: [PATCH 20/20] Add cluster hardening tests (#759) * Add first tests for cluster hardening standard Signed-off-by: michal.gubricky * Add test for Pod security admission controller Signed-off-by: michal.gubricky * Add test for authorization methods Signed-off-by: michal.gubricky * Add test for authentication methods Signed-off-by: michal.gubricky * Add test if communication with etcd is secured via TLS Signed-off-by: michal.gubricky * Add test whether the etcd is isolated from k8s cluster Signed-off-by: michal.gubricky * Rebase after current sonobuoy golang framework was restructured Signed-off-by: michal.gubricky * Adjust test for etcd tls communication Signed-off-by: michal.gubricky * Adjust test which checks if kubelet readonly port is disabled Signed-off-by: michal.gubricky * Remove unused functions Signed-off-by: michal.gubricky * Remove development artifact Signed-off-by: michal.gubricky * Restructure the code to make it a bit more clearly Signed-off-by: michal.gubricky * Update kind config Signed-off-by: michal.gubricky * Remove kubeadmConfigPatches section in kind_config.yaml Signed-off-by: michal.gubricky --------- Signed-off-by: michal.gubricky Co-authored-by: cah-patrickthiem <140701884+cah-patrickthiem@users.noreply.github.com> --- .../scs_0217_cluster_hardening_test.go | 837 ++++++++++++++++++ 1 file changed, 837 insertions(+) create mode 100644 Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0217_cluster_hardening_test.go diff --git a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0217_cluster_hardening_test.go b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0217_cluster_hardening_test.go new file mode 100644 index 000000000..3cca9b095 --- /dev/null +++ b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0217_cluster_hardening_test.go @@ -0,0 +1,837 @@ +package scs_k8s_tests + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "os" + "path" + "strconv" + "strings" + "testing" + "time" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" +) + +const ( + apiServerPort = "6443" + controllerManagerPort = "10257" + schedulerPort = "10259" + etcdPortRangeStart = 2379 + etcdPortRangeEnd = 2380 + kubeletApiPort = "10250" + kubeletReadOnlyPort = 10255 + connectionTimeout = 5 * time.Second + sonobuoyResultsDir = "/tmp/sonobuoy/results" +) + +type KubeletConfig struct { + KubeletConfig struct { + ReadOnlyPort int `json:"readOnlyPort"` + Authentication struct { + Anonymous struct { + Enabled bool `json:"enabled"` + } `json:"anonymous"` + } `json:"authentication"` + Authorization struct { + Mode string `json:"mode"` + } `json:"authorization"` + } `json:"kubeletconfig"` +} + +// ==================== Helper Functions ==================== + +// getNodeEndpoint returns the response from pinging a node endpoint. +func getNodeEndpoint(client rest.Interface, nodeName, endpoint string) (rest.Result, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + req := client.Get(). + Resource("nodes"). + Name(nodeName). + SubResource("proxy"). + Suffix(endpoint) + + result := req.Do(ctx) + if result.Error() != nil { + logrus.Warningf("Could not get %v endpoint for node %v: %v", endpoint, nodeName, result.Error()) + } + return result, result.Error() +} + +// fetchAndSaveEndpointData fetches data from a node endpoint and saves it to a file. +func fetchAndSaveEndpointData(client rest.Interface, nodeName, endpoint, filePath string) error { + result, err := getNodeEndpoint(client, nodeName, endpoint) + if err != nil { + return err + } + + resultBytes, err := result.Raw() + if err != nil { + return err + } + + if err := os.WriteFile(filePath, resultBytes, 0644); err != nil { + return err + } + + return nil +} + +// gatherNodeData collects non-resource information about a node. +func gatherNodeData(nodeNames []string, restclient rest.Interface, outputDir string) error { + for _, name := range nodeNames { + out := path.Join(outputDir, name) + logrus.Infof("Creating host results for %v under %v\n", name, out) + if err := os.MkdirAll(out, 0755); err != nil { + return err + } + + configzPath := path.Join(out, "configz.json") + if err := fetchAndSaveEndpointData(restclient, name, "configz", configzPath); err != nil { + return err + } + } + + return nil +} + +// readKubeletConfigFromFile reads and parses the Kubelet configuration from a file. +func readKubeletConfigFromFile(path string) (*KubeletConfig, error) { + fmt.Printf("Reading Kubelet config from file: %s\n", path) + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open Kubelet config file: %w", err) + } + defer file.Close() + + bytes, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("failed to read Kubelet config file: %w", err) + } + + var kubeletConfig KubeletConfig + if err := json.Unmarshal(bytes, &kubeletConfig); err != nil { + return nil, fmt.Errorf("failed to parse Kubelet config file: %w", err) + } + + return &kubeletConfig, nil +} + +// checkPortOpen tries to establish a TCP connection to the given IP and port. +// It returns true if the port is open and false if the connection is refused or times out. +func checkPortOpen(ip, port string, timeout time.Duration) bool { + address := net.JoinHostPort(ip, port) + conn, err := net.DialTimeout("tcp", address, timeout) + if err != nil { + // Connection failed, the port is likely closed + return false + } + // Connection succeeded, the port is open + conn.Close() + return true +} + +// checkPortAccessibility verifies if a port is accessible or not, logging the result. +func checkPortAccessibility(t *testing.T, ip, port string, shouldBeAccessible bool) { + isOpen := checkPortOpen(ip, port, connectionTimeout) + if isOpen && !shouldBeAccessible { + t.Errorf("Error: port %s on node %s should not be accessible, but it is open", port, ip) + } else if !isOpen && shouldBeAccessible { + t.Errorf("Error: port %s on node %s should be accessible, but it is not", port, ip) + } else if isOpen && shouldBeAccessible { + t.Logf("Port %s on node %s is accessible as expected", port, ip) + } else { + t.Logf("Port %s on node %s is correctly restricted", port, ip) + } +} + +// isPortSecuredWithHTTPS checks if a specific IP and port combination is secured via HTTPS. +func isPortSecuredWithHTTPS(ip string, port int32, timeout time.Duration) bool { + address := net.JoinHostPort(ip, fmt.Sprintf("%d", port)) + + conn, err := tls.DialWithDialer( + &net.Dialer{Timeout: timeout}, + "tcp", + address, + &tls.Config{InsecureSkipVerify: true}, + ) + if err != nil { + return false + } + defer conn.Close() + + return true +} + +// checkPodSecurityAdmissionControllerEnabled checks if the PodSecurity admission controller is enabled in kube-apiserver pods +func checkPodSecurityAdmissionControllerEnabled(t *testing.T, kubeClient *kubernetes.Clientset) { + // List all pods in the kube-system namespace with label "component=kube-apiserver" + podList, err := kubeClient.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil { + t.Fatal("failed to list kube-apiserver pods:", err) + } + + // Check each kube-apiserver pod + for _, pod := range podList.Items { + t.Logf("Checking pod: %s for PodSecurity admission controller", pod.Name) + for _, container := range pod.Spec.Containers { + admissionPluginsFound := false + // Look for the enable-admission-plugins flag in container command + for _, cmd := range container.Command { + if strings.Contains(cmd, "--enable-admission-plugins=") { + admissionPluginsFound = true + + // Extract the plugins list and check if PodSecurity is one of them + plugins := strings.Split(cmd, "=")[1] + if strings.Contains(plugins, "PodSecurity") { + t.Logf("PodSecurity admission plugin is enabled in container: %s of pod: %s", container.Name, pod.Name) + } else { + t.Errorf("Error: PodSecurity admission plugin is not enabled in container: %s of pod: %s", container.Name, pod.Name) + } + break + } + } + + if !admissionPluginsFound { + t.Errorf("Error: --enable-admission-plugins flag not found in container: %s of pod: %s", container.Name, pod.Name) + } + } + } +} + +// checkPodSecurityPoliciesEnforced checks if Baseline and Restricted policies are enforced on namespaces +func checkPodSecurityPoliciesEnforced(t *testing.T, kubeClient *kubernetes.Clientset) { + // List all namespaces + namespaceList, err := kubeClient.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) + if err != nil { + t.Fatal("failed to list namespaces:", err) + } + + // Check for the "pod-security.kubernetes.io/enforce" annotation in each namespace + for _, namespace := range namespaceList.Items { + annotations := namespace.Annotations + enforcePolicy, ok := annotations["pod-security.kubernetes.io/enforce"] + if !ok { + t.Logf("Warning: Namespace %s does not have an enforce policy annotation", namespace.Name) + continue + } + + // Check if the policy is either Baseline or Restricted + if enforcePolicy == "baseline" || enforcePolicy == "restricted" { + t.Logf("Namespace %s enforces the %s policy", namespace.Name, enforcePolicy) + } else { + t.Errorf("Error: Namespace %s does not enforce Baseline or Restricted policy, but has %s", namespace.Name, enforcePolicy) + } + } +} + +// checkAuthorizationmethods checks if authorization methods are correctly sets in k8s cluster based on standard +func checkAuthorizationmethods(t *testing.T, kubeClient *kubernetes.Clientset) { + podList, err := kubeClient.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil { + t.Fatal("failed to list kube-apiserver pods:", err) + } + + // Check each kube-apiserver pod + for _, pod := range podList.Items { + t.Logf("Checking pod: %s for authorization modes", pod.Name) + for _, container := range pod.Spec.Containers { + authModeFound := false + // Look for the --authorization-mode flag in container command + for _, cmd := range container.Command { + if strings.Contains(cmd, "--authorization-mode=") { + authModeFound = true + + modes := strings.Split(cmd, "=")[1] + authModes := strings.Split(modes, ",") + + nodeAuthEnabled := false + otherAuthEnabled := false + + for _, mode := range authModes { + mode = strings.TrimSpace(mode) + if mode == "Node" { + nodeAuthEnabled = true + } + if mode == "ABAC" || mode == "RBAC" || mode == "Webhook" { + otherAuthEnabled = true + } + } + + // Validate the presence of required authorization methods + if nodeAuthEnabled && otherAuthEnabled { + t.Logf("Node authorization is enabled and at least one method (ABAC, RBAC or Webhook) is enabled.") + } else if !nodeAuthEnabled { + t.Errorf("Error: Node authorization is not enabled in api-server pod: %s", pod.Name) + } else if !otherAuthEnabled { + t.Errorf("Error: None of ABAC, RBAC, or Webhook authorization methods are enabled in api-server pod: %s", pod.Name) + } + break + } + } + + // If the --authorization-mode flag is not found + if !authModeFound { + t.Errorf("Error: --authorization-mode flag not found in api-server pod: %s", pod.Name) + } + } + } +} + +// checkKubeAPIServerETCDTLS checks whether the kube-apiserver communicates with etcd over TLS. +func checkKubeAPIServerETCDTLS(t *testing.T, kubeClient *kubernetes.Clientset) { + // List kube-apiserver pods + podList, err := kubeClient.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil { + t.Fatal("failed to list kube-apiserver pods:", err) + } + + // Expected etcd TLS flags + requiredFlags := []string{ + "--etcd-certfile=", + "--etcd-keyfile=", + "--etcd-cafile=", + } + + // Check each kube-apiserver pod + for _, pod := range podList.Items { + for _, container := range pod.Spec.Containers { + // Gather all the commands into a single string for easier matching + cmdLine := strings.Join(container.Command, " ") + t.Logf("TEST: Checking container: %s of pod: %s", container.Name, pod.Name) + + // Check if all required etcd TLS flags are present + allFlagsPresent := true + for _, flag := range requiredFlags { + if !strings.Contains(cmdLine, flag) { + t.Errorf("Missing flag %s in container: %s of pod: %s", flag, container.Name, pod.Name) + allFlagsPresent = false + } + } + + if allFlagsPresent { + t.Logf("kube-apiserver communicates with etcd using TLS in container: %s of pod: %s", container.Name, pod.Name) + } else { + t.Errorf("Error: kube-apiserver does not use all required TLS flags for etcd communication in container: %s of pod: %s", container.Name, pod.Name) + } + } + } +} + +// checkIsolationETCD checks whether the etcd is isolated from k8s cluster. +func checkIsolationETCD(t *testing.T, kubeClient *kubernetes.Clientset) { + // List kube-apiserver pods + podList, err := kubeClient.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil { + t.Fatal("failed to list kube-apiserver pods:", err) + } + + // Check each kube-apiserver pod + for _, pod := range podList.Items { + for _, container := range pod.Spec.Containers { + etcdServersFound := false + for _, cmd := range container.Command { + if strings.Contains(cmd, "--etcd-servers=") { + etcdServersFound = true + etcdServers := strings.Split(cmd, "--etcd-servers=")[1] + etcdEndpoints := strings.Split(etcdServers, ",") + + // Verify that etcd is not running on localhost + for _, endpoint := range etcdEndpoints { + if strings.Contains(endpoint, "localhost") || strings.Contains(endpoint, "127.0.0.1") { + t.Logf("Warning: etcd should be isolated from k8s cluster, currently it is running on localhost: %s", endpoint) + } else { + t.Logf("ETCD is isolated at endpoint: %s", endpoint) + } + } + } + } + + if !etcdServersFound { + t.Errorf("Error: --etcd-servers flag is missing in kube-apiserver pod: %s", pod.Name) + } + } + } +} + +// ==================== Test Cases ==================== + +// Test_scs_0217_sonobuoy_Kubelet_ReadOnly_Port_Disabled checks +// if the Kubelet's read-only port (10255) is disabled on all nodes. +// If the port is open, it will log a warning. +func Test_scs_0217_sonobuoy_Kubelet_ReadOnly_Port_Disabled(t *testing.T) { + f := features.New("kubelet security").Assess( + "Kubelet read-only port (10255) should be disabled", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + nodeList, err := kubeClient.CoreV1().Nodes().List(context.Background(), v1.ListOptions{}) + if err != nil { + t.Fatal("failed to get node list:", err) + } + + nodeNames := make([]string, len(nodeList.Items)) + for i, node := range nodeList.Items { + nodeNames[i] = node.Name + } + + if err := gatherNodeData(nodeNames, kubeClient.CoreV1().RESTClient(), sonobuoyResultsDir); err != nil { + t.Fatal("failed to gather node data:", err) + } + + // Get kubelets configz file from each node + for _, nodeName := range nodeNames { + configzPath := path.Join(sonobuoyResultsDir, nodeName, "configz.json") + kubeletConfig, err := readKubeletConfigFromFile(configzPath) + if err != nil { + t.Errorf("Failed to read Kubelet config from file %s: %v", configzPath, err) + continue + } + + // Check if readonly port is enabled + if kubeletConfig.KubeletConfig.ReadOnlyPort == kubeletReadOnlyPort { + t.Logf("Warning: kubelet read-only port 10255 is open on node %s", nodeName) + } else { + t.Logf("Kubelet read-only port 10255 is correctly disabled on node %s", nodeName) + } + } + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_Control_Plane_Ports_Security checks if the control plane ports (other than API server and NodePorts) +// are not accessible from outside the internal network. +func Test_scs_0217_sonobuoy_Control_Plane_Ports_Security(t *testing.T) { + f := features.New("control plane security").Assess( + "Control plane ports (other than API server and NodePorts) should not be accessible externally", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + nodes := &corev1.NodeList{} + labelSelector := labels.Set{ + "node-role.kubernetes.io/control-plane": "", + }.AsSelector().String() + + err := cfg.Client().Resources().List(context.Background(), nodes, resources.WithLabelSelector(labelSelector)) + if err != nil { + t.Fatal("failed to list control plane nodes:", err) + } + + for _, node := range nodes.Items { + nodeIP := node.Status.Addresses[0].Address + + // Check that the API server port (6443) is accessible + checkPortAccessibility(t, nodeIP, apiServerPort, true) + + // Check that the control plane ports (other than API server) are not accessible + checkPortAccessibility(t, nodeIP, controllerManagerPort, false) + checkPortAccessibility(t, nodeIP, schedulerPort, false) + checkPortAccessibility(t, nodeIP, kubeletApiPort, false) + + // Check the etcd ports (2379-2380) are not accessible + for port := etcdPortRangeStart; port <= etcdPortRangeEnd; port++ { + checkPortAccessibility(t, nodeIP, fmt.Sprintf("%d", port), false) + } + } + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_K8s_Endpoints_HTTPS checks if all Kubernetes endpoints are secured via HTTPS. +func Test_scs_0217_sonobuoy_K8s_Endpoints_HTTPS(t *testing.T) { + f := features.New("Kubernetes endpoint security").Assess( + "All Kubernetes endpoints should be secured via HTTPS", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + requiredHTTPSPorts := map[string]bool{ + apiServerPort: true, + controllerManagerPort: true, + schedulerPort: true, + kubeletApiPort: true, + } + + // Add etcd ports + for port := etcdPortRangeStart; port <= etcdPortRangeEnd; port++ { + requiredHTTPSPorts[strconv.Itoa(port)] = true + } + + endpoints := &corev1.EndpointsList{} + if err := cfg.Client().Resources().List(context.Background(), endpoints); err != nil { + t.Fatal("failed to list endpoints:", err) + } + + for _, ep := range endpoints.Items { + for _, subset := range ep.Subsets { + for _, addr := range subset.Addresses { + for _, port := range subset.Ports { + portStr := strconv.Itoa(int(port.Port)) + if requiredHTTPSPorts[portStr] { + if isPortSecuredWithHTTPS(addr.IP, port.Port, connectionTimeout) { + t.Logf("Endpoint %s:%d (%s) is secured via HTTPS", addr.IP, port.Port, port.Name) + } else { + t.Errorf("Error: Endpoint %s:%d (%s) is not secured via HTTPS", addr.IP, port.Port, port.Name) + } + } + } + } + } + } + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_sonobuoy_Kubelet_HTTPS_Anonymous_Auth_Disabled checks +// if the Kubelet's anonymous access is disabled based on the configz file. +func Test_scs_0217_sonobuoy_Kubelet_HTTPS_Anonymous_Auth_Disabled(t *testing.T) { + f := features.New("kubelet security").Assess( + "Kubelet HTTPS anonymous access should be disabled based on the configz", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + nodeList, err := kubeClient.CoreV1().Nodes().List(context.Background(), v1.ListOptions{}) + if err != nil { + t.Fatal("failed to get node list:", err) + } + + nodeNames := make([]string, len(nodeList.Items)) + for i, node := range nodeList.Items { + nodeNames[i] = node.Name + } + + if err := gatherNodeData(nodeNames, kubeClient.CoreV1().RESTClient(), sonobuoyResultsDir); err != nil { + t.Fatal("failed to gather node data:", err) + } + + // Get kubelets configz file from each node + for _, nodeName := range nodeNames { + configzPath := path.Join(sonobuoyResultsDir, nodeName, "configz.json") + kubeletConfig, err := readKubeletConfigFromFile(configzPath) + if err != nil { + t.Errorf("Failed to read Kubelet config from file %s: %v", configzPath, err) + continue + } + + // Check if anonymous authentication is enabled + if kubeletConfig.KubeletConfig.Authentication.Anonymous.Enabled { + t.Errorf("ERROR: Kubelet anonymous authentication is enabled at %s", nodeName) + } else { + t.Logf("Kubelet anonymous authentication is correctly disabled at %s", nodeName) + } + } + + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_sonobuoy_Kubelet_Webhook_Authorization_Enabled checks +// if the Kubelet's authorization mode is Webhook by accessing the Kubelet's /configz endpoint. +func Test_scs_0217_sonobuoy_Kubelet_Webhook_Authorization_Enabled(t *testing.T) { + f := features.New("kubelet security").Assess( + "Kubelet authorization mode should be Webhook from Sonobuoy results", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + nodeList, err := kubeClient.CoreV1().Nodes().List(context.Background(), v1.ListOptions{}) + if err != nil { + t.Fatal("failed to get node list:", err) + } + + nodeNames := make([]string, len(nodeList.Items)) + for i, node := range nodeList.Items { + nodeNames[i] = node.Name + } + + if err := gatherNodeData(nodeNames, kubeClient.CoreV1().RESTClient(), sonobuoyResultsDir); err != nil { + t.Fatal("failed to gather node data:", err) + } + + // Get kubelets configz file from each node + for _, nodeName := range nodeNames { + configzPath := path.Join(sonobuoyResultsDir, nodeName, "configz.json") + kubeletConfig, err := readKubeletConfigFromFile(configzPath) + if err != nil { + t.Errorf("failed to read Kubelet config from file %s: %v", configzPath, err) + continue + } + + // Check if the authorization mode is set to Webhook + if kubeletConfig.KubeletConfig.Authorization.Mode != "Webhook" { + t.Errorf("Error: Kubelet authorization mode is not webhook, got %s", kubeletConfig.KubeletConfig.Authorization.Mode) + } else { + t.Logf("kubelet authorization mode is correctly set to Webhook") + } + } + + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_sonobuoy_NodeRestriction_Admission_Controller_Enabled_in_KubeAPIServer +// checks if the NodeRestriction admission controller is enabled. +func Test_scs_0217_sonobuoy_NodeRestriction_Admission_Controller_Enabled_in_KubeAPIServer(t *testing.T) { + f := features.New("kube-apiserver admission plugins").Assess( + "Check if NodeRestriction admission plugin is enabled in kube-apiserver pods", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + // Create a Kubernetes client + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + // List all pods in the kube-system namespace with label "component=kube-apiserver" + podList, err := kubeClient.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil { + t.Fatal("failed to list kube-apiserver pods:", err) + } + + // Check each kube-apiserver pod + for _, pod := range podList.Items { + for _, container := range pod.Spec.Containers { + cmdFound := false + for _, cmd := range container.Command { + if strings.Contains(cmd, "--enable-admission-plugins=NodeRestriction") { + t.Logf("NodeRestriction admission plugin is enabled in pod: %s", pod.Name) + cmdFound = true + break + } + } + + if !cmdFound { + t.Errorf("Error: NodeRestriction admission plugin is not enabled in pod: %s", pod.Name) + } + } + } + + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_sonobuoy_PodSecurity_Standards_And_Admission_Controller_Enabled +// checks if the PodSecurity admission controller is enabled and +// verify that if Pod Security Standards (Baseline/Restricted) are enforced on namespaces. +func Test_scs_0217_sonobuoy_PodSecurity_Standards_And_Admission_Controller_Enabled(t *testing.T) { + f := features.New("pod security standards").Assess( + "Pod security admission controller should be enabled and enforce Baseline/Restricted policies", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + // Create a Kubernetes client + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + // Check that the PodSecurity admission controller is enabled in the kube-apiserver pods + checkPodSecurityAdmissionControllerEnabled(t, kubeClient) + + // Verify that Pod Security Standards (Baseline/Restricted) are enforced on namespaces + checkPodSecurityPoliciesEnforced(t, kubeClient) + + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_sonobuoy_Authorization_Methods checks whether at least two authorization methods are sets in k8s cluster, +// one of which MUST be Node authorization and another one consisting of either ABAC, RBAC or Webhook authorization. +func Test_scs_0217_sonobuoy_Authorization_Methods(t *testing.T) { + f := features.New("authorization methods").Assess( + "At least two authorization methods must be set, one of which must be Node authorization "+ + "and another one consisting of either ABAC, RBAC, or Webhook authorization", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + // Create a Kubernetes client + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + // Check authorization methods in the kube-apiserver + checkAuthorizationmethods(t, kubeClient) + + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_sonobuoy_Authentication_Methods checks if at least two authentication methods are enabled in the cluster, +// one of which MUST be Service Account Tokens, to provide full functionality to Pods. +func Test_scs_0217_sonobuoy_Authentication_Methods(t *testing.T) { + f := features.New("authentication methods").Assess( + "At least two authentication methods must be enabled, one of which MUST be Service Account Tokens", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + podList, err := kubeClient.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil { + t.Fatal("failed to list kube-apiserver pods:", err) + } + + // Check each kube-apiserver pod for authentication modes + for _, pod := range podList.Items { + t.Logf("Checking pod: %s for authentication methods", pod.Name) + + for _, container := range pod.Spec.Containers { + // Check for the --authentication-token-webhook and --service-account-issuer flags in the container's command + authMethodsFound := map[string]bool{ + "ServiceAccountTokens": false, + "OtherAuthMethod": false, + } + + for _, cmd := range container.Command { + // Check for Service Account Tokens (--service-account-issuer) + if strings.Contains(cmd, "--service-account-issuer=") { + authMethodsFound["ServiceAccountTokens"] = true + } + // Check for other authentication methods like --authentication-token-webhook-config-file or --oidc-issuer-url + if strings.Contains(cmd, "--authentication-token-webhook-config-file=") || strings.Contains(cmd, "--oidc-issuer-url=") { + authMethodsFound["OtherAuthMethod"] = true + } + } + + // Check if both authentication methods are present + if authMethodsFound["ServiceAccountTokens"] && authMethodsFound["OtherAuthMethod"] { + t.Logf("Both authentication methods (Service Account Tokens and another method) are enabled") + } else { + if !authMethodsFound["ServiceAccountTokens"] { + t.Errorf("Error: Service Account Tokens are not enabled") + } + if !authMethodsFound["OtherAuthMethod"] { + t.Errorf("Error: No other authentication method (Token Webhook or OIDC) is enabled") + } + } + } + } + + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_etcd_tls checks if communication with etcd is secured with TLS for both peer- and cluster-communication. +func Test_scs_0217_etcd_tls_communication(t *testing.T) { + f := features.New("etcd security").Assess( + "Communication with etcd MUST be secured with TLS for both peer- and cluster-communication", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + // Check kube-apiserver communication with etcd + checkKubeAPIServerETCDTLS(t, kubeClient) + + return ctx + }) + + testenv.Test(t, f.Feature()) +} + +// Test_scs_0217_etcd_isolation checks ETCD is isolated from k8s cluster by checking the etcd server endpoints. +func Test_scs_0217_etcd_isolation(t *testing.T) { + f := features.New("etcd security").Assess( + "ETCD should be isolated from the Kubernetes cluster", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + restConf, err := rest.InClusterConfig() + if err != nil { + t.Fatal("failed to create rest config:", err) + } + + kubeClient, err := kubernetes.NewForConfig(restConf) + if err != nil { + t.Fatal("failed to create Kubernetes client:", err) + } + + // Check if etcd is isolated from k8s cluster + checkIsolationETCD(t, kubeClient) + + return ctx + }) + + testenv.Test(t, f.Feature()) +}