From 40f6a9a1e2eda466df407a7559f08ca0168c75bf Mon Sep 17 00:00:00 2001 From: Denis Otkidach <denis.otkidach@gmail.com> Date: Mon, 27 Jun 2022 18:18:52 +0300 Subject: [PATCH] Fix certificates generation (#837) --- tests/conftest.py | 91 +++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b5ab6f59..8d8767a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,13 +3,13 @@ import gc import docker as libdocker import logging +import os import pytest import socket import uuid import sys import pathlib import shutil -import subprocess from aiokafka.record.legacy_records import ( LegacyRecordBatchBuilder, _LegacyRecordBatchBuilderPy) @@ -94,30 +94,66 @@ def kerberos_utils(): return +if sys.platform != 'win32': + + @pytest.fixture(scope='session') + def kafka_image(request, docker): + image = request.config.getoption('--docker-image') + if not image: + pytest.skip( + "Skipping functional test as `--docker-image` not provided") + return + if not request.config.getoption('--no-pull'): + docker.images.pull(image) + return image + +else: + + @pytest.fixture(scope='session') + def kafka_image(): + pytest.skip("Only unit tests on windows for now =(") + + @pytest.fixture(scope='session') -def ssl_folder(docker_ip_address): +def ssl_folder(docker_ip_address, docker, kafka_image): ssl_dir = pathlib.Path('tests/ssl_cert') if ssl_dir.exists(): shutil.rmtree(str(ssl_dir)) ssl_dir.mkdir() - p = subprocess.Popen( - f"bash ../../gen-ssl-certs.sh ca ca-cert {docker_ip_address}", - shell=True, stdout=subprocess.DEVNULL, - cwd=str(ssl_dir), stderr=subprocess.DEVNULL) - p.wait() - p = subprocess.Popen( - "bash ../../gen-ssl-certs.sh -k server ca-cert br_ {}".format( - docker_ip_address), - shell=True, stdout=subprocess.DEVNULL, - cwd=str(ssl_dir), stderr=subprocess.DEVNULL,) - p.wait() - p = subprocess.Popen( - "bash ../../gen-ssl-certs.sh client ca-cert cl_ {}".format( - docker_ip_address), - shell=True, stdout=subprocess.DEVNULL, - cwd=str(ssl_dir), stderr=subprocess.DEVNULL,) - p.wait() + + container = docker.containers.run( + image=kafka_image, + command="sleep 120", + volumes={ + pathlib.Path("gen-ssl-certs.sh").resolve(): { + "bind": "/gen-ssl-certs.sh", + }, + str(ssl_dir.resolve()): { + "bind": "/ssl_cert", + }, + }, + working_dir="/ssl_cert", + tty=True, + detach=True, + remove=True) + + try: + for args in [ + ["ca", "ca-cert", docker_ip_address], + ["-k", "server", "ca-cert", "br_", docker_ip_address], + ["client", "ca-cert", "cl_", docker_ip_address], + ]: + exit_code, output = container.exec_run( + ["bash", "/gen-ssl-certs.sh"] + args, + user=f"{os.getuid()}:{os.getgid()}" + ) + if exit_code != 0: + print(output.decode(), file=sys.stderr) + pytest.exit("Could not generate certificates") + + finally: + container.stop() return ssl_dir @@ -159,15 +195,8 @@ def hosts(self): if sys.platform != 'win32': @pytest.fixture(scope='session') - def kafka_server(request, docker, docker_ip_address, + def kafka_server(kafka_image, docker, docker_ip_address, unused_port, session_id, ssl_folder): - image = request.config.getoption('--docker-image') - if not image: - pytest.skip( - "Skipping functional test as `--docker-image` not provided") - return - if not request.config.getoption('--no-pull'): - docker.images.pull(image) kafka_host = docker_ip_address kafka_port = unused_port() kafka_ssl_port = unused_port() @@ -181,7 +210,7 @@ def kafka_server(request, docker, docker_ip_address, 'ADVERTISED_SASL_SSL_PORT': kafka_sasl_ssl_port, 'NUM_PARTITIONS': 2 } - kafka_version = image.split(":")[-1].split("_")[-1] + kafka_version = kafka_image.split(":")[-1].split("_")[-1] kafka_version = tuple(int(x) for x in kafka_version.split('.')) if kafka_version >= (0, 10, 2): environment['SASL_MECHANISMS'] = ( @@ -196,7 +225,7 @@ def kafka_server(request, docker, docker_ip_address, environment['SASL_JAAS_FILE'] = "kafka_server_gssapi_jaas.conf" container = docker.containers.run( - image=image, + image=kafka_image, name='aiokafka-tests', ports={ 2181: 2181, @@ -220,11 +249,11 @@ def kafka_server(request, docker, docker_ip_address, try: if not wait_kafka(kafka_host, kafka_port): exit_code, output = container.exec_run( - ["supervisorctl", "tail", "-10000", "kafka"]) + ["supervisorctl", "tail", "-20000", "kafka"]) print("Kafka failed to start. \n--- STDOUT:") print(output.decode(), file=sys.stdout) exit_code, output = container.exec_run( - ["supervisorctl", "tail", "-10000", "kafka", "stderr"]) + ["supervisorctl", "tail", "-20000", "kafka", "stderr"]) print("--- STDERR:") print(output.decode(), file=sys.stderr) pytest.exit("Could not start Kafka Server")