[DOP-7507] Rebuild fixtures

setup.cfg

@@ -258,6 +258,8 @@ ignore =
     WPS601,
 # WPS604 Found incorrect node inside `class` body: pass
     WPS604,
+# WPS100 Found wrong module name: util
+    WPS100
 
 # http://flake8.pycqa.org/en/latest/user/options.html?highlight=per-file-ignores#cmdoption-flake8-per-file-ignores
 per-file-ignores =

tests/conftest.py

@@ -0,0 +1,20 @@
+import os
+
+# disable failing plugin import
+os.environ["ONETL_PLUGINS_BLACKLIST"] = "failing-plugin"
+
+pytest_plugins = [
+    "tests.fixtures.file_connections.fixtures",
+    "tests.fixtures.file_connections.ftp",
+    "tests.fixtures.file_connections.ftps",
+    "tests.fixtures.file_connections.hdfs",
+    "tests.fixtures.file_connections.s3",
+    "tests.fixtures.file_connections.sftp",
+    "tests.fixtures.file_connections.webdav",
+    "tests.fixtures.processing.fixtures",
+    "tests.fixtures.create_keytab",
+    "tests.fixtures.global_hwm_store",
+    "tests.fixtures.hwm_delta",
+    "tests.fixtures.spark_mock",
+    "tests.fixtures.spark",
+]

tests/fixtures/create_keytab.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture()
+def create_keytab(tmp_path_factory):
+    path = Path(tmp_path_factory.mktemp("data") / "keytab")
+    path.write_text("content")
+
+    return path

tests/fixtures/file_connections/fixtures.py

@@ -0,0 +1,70 @@
+import secrets
+import shutil
+from pathlib import Path
+
+import pytest
+from pytest_lazyfixture import lazy_fixture
+
+
+@pytest.fixture(scope="session")
+def resource_path_original():
+    path = Path(__file__).parent.parent.parent / "resources"
+    assert path.exists()
+    return path
+
+
+@pytest.fixture()
+def resource_path(resource_path_original, tmp_path_factory):
+    temp_dir = tmp_path_factory.mktemp("test_files") / secrets.token_hex(5)
+    shutil.copytree(resource_path_original, temp_dir)
+    return temp_dir
+
+
+@pytest.fixture()
+def test_files(resource_path):
+    return [
+        resource_path / "raw/ascii.txt",
+        resource_path / "raw/utf-8.txt",
+    ]
+
+
+@pytest.fixture(
+    params=[
+        lazy_fixture("ftp_file_connection"),
+        lazy_fixture("ftps_file_connection"),
+        lazy_fixture("hdfs_file_connection"),
+        lazy_fixture("s3_file_connection"),
+        lazy_fixture("sftp_file_connection"),
+        lazy_fixture("webdav_file_connection"),
+    ],
+)
+def file_connection(request):
+    return request.param
+
+
+@pytest.fixture(
+    params=[
+        lazy_fixture("ftp_file_connection_with_path"),
+        lazy_fixture("ftps_file_connection_with_path"),
+        lazy_fixture("hdfs_file_connection_with_path"),
+        lazy_fixture("s3_file_connection_with_path"),
+        lazy_fixture("sftp_file_connection_with_path"),
+        lazy_fixture("webdav_file_connection_with_path"),
+    ],
+)
+def file_connection_with_path(request):
+    return request.param
+
+
+@pytest.fixture(
+    params=[
+        lazy_fixture("ftp_file_connection_with_path_and_files"),
+        lazy_fixture("ftps_file_connection_with_path_and_files"),
+        lazy_fixture("hdfs_file_connection_with_path_and_files"),
+        lazy_fixture("s3_file_connection_with_path_and_files"),
+        lazy_fixture("sftp_file_connection_with_path_and_files"),
+        lazy_fixture("webdav_file_connection_with_path_and_files"),
+    ],
+)
+def file_connection_with_path_and_files(request):
+    return request.param

tests/fixtures/file_connections/ftp.py

@@ -0,0 +1,60 @@
+import os
+from collections import namedtuple
+from pathlib import PurePosixPath
+
+import pytest
+
+from tests.fixtures.file_connections.util import upload_files
+
+
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param("real", marks=[pytest.mark.ftp, pytest.mark.file_connection, pytest.mark.connection]),
+    ],
+)
+def ftp_server():
+    FTPServer = namedtuple("FTPServer", ["host", "port", "user", "password"])
+
+    return FTPServer(
+        host=os.getenv("ONETL_FTP_HOST"),
+        port=os.getenv("ONETL_FTP_PORT"),
+        user=os.getenv("ONETL_FTP_USER"),
+        password=os.getenv("ONETL_FTP_PASSWORD"),
+    )
+
+
+@pytest.fixture()
+def ftp_file_connection(ftp_server):
+    from onetl.connection import FTP
+
+    return FTP(
+        host=ftp_server.host,
+        port=ftp_server.port,
+        user=ftp_server.user,
+        password=ftp_server.password,
+    )
+
+
+@pytest.fixture()
+def ftp_file_connection_with_path(request, ftp_file_connection):
+    connection = ftp_file_connection
+    root = PurePosixPath("/data/")
+
+    def finalizer():
+        connection.remove_dir(root, recursive=True)
+
+    request.addfinalizer(finalizer)
+
+    connection.remove_dir(root, recursive=True)
+    connection.create_dir(root)
+
+    return connection, root
+
+
+@pytest.fixture()
+def ftp_file_connection_with_path_and_files(resource_path_original, ftp_file_connection_with_path):
+    connection, upload_to = ftp_file_connection_with_path
+    upload_from = resource_path_original
+    files = upload_files(upload_from, upload_to, connection)
+    return connection, upload_to, files

tests/fixtures/file_connections/ftps.py

@@ -0,0 +1,60 @@
+import os
+from collections import namedtuple
+from pathlib import PurePosixPath
+
+import pytest
+
+from tests.fixtures.file_connections.util import upload_files
+
+
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param("real", marks=[pytest.mark.ftps, pytest.mark.file_connection, pytest.mark.connection]),
+    ],
+)
+def ftps_server():
+    FTPSServer = namedtuple("FTPSServer", ["host", "port", "user", "password"])
+
+    return FTPSServer(
+        host=os.getenv("ONETL_FTPS_HOST"),
+        port=os.getenv("ONETL_FTPS_PORT"),
+        user=os.getenv("ONETL_FTPS_USER"),
+        password=os.getenv("ONETL_FTPS_PASSWORD"),
+    )
+
+
+@pytest.fixture()
+def ftps_file_connection(ftps_server):
+    from onetl.connection import FTPS
+
+    return FTPS(
+        host=ftps_server.host,
+        port=ftps_server.port,
+        user=ftps_server.user,
+        password=ftps_server.password,
+    )
+
+
+@pytest.fixture()
+def ftps_file_connection_with_path(request, ftps_file_connection):
+    connection = ftps_file_connection
+    root = PurePosixPath("/data/")
+
+    def finalizer():
+        connection.remove_dir(root, recursive=True)
+
+    request.addfinalizer(finalizer)
+
+    connection.remove_dir(root, recursive=True)
+    connection.create_dir(root)
+
+    return connection, root
+
+
+@pytest.fixture()
+def ftps_file_connection_with_path_and_files(resource_path_original, ftps_file_connection_with_path):
+    connection, upload_to = ftps_file_connection_with_path
+    upload_from = resource_path_original
+    files = upload_files(upload_from, upload_to, connection)
+    return connection, upload_to, files

tests/fixtures/file_connections/hdfs.py

@@ -0,0 +1,52 @@
+import os
+from collections import namedtuple
+from pathlib import PurePosixPath
+
+import pytest
+
+from tests.fixtures.file_connections.util import upload_files
+
+
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param("real", marks=[pytest.mark.hdfs, pytest.mark.file_connection, pytest.mark.connection]),
+    ],
+)
+def hdfs_server():
+    HDFSServer = namedtuple("HDFSServer", ["host", "port"])
+    return HDFSServer(
+        host=os.getenv("ONETL_HDFS_HOST"),
+        port=os.getenv("ONETL_HDFS_PORT"),
+    )
+
+
+@pytest.fixture()
+def hdfs_file_connection(hdfs_server):
+    from onetl.connection import HDFS
+
+    return HDFS(host=hdfs_server.host, port=hdfs_server.port)
+
+
+@pytest.fixture()
+def hdfs_file_connection_with_path(request, hdfs_file_connection):
+    connection = hdfs_file_connection
+    root = PurePosixPath("/data/")
+
+    def finalizer():
+        connection.remove_dir(root, recursive=True)
+
+    request.addfinalizer(finalizer)
+
+    connection.remove_dir(root, recursive=True)
+    connection.create_dir(root)
+
+    return connection, root
+
+
+@pytest.fixture()
+def hdfs_file_connection_with_path_and_files(resource_path_original, hdfs_file_connection_with_path):
+    connection, upload_to = hdfs_file_connection_with_path
+    upload_from = resource_path_original
+    files = upload_files(upload_from, upload_to, connection)
+    return connection, upload_to, files

tests/fixtures/file_connections/s3.py

@@ -0,0 +1,67 @@
+import os
+from collections import namedtuple
+from pathlib import PurePosixPath
+
+import pytest
+
+from tests.fixtures.file_connections.util import upload_files
+
+
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param("real", marks=[pytest.mark.s3, pytest.mark.file_connection, pytest.mark.connection]),
+    ],
+)
+def s3_server():
+    S3Server = namedtuple("S3Server", ["host", "port", "bucket", "access_key", "secret_key", "protocol"])
+
+    return S3Server(
+        host=os.getenv("ONETL_S3_HOST"),
+        port=os.getenv("ONETL_S3_PORT"),
+        bucket=os.getenv("ONETL_S3_BUCKET"),
+        access_key=os.getenv("ONETL_S3_ACCESS_KEY"),
+        secret_key=os.getenv("ONETL_S3_SECRET_KEY"),
+        protocol=os.getenv("ONETL_S3_PROTOCOL", "http").lower(),
+    )
+
+
+@pytest.fixture()
+def s3_file_connection(s3_server):
+    from onetl.connection import S3
+
+    return S3(
+        host=s3_server.host,
+        port=s3_server.port,
+        bucket=s3_server.bucket,
+        access_key=s3_server.access_key,
+        secret_key=s3_server.secret_key,
+        protocol=s3_server.protocol,
+    )
+
+
+@pytest.fixture()
+def s3_file_connection_with_path(request, s3_file_connection):
+    connection = s3_file_connection
+    root = PurePosixPath("/data/")
+
+    if not connection.client.bucket_exists(connection.bucket):
+        connection.client.make_bucket(connection.bucket)
+
+    def finalizer():
+        connection.remove_dir(root, recursive=True)
+
+    request.addfinalizer(finalizer)
+
+    connection.remove_dir(root, recursive=True)
+    connection.create_dir(root)
+
+    return connection, root
+
+
+@pytest.fixture()
+def s3_file_connection_with_path_and_files(resource_path_original, s3_file_connection_with_path):
+    connection, upload_to = s3_file_connection_with_path
+    upload_from = resource_path_original
+    files = upload_files(upload_from, upload_to, connection)
+    return connection, upload_to, files