-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
53 changed files
with
1,577 additions
and
1,495 deletions.
There are no files selected for viewing
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
import os | ||
|
||
# disable failing plugin import | ||
os.environ["ONETL_PLUGINS_BLACKLIST"] = "failing-plugin" | ||
|
||
pytest_plugins = [ | ||
"tests.fixtures.file_connections.fixtures", | ||
"tests.fixtures.file_connections.ftp", | ||
"tests.fixtures.file_connections.ftps", | ||
"tests.fixtures.file_connections.hdfs", | ||
"tests.fixtures.file_connections.s3", | ||
"tests.fixtures.file_connections.sftp", | ||
"tests.fixtures.file_connections.webdav", | ||
"tests.fixtures.processing.fixtures", | ||
"tests.fixtures.create_keytab", | ||
"tests.fixtures.global_hwm_store", | ||
"tests.fixtures.hwm_delta", | ||
"tests.fixtures.spark_mock", | ||
"tests.fixtures.spark", | ||
] |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
from pathlib import Path | ||
|
||
import pytest | ||
|
||
|
||
@pytest.fixture() | ||
def create_keytab(tmp_path_factory): | ||
path = Path(tmp_path_factory.mktemp("data") / "keytab") | ||
path.write_text("content") | ||
|
||
return path |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
import secrets | ||
import shutil | ||
from pathlib import Path | ||
|
||
import pytest | ||
from pytest_lazyfixture import lazy_fixture | ||
|
||
|
||
@pytest.fixture(scope="session") | ||
def resource_path_original(): | ||
path = Path(__file__).parent.parent.parent / "resources" | ||
assert path.exists() | ||
return path | ||
|
||
|
||
@pytest.fixture() | ||
def resource_path(resource_path_original, tmp_path_factory): | ||
temp_dir = tmp_path_factory.mktemp("test_files") / secrets.token_hex(5) | ||
shutil.copytree(resource_path_original, temp_dir) | ||
return temp_dir | ||
|
||
|
||
@pytest.fixture() | ||
def test_files(resource_path): | ||
return [ | ||
resource_path / "raw/ascii.txt", | ||
resource_path / "raw/utf-8.txt", | ||
] | ||
|
||
|
||
@pytest.fixture( | ||
params=[ | ||
lazy_fixture("ftp_file_connection"), | ||
lazy_fixture("ftps_file_connection"), | ||
lazy_fixture("hdfs_file_connection"), | ||
lazy_fixture("s3_file_connection"), | ||
lazy_fixture("sftp_file_connection"), | ||
lazy_fixture("webdav_file_connection"), | ||
], | ||
) | ||
def file_connection(request): | ||
return request.param | ||
|
||
|
||
@pytest.fixture( | ||
params=[ | ||
lazy_fixture("ftp_file_connection_with_path"), | ||
lazy_fixture("ftps_file_connection_with_path"), | ||
lazy_fixture("hdfs_file_connection_with_path"), | ||
lazy_fixture("s3_file_connection_with_path"), | ||
lazy_fixture("sftp_file_connection_with_path"), | ||
lazy_fixture("webdav_file_connection_with_path"), | ||
], | ||
) | ||
def file_connection_with_path(request): | ||
return request.param | ||
|
||
|
||
@pytest.fixture( | ||
params=[ | ||
lazy_fixture("ftp_file_connection_with_path_and_files"), | ||
lazy_fixture("ftps_file_connection_with_path_and_files"), | ||
lazy_fixture("hdfs_file_connection_with_path_and_files"), | ||
lazy_fixture("s3_file_connection_with_path_and_files"), | ||
lazy_fixture("sftp_file_connection_with_path_and_files"), | ||
lazy_fixture("webdav_file_connection_with_path_and_files"), | ||
], | ||
) | ||
def file_connection_with_path_and_files(request): | ||
return request.param |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
import os | ||
from collections import namedtuple | ||
from pathlib import PurePosixPath | ||
|
||
import pytest | ||
|
||
from tests.fixtures.file_connections.util import upload_files | ||
|
||
|
||
@pytest.fixture( | ||
scope="session", | ||
params=[ | ||
pytest.param("real", marks=[pytest.mark.ftp, pytest.mark.file_connection, pytest.mark.connection]), | ||
], | ||
) | ||
def ftp_server(): | ||
FTPServer = namedtuple("FTPServer", ["host", "port", "user", "password"]) | ||
|
||
return FTPServer( | ||
host=os.getenv("ONETL_FTP_HOST"), | ||
port=os.getenv("ONETL_FTP_PORT"), | ||
user=os.getenv("ONETL_FTP_USER"), | ||
password=os.getenv("ONETL_FTP_PASSWORD"), | ||
) | ||
|
||
|
||
@pytest.fixture() | ||
def ftp_file_connection(ftp_server): | ||
from onetl.connection import FTP | ||
|
||
return FTP( | ||
host=ftp_server.host, | ||
port=ftp_server.port, | ||
user=ftp_server.user, | ||
password=ftp_server.password, | ||
) | ||
|
||
|
||
@pytest.fixture() | ||
def ftp_file_connection_with_path(request, ftp_file_connection): | ||
connection = ftp_file_connection | ||
root = PurePosixPath("/data/") | ||
|
||
def finalizer(): | ||
connection.remove_dir(root, recursive=True) | ||
|
||
request.addfinalizer(finalizer) | ||
|
||
connection.remove_dir(root, recursive=True) | ||
connection.create_dir(root) | ||
|
||
return connection, root | ||
|
||
|
||
@pytest.fixture() | ||
def ftp_file_connection_with_path_and_files(resource_path_original, ftp_file_connection_with_path): | ||
connection, upload_to = ftp_file_connection_with_path | ||
upload_from = resource_path_original | ||
files = upload_files(upload_from, upload_to, connection) | ||
return connection, upload_to, files |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
import os | ||
from collections import namedtuple | ||
from pathlib import PurePosixPath | ||
|
||
import pytest | ||
|
||
from tests.fixtures.file_connections.util import upload_files | ||
|
||
|
||
@pytest.fixture( | ||
scope="session", | ||
params=[ | ||
pytest.param("real", marks=[pytest.mark.ftps, pytest.mark.file_connection, pytest.mark.connection]), | ||
], | ||
) | ||
def ftps_server(): | ||
FTPSServer = namedtuple("FTPSServer", ["host", "port", "user", "password"]) | ||
|
||
return FTPSServer( | ||
host=os.getenv("ONETL_FTPS_HOST"), | ||
port=os.getenv("ONETL_FTPS_PORT"), | ||
user=os.getenv("ONETL_FTPS_USER"), | ||
password=os.getenv("ONETL_FTPS_PASSWORD"), | ||
) | ||
|
||
|
||
@pytest.fixture() | ||
def ftps_file_connection(ftps_server): | ||
from onetl.connection import FTPS | ||
|
||
return FTPS( | ||
host=ftps_server.host, | ||
port=ftps_server.port, | ||
user=ftps_server.user, | ||
password=ftps_server.password, | ||
) | ||
|
||
|
||
@pytest.fixture() | ||
def ftps_file_connection_with_path(request, ftps_file_connection): | ||
connection = ftps_file_connection | ||
root = PurePosixPath("/data/") | ||
|
||
def finalizer(): | ||
connection.remove_dir(root, recursive=True) | ||
|
||
request.addfinalizer(finalizer) | ||
|
||
connection.remove_dir(root, recursive=True) | ||
connection.create_dir(root) | ||
|
||
return connection, root | ||
|
||
|
||
@pytest.fixture() | ||
def ftps_file_connection_with_path_and_files(resource_path_original, ftps_file_connection_with_path): | ||
connection, upload_to = ftps_file_connection_with_path | ||
upload_from = resource_path_original | ||
files = upload_files(upload_from, upload_to, connection) | ||
return connection, upload_to, files |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
import os | ||
from collections import namedtuple | ||
from pathlib import PurePosixPath | ||
|
||
import pytest | ||
|
||
from tests.fixtures.file_connections.util import upload_files | ||
|
||
|
||
@pytest.fixture( | ||
scope="session", | ||
params=[ | ||
pytest.param("real", marks=[pytest.mark.hdfs, pytest.mark.file_connection, pytest.mark.connection]), | ||
], | ||
) | ||
def hdfs_server(): | ||
HDFSServer = namedtuple("HDFSServer", ["host", "port"]) | ||
return HDFSServer( | ||
host=os.getenv("ONETL_HDFS_HOST"), | ||
port=os.getenv("ONETL_HDFS_PORT"), | ||
) | ||
|
||
|
||
@pytest.fixture() | ||
def hdfs_file_connection(hdfs_server): | ||
from onetl.connection import HDFS | ||
|
||
return HDFS(host=hdfs_server.host, port=hdfs_server.port) | ||
|
||
|
||
@pytest.fixture() | ||
def hdfs_file_connection_with_path(request, hdfs_file_connection): | ||
connection = hdfs_file_connection | ||
root = PurePosixPath("/data/") | ||
|
||
def finalizer(): | ||
connection.remove_dir(root, recursive=True) | ||
|
||
request.addfinalizer(finalizer) | ||
|
||
connection.remove_dir(root, recursive=True) | ||
connection.create_dir(root) | ||
|
||
return connection, root | ||
|
||
|
||
@pytest.fixture() | ||
def hdfs_file_connection_with_path_and_files(resource_path_original, hdfs_file_connection_with_path): | ||
connection, upload_to = hdfs_file_connection_with_path | ||
upload_from = resource_path_original | ||
files = upload_files(upload_from, upload_to, connection) | ||
return connection, upload_to, files |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
import os | ||
from collections import namedtuple | ||
from pathlib import PurePosixPath | ||
|
||
import pytest | ||
|
||
from tests.fixtures.file_connections.util import upload_files | ||
|
||
|
||
@pytest.fixture( | ||
scope="session", | ||
params=[ | ||
pytest.param("real", marks=[pytest.mark.s3, pytest.mark.file_connection, pytest.mark.connection]), | ||
], | ||
) | ||
def s3_server(): | ||
S3Server = namedtuple("S3Server", ["host", "port", "bucket", "access_key", "secret_key", "protocol"]) | ||
|
||
return S3Server( | ||
host=os.getenv("ONETL_S3_HOST"), | ||
port=os.getenv("ONETL_S3_PORT"), | ||
bucket=os.getenv("ONETL_S3_BUCKET"), | ||
access_key=os.getenv("ONETL_S3_ACCESS_KEY"), | ||
secret_key=os.getenv("ONETL_S3_SECRET_KEY"), | ||
protocol=os.getenv("ONETL_S3_PROTOCOL", "http").lower(), | ||
) | ||
|
||
|
||
@pytest.fixture() | ||
def s3_file_connection(s3_server): | ||
from onetl.connection import S3 | ||
|
||
return S3( | ||
host=s3_server.host, | ||
port=s3_server.port, | ||
bucket=s3_server.bucket, | ||
access_key=s3_server.access_key, | ||
secret_key=s3_server.secret_key, | ||
protocol=s3_server.protocol, | ||
) | ||
|
||
|
||
@pytest.fixture() | ||
def s3_file_connection_with_path(request, s3_file_connection): | ||
connection = s3_file_connection | ||
root = PurePosixPath("/data/") | ||
|
||
if not connection.client.bucket_exists(connection.bucket): | ||
connection.client.make_bucket(connection.bucket) | ||
|
||
def finalizer(): | ||
connection.remove_dir(root, recursive=True) | ||
|
||
request.addfinalizer(finalizer) | ||
|
||
connection.remove_dir(root, recursive=True) | ||
connection.create_dir(root) | ||
|
||
return connection, root | ||
|
||
|
||
@pytest.fixture() | ||
def s3_file_connection_with_path_and_files(resource_path_original, s3_file_connection_with_path): | ||
connection, upload_to = s3_file_connection_with_path | ||
upload_from = resource_path_original | ||
files = upload_files(upload_from, upload_to, connection) | ||
return connection, upload_to, files |
Oops, something went wrong.