Skip to content

Commit

Permalink
Source Azure Blob Storage: add integration tests (#36542)
Browse files Browse the repository at this point in the history 
Signed-off-by: Artem Inzhyyants <artem.inzhyyants@gmail.com>
Co-authored-by: Catherine Noll <noll.catherine@gmail.com>
Co-authored-by: Serhii Lazebnyi <serhii.lazebnyi@globallogic.com>
  • Loading branch information
@artem1205 @clnoll @lazebnyi
3 people authored Apr 3, 2024
1 parent 33d6c50 commit 28dd34d
Show file tree
Hide file tree
Showing 23 changed files with 1,223 additions and 186 deletions.
3 changes: 3 additions & 0 deletions airbyte-integrations/connectors/source-azure-blob-storage/.coveragerc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[run]
omit =
source_azure_blob_storage/run.py
23 changes: 23 additions & 0 deletions airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.

from __future__ import annotations

from typing import TYPE_CHECKING

if TYPE_CHECKING:
from dagger import Container


async def pre_connector_install(base_image_container: Container) -> Container:
"""
Docker compose is required to run the integration tests so we install Docker on top of the base image.
"""
return (
base_image_container.with_exec(["sh", "-c", "apt-get update && apt-get install -y curl jq"])
# Download install-docker.sh script
.with_exec(["curl", "-fsSL", "https://get.docker.com", "-o", "/tmp/install-docker.sh"])
# Run the install-docker.sh script with a pinned Docker version
.with_exec(["sh", "/tmp/install-docker.sh", "--version", "25.0"])
# Remove the install-docker.sh script
.with_exec(["rm", "/tmp/install-docker.sh"])
)
35 changes: 35 additions & 0 deletions ...nnectors/source-azure-blob-storage/integration_tests/configs/config_integration_avro.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
"azure_blob_storage_account_name": "account1",
"azure_blob_storage_account_key": "key1",
"azure_blob_storage_container_name": "testcontainer",
"streams": [
{
"name": "users",
"file_type": "avro",
"globs": ["**/test_avro_users*.avro"],
"validation_policy": "Emit Record",
"format": {
"filetype": "avro"
}
},
{
"name": "purchases",
"file_type": "avro",
"globs": ["**/test_avro_purchases*.avro"],
"validation_policy": "Emit Record",
"format": {
"filetype": "avro"
}
},
{
"name": "products",
"file_type": "avro",
"globs": ["**/test_avro_products*.avro"],
"validation_policy": "Emit Record",
"format": {
"filetype": "avro"
}
}
]
}
128 changes: 128 additions & 0 deletions ...onnectors/source-azure-blob-storage/integration_tests/configs/config_integration_csv.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
{
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
"azure_blob_storage_account_name": "account1",
"azure_blob_storage_account_key": "key1",
"azure_blob_storage_container_name": "testcontainer",
"streams": [
{
"name": "users",
"file_type": "csv",
"globs": ["**/test_csv_users*.csv"],
"legacy_prefix": "",
"validation_policy": "Emit Record",
"format": {
"filetype": "csv",
"delimiter": ",",
"quote_char": "\"",
"double_quote": true,
"null_values": [
"",
"#N/A",
"#N/A N/A",
"#NA",
"-1.#IND",
"-1.#QNAN",
"-NaN",
"-nan",
"1.#IND",
"1.#QNAN",
"N/A",
"NA",
"NULL",
"NaN",
"n/a",
"nan",
"null"
],
"true_values": ["1", "True", "TRUE", "true"],
"false_values": ["0", "False", "FALSE", "false"],
"inference_type": "Primitive Types Only",
"strings_can_be_null": false,
"encoding": "utf8",
"header_definition": {
"header_definition_type": "From CSV"
}
}
},
{
"name": "purchases",
"file_type": "csv",
"globs": ["**/test_csv_purchases*.csv"],
"legacy_prefix": "",
"validation_policy": "Emit Record",
"format": {
"filetype": "csv",
"delimiter": ",",
"quote_char": "\"",
"double_quote": true,
"null_values": [
"",
"#N/A",
"#N/A N/A",
"#NA",
"-1.#IND",
"-1.#QNAN",
"-NaN",
"-nan",
"1.#IND",
"1.#QNAN",
"N/A",
"NA",
"NULL",
"NaN",
"n/a",
"nan",
"null"
],
"true_values": ["1", "True", "TRUE", "true"],
"false_values": ["0", "False", "FALSE", "false"],
"inference_type": "Primitive Types Only",
"strings_can_be_null": false,
"encoding": "utf8",
"header_definition": {
"header_definition_type": "From CSV"
}
}
},
{
"name": "products",
"file_type": "csv",
"globs": ["**/test_csv_products*.csv"],
"legacy_prefix": "",
"validation_policy": "Emit Record",
"format": {
"filetype": "csv",
"delimiter": ",",
"quote_char": "\"",
"double_quote": true,
"null_values": [
"",
"#N/A",
"#N/A N/A",
"#NA",
"-1.#IND",
"-1.#QNAN",
"-NaN",
"-nan",
"1.#IND",
"1.#QNAN",
"N/A",
"NA",
"NULL",
"NaN",
"n/a",
"nan",
"null"
],
"true_values": ["1", "True", "TRUE", "true"],
"false_values": ["0", "False", "FALSE", "false"],
"inference_type": "Primitive Types Only",
"strings_can_be_null": false,
"encoding": "utf8",
"header_definition": {
"header_definition_type": "From CSV"
}
}
}
]
}
41 changes: 41 additions & 0 deletions ...nectors/source-azure-blob-storage/integration_tests/configs/config_integration_jsonl.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
{
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
"azure_blob_storage_account_name": "account1",
"azure_blob_storage_account_key": "key1",
"azure_blob_storage_container_name": "testcontainer",
"streams": [
{
"name": "users",
"file_type": "jsonl",
"globs": ["**/test_jsonl_users*.jsonl"],
"legacy_prefix": "",
"newlines_in_values": true,
"validation_policy": "Emit Record",
"format": {
"filetype": "jsonl"
}
},
{
"name": "purchases",
"file_type": "jsonl",
"globs": ["**/test_jsonl_purchases*.jsonl"],
"legacy_prefix": "",
"newlines_in_values": true,
"validation_policy": "Emit Record",
"format": {
"filetype": "jsonl"
}
},
{
"name": "products",
"file_type": "jsonl",
"globs": ["**/test_jsonl_products*.jsonl"],
"legacy_prefix": "",
"newlines_in_values": true,
"validation_policy": "Emit Record",
"format": {
"filetype": "jsonl"
}
}
]
}
35 changes: 35 additions & 0 deletions ...ctors/source-azure-blob-storage/integration_tests/configs/config_integration_parquet.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"azure_blob_storage_endpoint": "http://localhost:10000/account1",
"azure_blob_storage_account_name": "account1",
"azure_blob_storage_account_key": "key1",
"azure_blob_storage_container_name": "testcontainer",
"streams": [
{
"name": "users",
"file_type": "jsonl",
"globs": ["**/test_parquet_users*.parquet"],
"validation_policy": "Emit Record",
"format": {
"filetype": "parquet"
}
},
{
"name": "purchases",
"file_type": "jsonl",
"globs": ["**/test_parquet_purchases*.parquet"],
"validation_policy": "Emit Record",
"format": {
"filetype": "parquet"
}
},
{
"name": "products",
"file_type": "jsonl",
"globs": ["**/test_parquet_products*.parquet"],
"validation_policy": "Emit Record",
"format": {
"filetype": "parquet"
}
}
]
}
Loading

0 comments on commit 28dd34d

Please sign in to comment.