diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..7d09d79 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,25 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Python 3", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:0-3.10", + "features": { + "ghcr.io/devcontainers-contrib/features/black:2": {}, + "ghcr.io/devcontainers-contrib/features/flake8:2": {}, + "ghcr.io/devcontainers-contrib/features/isort:2": {}, + "ghcr.io/devcontainers-contrib/features/meltano:2": {}, + "ghcr.io/devcontainers-contrib/features/mypy:2": {}, + "ghcr.io/devcontainers-contrib/features/poetry:2": {}, + "ghcr.io/devcontainers-contrib/features/twine:2": {} + }, + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "poetry install" + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/poetry.lock b/poetry.lock index 8bb9ad5..b785d6b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -611,46 +611,55 @@ files = [ [[package]] name = "mypy" -version = "0.910" +version = "0.991" description = "Optional static typing for Python" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" files = [ - {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"}, - {file = "mypy-0.910-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb"}, - {file = "mypy-0.910-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9"}, - {file = "mypy-0.910-cp35-cp35m-win_amd64.whl", hash = "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e"}, - {file = "mypy-0.910-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921"}, - {file = "mypy-0.910-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6"}, - {file = "mypy-0.910-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212"}, - {file = "mypy-0.910-cp36-cp36m-win_amd64.whl", hash = "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885"}, - {file = "mypy-0.910-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0"}, - {file = "mypy-0.910-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de"}, - {file = "mypy-0.910-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703"}, - {file = "mypy-0.910-cp37-cp37m-win_amd64.whl", hash = "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a"}, - {file = "mypy-0.910-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504"}, - {file = "mypy-0.910-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9"}, - {file = "mypy-0.910-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072"}, - {file = "mypy-0.910-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811"}, - {file = "mypy-0.910-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e"}, - {file = "mypy-0.910-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b"}, - {file = "mypy-0.910-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2"}, - {file = "mypy-0.910-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97"}, - {file = "mypy-0.910-cp39-cp39-win_amd64.whl", hash = "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8"}, - {file = "mypy-0.910-py3-none-any.whl", hash = "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"}, - {file = "mypy-0.910.tar.gz", hash = "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150"}, + {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, + {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, + {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, + {file = "mypy-0.991-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9ec663ed6c8f15f4ae9d3c04c989b744436c16d26580eaa760ae9dd5d662eb"}, + {file = "mypy-0.991-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4307270436fd7694b41f913eb09210faff27ea4979ecbcd849e57d2da2f65305"}, + {file = "mypy-0.991-cp310-cp310-win_amd64.whl", hash = "sha256:901c2c269c616e6cb0998b33d4adbb4a6af0ac4ce5cd078afd7bc95830e62c1c"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d13674f3fb73805ba0c45eb6c0c3053d218aa1f7abead6e446d474529aafc372"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c8cd4fb70e8584ca1ed5805cbc7c017a3d1a29fb450621089ffed3e99d1857f"}, + {file = "mypy-0.991-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:209ee89fbb0deed518605edddd234af80506aec932ad28d73c08f1400ef80a33"}, + {file = "mypy-0.991-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37bd02ebf9d10e05b00d71302d2c2e6ca333e6c2a8584a98c00e038db8121f05"}, + {file = "mypy-0.991-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:26efb2fcc6b67e4d5a55561f39176821d2adf88f2745ddc72751b7890f3194ad"}, + {file = "mypy-0.991-cp311-cp311-win_amd64.whl", hash = "sha256:3a700330b567114b673cf8ee7388e949f843b356a73b5ab22dd7cff4742a5297"}, + {file = "mypy-0.991-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1f7d1a520373e2272b10796c3ff721ea1a0712288cafaa95931e66aa15798813"}, + {file = "mypy-0.991-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:641411733b127c3e0dab94c45af15fea99e4468f99ac88b39efb1ad677da5711"}, + {file = "mypy-0.991-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3d80e36b7d7a9259b740be6d8d906221789b0d836201af4234093cae89ced0cd"}, + {file = "mypy-0.991-cp37-cp37m-win_amd64.whl", hash = "sha256:e62ebaad93be3ad1a828a11e90f0e76f15449371ffeecca4a0a0b9adc99abcef"}, + {file = "mypy-0.991-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b86ce2c1866a748c0f6faca5232059f881cda6dda2a893b9a8373353cfe3715a"}, + {file = "mypy-0.991-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac6e503823143464538efda0e8e356d871557ef60ccd38f8824a4257acc18d93"}, + {file = "mypy-0.991-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cca5adf694af539aeaa6ac633a7afe9bbd760df9d31be55ab780b77ab5ae8bf"}, + {file = "mypy-0.991-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12c56bf73cdab116df96e4ff39610b92a348cc99a1307e1da3c3768bbb5b135"}, + {file = "mypy-0.991-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:652b651d42f155033a1967739788c436491b577b6a44e4c39fb340d0ee7f0d70"}, + {file = "mypy-0.991-cp38-cp38-win_amd64.whl", hash = "sha256:4175593dc25d9da12f7de8de873a33f9b2b8bdb4e827a7cae952e5b1a342e243"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98e781cd35c0acf33eb0295e8b9c55cdbef64fcb35f6d3aa2186f289bed6e80d"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d7464bac72a85cb3491c7e92b5b62f3dcccb8af26826257760a552a5e244aa5"}, + {file = "mypy-0.991-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c9166b3f81a10cdf9b49f2d594b21b31adadb3d5e9db9b834866c3258b695be3"}, + {file = "mypy-0.991-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8472f736a5bfb159a5e36740847808f6f5b659960115ff29c7cecec1741c648"}, + {file = "mypy-0.991-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e80e758243b97b618cdf22004beb09e8a2de1af481382e4d84bc52152d1c476"}, + {file = "mypy-0.991-cp39-cp39-win_amd64.whl", hash = "sha256:74e259b5c19f70d35fcc1ad3d56499065c601dfe94ff67ae48b85596b9ec1461"}, + {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, + {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, ] [package.dependencies] -mypy-extensions = ">=0.4.3,<0.5.0" -toml = "*" -typed-ast = {version = ">=1.4.0,<1.5.0", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.7.4" +mypy-extensions = ">=0.4.3" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} +typing-extensions = ">=3.10" [package.extras] dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<1.5.0)"] +install-types = ["pip"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] [[package]] name = "mypy-extensions" @@ -1152,14 +1161,14 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "setuptools" -version = "67.0.0" +version = "67.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.0.0-py3-none-any.whl", hash = "sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d"}, - {file = "setuptools-67.0.0.tar.gz", hash = "sha256:883131c5b6efa70b9101c7ef30b2b7b780a4283d5fc1616383cdf22c83cbefe6"}, + {file = "setuptools-67.1.0-py3-none-any.whl", hash = "sha256:a7687c12b444eaac951ea87a9627c4f904ac757e7abdc5aac32833234af90378"}, + {file = "setuptools-67.1.0.tar.gz", hash = "sha256:e261cdf010c11a41cb5cb5f1bf3338a7433832029f559a6a7614bd42a967c300"}, ] [package.extras] @@ -1293,8 +1302,8 @@ testing = ["pytest (>=7.2.1,<8.0.0)", "pytest-durations (>=1.2.0,<2.0.0)"] [package.source] type = "git" url = "https://github.com/meltano/sdk.git" -reference = "kgpayne/fix-test-tap-discovery" -resolved_reference = "44b4b020639506964db20d064ec22595c0aa92ca" +reference = "main" +resolved_reference = "5db1b505abd2942645104fd3d80a6279e7c0b415" [[package]] name = "six" @@ -1470,18 +1479,6 @@ postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] sqlcipher = ["sqlcipher3-binary"] -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] - [[package]] name = "tomli" version = "2.0.1" @@ -1523,42 +1520,36 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psu [[package]] name = "typed-ast" -version = "1.4.3" +version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, - {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, - {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, - {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, - {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, - {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, - {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, - {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, - {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, - {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, + {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, + {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, + {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, + {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, + {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, + {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, + {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, + {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, + {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, + {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, ] [[package]] @@ -1658,4 +1649,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = "<3.11,>=3.7.1" -content-hash = "cfb446dcc89f7bd25d7169b323e659a187834ad45e789622c5e71ed0a7f0c681" +content-hash = "8a2ecfb051a4f767f02acc48315be9aa92032be6bfe6ed3e8da5cd7be425da9f" diff --git a/pyproject.toml b/pyproject.toml index 0c46531..6ab1699 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ tox = "^3.24.4" flake8 = "^3.9.2" black = "^22.3.0" pydocstyle = "^6.1.1" -mypy = "^0.910" +mypy = "^0.991" types-requests = "^2.26.1" isort = "^5.10.1" diff --git a/tap_snowflake/client.py b/tap_snowflake/client.py index 1061893..942dbf6 100644 --- a/tap_snowflake/client.py +++ b/tap_snowflake/client.py @@ -6,18 +6,50 @@ from __future__ import annotations import os +from dataclasses import dataclass +from enum import Enum, auto from pathlib import Path -from typing import Iterable, List, Tuple +from typing import Any, Iterable, List, Tuple from uuid import uuid4 import sqlalchemy -from singer_sdk import SQLConnector, SQLStream +from singer_sdk import SQLConnector, SQLStream, metrics from singer_sdk.helpers._batch import BaseBatchFileEncoding, BatchConfig from singer_sdk.streams.core import REPLICATION_FULL_TABLE, REPLICATION_INCREMENTAL from snowflake.sqlalchemy import URL from sqlalchemy.sql import text +class ProfileStats(Enum): + """Profile Statistics Enum.""" + + TABLE_SIZE_IN_MB = auto() + TABLE_ROW_COUNT = auto() + COLUMN_MIN_VALUE = auto() + COLUMN_MAX_VALUE = auto() + COLUMN_NULL_VALUES = auto() + COLUMN_NONNULL_VALUES = auto() + + +@dataclass +class ColumnProfile: + """Column Profile.""" + + min_value: int | None + max_value: int | None + null_values: int | None + nonnull_values: int | None + + +@dataclass +class TableProfile: + """Table Profile.""" + + column_profiles: dict[str, ColumnProfile] + row_count: int | None + size_in_mb: int | None + + class SnowflakeConnector(SQLConnector): """Connects to the Snowflake SQL source.""" @@ -76,12 +108,148 @@ def discover_catalog_entries(self) -> list[dict]: return result + def get_table_profile( + self, + full_table_name: str, + stats: set[ProfileStats], + profile_columns: list[str] | None = None, + ) -> TableProfile: + """Scan source system for a set of profile stats. + + This is a proposed implementation to eventually be included in the SDK. + + This generic implementation should be compatible with most/all SQL providers, as + it only requires min(), max(), and count() support, which are all part of ANSI + SQL. + + Developers may override this for performance improvements on their native SQL + implementation. + + Any stats not requested but available 'for free' while collecting other stats + may be included in the returned profile. For example, the base implementation + cannot calculate COLUMN_NULL_VALUES without also calculating TABLE_ROW_COUNT and + COLUMN_NONNULL_VALUES. The additional stats therefore would be returned along + with the requested stats. The consumer should ignore or disregard any stats not + needed. + + Note: Gathering stats can take a long time. The implementation should attempt + to combine stats gathering into fewer tables scans where possible and only + spend time pulling in requested stats. + + Returns: + A TableProfile object. Stats may be left null if not requested, not + available, or not implemented. Callers should check each field for null + values and handle null as 'not available'. + """ + profile_columns = profile_columns or [] + expressions: list[str] = [] + if ( + ProfileStats.TABLE_ROW_COUNT in stats + or ProfileStats.COLUMN_NULL_VALUES in stats + ): + expressions.append("count(1) as row_count") + if ProfileStats.TABLE_SIZE_IN_MB in stats: + self.logger.debug( + "TABLE_SIZE_IN_MB stats not implemented for this provider." + ) + for col in profile_columns or []: + if ProfileStats.COLUMN_MIN_VALUE in stats: + expressions.append(f"min({col}) as min__{col}") + if ProfileStats.COLUMN_MAX_VALUE in stats: + expressions.append(f"max({col}) as max__{col}") + if ( + ProfileStats.COLUMN_NONNULL_VALUES in stats + or ProfileStats.COLUMN_NULL_VALUES in stats + ): + expressions.append(f"count({col}) as nonnull__{col}") + if ProfileStats.COLUMN_NULL_VALUES in stats: + expressions.append(f"count(1) - count({col}) as null__{col}") + result_dict = ( + self.connection.execute( + text(f"SELECT {', '.join(expressions)} FROM {full_table_name}") + ) + .one() + ._asdict() + ) + return TableProfile( + row_count=result_dict.get("row_count", None), + size_in_mb=result_dict.get("size_in_mb", None), + column_profiles={ + col: ColumnProfile( + min_value=result_dict.get(f"min__{col}", None), + max_value=result_dict.get(f"max__{col}", None), + null_values=result_dict.get(f"null__{col}", None), + nonnull_values=result_dict.get(f"nonnull__{col}", None), + ) + for col in profile_columns + }, + ) + class SnowflakeStream(SQLStream): """Stream class for Snowflake streams.""" connector_class = SnowflakeConnector + @property + def is_sorted(self): + """Is sorted.""" + return bool(self.replication_key) + + def _sync_batches( + self, + batch_config: BatchConfig, + context: dict | None = None, + ) -> None: + """Sync batches, emitting BATCH messages. + + This is a proposed replacement for the SDK internal SQLStream._sync_baches. + Per: https://github.com/meltano/sdk/issues/976 + + This version stores the max replication value before batch sync starts, and then + increments the stream state with this value after the sync operation completes. + + Since any FULL_TABLE sync operations may subsequently be run as INCREMENTAL, + the querying of the max value is not dependent upon running in INCREMENTAL mode. + + Args: + batch_config: The batch configuration. + context: Stream partition or context dictionary. + """ + self._write_starting_replication_value(context) + + # New: Collect the max value for the replication column. + max_replication_key_value = None + if self.replication_key: + table_profile: TableProfile = ( + self.connector.get_table_profile( # type: ignore + full_table_name=self.fully_qualified_name, + stats={ProfileStats.COLUMN_MAX_VALUE}, + profile_columns=[self.replication_key], + ) + ) + max_replication_key_value = table_profile.column_profiles[ + self.replication_key + ].max_value + + # Not chanded: Note that the STATE messages will not have an incremented + # replication key value at this point. + with metrics.batch_counter(self.name, context=context) as counter: + for encoding, manifest in self.get_batches(batch_config, context): + counter.increment() + self._write_batch_message(encoding=encoding, manifest=manifest) + self._write_state_message() + + # New: Increment and emit the final STATE message after sync has completed. + if max_replication_key_value: + self._increment_stream_state( + latest_record={ + self.replication_key: max_replication_key_value # type: ignore + }, + context=context, + ) + self._write_state_message() + def get_batches( self, batch_config: BatchConfig, context: dict | None = None ) -> Iterable[tuple[BaseBatchFileEncoding, list[str]]]: @@ -99,19 +267,27 @@ def get_batches( raise NotImplementedError( f"Stream '{self.name}' does not support partitioning." ) - return self.get_batches_from_internal_user_stage(batch_config, context) + yield from self.get_batches_from_internal_user_stage(batch_config, context) - @staticmethod def _get_full_table_copy_statement( - sync_id: str, prefix: str, objects: List[str], table_name: str + self, sync_id: str, prefix: str, objects: List[str], table_name: str ) -> Tuple[text, dict]: """Get FULL_TABLE copy statement and key bindings.""" + statement = [f"copy into '@~/tap-snowflake/{sync_id}/{prefix}' from "] + if self.replication_key: + statement.append( + f"(select object_construct({', '.join(objects)}) from {table_name} " + f"order by {self.replication_key}) " + ) + else: + statement.append( + f"(select object_construct({', '.join(objects)}) from {table_name}) " + ) + statement.append( + "file_format = (type='JSON', compression='GZIP') overwrite = TRUE" + ) return ( - text( - f"copy into '@~/tap-snowflake/{sync_id}/{prefix}' from " - + f"(select object_construct({', '.join(objects)}) from {table_name}) " - + "file_format = (type='JSON', compression='GZIP') overwrite = TRUE" - ), + text("".join(statement)), {}, ) @@ -202,11 +378,11 @@ def get_batches_from_internal_user_stage( copy_statement, kwargs = self._get_copy_statement( sync_id=sync_id, prefix=prefix, context=context ) - self.connector.connection.execute(copy_statement, **kwargs) + self.connector.connection.execute(copy_statement, **kwargs).all() # list available files results = self.connector.connection.execute( text(f"list '@~/tap-snowflake/{sync_id}/'") - ) + ).all() # download available files local_path = f"{root.replace('file://', '')}/{sync_id}" Path(local_path).mkdir(parents=True, exist_ok=True) @@ -223,3 +399,49 @@ def get_batches_from_internal_user_stage( text(f"remove '@~/tap-snowflake/{sync_id}/'") ) yield (batch_config.encoding, files) + + # Get records from stream + # Overridden to use native objects under `if start_val:` + def get_records(self, context: dict | None) -> Iterable[dict[str, Any]]: + """Return a generator of record-type dictionary objects. + + If the stream has a replication_key value defined, records will be sorted by the + incremental key. If the stream also has an available starting bookmark, the + records will be filtered for values greater than or equal to the bookmark value. + + Args: + context: If partition context is provided, will read specifically from this + data slice. + + Yields: + One dict per record. + + Raises: + NotImplementedError: If partition is passed in context and the stream does + not support partitioning. + """ + if context: + raise NotImplementedError( + f"Stream '{self.name}' does not support partitioning." + ) + + selected_column_names = self.get_selected_schema()["properties"].keys() + table = self.connector.get_table( + full_table_name=self.fully_qualified_name, + column_names=selected_column_names, + ) + query = table.select() + + if self.replication_key: + replication_key_col = table.columns[self.replication_key] + query = query.order_by(replication_key_col) + + start_val = self.get_starting_replication_key_value(context) + if start_val: + query = query.where(replication_key_col >= start_val) + + if self._MAX_RECORDS_LIMIT is not None: + query = query.limit(self._MAX_RECORDS_LIMIT) + + for record in self.connector.connection.execute(query): + yield dict(record) diff --git a/tests/catalog-incremental.json b/tests/catalog-incremental.json new file mode 100644 index 0000000..0fbf36e --- /dev/null +++ b/tests/catalog-incremental.json @@ -0,0 +1,222 @@ +{ + "streams": [ + { + "tap_stream_id": "tpch_sf1-supplier", + "table_name": "supplier", + "replication_method": "INCREMENTAL", + "replication_key": "s_suppkey", + "key_properties": ["s_suppkey"], + "schema": { + "properties": { + "s_suppkey": { + "type": ["number"] + }, + "s_name": { + "type": ["string"] + }, + "s_address": { + "type": ["string"] + }, + "s_nationkey": { + "type": ["number"] + }, + "s_phone": { + "type": ["string"] + }, + "s_acctbal": { + "type": ["number"] + }, + "s_comment": { + "type": ["string", "null"] + } + }, + "type": "object", + "required": [ + "s_suppkey", + "s_name", + "s_address", + "s_nationkey", + "s_phone", + "s_acctbal" + ] + }, + "is_view": false, + "stream": "tpch_sf1-supplier", + "metadata": [ + { + "breadcrumb": ["properties", "s_suppkey"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "s_name"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "s_address"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "s_nationkey"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "s_phone"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "s_acctbal"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "s_comment"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "table-key-properties": ["s_suppkey"], + "valid-replication-keys": ["s_suppkey"], + "forced-replication-method": "", + "schema-name": "tpch_sf1", + "selected": true + } + } + ] + }, + { + "tap_stream_id": "tpch_sf1-orders", + "table_name": "orders", + "replication_method": "INCREMENTAL", + "replication_key": "o_orderkey", + "key_properties": ["o_orderkey"], + "schema": { + "properties": { + "o_orderkey": { + "type": ["number"] + }, + "o_custkey": { + "type": ["number"] + }, + "o_orderstatus": { + "type": ["string"] + }, + "o_totalprice": { + "type": ["number"] + }, + "o_orderdate": { + "format": "date", + "type": ["string"] + }, + "o_orderpriority": { + "type": ["string"] + }, + "o_clerk": { + "type": ["string"] + }, + "o_shippriority": { + "type": ["number"] + }, + "o_comment": { + "type": ["string"] + } + }, + "type": "object", + "required": [ + "o_orderkey", + "o_custkey", + "o_orderstatus", + "o_totalprice", + "o_orderdate", + "o_orderpriority", + "o_clerk", + "o_shippriority", + "o_comment" + ] + }, + "is_view": false, + "stream": "tpch_sf1-orders", + "metadata": [ + { + "breadcrumb": ["properties", "o_orderkey"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_custkey"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_orderstatus"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_totalprice"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_orderdate"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_orderpriority"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_clerk"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_shippriority"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": ["properties", "o_comment"], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "table-key-properties": ["o_orderkey"], + "valid-replication-keys": ["o_orderkey", "o_orderdate"], + "forced-replication-method": "", + "schema-name": "tpch_sf1", + "selected": true + } + } + ] + } + ] +} diff --git a/tests/catalog.json b/tests/catalog.json index 80a89ad..62e5e69 100644 --- a/tests/catalog.json +++ b/tests/catalog.json @@ -4,7 +4,7 @@ "tap_stream_id": "tpch_sf1-customer", "table_name": "customer", "replication_method": "", - "key_properties": [], + "key_properties": ["c_custkey"], "schema": { "properties": { "c_custkey": { @@ -97,7 +97,8 @@ "breadcrumb": [], "metadata": { "inclusion": "available", - "table-key-properties": [], + "table-key-properties": ["c_custkey"], + "valid-replication-keys": ["c_custkey"], "forced-replication-method": "", "schema-name": "tpch_sf1", "selected": true @@ -299,7 +300,7 @@ "tap_stream_id": "tpch_sf1-nation", "table_name": "nation", "replication_method": "", - "key_properties": [], + "key_properties": ["n_nationkey"], "schema": { "properties": { "n_nationkey": { @@ -361,7 +362,7 @@ "tap_stream_id": "tpch_sf1-orders", "table_name": "orders", "replication_method": "", - "key_properties": [], + "key_properties": ["o_orderkey"], "schema": { "properties": { "o_orderkey": { @@ -467,7 +468,8 @@ "breadcrumb": [], "metadata": { "inclusion": "available", - "table-key-properties": [], + "table-key-properties": ["o_orderkey"], + "valid-replication-keys": ["o_orderkey"], "forced-replication-method": "", "schema-name": "tpch_sf1", "selected": true