diff --git a/.github/workflows/test_tap.yml b/.github/workflows/test_tap.yml index c7efef4a..76567dc8 100644 --- a/.github/workflows/test_tap.yml +++ b/.github/workflows/test_tap.yml @@ -56,9 +56,10 @@ jobs: run: | poetry install if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - - name: Check formatting with black + - name: Check formatting with black and isort run: | poetry run black --check . + poetry run isort --check . - name: Check typing annotations with mypy run: | poetry run mypy . --ignore-missing-imports diff --git a/poetry.lock b/poetry.lock index baf23304..28eef0c0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -237,6 +237,20 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "isort" +version = "5.10.1" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.6.1,<4.0" + +[package.extras] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] +plugins = ["setuptools"] + [[package]] name = "joblib" version = "1.1.0" @@ -771,7 +785,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "<3.11,>=3.7.2" -content-hash = "25c20e3fbddd289d382ccffc00c9e7eeff4b4ad191e2aed5993551da3bfedc90" +content-hash = "d755f943eb3f9d2839f43d94a4bb59271b70b8460c36f5492f14c3f5ade71309" [metadata.files] appdirs = [ @@ -996,6 +1010,10 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +isort = [ + {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, + {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, +] joblib = [ {file = "joblib-1.1.0-py2.py3-none-any.whl", hash = "sha256:f21f109b3c7ff9d95f8387f752d0d9c34a02aa2f7060c2135f465da0e5160ff6"}, {file = "joblib-1.1.0.tar.gz", hash = "sha256:4158fcecd13733f8be669be0683b96ebdbbd38d23559f54dca7205aea1bf1e35"}, diff --git a/pyproject.toml b/pyproject.toml index d8674a84..34e125b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ types-beautifulsoup4 = "^4.11.1" types-requests = "^2.25.6" types-python-dateutil = "^2.8.9" requests-cache = "^0.9.1" +isort = "^5.10.1" [[tool.mypy.overrides]] module = [ diff --git a/tap_github/client.py b/tap_github/client.py index d443b01f..74714f68 100644 --- a/tap_github/client.py +++ b/tap_github/client.py @@ -1,16 +1,15 @@ """REST client handling, including GitHubStream base class.""" +import collections +import inspect +import re from types import FrameType from typing import Any, Dict, Iterable, List, Optional, cast +from urllib.parse import parse_qs, urlparse -import inspect import requests - from dateutil.parser import parse -from urllib.parse import parse_qs, urlparse - from nested_lookup import nested_lookup - from singer_sdk.exceptions import FatalAPIError, RetriableAPIError from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import GraphQLStream, RESTStream diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index 82346551..2a751359 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -1,6 +1,6 @@ """User Stream types classes for tap-github.""" -from typing import Dict, List, Optional, Iterable, Any +from typing import Any, Dict, Iterable, List, Optional from singer_sdk import typing as th # JSON Schema typing helpers diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 703311d4..d599b256 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1,20 +1,19 @@ """Repository Stream types classes for tap-github.""" from typing import Any, Dict, Iterable, List, Optional, Tuple +from urllib.parse import parse_qs, urlparse import requests +from dateutil.parser import parse from singer_sdk import typing as th # JSON Schema typing helpers from singer_sdk.helpers.jsonpath import extract_jsonpath -from dateutil.parser import parse -from urllib.parse import parse_qs, urlparse - from tap_github.client import GitHubGraphqlStream, GitHubRestStream from tap_github.schema_objects import ( - user_object, label_object, - reactions_object, milestone_object, + reactions_object, + user_object, ) from tap_github.scraping import scrape_dependents diff --git a/tap_github/scraping.py b/tap_github/scraping.py index 4604b709..3f6e52c8 100644 --- a/tap_github/scraping.py +++ b/tap_github/scraping.py @@ -3,12 +3,12 @@ Inspired by https://github.com/dogsheep/github-to-sqlite/pull/70 """ import logging -import requests import time - from typing import Any, Dict, Iterable, Optional from urllib.parse import urlparse +import requests + def scrape_dependents( response: requests.Response, logger: Optional[logging.Logger] = None @@ -36,8 +36,7 @@ def scrape_dependents( def _scrape_dependents(url: str, logger: logging.Logger) -> Iterable[Dict[str, Any]]: # Optional dependency: - from bs4 import BeautifulSoup - from bs4 import Tag + from bs4 import BeautifulSoup, Tag s = requests.Session() diff --git a/tap_github/streams.py b/tap_github/streams.py index a5763d43..545d7c7d 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -1,8 +1,14 @@ from enum import Enum -from typing import Type, Set, List +from typing import List, Set, Type from singer_sdk.streams.core import Stream +from tap_github.organization_streams import ( + OrganizationStream, + TeamMembersStream, + TeamRolesStream, + TeamsStream, +) from tap_github.repository_streams import ( AnonymousContributorsStream, AssigneesStream, @@ -36,17 +42,7 @@ WorkflowRunsStream, WorkflowsStream, ) -from tap_github.user_streams import ( - StarredStream, - UserContributedToStream, - UserStream, -) -from tap_github.organization_streams import ( - OrganizationStream, - TeamMembersStream, - TeamRolesStream, - TeamsStream, -) +from tap_github.user_streams import StarredStream, UserContributedToStream, UserStream class Streams(Enum): diff --git a/tap_github/tap.py b/tap_github/tap.py index dd0ac488..5dc028c5 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -1,9 +1,8 @@ """GitHub tap class.""" -from typing import List - -import os import logging +import os +from typing import List from singer_sdk import Stream, Tap from singer_sdk import typing as th # JSON schema typing helpers diff --git a/tap_github/tests/fixtures.py b/tap_github/tests/fixtures.py index 77d44f43..20e76399 100644 --- a/tap_github/tests/fixtures.py +++ b/tap_github/tests/fixtures.py @@ -1,9 +1,10 @@ -import os -import logging import datetime +import logging +import os import sys import pytest + from ..utils.filter_stdout import FilterStdOutput # Filter out singer output during tests diff --git a/tap_github/tests/test_core.py b/tap_github/tests/test_core.py index fc72269b..edb5fffb 100644 --- a/tap_github/tests/test_core.py +++ b/tap_github/tests/test_core.py @@ -1,22 +1,20 @@ """Tests standard tap features using the built-in SDK tests library.""" -import os import logging - +import os from unittest import mock from unittest.mock import patch -from tap_github.utils.filter_stdout import nostdout - -from .fixtures import alternative_sync_chidren from singer_sdk.testing import get_standard_tap_tests from tap_github.tap import TapGitHub +from tap_github.utils.filter_stdout import nostdout from .fixtures import ( + alternative_sync_chidren, + organization_list_config, repo_list_config, search_config, username_list_config, - organization_list_config, ) diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index c79c938e..135a550c 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -1,12 +1,12 @@ -import os import logging -import pytest +import os from typing import Optional - from unittest.mock import patch -from singer_sdk.helpers._singer import Catalog +import pytest from singer_sdk.helpers import _catalog as cat_helpers +from singer_sdk.helpers._singer import Catalog + from tap_github.tap import TapGitHub from .fixtures import alternative_sync_chidren, repo_list_config, username_list_config diff --git a/tap_github/user_streams.py b/tap_github/user_streams.py index f1daa5df..6cff4af3 100644 --- a/tap_github/user_streams.py +++ b/tap_github/user_streams.py @@ -1,7 +1,7 @@ """User Stream types classes for tap-github.""" import re -from typing import Dict, List, Optional, Iterable, Any +from typing import Any, Dict, Iterable, List, Optional from singer_sdk import typing as th # JSON Schema typing helpers from singer_sdk.exceptions import FatalAPIError