From 8f1f90af5b113c251fb779a7f56ab8368a825f83 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Thu, 15 Dec 2022 13:01:33 +0200 Subject: [PATCH] Source Gitlab: unit tests (#20479) * #20371 source gitlab: unit tests * #20371 source gitlab: upd changelog * #20371 source gitlab: revert httpAvailabilityStrategy * #20371 source gitlab: rm availability strategy * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-gitlab/Dockerfile | 2 +- .../integration_tests/expected_records.txt | 2 +- .../connectors/source-gitlab/setup.py | 7 +- .../source-gitlab/source_gitlab/streams.py | 6 + .../source-gitlab/unit_tests/conftest.py | 14 ++ .../source-gitlab/unit_tests/test_source.py | 36 +++++ .../source-gitlab/unit_tests/test_streams.py | 135 ++++++++++++++++++ .../source-gitlab/unit_tests/unit_test.py | 7 - docs/integrations/sources/gitlab.md | 1 + 11 files changed, 198 insertions(+), 16 deletions(-) create mode 100644 airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py delete mode 100644 airbyte-integrations/connectors/source-gitlab/unit_tests/unit_test.py diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2c111dc09c0f..daf0c819a097 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -562,7 +562,7 @@ - name: Gitlab sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 dockerRepository: airbyte/source-gitlab - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab icon: gitlab.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ef1840396805..470926107b2f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4683,7 +4683,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-gitlab:0.1.10" +- dockerImage: "airbyte/source-gitlab:0.1.11" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/gitlab" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index 9644fab5adcf..64a7fa5b6e76 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt index e68e1ae2e4ef..ecc417c6b7fb 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt @@ -1,4 +1,4 @@ -{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "assignees": [], "type": "ISSUE", "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "17 days remaining"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1670761695320} +{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "assignees": [], "type": "ISSUE", "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "15 days remaining"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1670761695320} {"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1670873255735} {"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873256216} {"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1670873256571} diff --git a/airbyte-integrations/connectors/source-gitlab/setup.py b/airbyte-integrations/connectors/source-gitlab/setup.py index c9fd74b8641a..5b1e107930b1 100644 --- a/airbyte-integrations/connectors/source-gitlab/setup.py +++ b/airbyte-integrations/connectors/source-gitlab/setup.py @@ -5,12 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.12.4", "vcrpy==4.1.1"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "vcrpy==4.1.1"] -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "source-acceptance-test", -] +TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests_mock"] setup( name="source_gitlab", diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index 4e82f1329864..10aafd8c213b 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -15,6 +15,7 @@ class GitlabStream(HttpStream, ABC): primary_key = "id" raise_on_http_errors = True + availability_strategy = None stream_base_params = {} flatten_id_keys = [] flatten_list_keys = [] @@ -43,6 +44,9 @@ def url_base(self) -> str: def should_retry(self, response: requests.Response) -> bool: # Gitlab API returns a 403 response in case a feature is disabled in a project (pipelines/jobs for instance). + # This code is not equivalent of what HttpAvailabilityStrategy does. + # Different stream slices (different projects or groups) - may or may not result in a 403 error. + # HttpAvailabilityStrategy skips all the slices in case it faces error when reading the first record. if response.status_code == 403: setattr(self, "raise_on_http_errors", False) self.logger.warning( @@ -53,6 +57,8 @@ def should_retry(self, response: requests.Response) -> bool: return super().should_retry(response) def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + if response.status_code != 200: + return response_data = response.json() if isinstance(response_data, dict): return None diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py new file mode 100644 index 000000000000..e83bbf7a7c3a --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest + + +@pytest.fixture +def config(mocker): + return { + "start_date": "2021-01-01T00:00:00Z", + "api_url": "gitlab.com", + "private_token": "secret_token" + } diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py new file mode 100644 index 000000000000..5981c6985454 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging + +from source_gitlab import SourceGitlab +from source_gitlab.streams import GitlabStream + + +def test_streams(config, requests_mock): + requests_mock.get("/api/v4/groups", json=[{"id": "g1"}, {"id": "g256"}]) + source = SourceGitlab() + streams = source.streams(config) + assert len(streams) == 22 + assert all([isinstance(stream, GitlabStream) for stream in streams]) + groups, projects, *_ = streams + assert groups.group_ids == ["g1", "g256"] + assert projects.project_ids == [] + + +def test_connection_success(config, requests_mock): + requests_mock.get("/api/v4/groups", json=[{"id": "g1"}]) + requests_mock.get("/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}]) + requests_mock.get("/api/v4/projects/p1", json={"id": "p1"}) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger(), config) + assert (status, msg) == (True, None) + + +def test_connection_fail(config, mocker, requests_mock): + mocker.patch("time.sleep") + requests_mock.get("/api/v4/groups", status_code=500) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger(), config) + assert status is False, msg.startswith('Unable to connect to Gitlab API with the provided credentials - "DefaultBackoffException"') diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py new file mode 100644 index 000000000000..7f2b795ce973 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py @@ -0,0 +1,135 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.streams.http.auth import NoAuth +from source_gitlab.streams import Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags + +auth_params = {"authenticator": NoAuth(), "api_url": "gitlab.com"} + + +projects = Projects(project_ids=["p_1"], **auth_params) +pipelines = Pipelines(parent_stream=projects, start_date="2021-01-01T00:00:00Z", **auth_params) +merge_requests = MergeRequests(parent_stream=projects, start_date="2021-01-01T00:00:00Z", **auth_params) +tags = Tags(parent_stream=projects, repository_part=True, **auth_params) +releases = Releases(parent_stream=projects, **auth_params) +jobs = Jobs(parent_stream=pipelines, **auth_params) +merge_request_commits = MergeRequestCommits(parent_stream=merge_requests, **auth_params) +commits = Commits(parent_stream=projects, repository_part=True, start_date="2021-01-01T00:00:00Z", **auth_params) + + +def test_should_retry(mocker, requests_mock): + mocker.patch("time.sleep") + requests_mock.get("/api/v4/projects/p_1", status_code=403) + for stream_slice in projects.stream_slices(sync_mode="full_refresh"): + records = list(projects.read_records(sync_mode="full_refresh", stream_slice=stream_slice)) + assert records == [] + assert requests_mock.call_count == 1 + + +test_cases = ( + ( + jobs, + ( + ("/api/v4/projects/p_1/pipelines", [{"project_id": "p_1", "id": "build_project_p1"}],), + ( + "/api/v4/projects/p_1/pipelines/build_project_p1/jobs", + [ + {"id": "j_1", "user": {"id": "u_1"}, "pipeline": {"id": "p_17"}, "runner": None, "commit": {"id": "c_23"}} + ] + ), + ), + [{"commit_id": "c_23", "id": "j_1", "pipeline_id": "p_17", "project_id": "p_1", "runner_id": None, "user_id": "u_1"}] + ), + ( + tags, + ( + ("/api/v4/projects/p_1/repository/tags", [{"commit": {"id": "c_1"}, "name": "t_1", "target": "ddc89"}],), + ), + [{"commit_id": "c_1", "project_id": "p_1", "name": "t_1", "target": "ddc89"}] + ), + ( + releases, + ( + ( + "/api/v4/projects/p_1/releases", + [ + { + "id": "r_1", + "author": {"name": "John", "id": "666"}, + "commit": {"id": "abcd689"}, + "milestones": [{"id": "m1", "title": "Q1"}, {"id": "m2", "title": "Q2"}] + } + ], + ), + ), + [{"author_id": "666", "commit_id": "abcd689", "id": "r_1", "milestones": ["m1", "m2"], "project_id": "p_1"}] + ), + ( + merge_request_commits, + ( + ("/api/v4/projects/p_1/merge_requests", [{"id": "mr_1", "iid": "mr_1", "project_id": "p_1"}],), + ("/api/v4/projects/p_1/merge_requests/mr_1", [{"id": "mrc_1",}],), + ), + [{"id": "mrc_1", "project_id": "p_1", "merge_request_iid": "mr_1"}] + ) +) + + +@pytest.mark.parametrize("stream, response_mocks, expected_records", test_cases) +def test_transform(requests_mock, stream, response_mocks, expected_records): + requests_mock.get("/api/v4/projects/p_1", json=[{"id": "p_1"}]) + + for url, json in response_mocks: + requests_mock.get(url, json=json) + + records_iter = iter(expected_records) + for stream_slice in stream.stream_slices(sync_mode="full_refresh"): + for record in stream.read_records(sync_mode="full_refresh", stream_slice=stream_slice): + assert record == next(records_iter) + + +@pytest.mark.parametrize( + "stream, current_state, latest_record, new_state", + ( + ( + pipelines, + {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}}, + {"project_id": "219445", "updated_at": "2022-12-16T00:12:41.005675+02:00"}, + {"219445": {"updated_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}} + ), + ( + pipelines, + {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}}, + {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.011+02:00"}, + {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}} + ), + ( + pipelines, + {}, + {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.010001+02:00"}, + {"211378": {"updated_at": "2021-03-10T23:58:58.010001+02:00"}} + ), + ( + commits, + {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}}, + {"project_id": "219445", "created_at": "2022-12-16T00:12:41.005675+02:00"}, + {"219445": {"created_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}} + ), + ( + commits, + {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}}, + {"project_id": "211378", "created_at": "2021-03-10T23:58:58.011+02:00"}, + {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}} + ), + ( + commits, + {}, + {"project_id": "211378", "created_at": "2021-03-10T23:58:58.010001+02:00"}, + {"211378": {"created_at": "2021-03-10T23:58:58.010001+02:00"}} + ) + ) +) +def test_updated_state(stream, current_state, latest_record, new_state): + assert stream.get_updated_state(current_state, latest_record) == new_state diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/unit_test.py deleted file mode 100644 index dddaea0060fa..000000000000 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/unit_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index d538c712b89b..33dc1f4a98b6 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -63,6 +63,7 @@ GitLab source is working with GitLab API v4. It can also work with self-hosted G | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------| +| 0.1.11 | 2022-12-14 | [20479](https://github.com/airbytehq/airbyte/pull/20479) | Use HttpAvailabilityStrategy + add unit tests | | 0.1.10 | 2022-12-12 | [20384](https://github.com/airbytehq/airbyte/pull/20384) | Fetch groups along with their subgroups | | 0.1.9 | 2022-12-11 | [20348](https://github.com/airbytehq/airbyte/pull/20348) | Fix 403 error when syncing `EpicIssues` stream | | 0.1.8 | 2022-12-02 | [20023](https://github.com/airbytehq/airbyte/pull/20023) | Fix duplicated records issue for `Projects` stream |