Skip to content

Commit

Permalink
test fixes so far
Browse files Browse the repository at this point in the history
  • Loading branch information
@leslievandemark
leslievandemark committed Dec 12, 2024
1 parent d14a435 commit 75d3abd
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 57 deletions.
7 changes: 4 additions & 3 deletions tap_github/sync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,10 @@ def translate_state(state, catalog, repositories):

# If the state contains a bookmark for `repo_a` and `repo_b` and the user deselects these both repos and adds another repo
# then in that case this function was returning an empty state. Now this change will return the existing state instead of the empty state.
if key not in stream_names and key not in repositories:
# Return the existing state if all repos from the previous state are deselected(not found) in the current sync.
return state
for repo in state['bookmarks'][key].keys():
if repo not in stream_names and repo not in repositories:
# Return the existing state if all repos from the previous state are deselected(not found) in the current sync.
return state

for stream in catalog['streams']:
stream_name = stream['tap_stream_id']
Expand Down
1 change: 0 additions & 1 deletion tests/test_github_all_fields.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
},
'issues': {
'body_text',
'closed_by',
'body_html'
},
'releases': {
Expand Down
35 changes: 18 additions & 17 deletions tests/test_github_bookmarks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,23 @@ def calculated_states_by_stream(self, current_state, synced_records, replication
timedelta_by_stream["commits"] = [7, 0, 0]

repo = self.get_properties().get('repository')
#stream_to_calculated_state = {repo: {stream: "" for stream in current_state['bookmarks'][repo].keys()}}
stream_to_calculated_state = current_state.copy()['bookmarks']

stream_to_calculated_state = {repo: {stream: "" for stream in current_state['bookmarks'][repo].keys()}}
for stream, state in current_state['bookmarks'][repo].items():
state_key, state_value = next(iter(state.keys())), next(iter(state.values()))
state_as_datetime = dateutil.parser.parse(state_value)
for stream in current_state['bookmarks'].keys():
for repo, state in current_state['bookmarks'][stream].items():
state_key, state_value = next(iter(state.keys())), next(iter(state.values()))
state_as_datetime = dateutil.parser.parse(state_value)

days, hours, minutes = timedelta_by_stream[stream]
days, hours, minutes = timedelta_by_stream[stream]

start_date_as_datetime = dateutil.parser.parse(start_date)
calculated_state_as_datetime = start_date_as_datetime + datetime.timedelta(days=days, hours=hours, minutes=minutes)
start_date_as_datetime = dateutil.parser.parse(start_date)
calculated_state_as_datetime = start_date_as_datetime + datetime.timedelta(days=days, hours=hours, minutes=minutes)

state_format = '%Y-%m-%dT%H:%M:%SZ'
calculated_state_formatted = datetime.datetime.strftime(calculated_state_as_datetime, state_format)
state_format = '%Y-%m-%dT%H:%M:%SZ'
calculated_state_formatted = datetime.datetime.strftime(calculated_state_as_datetime, state_format)

stream_to_calculated_state[repo][stream] = {state_key: calculated_state_formatted}
stream_to_calculated_state[stream][repo] = {state_key: calculated_state_formatted}

return stream_to_calculated_state

Expand Down Expand Up @@ -69,7 +71,6 @@ def test_run(self):
##########################################################################
### First Sync
##########################################################################

conn_id = connections.ensure_connection(self, original_properties=True)

# Run in check mode
Expand All @@ -94,8 +95,8 @@ def test_run(self):
first_sync_records,
expected_replication_keys,
first_sync_start_date)
for repo, new_state in simulated_states.items():
new_states['bookmarks'][repo] = new_state
for stream, new_state in simulated_states.items():
new_states['bookmarks'][stream] = new_state
menagerie.set_state(conn_id, new_states)

##########################################################################
Expand Down Expand Up @@ -125,20 +126,21 @@ def test_run(self):
second_sync_messages = [record.get('data') for record in
second_sync_records.get(stream, {'messages': []}).get('messages')
if record.get('action') == 'upsert']
first_bookmark_key_value = first_sync_bookmarks.get('bookmarks', {}).get(repo, {stream: None}).get(stream)
second_bookmark_key_value = second_sync_bookmarks.get('bookmarks', {}).get(repo, {stream: None}).get(stream)
first_bookmark_key_value = first_sync_bookmarks.get('bookmarks', {}).get(stream, {repo: None}).get(repo)
second_bookmark_key_value = second_sync_bookmarks.get('bookmarks', {}).get(stream, {repo: None}).get(repo)


if expected_replication_method == self.INCREMENTAL:
# Collect information specific to incremental streams from syncs 1 & 2
replication_key = next(iter(expected_replication_keys[stream]))

first_bookmark_value = first_bookmark_key_value.get('since')
second_bookmark_value = second_bookmark_key_value.get('since')

first_bookmark_value_ts = self.dt_to_ts(first_bookmark_value, self.BOOKMARK_FORMAT)
second_bookmark_value_ts = self.dt_to_ts(second_bookmark_value, self.BOOKMARK_FORMAT)

simulated_bookmark_value = self.dt_to_ts(new_states['bookmarks'][repo][stream]['since'], self.BOOKMARK_FORMAT)
simulated_bookmark_value = self.dt_to_ts(new_states['bookmarks'][stream][repo]['since'], self.BOOKMARK_FORMAT)

# Verify the first sync sets a bookmark of the expected form
self.assertIsNotNone(first_bookmark_key_value)
Expand All @@ -159,7 +161,6 @@ def test_run(self):
for record in first_sync_messages:
# Verify the first sync bookmark value is the max replication key value for a given stream
replication_key_value = self.dt_to_ts(record.get(replication_key), replication_key_format)

self.assertLessEqual(
replication_key_value, first_bookmark_value_ts,
msg="First sync bookmark was set incorrectly, a record with a greater replication-key value was synced."
Expand Down
3 changes: 2 additions & 1 deletion tests/unittests/test_get_streams_and_state_translate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ def test_state_with_no_previous_repo_name_newer_format_bookmark(self):
newer_format_state = {
"bookmarks": {
"comments" : {
"org/test-repo": {"since": "2019-01-01T00:00:00Z"}
"org/test-repo": {"since": "2019-01-01T00:00:00Z"},
"org/test-repo2": {"since": "2019-01-01T00:00:00Z"}
},
}
}
Expand Down
62 changes: 30 additions & 32 deletions tests/unittests/test_stream.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def test_get_schema(self):
]
expected_schema = {"tap_stream_id": "comments"}

# Verify returned schema is same as exected schema
# Verify returned schema is same as exected schema
self.assertEqual(get_schema(catalog, "comments"), expected_schema)


Expand All @@ -29,26 +29,28 @@ class TestGetBookmark(unittest.TestCase):

test_stream = Comments()

def test_with_out_repo_path(self):
def test_without_stream_key(self):
"""
Test if the state does not contain a repo path
"""
state = {
"bookmarks": {
"projects": {"since": "2022-01-01T00:00:00Z"}
"org/test-repo": {
"projects" : {"since": "2022-01-01T00:00:00Z"}
}
}
}
returned_bookmark = get_bookmark(state, "org/test-repo", "projects", "since", "2021-01-01T00:00:00Z")
self.assertEqual(returned_bookmark, "2021-01-01T00:00:00Z")

def test_with_repo_path(self):
def test_with_streams_key(self):
"""
Test if the state does contains a repo path
"""
state = {
"bookmarks": {
"org/test-repo": {
"projects": {"since": "2022-01-01T00:00:00Z"}
"projects": {
"org/test-repo": {"since": "2022-01-01T00:00:00Z"}
}
}
}
Expand Down Expand Up @@ -83,15 +85,13 @@ class GetMinBookmark(unittest.TestCase):
start_date = "2020-04-01T00:00:00Z"
state = {
"bookmarks": {
"org/test-repo": {
"projects": {"since": "2022-03-29T00:00:00Z"},
"project_columns": {"since": "2022-03-01T00:00:00Z"},
"project_cards": {"since": "2022-03-14T00:00:00Z"},
"pull_requests": {"since": "2022-04-01T00:00:00Z"},
"review_comments": {"since": "2022-03-01T00:00:00Z"},
"pr_commits": {"since": "2022-02-01T00:00:00Z"},
"reviews": {"since": "2022-05-01T00:00:00Z"}
}
"projects": {"org/test-repo" : {"since": "2022-03-29T00:00:00Z"}},
"project_columns": {"org/test-repo" : {"since": "2022-03-01T00:00:00Z"}},
"project_cards": {"org/test-repo" : {"since": "2022-03-14T00:00:00Z"}},
"pull_requests": {"org/test-repo" : {"since": "2022-04-01T00:00:00Z"}},
"review_comments": {"org/test-repo" : {"since": "2022-03-01T00:00:00Z"}},
"pr_commits": {"org/test-repo" : {"since": "2022-02-01T00:00:00Z"}},
"reviews": {"org/test-repo" : {"since": "2022-05-01T00:00:00Z"}}
}
}

Expand All @@ -103,7 +103,7 @@ class GetMinBookmark(unittest.TestCase):
])
def test_multiple_children(self, name, stream_class, stream_name, stream_to_sync, current_date, expected_bookmark):
"""
Test that `get_min_bookmark` method returns the minimum bookmark from the parent and its corresponding child bookmarks.
Test that `get_min_bookmark` method returns the minimum bookmark from the parent and its corresponding child bookmarks.
"""
test_stream = stream_class()
bookmark = test_stream.get_min_bookmark(stream_name, stream_to_sync,
Expand All @@ -121,15 +121,13 @@ class TestWriteBookmark(unittest.TestCase):

state = {
"bookmarks": {
"org/test-repo": {
"projects": {"since": "2021-03-29T00:00:00Z"},
"project_columns": {"since": "2021-03-01T00:00:00Z"},
"project_cards": {"since": "2021-03-14T00:00:00Z"},
"pull_requests": {"since": "2021-04-01T00:00:00Z"},
"review_comments": {"since": "2021-03-01T00:00:00Z"},
"pr_commits": {"since": "2021-02-01T00:00:00Z"},
"reviews": {"since": "2021-05-01T00:00:00Z"}
}
"projects": {"org/test-repo" : {"since": "2021-03-29T00:00:00Z"}},
"project_columns": {"org/test-repo" : {"since": "2021-03-01T00:00:00Z"}},
"project_cards": {"org/test-repo" : {"since": "2021-03-14T00:00:00Z"}},
"pull_requests": {"org/test-repo" : {"since": "2021-04-01T00:00:00Z"}},
"review_comments": {"org/test-repo" : {"since": "2021-03-01T00:00:00Z"}},
"pr_commits": {"org/test-repo" : {"since": "2021-02-01T00:00:00Z"}},
"reviews": {"org/test-repo" : {"since": "2021-05-01T00:00:00Z"}}
}
}

Expand All @@ -142,12 +140,12 @@ def test_multiple_child(self, mock_write_bookmark):
"2022-04-01T00:00:00Z", "org/test-repo", self.state)

expected_calls = [
mock.call(mock.ANY, mock.ANY, "pull_requests", {"since": "2022-04-01T00:00:00Z"}),
mock.call(mock.ANY, mock.ANY, "pr_commits", {"since": "2022-04-01T00:00:00Z"}),
mock.call(mock.ANY, mock.ANY, "review_comments", {"since": "2022-04-01T00:00:00Z"}),
mock.call(mock.ANY, "pull_requests", mock.ANY, {"since": "2022-04-01T00:00:00Z"}),
mock.call(mock.ANY, "pr_commits", mock.ANY, {"since": "2022-04-01T00:00:00Z"}),
mock.call(mock.ANY, "review_comments", mock.ANY, {"since": "2022-04-01T00:00:00Z"}),
]

# Verify `write_bookmark` is called for all selected streams
# Verify `write_bookmark` is called for all selected streams
self.assertEqual(mock_write_bookmark.call_count, 3)

self.assertIn(mock_write_bookmark.mock_calls[0], expected_calls)
Expand All @@ -162,10 +160,10 @@ def test_nested_child(self, mock_write_bookmark):
test_stream.write_bookmarks("projects", ["project_cards"],
"2022-04-01T00:00:00Z", "org/test-repo", self.state)

# Verify `write_bookmark` is called for all selected streams
# Verify `write_bookmark` is called for all selected streams
self.assertEqual(mock_write_bookmark.call_count, 1)
mock_write_bookmark.assert_called_with(mock.ANY, mock.ANY,
"project_cards", {"since": "2022-04-01T00:00:00Z"})
mock_write_bookmark.assert_called_with(mock.ANY, "project_cards",
mock.ANY, {"since": "2022-04-01T00:00:00Z"})


class TestGetChildUrl(unittest.TestCase):
Expand Down
6 changes: 3 additions & 3 deletions tests/unittests/test_sync_endpoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def test_sync_without_state(self, mock_write_records, mock_authed_all_pages, moc
{"id": 2, "created_at": "2019-01-04T00:00:00Z"}]),
MockResponse([{"id": 3, "created_at": "2019-01-03T00:00:00Z"},
{"id": 4, "created_at": "2019-01-02T00:00:00Z"}])]
expected_state = {'bookmarks': {'tap-github': {'events': {'since': '2019-01-04T00:00:00Z'}}}}
expected_state = {'bookmarks': {'events': {'tap-github': {'since': '2019-01-04T00:00:00Z'}}}}
test_client = GithubClient(self.config)
final_state = test_stream.sync_endpoint(test_client, {}, self.catalog, "tap-github", "2018-01-02T00:00:00Z", ["events"], ['events'])

Expand All @@ -52,9 +52,9 @@ def test_sync_with_state(self, mock_write_records, mock_authed_all_pages, mock_v
{"id": 2, "created_at": "2019-01-04T00:00:00Z"}]),
MockResponse([{"id": 3, "created_at": "2019-01-03T00:00:00Z"},
{"id": 4, "created_at": "2019-01-02T00:00:00Z"}])]
mock_state = {'bookmarks': {'tap-github': {'events': {'since': '2019-01-02T00:00:00Z'}}}}
mock_state = {'bookmarks': {'events': {'tap-github': {'since': '2019-01-02T00:00:00Z'}}}}

expected_state = {'bookmarks': {'tap-github': {'events': {'since': '2019-01-04T00:00:00Z'}}}}
expected_state = {'bookmarks': {'events': {'tap-github': {'since': '2019-01-04T00:00:00Z'}}}}
test_client = GithubClient(self.config)
final_state = test_stream.sync_endpoint(test_client, mock_state, self.catalog, "tap-github", "2018-01-02T00:00:00Z", ["events"], ['events'])

Expand Down

0 comments on commit 75d3abd

Please sign in to comment.