Skip to content

Commit

Permalink
Round duration for provider ingestion completion message (#422)
Browse files Browse the repository at this point in the history
* Round duration for provider ingestion completion message

* Report duration in humanized format

* Add case for one interval unit
  • Loading branch information
AetherUnbound authored Mar 21, 2022
1 parent 0e3675c commit c4d381f
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 1 deletion.
25 changes: 25 additions & 0 deletions openverse_catalog/dags/common/loader/reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,37 @@
logger = logging.getLogger(__name__)


# Shamelessly lifted from:
# https://gist.github.com/borgstrom/936ca741e885a1438c374824efb038b3
TIME_DURATION_UNITS = (
("week", 60 * 60 * 24 * 7),
("day", 60 * 60 * 24),
("hour", 60 * 60),
("min", 60),
("sec", 1),
)


def humanize_time_duration(seconds):
if seconds == 0:
return "inf"
parts = []
for unit, div in TIME_DURATION_UNITS:
amount, seconds = divmod(int(seconds), div)
if amount > 0:
parts.append("{} {}{}".format(amount, unit, "" if amount == 1 else "s"))
return ", ".join(parts)


def report_completion(provider_name, media_type, duration, record_count):
"""
Send a Slack notification when the load_data task has completed.
Messages are only sent out in production and if a Slack connection is defined.
In all cases the data is logged.
"""
# Truncate the duration value if it's provided
if isinstance(duration, float):
duration = humanize_time_duration(duration)

message = f"""
*Provider*: `{provider_name}`
Expand Down
20 changes: 19 additions & 1 deletion tests/dags/common/loader/test_reporting.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from unittest import mock

import pytest
from common.loader.reporting import report_completion
from common.loader.reporting import humanize_time_duration, report_completion


@pytest.fixture(autouse=True)
Expand All @@ -21,3 +21,21 @@ def test_report_completion(should_send_message):
report_completion("Jamendo", "Audio", None, 100)
# Send message is only called if `should_send_message` is True.
send_message_mock.called = should_send_message


@pytest.mark.parametrize(
"seconds, expected",
[
(1, "1 sec"),
(10, "10 secs"),
(100, "1 min, 40 secs"),
(1000, "16 mins, 40 secs"),
(10000, "2 hours, 46 mins, 40 secs"),
(100000, "1 day, 3 hours, 46 mins, 40 secs"),
(1000000, "1 week, 4 days, 13 hours, 46 mins, 40 secs"),
(10000000, "16 weeks, 3 days, 17 hours, 46 mins, 40 secs"),
],
)
def test_humanize_time_duration(seconds, expected):
actual = humanize_time_duration(seconds)
assert actual == expected

0 comments on commit c4d381f

Please sign in to comment.