Monthly usage (#20)

* Re-instate the monthly function app

* Fix typing failures

* Add tests for retrieve_usage

* Simplify retrieve_data function

* Update Dockerfile

* Use right date type for Usage.monthly_usage

* Remove old comment

* Make schedule clearer

* WIP

* Add typing to usage function

* Fix typing errors

* Fix typing errors

* Update usage.rst

docs/content/usage.rst

@@ -4,11 +4,11 @@ Usage Function
 `usage` is an Azure function for deployment to an Azure Function App.
 It will get all the usage data in a management group or billing account and send it to an instance of the RCTab web server API.
 
+`monthly_usage` uses similar code as `usage` but runs on the 7th and 8th day of each month to get the previous month's usage.
+
 ..
     `costmanagement` is also an Azure function. It can be deployed to the same function app as the `usage` function.
 
-    `monthly_usage` uses similar code as `usage` but runs on the 7th day of each month to get the previous month's usage.
-
 Running Locally
 +++++++++++++++
 

usage_function/.dockerignore

@@ -7,3 +7,5 @@
 !usage/*.py
 !usage/function.json
 !utils/*.py
+!monthly_usage/*.py
+!monthly_usage/function.json

usage_function/Dockerfile

@@ -12,6 +12,10 @@ COPY usage/function.json /home/site/wwwroot/usage/
 RUN mkdir -p /home/site/wwwroot/utils
 COPY utils/*.py /home/site/wwwroot/utils/
 
+RUN mkdir -p /home/site/wwwroot/monthly_usage
+COPY monthly_usage/*.py /home/site/wwwroot/monthly_usage/
+COPY monthly_usage/function.json /home/site/wwwroot/monthly_usage/
+
 WORKDIR /home/site/wwwroot
 
 RUN ~/.local/share/pypoetry/venv/bin/poetry config virtualenvs.create false

usage_function/README.md

@@ -5,6 +5,6 @@ It will get all the usage data for subscriptions in a management group or billin
 
 `costmanagement` is also an Azure function. It should be deployed to the same function app as the `usage` function.
 
-`monthly_usage` uses similar code as `usage` but runs on the 7th day of each month to get the previous month's usage.
+`monthly_usage` uses similar code as `usage` but runs bi-hourly on the 7th and 8th day of each month to get the previous month's usage.
 
 See the docs for more.

usage_function/monthly_usage/__init__.py

@@ -1,18 +1,49 @@
 """An Azure Function App to collect usage information for the previous month."""
 import logging
 import time
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta
+from typing import Tuple, Union
 
 import azure.functions as func
 from azure.core.exceptions import HttpResponseError
 
 import utils.settings
 from utils.logutils import add_log_handler_once
-from utils.usage import date_range, get_all_usage, retrieve_usage, send_usage
+from utils.usage import get_all_usage, retrieve_usage, send_usage
 
 MAX_ATTEMPTS = 5
 
 
+def get_dates() -> Union[None, Tuple[date], Tuple[date, date]]:
+    """Get up to two dates to process.
+
+    Assuming we are called bi-hourly on the 7th and 8th of the month,
+    return up to two days from the previous month to process.
+    """
+    now = datetime.now()
+
+    # Map our day (7 or 8) and hour (0-22) to a day.
+    day_of_month = ((now.day - 7) * 24) + now.hour + 1
+
+    end_of_last_month = now - timedelta(days=now.day)
+    try:
+        day1 = date(
+            year=end_of_last_month.year, month=end_of_last_month.month, day=day_of_month
+        )
+    except ValueError:
+        return None
+
+    try:
+        day2 = date(
+            year=end_of_last_month.year,
+            month=end_of_last_month.month,
+            day=day_of_month + 1,
+        )
+    except ValueError:
+        return (day1,)
+    return day1, day2
+
+
 def main(mytimer: func.TimerRequest) -> None:
     """Collect usage information for the previous month."""
     # If incorrect settings have been given,
@@ -31,58 +62,51 @@ def main(mytimer: func.TimerRequest) -> None:
     if mytimer.past_due:
         logger.info("The timer is past due.")
 
-    # The end of the previous month is today minus the number of days we are
-    # into the month
-    now = datetime.now()
-    end_of_last_month = now - timedelta(days=now.day)
-
-    end_datetime = datetime(
-        end_of_last_month.year, end_of_last_month.month, end_of_last_month.day
-    )
-    start_datetime = datetime(end_datetime.year, end_datetime.month, 1)
-
-    logger.warning(
-        "Requesting all data between %s and %s in reverse order",
-        start_datetime,
-        end_datetime,
-    )
-
-    usage = []
-    for usage_date in reversed(list(date_range(start_datetime, end_datetime))):
-        # Try up to 5 times to get usage and send to the API
-        for cnt in range(MAX_ATTEMPTS):
-            logger.warning("Requesting all usage data for %s", usage_date)
-            usage_day = get_all_usage(
-                usage_date,
-                usage_date,
-                billing_account_id=config.BILLING_ACCOUNT_ID,
-                mgmt_group=config.MGMT_GROUP,
-            )
-
-            try:
-                usage_day_list = retrieve_usage(usage_day)
-
-                for usage_day in usage_day_list:
-                    usage_day.monthly_upload = now.date()
-
-                usage += usage_day_list
-                break
-            except HttpResponseError as e:
-                logger.error("Request to azure failed. Trying again in 60 seconds")
-                logger.error(e)
-                time.sleep(60)
-
-        if cnt == MAX_ATTEMPTS - 1:
-            logger.error("Could not retrieve usage data.")
-            raise RuntimeError("Could not retrieve usage data.")
+    dates = get_dates()
+    if not dates:
+        logger.warning("No dates to process.")
+        return
 
     logger.warning(
-        "Retrieved %d usage records for %s to %s",
-        len(usage),
-        start_datetime,
-        end_datetime,
+        "Requesting all data for %s",
+        dates,
     )
 
-    send_usage(config.API_URL, usage, monthly_usage_upload=True)
-
-    logger.warning("Monthly usage function finished.")
+    # Try up to 5 times to get usage and send to the API
+    for attempt in range(MAX_ATTEMPTS):
+        logger.warning("Attempt %d", attempt + 1)
+
+        date_from = datetime(dates[0].year, dates[0].month, dates[0].day)
+        date_to = (
+            datetime(dates[1].year, dates[1].month, dates[1].day)
+            if len(dates) == 2
+            else date_from
+        )
+        usage_query = get_all_usage(
+            date_from,
+            date_to,
+            billing_account_id=config.BILLING_ACCOUNT_ID,
+            mgmt_group=config.MGMT_GROUP,
+        )
+
+        try:
+            usage_items = retrieve_usage(usage_query)
+
+            today = date.today()
+            for usage_item in usage_items:
+                usage_item.monthly_upload = today
+
+            logger.warning("Sending usage for %s", dates)
+            send_usage(config.API_URL, usage_items, monthly_usage_upload=True)
+
+            logger.warning("Monthly usage function finished.")
+            return
+
+        except HttpResponseError as e:
+            if attempt == MAX_ATTEMPTS - 1:
+                logger.error("Could not retrieve usage data.")
+                raise RuntimeError("Could not retrieve usage data.")
+
+            logger.error("Request to azure failed. Trying again in 60 seconds")
+            logger.error(e)
+            time.sleep(60)

usage_function/monthly_usage/function.json

@@ -2,8 +2,8 @@
   "bindings": [
     {
       "direction": "in",
-      "name": "mytimer",
-      "schedule": "1 1 1 7 * *",
+      "name": "every_other_hour_on_7th_and_8th",
+      "schedule": "10 0,2,4,6,8,10,12,14,16,18,20,22 7,8 * *",
       "type": "timerTrigger"
     }
   ],

usage_function/pyproject.toml

@@ -6,8 +6,8 @@ authors = []
 packages = [
     {include = "usage"},
     {include = "utils"},
-#    {include = "costmanagement"},
-#    {include = "monthly_usage"},
+    {include = "monthly_usage"},
+    #    {include = "costmanagement"},
 ]
 
 [tool.poetry.dependencies]

usage_function/run_tests.sh

@@ -44,7 +44,7 @@ status=$((status+$?))
 
 # Check types with MyPy
 echo "Running type checking..."
-python -m mypy --config-file tests/mypy.ini usage/ tests/
+python -m mypy --config-file tests/mypy.ini utils/ usage/ monthly_usage/ tests/
 status=$((status+$?))
 
 # [optional] Check Markdown coding style with Ruby's markdown lint

usage_function/tests/test_function_app.py

@@ -1,4 +1,5 @@
 """Tests for Azure functions."""
+
 from datetime import date, datetime, timedelta
 from typing import Final
 from unittest import TestCase, main
@@ -117,19 +118,78 @@ def test_main(self) -> None:
 
         with patch("monthly_usage.get_all_usage") as mock_get_all, patch(
             "monthly_usage.retrieve_usage"
-        ) as mock_retrieve, patch("monthly_usage.date_range") as mock_date_range, patch(
+        ) as mock_retrieve, patch("monthly_usage.get_dates") as mock_get_dates, patch(
             "monthly_usage.send_usage"
         ) as mock_send, patch(
             "utils.settings.get_settings"
         ) as mock_get_settings:
-            mock_date_range.return_value = [0]
+            mock_get_dates.return_value = date(2024, 1, 1), date(2024, 1, 2)
             mock_get_settings.return_value = settings
             monthly_usage.main(mock_timer)
 
-            mock_get_all.assert_called_once()
+            mock_get_all.assert_called_once_with(
+                datetime(2024, 1, 1),
+                datetime(2024, 1, 2),
+                billing_account_id="88111111",
+                mgmt_group=None,
+            )
             mock_retrieve.assert_called_once()
             mock_send.assert_called_once()
 
+    def test_get_date_range(self) -> None:
+        """Test that the get_date_range function returns the expected dates."""
+
+        with patch("monthly_usage.datetime") as mock_datetime:
+            # On hour 0 of the 7th day, we expect to get dates 1 and 2.
+            mock_datetime.now.return_value = datetime(2024, 2, 7, 0, 4, 56)
+
+            expected_dates: tuple[date, ...] = (date(2024, 1, 1), date(2024, 1, 2))
+
+            actual_dates = monthly_usage.get_dates()
+            assert actual_dates is not None
+
+            self.assertTupleEqual(expected_dates, actual_dates)
+
+        with patch("monthly_usage.datetime") as mock_datetime:
+            # On hour 2 of the 7th day, we expect to get dates 3 and 4.
+            mock_datetime.now.return_value = datetime(2024, 2, 7, 2, 6, 0)
+
+            expected_dates = (date(2024, 1, 3), date(2024, 1, 4))
+
+            actual_dates = monthly_usage.get_dates()
+            assert actual_dates is not None
+
+            self.assertTupleEqual(expected_dates, actual_dates)
+
+        with patch("monthly_usage.datetime") as mock_datetime:
+            # Some hours of the 8th day don't map to valid dates.
+            mock_datetime.now.return_value = datetime(2024, 2, 8, 22, 0, 0)
+
+            actual_dates = monthly_usage.get_dates()
+
+            self.assertIsNone(actual_dates)
+
+        with patch("monthly_usage.datetime") as mock_datetime:
+            # For leap year February, we only expect one final date.
+            mock_datetime.now.return_value = datetime(2024, 3, 8, 4, 0, 0)
+
+            expected_dates = (date(2024, 2, 29),)
+
+            actual_dates = monthly_usage.get_dates()
+            assert actual_dates is not None
+
+            self.assertTupleEqual(expected_dates, actual_dates)
+
+        with patch("monthly_usage.datetime") as mock_datetime:
+            # For months with 31 days, we only expect one final date.
+            mock_datetime.now.return_value = datetime(2024, 2, 8, 6, 0, 0)
+
+            expected_dates = (date(2024, 1, 31),)
+
+            actual_dates = monthly_usage.get_dates()
+            assert actual_dates is not None
+            self.assertTupleEqual(expected_dates, actual_dates)
+
 
 class TestCostManagement(TestCase):
     """Tests for the costmanagement/__init__.py file."""