Merge pull request #218 from Amsterdam/feature/add-recalculate-histor…

…y-since

Added a recalculate-history-since flag

requirements.txt

@@ -1,12 +1,12 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile --output-file=requirements.txt requirements.in
 #
 asgiref==3.5.2
     # via django
-certifi==2022.6.15
+certifi==2022.9.24
     # via
     #   requests
     #   sentry-sdk
@@ -20,7 +20,7 @@ coreschema==0.0.4
     # via
     #   coreapi
     #   drf-yasg
-cryptography==37.0.4
+cryptography==38.0.1
     # via jwcrypto
 datapunt-authorization-django==1.3.2
     # via -r requirements.in
@@ -49,17 +49,17 @@ django==3.2.15
     #   drf-yasg
 django-datetime-utc==1.0.3
     # via -r requirements.in
-django-debug-toolbar==3.6.0
+django-debug-toolbar==3.7.0
     # via -r requirements.in
-django-extensions==3.2.0
+django-extensions==3.2.1
     # via -r requirements.in
 django-filter==22.1
     # via -r requirements.in
-django-import-export==2.8.0
+django-import-export==2.9.0
     # via -r requirements.in
 django-leaflet==0.28.3
     # via -r requirements.in
-djangorestframework==3.13.1
+djangorestframework==3.14.0
     # via
     #   -r requirements.in
     #   datapunt-data-ingress
@@ -80,19 +80,19 @@ drf-amsterdam==0.4.1
     # via -r requirements.in
 drf-extensions==0.7.1
     # via drf-amsterdam
-drf-yasg==1.21.3
+drf-yasg==1.21.4
     # via -r requirements.in
 et-xmlfile==1.1.0
     # via openpyxl
-idna==3.3
+idna==3.4
     # via requests
 inflection==0.5.1
     # via drf-yasg
 itypes==1.2.0
     # via coreapi
 jinja2==3.1.2
     # via coreschema
-jwcrypto==1.3.1
+jwcrypto==1.4.2
     # via datapunt-authorization-django
 markuppy==1.14
     # via tablib
@@ -129,13 +129,13 @@ ruamel-yaml==0.17.21
     # via drf-yasg
 ruamel-yaml-clib==0.2.6
     # via ruamel-yaml
-sentry-sdk==1.9.5
+sentry-sdk==1.9.9
     # via -r requirements.in
 six==1.16.0
     # via
     #   djangorestframework-csv
     #   python-dateutil
-sqlparse==0.4.2
+sqlparse==0.4.3
     # via
     #   django
     #   django-debug-toolbar
@@ -147,7 +147,7 @@ uritemplate==4.1.1
     # via
     #   coreapi
     #   drf-yasg
-urllib3==1.26.11
+urllib3==1.26.12
     # via
     #   requests
     #   sentry-sdk

requirements_dev.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile --output-file=requirements_dev.txt requirements_dev.in
@@ -18,7 +18,7 @@ brotli==1.0.9
     # via geventhttpclient
 build==0.8.0
     # via pip-tools
-certifi==2022.6.15
+certifi==2022.9.24
     # via
     #   -r ./requirements.txt
     #   geventhttpclient
@@ -49,7 +49,7 @@ coreschema==0.0.4
     #   drf-yasg
 coverage[toml]==6.4.4
     # via pytest-cov
-cryptography==37.0.4
+cryptography==38.0.1
     # via
     #   -r ./requirements.txt
     #   jwcrypto
@@ -88,17 +88,17 @@ django==3.2.15
     #   model-bakery
 django-datetime-utc==1.0.3
     # via -r ./requirements.txt
-django-debug-toolbar==3.6.0
+django-debug-toolbar==3.7.0
     # via -r ./requirements.txt
-django-extensions==3.2.0
+django-extensions==3.2.1
     # via -r ./requirements.txt
 django-filter==22.1
     # via -r ./requirements.txt
-django-import-export==2.8.0
+django-import-export==2.9.0
     # via -r ./requirements.txt
 django-leaflet==0.28.3
     # via -r ./requirements.txt
-djangorestframework==3.13.1
+djangorestframework==3.14.0
     # via
     #   -r ./requirements.txt
     #   datapunt-data-ingress
@@ -123,19 +123,19 @@ drf-extensions==0.7.1
     # via
     #   -r ./requirements.txt
     #   drf-amsterdam
-drf-yasg==1.21.3
+drf-yasg==1.21.4
     # via -r ./requirements.txt
 et-xmlfile==1.1.0
     # via
     #   -r ./requirements.txt
     #   openpyxl
-executing==0.10.0
+executing==1.1.0
     # via stack-data
 factory-boy==3.2.1
     # via
     #   -r requirements_dev.in
     #   pytest-factoryboy
-faker==14.1.0
+faker==15.0.0
     # via factory-boy
 flask==2.2.2
     # via
@@ -150,24 +150,22 @@ gevent==21.12.0
     # via
     #   geventhttpclient
     #   locust
-geventhttpclient==2.0
+geventhttpclient==2.0.2
     # via locust
-greenlet==1.1.2
+greenlet==1.1.3
     # via gevent
-idna==3.3
+idna==3.4
     # via
     #   -r ./requirements.txt
     #   requests
-importlib-metadata==4.12.0
-    # via flask
 inflection==0.5.1
     # via
     #   -r ./requirements.txt
     #   drf-yasg
     #   pytest-factoryboy
 iniconfig==1.1.1
     # via pytest
-ipython==8.4.0
+ipython==8.5.0
     # via -r requirements_dev.in
 itsdangerous==2.1.2
     # via flask
@@ -182,11 +180,11 @@ jinja2==3.1.2
     #   -r ./requirements.txt
     #   coreschema
     #   flask
-jwcrypto==1.3.1
+jwcrypto==1.4.2
     # via
     #   -r ./requirements.txt
     #   datapunt-authorization-django
-locust==2.11.0
+locust==2.12.1
     # via -r requirements_dev.in
 markuppy==1.14
     # via
@@ -229,9 +227,9 @@ pip-tools==6.8.0
     # via -r requirements_dev.in
 pluggy==1.0.0
     # via pytest
-prompt-toolkit==3.0.30
+prompt-toolkit==3.0.31
     # via ipython
-psutil==5.9.1
+psutil==5.9.2
     # via locust
 psycopg2-binary==2.9.3
     # via -r ./requirements.txt
@@ -251,7 +249,7 @@ pyparsing==3.0.9
     # via
     #   -r ./requirements.txt
     #   packaging
-pytest==7.1.2
+pytest==7.1.3
     # via
     #   -r requirements_dev.in
     #   pytest-cov
@@ -279,7 +277,7 @@ pyyaml==6.0
     # via
     #   -r ./requirements.txt
     #   tablib
-pyzmq==22.3.0
+pyzmq==24.0.1
     # via locust
 requests==2.28.1
     # via
@@ -297,7 +295,7 @@ ruamel-yaml-clib==0.2.6
     # via
     #   -r ./requirements.txt
     #   ruamel-yaml
-sentry-sdk==1.9.5
+sentry-sdk==1.9.9
     # via -r ./requirements.txt
 six==1.16.0
     # via
@@ -307,12 +305,12 @@ six==1.16.0
     #   flask-cors
     #   geventhttpclient
     #   python-dateutil
-sqlparse==0.4.2
+sqlparse==0.4.3
     # via
     #   -r ./requirements.txt
     #   django
     #   django-debug-toolbar
-stack-data==0.4.0
+stack-data==0.5.1
     # via ipython
 tablib[html,ods,xls,xlsx,yaml]==3.2.1
     # via
@@ -326,7 +324,7 @@ tomli==2.0.1
     #   coverage
     #   pep517
     #   pytest
-traitlets==5.3.0
+traitlets==5.4.0
     # via
     #   ipython
     #   matplotlib-inline
@@ -343,7 +341,7 @@ uritemplate==4.1.1
     #   -r ./requirements.txt
     #   coreapi
     #   drf-yasg
-urllib3==1.26.11
+urllib3==1.26.12
     # via
     #   -r ./requirements.txt
     #   requests
@@ -368,8 +366,6 @@ xlwt==1.3.0
     # via
     #   -r ./requirements.txt
     #   tablib
-zipp==3.8.1
-    # via importlib-metadata
 zope-event==4.5.0
     # via gevent
 zope-interface==5.4.0

src/continuousaggregate/management/commands/complete_aggregate.py

@@ -1,6 +1,5 @@
 import logging
-import time
-from datetime import datetime
+from datetime import date, datetime
 
 from django.core.management.base import BaseCommand
 from django.db import connection, transaction
@@ -16,16 +15,25 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument('table_name', type=str)
         parser.add_argument('--recalculate-history', default=False)
+        parser.add_argument('--recalculate-history-since', type=date.fromisoformat)
 
     def handle(self, *args, **options):
         table_name = options['table_name']
         recalculate_history = options['recalculate_history']
+        recalculate_history_since = options['recalculate_history_since']
 
         if recalculate_history:
             self.stdout.write(f"Start deleting full aggregation table {Cmsa15Min._meta.db_table}")
             Cmsa15Min.objects.all().delete()
             self.stdout.write(f"Finished deleting full aggregation table {Cmsa15Min._meta.db_table}")
-
+
+        if recalculate_history_since:
+            self.stdout.write(
+                f"Start deleting aggregation table {Cmsa15Min._meta.db_table} since {recalculate_history_since}")
+            Cmsa15Min.objects.filter(timestamp_rounded__gt=recalculate_history_since).delete()
+            self.stdout.write(
+                f"Finished deleting aggregation table {Cmsa15Min._meta.db_table} since {recalculate_history_since}")
+
         # calculate run_id based on the latest run_id from execution_log table
         with connection.cursor() as cursor:
             self.stdout.write(f"Start calculate next value for run_id based on execution_log table ")

tests/test_continuousaggregate.py

@@ -32,14 +32,13 @@ def setup(self):
         for i, sensor_name in enumerate(self.sensor_names):
             Sensors.objects.create(objectnummer=sensor_name, gid=i)
 
-    def test_vanilla(self, client):
+    def add_test_records(self, client, test_days=2):
         # Add records every 5min for multiple days
         Message.objects.all().delete()
-        test_days = 2
         today = date.today()
-        start_date = today - timedelta(days=test_days)
+        start_date = today - timedelta(days=test_days-1)
         the_dt = datetime(start_date.year, start_date.month, start_date.day)
-        while the_dt < datetime(today.year, today.month, today.day):
+        while the_dt < (datetime(today.year, today.month, today.day)+timedelta(days=1)):
             for sensor_name in self.sensor_names:
                 test_post = json.loads(TEST_POST)
                 test_post['data'][0]['sensor'] = sensor_name
@@ -51,6 +50,11 @@ def test_vanilla(self, client):
         parser = TelcameraParser()
         parser.consume(end_at_empty_queue=True)
 
+        return today
+
+    def test_vanilla(self, client):
+        today = self.add_test_records(client)
+
         # Make sure we've got source data
         assert Observation.objects.all().count() > 100
 
@@ -62,9 +66,30 @@ def test_vanilla(self, client):
 
         # Take a record in the middle of the data in the continuous aggregate table
         # and check whether the record is made up of exactly 3 messages (one every 5 min)
-        last_record = Cmsa15Min.objects\
+        middle_record = Cmsa15Min.objects\
             .filter(sensor=self.sensor_names[0])\
             .filter(timestamp_rounded__gte=(today - timedelta(days=1)).isoformat())\
             .order_by('timestamp_rounded')\
             .first()
-        assert last_record.basedonxmessages == 3
+        assert middle_record.basedonxmessages == 3
+
+    def test_aggregate_recalculation(self, client):
+        today = self.add_test_records(client, test_days=3)
+
+        # Run the aggregator
+        call_man_command('complete_aggregate', 'continuousaggregate_cmsa15min')
+
+        # Remove underlying source data
+        Observation.objects.all().delete()
+
+        # Recalculate the history since yesterday
+        a_day_ago = (date.today() - timedelta(days=1)).isoformat()
+        call_man_command('complete_aggregate', 'continuousaggregate_cmsa15min', f'--recalculate-history-since={a_day_ago}')
+
+        # We removed source records from yesterday onwards. That means we now expect the
+        # latest date in the continuous aggregate to be the day before yesterday
+        last_record = Cmsa15Min.objects \
+            .filter(sensor=self.sensor_names[0]) \
+            .order_by('-timestamp_rounded') \
+            .first()
+        assert today - timedelta(days=2) == last_record.timestamp_rounded.date()