Added 'mau' collection to Figures pipeline

The purpose of this is to improve API performance. See the previous
commit with SHA 92ea8c4 for an
explanation.

This commit adds the pipeline code to populate 'SiteDailyMetrics.mau'
with the latest numbers for the month as of yesterday

figures/metrics.py

@@ -253,8 +253,10 @@ def get_site_mau_history_metrics(site, months_back):
                                          month=str(rec.month_for.month).zfill(2))
         history.append(dict(period=period, value=rec.active_user_count))
 
-    # Hack to set current month data in the history list
-    current_month_active = get_site_mau_current_month(site)
+    # Get our latest stored site MAU count
+    sdm = SiteDailyMetrics.latest_previous_record(site=site)
+    current_month_active = sdm.mau if sdm else 0
+
     if history:
         # reverse the list because it is currently in reverser chronological order
         history.reverse()
@@ -278,7 +280,7 @@ def get_site_mau_current_month(site):
     site_sm = figures.sites.get_student_modules_for_site(site)
     curr_sm = site_sm.filter(modified__year=month_for.year,
                              modified__month=month_for.month)
-    return curr_sm.values('student__id').distinct().count()
+    return curr_sm.values('student__id').distinct()
 
 
 def get_active_users_for_time_period(site, start_date, end_date, course_ids=None):

figures/pipeline/site_daily_metrics.py

@@ -12,6 +12,7 @@
 from django.db.models import Sum
 
 from figures.helpers import as_course_key, as_datetime, next_day, prev_day
+from figures.metrics import get_site_mau_current_month
 from figures.models import CourseDailyMetrics, SiteDailyMetrics
 from figures.sites import (
     get_courses_for_site,
@@ -139,6 +140,7 @@ def extract(self, site, date_for=None, **kwargs):  # pylint: disable=unused-argu
         data['total_user_count'] = user_count
         data['course_count'] = course_count
         data['total_enrollment_count'] = get_total_enrollment_count(site, date_for)
+        data['mau'] = get_site_mau_current_month(site).count()
         return data
 
 
@@ -186,6 +188,7 @@ def load(self, site, date_for=None, force_update=False, **_kwargs):
                 total_user_count=data['total_user_count'],
                 course_count=data['course_count'],
                 total_enrollment_count=data['total_enrollment_count'],
+                mau=data['mau'],
             )
         )
         return site_metrics, created

tests/metrics/test_site_monthly_metrics.py

@@ -20,6 +20,7 @@
     CourseOverviewFactory,
     OrganizationFactory,
     OrganizationCourseFactory,
+    SiteDailyMetricsFactory,
     SiteMonthlyMetricsFactory,
     SiteFactory,
     StudentModuleFactory,
@@ -68,8 +69,9 @@ def test_get_site_mau_history_metrics_basic(db, monkeypatch):
                                                  active_user_count=counter))
 
     current_month_active = 42
-    monkeypatch.setattr('figures.metrics.get_site_mau_current_month',
-                        lambda n: current_month_active)
+    SiteDailyMetricsFactory(site=our_site,
+                            date_for=mock_today - relativedelta(day=2),
+                            mau=current_month_active)
 
     data = get_site_mau_history_metrics(site=our_site, months_back=months_back)
 
@@ -112,6 +114,7 @@ def test_get_site_mau_current_month(db):
             UserOrganizationMappingFactory(user=user,
                                            organization=org)
 
-    active_user_count = get_site_mau_current_month(site)
+    active_user_ids = get_site_mau_current_month(site)
+    # active_user_count = active_user_ids.count()
     freezer.stop()
-    assert active_user_count == len(users)
+    assert active_user_ids.count() == len(users)

tests/pipeline/test_site_daily_metrics.py

@@ -1,4 +1,7 @@
-'''Tests figures.pipeline.
+'''Tests figures.pipeline.site_daily_metrics module
+
+IMPORTANT: We need to refactor the test data in this test module as this was
+early work and we've learned a lot since then.
 
 TODO:
 
@@ -71,6 +74,7 @@
         total_user_count=200,
         course_count=len(CDM_INPUT_TEST_DATA),
         total_enrollment_count=100,
+        mau=55,
     )
 ]
 
@@ -81,6 +85,7 @@
     total_user_count=200,
     course_count=len(CDM_INPUT_TEST_DATA),
     total_enrollment_count=150,
+    mau=56,
     )
 
 
@@ -232,12 +237,15 @@ def setup(self, db):
                                                    organization=self.organization)
 
     def test_extract(self, monkeypatch):
+        previous_cumulative_active_user_count = 50
+
         expected_results = dict(
             cumulative_active_user_count=52,  # previous cumulative is 50
             todays_active_user_count=2,
             total_user_count=len(self.users),
             course_count=len(CDM_INPUT_TEST_DATA),
             total_enrollment_count=150,
+            mau=len(self.users),  # expect 3
         )
 
         assert not StudentModule.objects.count()
@@ -253,6 +261,19 @@ def mock_student_modules_for_site(site):
         monkeypatch.setattr(pipeline_sdm, 'get_student_modules_for_site',
                             mock_student_modules_for_site)
 
+        def mock_get_site_mau_current_month(site):
+            return get_user_model().objects.filter(
+                id__in=[user.id for user in self.users]).values('id')
+
+        monkeypatch.setattr(pipeline_sdm, 'get_site_mau_current_month',
+                            mock_get_site_mau_current_month)
+
+        def mock_get_previous_cumulative_active_user_count(site, date_for):
+            return previous_cumulative_active_user_count
+
+        monkeypatch.setattr(pipeline_sdm, 'get_previous_cumulative_active_user_count',
+                            mock_get_previous_cumulative_active_user_count)
+
         for course in figures.sites.get_courses_for_site(self.site):
             assert course.created.date() < self.date_for
         for user in figures.sites.get_users_for_site(self.site):
@@ -278,6 +299,7 @@ class TestSiteDailyMetricsLoader(object):
                 total_user_count=3,
                 course_count=4,
                 total_enrollment_count=5,
+                mau=6,
                 )
 
     class MockExtractor(object):