Use django-qsstats for dashboard computations.

This is like 10000000000 times faster, to say the least.  The only thing that's lacking is a really fast page content calculation, but testing with santa cruz (~6000 pages) on my laptop the whole thing only takes ~20s to generate, so I'd say it's good for now.

Author: philipn

sapling/dashboard/views.py

@@ -6,6 +6,7 @@
 from django.utils.translation import ugettext as _
 
 import pyflot
+import qsstats
 
 from pages.models import Page, PageFile
 from maps.models import MapData
@@ -50,12 +51,11 @@ def get_context_data(self, **kwargs):
                 return {'generated': False}
 
             cache.set('dashboard_generating', True)
-            now = datetime.now()
             context = {
-                'num_items_over_time': items_over_time(now),
-                'num_edits_over_time': edits_over_time(now),
-                'page_content_over_time': page_content_over_time(now),
-                'users_registered_over_time': users_registered_over_time(now),
+                'num_items_over_time': items_over_time(),
+                'num_edits_over_time': edits_over_time(),
+                'page_content_over_time': page_content_over_time(),
+                'users_registered_over_time': users_registered_over_time(),
             }
             context.update(self.get_nums())
 
@@ -66,49 +66,66 @@ def get_context_data(self, **kwargs):
         return context
 
 
-def items_over_time(now):
-    def _total_count_as_of(d, M, total_count):
-        # Figure out which instances of M were added, deleted on this day.
-        num_added = len(M.versions.filter(
-            version_info__date__gte=d, version_info__date__lt=next_d,
-            version_info__type__in=ADDED_TYPES))
-        num_deleted = len(M.versions.filter(
-            version_info__date__gte=d, version_info__date__lt=next_d,
-            version_info__type__in=DELETED_TYPES))
+def _summed_series(series):
+    """
+    Take a time series, ``series``, and turn it into a summed-by-term series.
+    """
+    sum = 0
+    l = []
+    for (d, num) in series:
+        sum += num
+        l.append((d, sum))
+    return l
 
-        total_count += num_added
-        total_count -= num_deleted
-        return total_count
 
+def _sum_from_add_del(added_series, deleted_series):
+    """
+    Sum the provided, aligned time series (added, deleted).
+    """
+    sum = 0
+    l = []
+    for ((d, added), (_, deleted)) in zip(added_series, deleted_series):
+        sum += (added - deleted)
+        l.append((d, sum))
+    return l
+
+
+def items_over_time():
     oldest_page = Page.versions.all().order_by(
         'history_date')[0].version_info.date
-
     graph = pyflot.Flot()
-    page_total_count, map_total_count, file_total_count, \
-        redirect_total_count = 0, 0, 0, 0
-    num_pages_over_time, num_maps_over_time, num_files_over_time, \
-        num_redirects_over_time = [], [], [], []
-    # Start at the oldest page's date and then iterate, day-by-day, until
-    # current day, day by day.
-    d = datetime(oldest_page.year, oldest_page.month, oldest_page.day - 1)
-    while (now.year, now.month, now.day) != (d.year, d.month, d.day):
-        next_d = d + timedelta(days=1)
-
-        page_total_count = _total_count_as_of(d, Page, page_total_count)
-        num_pages_over_time.append((d, page_total_count))
-
-        map_total_count = _total_count_as_of(d, MapData, map_total_count)
-        num_maps_over_time.append((d, map_total_count))
 
-        file_total_count = _total_count_as_of(d, PageFile,
-            file_total_count)
-        num_files_over_time.append((d, file_total_count))
-
-        redirect_total_count = _total_count_as_of(d, Redirect,
-            redirect_total_count)
-        num_redirects_over_time.append((d, redirect_total_count))
-
-        d = next_d
+    pages_added = qsstats.QuerySetStats(
+        Page.versions.filter(version_info__type__in=ADDED_TYPES),
+        'history_date').time_series(oldest_page)
+    pages_deleted = qsstats.QuerySetStats(
+        Page.versions.filter(version_info__type__in=DELETED_TYPES),
+        'history_date').time_series(oldest_page)
+    num_pages_over_time = _sum_from_add_del(pages_added, pages_deleted)
+
+    maps_added = qsstats.QuerySetStats(
+        MapData.versions.filter(version_info__type__in=ADDED_TYPES),
+        'history_date').time_series(oldest_page)
+    maps_deleted = qsstats.QuerySetStats(
+        MapData.versions.filter(version_info__type__in=DELETED_TYPES),
+        'history_date').time_series(oldest_page)
+    num_maps_over_time = _sum_from_add_del(maps_added, maps_deleted)
+
+    files_added = qsstats.QuerySetStats(
+        PageFile.versions.filter(version_info__type__in=ADDED_TYPES),
+        'history_date').time_series(oldest_page)
+    files_deleted = qsstats.QuerySetStats(
+        PageFile.versions.filter(version_info__type__in=DELETED_TYPES),
+        'history_date').time_series(oldest_page)
+    num_files_over_time = _sum_from_add_del(files_added, files_deleted)
+
+    redir_added = qsstats.QuerySetStats(
+        Redirect.versions.filter(version_info__type__in=ADDED_TYPES),
+        'history_date').time_series(oldest_page)
+    redir_deleted = qsstats.QuerySetStats(
+        Redirect.versions.filter(version_info__type__in=DELETED_TYPES),
+        'history_date').time_series(oldest_page)
+    num_redirects_over_time = _sum_from_add_del(redir_added, redir_deleted)
 
     graph.add_time_series(num_pages_over_time, label=_("pages"))
     graph.add_time_series(num_maps_over_time, label=_("maps"))
@@ -119,56 +136,37 @@ def _total_count_as_of(d, M, total_count):
     return [graph.prepare_series(s) for s in graph._series]
 
 
-def edits_over_time(now):
+def edits_over_time():
     oldest_page = Page.versions.all().order_by(
         'history_date')[0].version_info.date
-
     graph = pyflot.Flot()
-    page_edits = []
-    map_edits = []
-    file_edits = []
-    redirect_edits = []
-    d = datetime(oldest_page.year, oldest_page.month, oldest_page.day)
-    while (now.year, now.month, now.day) != (d.year, d.month, d.day):
-        next_d = d + timedelta(days=1)
-
-        # Page edits
-        page_edits_this_day = len(Page.versions.filter(
-            version_info__date__gte=d, version_info__date__lt=next_d))
-        page_edits.append((d, page_edits_this_day))
-
-        # Map edits
-        map_edits_this_day = len(MapData.versions.filter(
-            version_info__date__gte=d, version_info__date__lt=next_d))
-        map_edits.append((d, map_edits_this_day))
 
-        # File edits
-        file_edits_this_day = len(PageFile.versions.filter(
-            version_info__date__gte=d, version_info__date__lt=next_d))
-        file_edits.append((d, file_edits_this_day))
+    qss = qsstats.QuerySetStats(Page.versions.all(), 'history_date')
+    graph.add_time_series(qss.time_series(oldest_page), label=_("pages"))
 
-        # Redirect edits
-        redirect_edits_this_day = len(Redirect.versions.filter(
-            version_info__date__gte=d, version_info__date__lt=next_d))
-        redirect_edits.append((d, redirect_edits_this_day))
+    qss = qsstats.QuerySetStats(MapData.versions.all(), 'history_date')
+    graph.add_time_series(qss.time_series(oldest_page), label=_("maps"))
 
-        d = next_d
+    qss = qsstats.QuerySetStats(PageFile.versions.all(), 'history_date')
+    graph.add_time_series(qss.time_series(oldest_page), label=_("files"))
 
-    graph.add_time_series(page_edits, label=_("pages"))
-    graph.add_time_series(map_edits, label=_("maps"))
-    graph.add_time_series(file_edits, label=_("files"))
-    graph.add_time_series(redirect_edits, label=_("redirects"))
+    qss = qsstats.QuerySetStats(Redirect.versions.all(), 'history_date')
+    graph.add_time_series(qss.time_series(oldest_page), label=_("redirects"))
 
     return [graph.prepare_series(s) for s in graph._series]
 
 
-def page_content_over_time(now):
+def page_content_over_time():
     oldest_page = Page.versions.all().order_by(
         'history_date')[0].version_info.date
 
+    # TODO: There's probably a much faster way to do this.  But it's
+    # fast enough for now.
+
     graph = pyflot.Flot()
     page_dict = {}
     page_contents = []
+    now = datetime.now()
 
     d = datetime(oldest_page.year, oldest_page.month, oldest_page.day)
     while (now.year, now.month, now.day) != (d.year, d.month, d.day):
@@ -178,14 +176,11 @@ def page_content_over_time(now):
             version_info__date__gte=d, version_info__date__lt=next_d)
         # Group the edits by slug and annotate with the max history date
         # for the associated page.
-        slugs_with_date = page_edits_this_day.values('slug').annotate(
-            Max('history_date')).order_by()
+        slugs_with_date = page_edits_this_day.values(
+            'slug', 'content').annotate(Max('history_date'))
 
         for item in slugs_with_date:
-            p = Page(slug=item['slug'])
-            # Grab the historical version at this date.
-            p_h = p.versions.as_of(date=item['history_date__max'])
-            page_dict[item['slug']] = len(p_h.content)
+            page_dict[item['slug']] = len(item['content'])
 
         total_content_today = 0
         for slug, length in page_dict.iteritems():
@@ -199,24 +194,11 @@ def page_content_over_time(now):
     return [graph.prepare_series(s) for s in graph._series]
 
 
-def users_registered_over_time(now):
+def users_registered_over_time():
     oldest_user = User.objects.order_by('date_joined')[0].date_joined
-
     graph = pyflot.Flot()
-    users_registered = []
-    num_users = 0
-    d = datetime(oldest_user.year, oldest_user.month, oldest_user.day)
-    while (now.year, now.month, now.day) != (d.year, d.month, d.day):
-        next_d = d + timedelta(days=1)
-
-        users_this_day = len(User.objects.filter(
-            date_joined__gte=d, date_joined__lt=next_d))
-
-        num_users += users_this_day
-
-        users_registered.append((d, num_users))
-        d = next_d
 
-    graph.add_time_series(users_registered)
+    qss = qsstats.QuerySetStats(User.objects.all(), 'date_joined')
+    graph.add_time_series(_summed_series(qss.time_series(oldest_user)))
 
     return [graph.prepare_series(s) for s in graph._series]

sapling/etc/install_config/requirements.txt

@@ -10,5 +10,6 @@ South==0.7.4
 python-flot-utils==0.2.1
 django-staticfiles==1.2.1
 django-registration==0.8.0
+django-qsstats-magic==0.7
 -e git+git://github.com/philipn/olwidget.git@72de64cd25ffdc3644bc92d14668c145533d6d45#egg=django-olwidget
 -e git+git://github.com/mivanov/django-honeypot.git#egg=django-honeypot

setup.py

@@ -129,6 +129,7 @@ def gen_data_files(*dirs):
     'django-registration==0.8.0',
     'django-olwidget==0.46-custom1',
     'django-honeypot==0.3.0-custom',
+    'django-qsstats-magic==0.7',
     # Actually optional:
     'Sphinx==1.1.3',
 ]