analytics: Add APIs for submitting analytics to another server.

This adds a new API for sending basic analytics data (number of users,
number of messages sent) from a Zulip server to the Zulip Cloud
central analytics database, which will make it possible for servers to
elect to have their usage numbers counted in published stats on the
size of the Zulip ecosystem.

zerver/lib/export.py

@@ -214,6 +214,7 @@
     'zerver_userprofile': ['date_joined', 'last_login', 'last_reminder'],
     'zerver_realmauditlog': ['event_time'],
     'zerver_userhotspot': ['timestamp'],
+    'analytics_installationcount': ['end_time'],
     'analytics_realmcount': ['end_time'],
     'analytics_usercount': ['end_time'],
     'analytics_streamcount': ['end_time'],

zerver/lib/remote_server.py

@@ -1,13 +1,16 @@
 import requests
 import ujson
 import urllib
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from django.conf import settings
+from django.forms.models import model_to_dict
 from django.utils.translation import ugettext as _
 
+from analytics.models import InstallationCount, RealmCount
 from version import ZULIP_VERSION
 from zerver.lib.exceptions import JsonableError
+from zerver.lib.export import floatify_datetime_fields
 
 class PushNotificationBouncerException(Exception):
     pass
@@ -85,3 +88,41 @@ def send_json_to_push_bouncer(method: str, endpoint: str, post_data: Dict[str, A
         ujson.dumps(post_data),
         extra_headers={"Content-type": "application/json"},
     )
+
+def build_analytics_data(realm_count_query: Any,
+                         installation_count_query: Any) -> Tuple[List[Dict[str, Any]],
+                                                                 List[Dict[str, Any]]]:
+    data = {}
+    data['analytics_realmcount'] = [
+        model_to_dict(realm_count) for realm_count in realm_count_query.order_by("id")
+    ]
+    data['analytics_installationcount'] = [
+        model_to_dict(count) for count in installation_count_query.order_by("id")
+    ]
+
+    floatify_datetime_fields(data, 'analytics_realmcount')
+    floatify_datetime_fields(data, 'analytics_installationcount')
+    return (data['analytics_realmcount'], data['analytics_installationcount'])
+
+def send_analytics_to_remote_server() -> None:
+    # first, check what's latest
+    result = send_to_push_bouncer("GET", "server/analytics/status", {})
+    last_acked_realm_count_id = result['last_realm_count_id']
+    last_acked_installation_count_id = result['last_installation_count_id']
+
+    (realm_count_data, installation_count_data) = build_analytics_data(
+        realm_count_query=RealmCount.objects.filter(
+            id__gt=last_acked_realm_count_id),
+        installation_count_query=InstallationCount.objects.filter(
+            id__gt=last_acked_installation_count_id))
+
+    if len(realm_count_data) == 0 and len(installation_count_data) == 0:
+        return
+
+    request = {
+        'realm_counts': ujson.dumps(realm_count_data),
+        'installation_counts': ujson.dumps(installation_count_data),
+    }
+
+    # Gather only entries with an ID greater than last_realm_count_id
+    send_to_push_bouncer("POST", "server/analytics", request)

zerver/tests/test_push_notifications.py

@@ -1,5 +1,6 @@
 
 from contextlib import contextmanager
+import datetime
 import itertools
 import requests
 import mock
@@ -18,7 +19,11 @@
 from django.conf import settings
 from django.http import HttpResponse
 from django.utils.crypto import get_random_string
+from django.utils.timezone import now as timezone_now
+from django.utils.timezone import utc as timezone_utc
 
+from analytics.lib.counts import CountStat, LoggingCountStat
+from analytics.models import InstallationCount, RealmCount
 from zerver.models import (
     PushDeviceToken,
     UserProfile,
@@ -39,12 +44,16 @@
 from zerver.lib import push_notifications as apn
 from zerver.lib.push_notifications import get_mobile_push_content, \
     DeviceToken, PushNotificationBouncerException, get_apns_client
+from zerver.lib.remote_server import send_analytics_to_remote_server, \
+    build_analytics_data
+from zerver.lib.request import JsonableError
 from zerver.lib.response import json_success
 from zerver.lib.test_classes import (
     ZulipTestCase,
 )
 
-from zilencer.models import RemoteZulipServer, RemotePushDeviceToken
+from zilencer.models import RemoteZulipServer, RemotePushDeviceToken, \
+    RemoteRealmCount, RemoteInstallationCount
 from django.utils.timezone import now
 
 ZERVER_DIR = os.path.dirname(os.path.dirname(__file__))
@@ -74,6 +83,11 @@ def bounce_request(self, *args: Any, **kwargs: Any) -> HttpResponse:
                                    local_url,
                                    kwargs['data'],
                                    subdomain="")
+        elif args[0] == "GET":
+            result = self.api_get(self.server_uuid,
+                                  local_url,
+                                  kwargs['data'],
+                                  subdomain="")
         else:
             raise AssertionError("Unsupported method for bounce_request")
         return result
@@ -278,6 +292,90 @@ def test_push_bouncer_api(self, mock: Any) -> None:
                                                                server=server))
             self.assertEqual(len(tokens), 0)
 
+class AnalyticsBouncerTest(BouncerTestCase):
+    TIME_ZERO = datetime.datetime(1988, 3, 14).replace(tzinfo=timezone_utc)
+
+    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL='https://push.zulip.org.example.com')
+    @mock.patch('zerver.lib.push_notifications.requests.request')
+    def test_analytics_api(self, mock: Any) -> None:
+        """This is a variant of the below test_push_api, but using the full
+        push notification bouncer flow
+        """
+        mock.side_effect = self.bounce_request
+        user = self.example_user('hamlet')
+        end_time = self.TIME_ZERO
+
+        realm_stat = LoggingCountStat('invites_sent::day', RealmCount, CountStat.DAY)
+        RealmCount.objects.create(
+            realm=user.realm, property=realm_stat.property, end_time=end_time, value=5)
+        InstallationCount.objects.create(
+            property=realm_stat.property, end_time=end_time, value=5)
+
+        self.assertEqual(RealmCount.objects.count(), 1)
+        self.assertEqual(InstallationCount.objects.count(), 1)
+
+        self.assertEqual(RemoteRealmCount.objects.count(), 0)
+        self.assertEqual(RemoteInstallationCount.objects.count(), 0)
+        send_analytics_to_remote_server()
+        self.assertEqual(mock.call_count, 2)
+        self.assertEqual(RemoteRealmCount.objects.count(), 1)
+        self.assertEqual(RemoteInstallationCount.objects.count(), 1)
+        send_analytics_to_remote_server()
+        self.assertEqual(mock.call_count, 3)
+        self.assertEqual(RemoteRealmCount.objects.count(), 1)
+        self.assertEqual(RemoteInstallationCount.objects.count(), 1)
+
+        RealmCount.objects.create(
+            realm=user.realm, property=realm_stat.property, end_time=end_time + datetime.timedelta(days=1), value=6)
+        RealmCount.objects.create(
+            realm=user.realm, property=realm_stat.property, end_time=end_time + datetime.timedelta(days=2), value=9)
+        self.assertEqual(RemoteRealmCount.objects.count(), 1)
+        self.assertEqual(mock.call_count, 3)
+        send_analytics_to_remote_server()
+        self.assertEqual(mock.call_count, 5)
+        self.assertEqual(RemoteRealmCount.objects.count(), 3)
+        self.assertEqual(RemoteInstallationCount.objects.count(), 1)
+
+        InstallationCount.objects.create(
+            property=realm_stat.property, end_time=end_time + datetime.timedelta(days=1), value=6)
+        InstallationCount.objects.create(
+            property=realm_stat.property, end_time=end_time + datetime.timedelta(days=2), value=9)
+        send_analytics_to_remote_server()
+        self.assertEqual(mock.call_count, 7)
+        self.assertEqual(RemoteRealmCount.objects.count(), 3)
+        self.assertEqual(RemoteInstallationCount.objects.count(), 3)
+
+        (realm_count_data,
+         installation_count_data) = build_analytics_data(RealmCount.objects.all(),
+                                                         InstallationCount.objects.all())
+        result = self.api_post(self.server_uuid,
+                               '/api/v1/remotes/server/analytics',
+                               {'realm_counts': ujson.dumps(realm_count_data),
+                                'installation_counts': ujson.dumps(installation_count_data)},
+                               subdomain="")
+        self.assert_json_error(result, "Data is out of order.")
+
+    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL='https://push.zulip.org.example.com')
+    @mock.patch('zerver.lib.push_notifications.requests.request')
+    def test_analytics_api_invalid(self, mock: Any) -> None:
+        """This is a variant of the below test_push_api, but using the full
+        push notification bouncer flow
+        """
+        mock.side_effect = self.bounce_request
+        user = self.example_user('hamlet')
+        end_time = self.TIME_ZERO
+
+        realm_stat = LoggingCountStat('invalid count stat', RealmCount, CountStat.DAY)
+        RealmCount.objects.create(
+            realm=user.realm, property=realm_stat.property, end_time=end_time, value=5)
+
+        self.assertEqual(RealmCount.objects.count(), 1)
+
+        self.assertEqual(RemoteRealmCount.objects.count(), 0)
+        with self.assertRaises(JsonableError):
+            send_analytics_to_remote_server()
+        self.assertEqual(RemoteRealmCount.objects.count(), 0)
+
 class PushNotificationTest(BouncerTestCase):
     def setUp(self) -> None:
         super().setUp()

zilencer/migrations/0016_remote_counts.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-02-02 06:02
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zilencer', '0015_delete_billing'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RemoteInstallationCount',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('property', models.CharField(max_length=32)),
+                ('subgroup', models.CharField(max_length=16, null=True)),
+                ('end_time', models.DateTimeField()),
+                ('value', models.BigIntegerField()),
+                ('remote_id', models.IntegerField(db_index=True)),
+                ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zilencer.RemoteZulipServer')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='RemoteRealmCount',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('property', models.CharField(max_length=32)),
+                ('subgroup', models.CharField(max_length=16, null=True)),
+                ('end_time', models.DateTimeField()),
+                ('value', models.BigIntegerField()),
+                ('realm_id', models.IntegerField(db_index=True)),
+                ('remote_id', models.IntegerField(db_index=True)),
+                ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zilencer.RemoteZulipServer')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='remoterealmcount',
+            unique_together=set([('server', 'realm_id', 'property', 'subgroup', 'end_time')]),
+        ),
+        migrations.AlterIndexTogether(
+            name='remoterealmcount',
+            index_together=set([('property', 'end_time')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='remoteinstallationcount',
+            unique_together=set([('server', 'property', 'subgroup', 'end_time')]),
+        ),
+    ]

zilencer/models.py

@@ -3,6 +3,7 @@
 from django.db import models
 
 from zerver.models import AbstractPushDeviceToken
+from analytics.models import BaseCount
 
 def get_remote_server_by_uuid(uuid: str) -> 'RemoteZulipServer':
     return RemoteZulipServer.objects.get(uuid=uuid)
@@ -33,3 +34,28 @@ class Meta:
 
     def __str__(self) -> str:
         return "<RemotePushDeviceToken %s %s>" % (self.server, self.user_id)
+
+class RemoteInstallationCount(BaseCount):
+    server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)  # type: RemoteZulipServer
+    # The remote_id field lets us deduplicate data from the remote server
+    remote_id = models.IntegerField(db_index=True)  # type: int
+
+    class Meta:
+        unique_together = ("server", "property", "subgroup", "end_time")
+
+    def __str__(self) -> str:
+        return "<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
+
+# We can't subclass RealmCount because we only have a realm_id here, not a foreign key.
+class RemoteRealmCount(BaseCount):
+    server = models.ForeignKey(RemoteZulipServer, on_delete=models.CASCADE)  # type: RemoteZulipServer
+    realm_id = models.IntegerField(db_index=True)  # type: int
+    # The remote_id field lets us deduplicate data from the remote server
+    remote_id = models.IntegerField(db_index=True)  # type: int
+
+    class Meta:
+        unique_together = ("server", "realm_id", "property", "subgroup", "end_time")
+        index_together = ["property", "end_time"]
+
+    def __str__(self) -> str:
+        return "%s %s %s %s %s" % (self.server, self.realm_id, self.property, self.subgroup, self.value)

zilencer/urls.py

@@ -18,6 +18,12 @@
 
     # Push signup doesn't use the REST API, since there's no auth.
     url('^remotes/server/register$', zilencer.views.register_remote_server),
+
+    # For receiving InstallationCount data and similar analytics.
+    url('^remotes/server/analytics$', rest_dispatch,
+        {'POST': 'zilencer.views.remote_server_post_analytics'}),
+    url('^remotes/server/analytics/status$', rest_dispatch,
+        {'GET': 'zilencer.views.remote_server_check_analytics'}),
 ]
 
 urlpatterns = [