retention: Add models to store expired messages.

This addresses part of zulip#106.

zerver/migrations/0067_archived_models.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-03-26 01:10
+from __future__ import unicode_literals
+
+import bitfield.models
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import zerver.lib.str_utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0066_realm_inline_url_embed_preview'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ArchivedAttachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file_name', models.TextField(db_index=True)),
+                ('path_id', models.TextField(db_index=True)),
+                ('is_realm_public', models.BooleanField(default=False)),
+                ('create_time', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                ('size', models.IntegerField(null=True)),
+                ('archive_timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='ArchivedMessage',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('subject', models.CharField(db_index=True, max_length=60)),
+                ('content', models.TextField()),
+                ('rendered_content', models.TextField(null=True)),
+                ('rendered_content_version', models.IntegerField(null=True)),
+                ('pub_date', models.DateTimeField(db_index=True, verbose_name='date published')),
+                ('last_edit_time', models.DateTimeField(null=True)),
+                ('edit_history', models.TextField(null=True)),
+                ('has_attachment', models.BooleanField(db_index=True, default=False)),
+                ('has_image', models.BooleanField(db_index=True, default=False)),
+                ('has_link', models.BooleanField(db_index=True, default=False)),
+                ('archive_timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                ('recipient', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Recipient')),
+                ('sender', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('sending_client', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Client')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='ArchivedUserMessage',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('flags', bitfield.models.BitField(['read', 'starred', 'collapsed', 'mentioned', 'wildcard_mentioned', 'summarize_in_home', 'summarize_in_stream', 'force_expand', 'force_collapse', 'has_alert_word', 'historical', 'is_me_message'], default=0)),
+                ('archive_timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                ('message', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.ArchivedMessage')),
+                ('user_profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
+        ),
+        migrations.AddField(
+            model_name='archivedattachment',
+            name='messages',
+            field=models.ManyToManyField(to='zerver.ArchivedMessage'),
+        ),
+        migrations.AddField(
+            model_name='archivedattachment',
+            name='owner',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='archivedattachment',
+            name='realm',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='archivedusermessage',
+            unique_together=set([('user_profile', 'message')]),
+        ),
+    ]

zerver/models.py

@@ -950,7 +950,7 @@ def sew_messages_and_reactions(messages, reactions):
     return list(converted_messages.values())
 
 
-class Message(ModelReprMixin, models.Model):
+class AbstractMessage(ModelReprMixin, models.Model):
     sender = models.ForeignKey(UserProfile) # type: UserProfile
     recipient = models.ForeignKey(Recipient) # type: Recipient
     subject = models.CharField(max_length=MAX_SUBJECT_LENGTH, db_index=True) # type: Text
@@ -965,6 +965,16 @@ class Message(ModelReprMixin, models.Model):
     has_image = models.BooleanField(default=False, db_index=True) # type: bool
     has_link = models.BooleanField(default=False, db_index=True) # type: bool
 
+    class Meta(object):
+        abstract = True
+
+
+class ArchivedMessage(AbstractMessage):
+    archive_timestamp = models.DateTimeField(default=timezone.now, db_index=True)  # type: datetime.datetime
+
+
+class Message(AbstractMessage):
+
     def topic_name(self):
         # type: () -> Text
         """
@@ -1139,9 +1149,8 @@ def get_raw_db_rows(needed_ids):
 #
 # UserMessage is the largest table in a Zulip installation, even
 # though each row is only 4 integers.
-class UserMessage(ModelReprMixin, models.Model):
+class AbstractUserMessage(ModelReprMixin, models.Model):
     user_profile = models.ForeignKey(UserProfile) # type: UserProfile
-    message = models.ForeignKey(Message) # type: Message
     # We're not using the archived field for now, but create it anyway
     # since this table will be an unpleasant one to do schema changes
     # on later
@@ -1151,16 +1160,27 @@ class UserMessage(ModelReprMixin, models.Model):
     flags = BitField(flags=ALL_FLAGS, default=0) # type: BitHandler
 
     class Meta(object):
+        abstract = True
         unique_together = ("user_profile", "message")
 
+    def flags_list(self):
+        # type: () -> List[str]
+        return [flag for flag in self.flags.keys() if getattr(self.flags, flag).is_set]
+
+
+class ArchivedUserMessage(AbstractUserMessage):
+    message = models.ForeignKey(ArchivedMessage)  # type: Message
+    archive_timestamp = models.DateTimeField(default=timezone.now, db_index=True)  # type: datetime.datetime
+
+
+class UserMessage(AbstractUserMessage):
+    message = models.ForeignKey(Message)  # type: Message
+
     def __unicode__(self):
         # type: () -> Text
         display_recipient = get_display_recipient(self.message.recipient)
         return u"<UserMessage: %s / %s (%s)>" % (display_recipient, self.user_profile.email, self.flags_list())
 
-    def flags_list(self):
-        # type: () -> List[str]
-        return [flag for flag in self.flags.keys() if getattr(self.flags, flag).is_set]
 
 def parse_usermessage_flags(val):
     # type: (int) -> List[str]
@@ -1172,18 +1192,31 @@ def parse_usermessage_flags(val):
         mask <<= 1
     return flags
 
-class Attachment(ModelReprMixin, models.Model):
-    file_name = models.TextField(db_index=True) # type: Text
+
+class AbstractAttachment(ModelReprMixin, models.Model):
+    file_name = models.TextField(db_index=True)  # type: Text
     # path_id is a storage location agnostic representation of the path of the file.
     # If the path of a file is http://localhost:9991/user_uploads/a/b/abc/temp_file.py
     # then its path_id will be a/b/abc/temp_file.py.
-    path_id = models.TextField(db_index=True) # type: Text
-    owner = models.ForeignKey(UserProfile) # type: UserProfile
-    realm = models.ForeignKey(Realm, blank=True, null=True) # type: Realm
-    is_realm_public = models.BooleanField(default=False) # type: bool
-    messages = models.ManyToManyField(Message) # type: Manager
-    create_time = models.DateTimeField(default=timezone.now, db_index=True) # type: datetime.datetime
-    size = models.IntegerField(null=True) # type: int
+    path_id = models.TextField(db_index=True)  # type: Text
+    owner = models.ForeignKey(UserProfile)  # type: UserProfile
+    realm = models.ForeignKey(Realm, blank=True, null=True)  # type: Realm
+    is_realm_public = models.BooleanField(default=False)  # type: bool
+    create_time = models.DateTimeField(default=timezone.now,
+                                       db_index=True)  # type: datetime.datetime
+    size = models.IntegerField(null=True)  # type: int
+
+    class Meta(object):
+        abstract = True
+
+
+class ArchivedAttachment(AbstractAttachment):
+    archive_timestamp = models.DateTimeField(default=timezone.now, db_index=True)  # type: datetime.datetime
+    messages = models.ManyToManyField(ArchivedMessage)  # type: Manager
+
+
+class Attachment(AbstractAttachment):
+    messages = models.ManyToManyField(Message)  # type: Manager
 
     def __unicode__(self):
         # type: () -> Text
@@ -1207,6 +1240,7 @@ def to_dict(self):
             } for m in self.messages.all()]
         }
 
+
 def get_old_unclaimed_attachments(weeks_ago):
     # type: (int) -> Sequence[Attachment]
     # TODO: Change return type to QuerySet[Attachment]