Always include Identity counts, calculate/store stats

activities/models/post.py

@@ -13,6 +13,7 @@
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import SearchVector
 from django.db import models, transaction
+from django.db.models.signals import post_delete, post_save
 from django.db.utils import IntegrityError
 from django.template import loader
 from django.template.defaultfilters import linebreaks_filter
@@ -1191,7 +1192,7 @@ def to_mastodon_json(self, interactions=None, bookmarks=None, identity=None):
             "id": str(self.pk),
             "uri": self.object_uri,
             "created_at": format_ld_date(self.published),
-            "account": self.author.to_mastodon_json(include_counts=False),
+            "account": self.author.to_mastodon_json(),
             "content": self.safe_content_remote(),
             "language": language,
             "visibility": visibility_mapping[self.visibility],
@@ -1244,3 +1245,16 @@ def to_mastodon_json(self, interactions=None, bookmarks=None, identity=None):
         if bookmarks:
             value["bookmarked"] = self.pk in bookmarks
         return value
+
+
+def post_created(sender, instance: Post, created, **kwargs):
+    if created:
+        instance.author.calculate_stats()
+
+
+def post_deleted(sender, instance: Post, **kwargs):
+    instance.author.calculate_stats()
+
+
+post_save.connect(post_created, sender=Post, dispatch_uid="activities.post.created")
+post_delete.connect(post_deleted, sender=Post, dispatch_uid="activities.post.deleted")

activities/models/post_interaction.py

@@ -562,7 +562,7 @@ def to_mastodon_status_json(self, interactions=None, identity=None):
             "id": f"{self.pk}",
             "uri": post_json["uri"],
             "created_at": format_ld_date(self.published),
-            "account": self.identity.to_mastodon_json(include_counts=False),
+            "account": self.identity.to_mastodon_json(),
             "content": "",
             "visibility": post_json["visibility"],
             "sensitive": post_json["sensitive"],

api/schemas.py

@@ -80,12 +80,9 @@ class Account(Schema):
     def from_identity(
         cls,
         identity: users_models.Identity,
-        include_counts: bool = True,
         source=False,
     ) -> "Account":
-        return cls(
-            **identity.to_mastodon_json(include_counts=include_counts, source=source)
-        )
+        return cls(**identity.to_mastodon_json(source=source))
 
 
 class MediaAttachment(Schema):
@@ -327,7 +324,7 @@ def from_follow(
         cls,
         follow: users_models.HashtagFollow,
     ) -> "FollowedTag":
-        return cls(id=follow.id, **follow.hashtag.to_mastodon_json(following=True))
+        return cls(id=str(follow.id), **follow.hashtag.to_mastodon_json(following=True))
 
     @classmethod
     def map_from_follows(

api/views/search.py

@@ -36,8 +36,7 @@ def search(
         type = None
     if type is None or type == "accounts":
         result["accounts"] = [
-            schemas.Account.from_identity(i, include_counts=False)
-            for i in search_result["identities"]
+            schemas.Account.from_identity(i) for i in search_result["identities"]
         ]
     if type is None or type == "hashtag":
         result["hashtags"] = [

api/views/statuses.py

@@ -4,7 +4,6 @@
 from django.http import HttpRequest
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
-from hatchway import ApiError, ApiResponse, Schema, api_view
 
 from activities.models import (
     Post,
@@ -18,6 +17,7 @@
 from api.decorators import scope_required
 from api.pagination import MastodonPaginator, PaginatingApiResponse, PaginationResult
 from core.models import Config
+from hatchway import ApiError, ApiResponse, Schema, api_view
 
 
 class PostPollSchema(Schema):
@@ -241,10 +241,7 @@ def favourited_by(
 
     return PaginatingApiResponse(
         [
-            schemas.Account.from_identity(
-                interaction.identity,
-                include_counts=False,
-            )
+            schemas.Account.from_identity(interaction.identity)
             for interaction in pager.results
         ],
         request=request,
@@ -283,10 +280,7 @@ def reblogged_by(
 
     return PaginatingApiResponse(
         [
-            schemas.Account.from_identity(
-                interaction.identity,
-                include_counts=False,
-            )
+            schemas.Account.from_identity(interaction.identity)
             for interaction in pager.results
         ],
         request=request,

users/management/commands/calculatestats.py

@@ -0,0 +1,60 @@
+from django.core.management.base import BaseCommand
+from django.db.models import (
+    Count,
+    DateTimeField,
+    IntegerField,
+    Max,
+    OuterRef,
+    Subquery,
+)
+
+from activities.models import Post
+from core.ld import format_ld_date
+from users.models import Follow, Identity
+
+
+class Command(BaseCommand):
+    help = "Recalculates Identity stats"
+
+    def handle(self, *args, **options):
+        posts = (
+            Post.objects.filter(author_id=OuterRef("id"))
+            .values("author_id")
+            .annotate(num=Count("id"))
+            .values("num")[:1]
+        )
+        latest = (
+            Post.objects.filter(author_id=OuterRef("id"))
+            .values("author_id")
+            .annotate(latest=Max("created"))
+            .values("latest")[:1]
+        )
+        followers = (
+            Follow.objects.filter(target_id=OuterRef("id"))
+            .values("target_id")
+            .annotate(num=Count("id"))
+            .values("num")[:1]
+        )
+        following = (
+            Follow.objects.filter(source_id=OuterRef("id"))
+            .values("source_id")
+            .annotate(num=Count("id"))
+            .values("num")[:1]
+        )
+
+        qs = Identity.objects.annotate(
+            statuses_count=Subquery(posts, output_field=IntegerField()),
+            last_status_at=Subquery(latest, output_field=DateTimeField()),
+            followers_count=Subquery(followers, output_field=IntegerField()),
+            following_count=Subquery(following, output_field=IntegerField()),
+        )
+
+        for i in qs:
+            latest = format_ld_date(i.last_status_at) if i.last_status_at else None
+            i.stats = {
+                "statuses_count": i.statuses_count or 0,
+                "last_status_at": latest,
+                "followers_count": i.followers_count or 0,
+                "following_count": i.following_count or 0,
+            }
+            i.save(update_fields=["stats"])

users/migrations/0027_identity_stats.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.11 on 2024-04-30 13:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("users", "0026_identity_featured_tags_uri_hashtagfeature"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="identity",
+            name="stats",
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

users/models/follow.py

@@ -3,6 +3,7 @@
 
 import httpx
 from django.db import models, transaction
+from django.db.models.signals import post_delete, post_save
 
 from core.ld import canonicalise, get_str_or_id
 from core.snowflake import Snowflake
@@ -436,3 +437,18 @@ def handle_undo_ap(cls, data):
             raise ValueError("Accept actor does not match its Follow object", data)
         # Delete the follow
         follow.transition_perform(FollowStates.pending_removal)
+
+
+def follow_created(sender, instance: Follow, created, **kwargs):
+    if created:
+        instance.source.calculate_stats()
+        instance.target.calculate_stats()
+
+
+def follow_deleted(sender, instance: Follow, **kwargs):
+    instance.source.calculate_stats()
+    instance.target.calculate_stats()
+
+
+post_save.connect(follow_created, sender=Follow, dispatch_uid="users.follow.created")
+post_delete.connect(follow_deleted, sender=Follow, dispatch_uid="users.follow.deleted")

users/models/identity.py

@@ -7,6 +7,7 @@
 import httpx
 import urlman
 from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import IntegrityError, models, transaction
 from django.utils import timezone
 from django.utils.functional import lazy
@@ -105,6 +106,8 @@ def handle_edited(cls, instance: "Identity"):
         if not instance.local:
             return cls.updated
 
+        # Not really necessary here, but is an easy way to force a recalculation.
+        instance.calculate_stats()
         cls.targets_fan_out(instance, FanOut.Types.identity_edited)
         return cls.updated
 
@@ -175,6 +178,7 @@ def handle_deleted(cls, instance: "Identity"):
     def handle_outdated(cls, identity: "Identity"):
         # Local identities never need fetching
         if identity.local:
+            identity.calculate_stats()
             return cls.updated
         # Run the actor fetch and progress to updated if it succeeds
         if identity.fetch_actor():
@@ -272,6 +276,9 @@ class Restriction(models.IntegerChoices):
     # the one URI it was moved to.
     aliases = models.JSONField(blank=True, null=True)
 
+    # Calculated (or fetched) statistics: follower/post counts, etc.
+    stats = models.JSONField(blank=True, null=True)
+
     # Admin-only moderation fields
     sensitive = models.BooleanField(default=False)
     restriction = models.IntegerField(
@@ -402,6 +409,21 @@ def ensure_uris(self):
             self.following_uri = self.actor_uri + "following/"
             self.shared_inbox_uri = f"https://{self.domain.uri_domain}/inbox/"
 
+    def calculate_stats(self, save=True):
+        try:
+            latest = format_ld_date(self.posts.latest("created").created)
+        except ObjectDoesNotExist:
+            latest = None
+        self.stats = {
+            "last_status_at": latest,
+            "statuses_count": self.posts.count(),
+            "followers_count": self.inbound_follows.count(),
+            "following_count": self.outbound_follows.count(),
+        }
+        logger.info("calculate_stats(%s): %s", self.handle, self.stats)
+        if save:
+            self.save()
+
     def add_alias(self, actor_uri: str):
         self.aliases = (self.aliases or []) + [actor_uri]
         self.save()
@@ -837,7 +859,7 @@ def fetch_webfinger(cls, handle: str) -> tuple[str | None, str | None]:
         return None, None
 
     @classmethod
-    def fetch_collection(cls, uri: str) -> list[dict]:
+    def fetch_collection(cls, uri: str, total_only=False) -> tuple[int, list[dict]]:
         with httpx.Client(
             timeout=settings.SETUP.REMOTE_TIMEOUT,
             headers={"User-Agent": settings.TAKAHE_USER_AGENT},
@@ -864,21 +886,26 @@ def fetch_collection(cls, uri: str) -> list[dict]:
                         f"Client error fetching featured collection: {response.status_code}",
                         response.content,
                     )
-                return []
+                return 0, []
 
         try:
             json_data = json_from_response(response)
             data = canonicalise(json_data, include_security=True)
+            total = data.get("totalItems", 0)
+            if total_only:
+                return total, []
             # canonicalise seems to turn single-item `items` list into a dict?? gross
             if value := data.get("orderedItems"):
-                return list(reversed(value)) if isinstance(value, list) else [value]
+                items = list(reversed(value)) if isinstance(value, list) else [value]
             elif value := data.get("items"):
-                return value if isinstance(value, list) else [value]
-            return []
+                items = value if isinstance(value, list) else [value]
+            else:
+                items = []
+            return total, items
         except ValueError:
             # Some servers return these with a 200 status code!
             if b"not found" in response.content.lower():
-                return []
+                return 0, []
             raise ValueError(
                 "JSON parse error fetching collection",
                 response.content,
@@ -889,7 +916,7 @@ def fetch_pinned_post_uris(cls, uri: str) -> list[str]:
         """
         Fetch an identity's featured collection (pins).
         """
-        items = cls.fetch_collection(uri)
+        total, items = cls.fetch_collection(uri)
         ids = []
         for item in items:
             if not isinstance(item, dict):
@@ -903,7 +930,7 @@ def fetch_featured_tags(cls, uri: str) -> list[str]:
         """
         Fetch an identity's featured tags.
         """
-        items = cls.fetch_collection(uri)
+        total, items = cls.fetch_collection(uri)
         names = []
         for item in items:
             if not isinstance(item, dict):
@@ -1032,6 +1059,8 @@ def fetch_actor(self) -> bool:
                 Emoji.by_ap_tag(self.domain, tag, create=True)
         # Mark as fetched
         self.fetched = timezone.now()
+        # Update the post stats
+        self.calculate_stats(save=False)
         try:
             with transaction.atomic():
                 # if we don't wrap this in its own transaction, the exception
@@ -1093,7 +1122,7 @@ def to_mastodon_mention_json(self):
             "acct": self.handle or "",
         }
 
-    def to_mastodon_json(self, source=False, include_counts=True):
+    def to_mastodon_json(self, source=False):
         from activities.models import Emoji, Post
 
         header_image = self.local_image_url()
@@ -1106,6 +1135,7 @@ def to_mastodon_json(self, source=False, include_counts=True):
             f"{self.name} {self.summary} {metadata_value_text}", self.domain
         )
         renderer = ContentRenderer(local=False)
+        stats = self.stats or {}
         result = {
             "id": str(self.pk),
             "username": self.username or "",
@@ -1140,10 +1170,10 @@ def to_mastodon_json(self, source=False, include_counts=True):
             "created_at": format_ld_date(
                 self.created.replace(hour=0, minute=0, second=0, microsecond=0)
             ),
-            "last_status_at": None,  # TODO: populate
-            "statuses_count": self.posts.count() if include_counts else 0,
-            "followers_count": self.inbound_follows.count() if include_counts else 0,
-            "following_count": self.outbound_follows.count() if include_counts else 0,
+            "last_status_at": stats.get("last_status_at"),
+            "statuses_count": stats.get("statuses_count", 0),
+            "followers_count": stats.get("followers_count", 0),
+            "following_count": stats.get("following_count", 0),
         }
         if source:
             privacy_map = {