[Structure] Calculer le nombre d'établissements (gip-inclusion#940)

* Add properties Siae.siren & Siae.nic

* New field Siae.etablissement_count

* Adapt existing update_counts command. Add cron

clevercloud/cron.json

@@ -7,6 +7,7 @@
     "20 7 * * 1 $ROOT/clevercloud/siaes_update_api_entreprise_fields.sh",
     "30 7 * * 1 $ROOT/clevercloud/siaes_update_api_qpv_fields.sh",
     "40 7 * * 1 $ROOT/clevercloud/siaes_update_api_zrr_fields.sh",
+    "50 7 * * 1 $ROOT/clevercloud/siaes_update_count_fields.sh",
     "0 7 * * 2 $ROOT/clevercloud/siaes_send_completion_reminder_emails.sh",
     "0 8 * * * $ROOT/clevercloud/siaes_send_user_request_reminder_emails.sh",
     "30 8 * * * $ROOT/clevercloud/tenders_send_author_transactioned_question_emails.sh",

clevercloud/siaes_update_count_fields.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -l
+
+# Update API ZRR fields for new siaes
+
+# Do not run if this env var is not set:
+if [[ -z "$CRON_UPDATE_COUNT_FIELDS_ENABLED" ]]; then
+    echo "CRON_UPDATE_COUNT_FIELDS_ENABLED not set. Exiting..."
+    exit 0
+fi
+
+# About clever cloud cronjobs:
+# https://www.clever-cloud.com/doc/tools/crons/
+
+if [[ "$INSTANCE_NUMBER" != "0" ]]; then
+    echo "Instance number is ${INSTANCE_NUMBER}. Stop here."
+    exit 0
+fi
+
+# $APP_HOME is set by default by clever cloud.
+cd $APP_HOME
+
+# django-admin update_count_fields
+django-admin update_count_fields --fields etablissement_count

lemarche/siaes/admin.py

@@ -164,6 +164,7 @@ class SiaeAdmin(FieldsetsInlineMixin, gis_admin.OSMGeoAdmin):
     autocomplete_fields = ["sectors", "networks", "groups"]
     # prepopulated_fields = {"slug": ("name",)}
     readonly_fields = [field for field in Siae.READONLY_FIELDS if field not in ("coords")] + [
+        "siren",
         "sector_count_with_link",
         "network_count_with_link",
         "group_count_with_link",
@@ -180,8 +181,6 @@ class SiaeAdmin(FieldsetsInlineMixin, gis_admin.OSMGeoAdmin):
         "tender_email_link_click_count_with_link",
         "tender_detail_display_count_with_link",
         "tender_detail_contact_click_count_with_link",
-        "signup_date",
-        "content_filled_basic_date",
         "logs_display",
         # "import_raw_object",
         "import_raw_object_display",
@@ -210,6 +209,8 @@ class SiaeAdmin(FieldsetsInlineMixin, gis_admin.OSMGeoAdmin):
                     "slug",
                     "brand",
                     "siret",
+                    "siren",
+                    "etablissement_count",
                     "naf",
                     "kind",
                     "nature",

lemarche/siaes/management/commands/update_count_fields.py

@@ -0,0 +1,84 @@
+from lemarche.siaes import utils as siae_utils
+from lemarche.siaes.models import Siae
+from lemarche.utils.apis import api_slack
+from lemarche.utils.commands import BaseCommand
+
+
+SIAE_COUNT_FIELDS = [
+    "user_count",
+    "sector_count",
+    "network_count",
+    "group_count",
+    "offer_count",
+    "client_reference_count",
+    "label_count",
+    "image_count",
+    "etablissement_count",
+]
+
+
+class Command(BaseCommand):
+    """
+    Goal: update the '_count' fields of each Siae
+
+    Note: these fields should be updated automatically on each Siae save()
+
+    Usage:
+    python manage.py update_count_fields
+    python manage.py update_count_fields --id 1
+    python manage.py update_count_fields --id 1 --fields user_count
+    python manage.py update_count_fields --id 1 --fields user_count --fields etablissement_count
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument("--id", type=int, default=None, help="Indiquer l'ID d'une structure")
+        parser.add_argument(
+            "--fields", action="append", default=[], help="Filtrer sur les champs count à mettre à jour"
+        )
+
+    def handle(self, *args, **options):
+        self.stdout_messages_info("Updating Siae count fields...")
+
+        # Step 1a: build Siae queryset
+        siae_queryset = Siae.objects.prefetch_related(
+            "users", "sectors", "networks", "groups", "offers", "client_references", "labels", "images"
+        ).all()
+        if options["id"]:
+            siae_queryset = siae_queryset.filter(id=options["id"])
+        self.stdout_messages_info(f"Found {siae_queryset.count()} Siae")
+
+        # Step 1b: init fields to update
+        update_fields = options["fields"] if options["fields"] else SIAE_COUNT_FIELDS
+        self.stdout_messages_info(f"Fields to update: {update_fields}")
+
+        # Step 2: loop on each Siae
+        progress = 0
+        for index, siae in enumerate(siae_queryset):
+            # M2M
+            siae.user_count = siae.users.count()
+            siae.sector_count = siae.sectors.count()
+            siae.network_count = siae.networks.count()
+            siae.group_count = siae.groups.count()
+            # FK
+            siae.offer_count = siae.offers.count()
+            siae.client_reference_count = siae.client_references.count()
+            siae.label_count = siae.labels_old.count()
+            siae.image_count = siae.images.count()
+            # etablissement_count
+            if siae.is_active and siae.siren:
+                siae.etablissement_count = siae_utils.calculate_etablissement_count(siae)
+
+            # Step 3: update count fields
+            siae.save(update_fields=update_fields)
+
+            progress += 1
+            if (progress % 500) == 0:
+                self.stdout_info(f"{progress}...")
+
+        msg_success = [
+            "----- Siae count fields -----",
+            f"Done! Processed {siae_queryset.count()} siaes",
+            f"Fields updated: {update_fields}",
+        ]
+        self.stdout_messages_success(msg_success)
+        api_slack.send_message_to_channel("\n".join(msg_success))

lemarche/siaes/migrations/0066_siae_etablissement_count.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.2 on 2023-10-10 08:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("siaes", "0065_siae_legal_form"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="siae",
+            name="etablissement_count",
+            field=models.IntegerField(default=0, verbose_name="Nombre d'établissements (à partir du Siren)"),
+        ),
+    ]

lemarche/siaes/models.py

@@ -502,12 +502,14 @@ class Siae(models.Model):
         "api_entreprise_ca_date_fin_exercice",
         "api_entreprise_exercice_last_sync_date",
     ]
+    READONLY_FIELDS_STATS = ["etablissement_count", "signup_date", "content_filled_basic_date"]
     READONLY_FIELDS = (
         READONLY_FIELDS_FROM_C1
         + READONLY_FIELDS_FROM_C2
         + READONLY_FIELDS_FROM_QPV
         + READONLY_FIELDS_FROM_ZRR
         + READONLY_FIELDS_FROM_API_ENTREPRISE
+        + READONLY_FIELDS_STATS
     )
 
     TRACK_UPDATE_FIELDS = [
@@ -759,6 +761,7 @@ class Siae(models.Model):
     client_reference_count = models.IntegerField("Nombre de références clients", default=0)
     label_count = models.IntegerField("Nombre de labels", default=0)
     image_count = models.IntegerField("Nombre d'images", default=0)
+    etablissement_count = models.IntegerField("Nombre d'établissements (à partir du Siren)", default=0)
     signup_date = models.DateTimeField(
         "Date d'inscription de la structure (premier utilisateur)", blank=True, null=True
     )
@@ -906,6 +909,18 @@ def siret_display(self):
             return f"{self.siret[0:3]} {self.siret[3:6]} {self.siret[6:9]}"
         return self.siret
 
+    @property
+    def siren(self):
+        return self.siret[:9]
+
+    @property
+    def nic(self):
+        """
+        The second part of SIRET is called the NIC (numéro interne de classement).
+        https://www.insee.fr/fr/metadonnees/definition/c1981
+        """
+        return self.siret[9:14]
+
     @property
     def year_constitution_display(self):
         if self.year_constitution:

lemarche/siaes/tests.py

@@ -5,7 +5,7 @@
 from lemarche.perimeters.factories import PerimeterFactory
 from lemarche.perimeters.models import Perimeter
 from lemarche.sectors.factories import SectorFactory
-from lemarche.siaes import constants as siae_constants
+from lemarche.siaes import constants as siae_constants, utils as siae_utils
 from lemarche.siaes.factories import SiaeFactory, SiaeGroupFactory, SiaeLabelOldFactory, SiaeOfferFactory
 from lemarche.siaes.models import Siae, SiaeGroup, SiaeLabel, SiaeUser
 from lemarche.users.factories import UserFactory
@@ -63,13 +63,18 @@ def test_name_display_property(self):
         self.assertEqual(siae_without_brand.name_display, "Ma raison sociale")
         self.assertEqual(siae_with_brand.name_display, "Mon enseigne")
 
-    def test_siret_display_property(self):
+    def test_siret_siren_nic_properties(self):
+        # siret_display
         siae_with_siret = SiaeFactory(siret="12312312312345")
         self.assertEqual(siae_with_siret.siret_display, "123 123 123 12345")
         siae_with_siren = SiaeFactory(siret="123123123")
         self.assertEqual(siae_with_siren.siret_display, "123 123 123")
         siae_with_anormal_siret = SiaeFactory(siret="123123123123")
         self.assertEqual(siae_with_anormal_siret.siret_display, "123123123123")
+        # siren
+        self.assertEqual(siae_with_siret.siren, "123123123")
+        # nic
+        self.assertEqual(siae_with_siret.nic, "12345")
 
     def test_geo_range_pretty_display_property(self):
         siae_country = SiaeFactory(geo_range=siae_constants.GEO_RANGE_COUNTRY)
@@ -464,3 +469,14 @@ def test_siae_labels_through(self):
         siae = SiaeFactory()
         SiaeLabel.objects.create(siae=siae, label=self.label_2)
         self.assertEqual(siae.labels.count(), 1)
+
+
+class SiaeUtilsTest(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.siae_with_siret_1 = SiaeFactory(siret="12312312312345", is_active=True)
+        cls.siae_with_siret_2 = SiaeFactory(siret="12312312312346", is_active=True)
+        cls.siae_with_siret_inactive = SiaeFactory(siret="12312312312347", is_active=False)
+
+    def test_calculate_etablissement_count(self):
+        self.assertEqual(siae_utils.calculate_etablissement_count(self.siae_with_siret_1), 2)

lemarche/siaes/utils.py

@@ -0,0 +1,7 @@
+from lemarche.siaes.models import Siae
+
+
+def calculate_etablissement_count(siae: Siae):
+    if siae.siren:
+        return Siae.objects.filter(is_active=True, siret__startswith=siae.siren).count()
+    return 0

lemarche/utils/commands.py

@@ -13,14 +13,12 @@ def stdout_info(self, message):
 
     def stdout_messages_info(self, messages):
         self.stdout_info("-" * 80)
-        messages = messages if (type(messages) == list) else [messages]
+        messages = messages if (type(messages) is list) else [messages]
         for message in messages:
             self.stdout_info(message)
-        self.stdout_info("-" * 80)
 
     def stdout_messages_success(self, messages):
         self.stdout_success("-" * 80)
-        messages = messages if (type(messages) == list) else [messages]
+        messages = messages if (type(messages) is list) else [messages]
         for message in messages:
             self.stdout_success(message)
-        self.stdout_success("-" * 80)