Fixing Development Fixtures

    - Ran into several dependency problems with the approach I was
      originally taking.  Punted and found a combination of dumpdata
      exclusions and a bit of Permission and ContentType deletion that
      works.

cms/management/commands/generate_dev_fixtures.py

@@ -1,14 +1,12 @@
 import gzip
+import json
+import io
 
-from collections import OrderedDict
 from optparse import make_option
 
-from django.apps import apps
-from django.core.management import BaseCommand, CommandError
-from django.core.management.commands.dumpdata import sort_dependencies
+from django.core.management import BaseCommand, call_command
 
-from django.core import serializers
-from django.db import router, DEFAULT_DB_ALIAS
+from django.contrib.auth.hashers import make_password
 
 
 class Command(BaseCommand):
@@ -22,7 +20,7 @@ class Command(BaseCommand):
     option_list = BaseCommand.option_list + (
         make_option(
             '--file',
-            default='/tmp/dev-fixtures.json',
+            default='/tmp/dev-fixtures.json.gz',
             dest='outputfile',
             help='Specifies the output file location of the fixtures.',
         ),
@@ -33,86 +31,31 @@ class Command(BaseCommand):
     def handle(self, **options):
         outputfile = options.get('outputfile')
 
-        app_labels = [
-            'auth',
-            'users',
-            'sitetree',
-            'boxes',
-            'pages',
-            'companies',
-            'jobs',
-            'sponsors',
-            'successstories',
-            'events',
-            'peps',
-            'blogs',
-            'downloads',
-            'codesamples',
-            'comments',
-        ]
-
-        # Get app label list for serialization
-        app_list = OrderedDict()
-        for label in app_labels:
-            try:
-                app_label, model_label = label.split('.')
-                try:
-                    app_config = apps.get_app_config(app_label)
-                except LookupError:
-                    raise CommandError("Unknown application: %s" % app_label)
-                if app_config.models_module is None:
-                    continue
-                try:
-                    model = app_config.get_model(model_label)
-                except LookupError:
-                    raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
-
-                app_list_value = app_list.setdefault(app_config, [])
-
-                # We may have previously seen a "all-models" request for
-                # this app (no model qualifier was given). In this case
-                # there is no need adding specific models to the list.
-                if app_list_value is not None:
-                    if model not in app_list_value:
-                        app_list_value.append(model)
-            except ValueError:
-                # This is just an app - no model qualifier
-                app_label = label
-                try:
-                    app_config = apps.get_app_config(app_label)
-                except LookupError:
-                    raise CommandError("Unknown application: %s" % app_label)
-                if app_config.models_module is None:
-                    continue
-                app_list[app_config] = None
-
-        def get_objects():
-            """
-            Collate the objects for serialization, taken from dumpdata command
-            and adjusted to sanitize User password hashes
-            """
-            # Collate the objects to be serialized.
-            for model in sort_dependencies(app_list.items()):
-                if not model._meta.proxy and router.allow_migrate(DEFAULT_DB_ALIAS, model):
-                    objects = model._base_manager
-
-                    queryset = objects.using(DEFAULT_DB_ALIAS).order_by(model._meta.pk.name)
-                    for obj in queryset.iterator():
-
-                        # Sanitize user objects
-                        if model._meta.model_name == 'user':
-                            obj.set_unusable_password()
-                        yield obj
-
-        # Create serializer
-        data = serializers.serialize(
+        content = io.StringIO()
+        call_command(
+            "dumpdata",
             format='json',
-            queryset=get_objects(),
             indent=4,
-            use_natural_foreign_keys=False,
-            use_natural_primary_keys=False,
+            exclude=[
+                "tastypie",
+                "sessions",
+                "feedbacks",
+                "account.emailconfirmation",
+            ],
+            stdout=content,
         )
 
-        data = bytes(data, 'utf-8')
+        content.seek(0)
+        raw_json = content.getvalue()
+        data = json.loads(raw_json)
+
+        # Scrub User passwords for security
+        for obj in data:
+            if obj['model'] != "users.user":
+                continue
+            obj['fields']['password'] = make_password(None)
+
         with gzip.open(outputfile, 'wb') as out:
-            out.write(data)
+            out.write(bytes(json.dumps(data, indent=4), 'UTF-8'))
+
+

cms/management/commands/load_dev_fixtures.py

@@ -5,6 +5,8 @@
 from django.core.management import call_command
 from django.core.management.base import NoArgsCommand
 from django.conf import settings
+from django.contrib.auth.models import Permission
+from django.contrib.contenttypes.models import ContentType
 from django.utils.six.moves import input
 
 
@@ -31,6 +33,10 @@ def handle_noargs(self, **options):
                 self.stderr.write("Unable to download file: Received status code {}".format(r.status_code))
                 sys.exit(1)
 
+            # Remove pesky objects that get in the way
+            Permission.objects.all().delete()
+            ContentType.objects.all().delete()
+
             with open('/tmp/dev-fixtures.json.gz', 'wb') as f:
                 for chunk in r.iter_content(chunk_size=1024):
                     f.write(chunk)