Merge pull request python#505 from justinabrahms/pep-image-import

Pull in images when doing a PEP import.

peps/converters.py

@@ -5,11 +5,12 @@
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
+from django.core.files import File
 
-from pages.models import Page
+from pages.models import Page, Image
 
 PEP_TEMPLATE = 'pages/pep-page.html'
-
+pep_url = lambda num: 'dev/peps/pep-{}/'.format(num)
 
 def check_paths():
     """ Checks to ensure our PEP_REPO_PATH is setup correctly """
@@ -153,8 +154,7 @@ def get_pep_page(pep_number, commit=True):
 
     pep_content = convert_pep_page(pep_number, open(pep_path).read())
 
-    pep_url = 'dev/peps/pep-{}/'.format(pep_number)
-    pep_page, _ = Page.objects.get_or_create(path=pep_url)
+    pep_page, _ = Page.objects.get_or_create(path=pep_url(pep_number))
 
     # Remove leading zeros from PEP number for display purposes
     pep_number_string = str(pep_number)
@@ -173,3 +173,31 @@ def get_pep_page(pep_number, commit=True):
         pep_page.save()
 
     return pep_page
+
+def add_pep_image(pep_number, path):
+    image_path = os.path.join(settings.PEP_REPO_PATH, path)
+    if not os.path.exists(image_path):
+        print("Image Path '{}' does not exist, skipping".format(image_path))
+
+    try:
+        page = Page.objects.get(path=pep_url(pep_number))
+    except Page.DoesNotExist:
+        print("Could not find backing PEP {}".format(pep_number))
+        return
+
+    image, created = Image.objects.get_or_create(page=page)
+    if created:
+        with open(image_path, 'rb') as image_obj:
+            image.image.save(path, File(image_obj))
+
+        # Old images used to live alongside html, but now they're in different
+        # places, so update the page accordingly.
+        soup = BeautifulSoup(page.content.raw)
+        for img_tag in soup.findAll('img'):
+            if img_tag['src'] == path:
+                img_tag['src'] = os.path.join(settings.MEDIA_URL, page.path, path)
+
+        page.content.raw = soup.prettify()
+        page.save()
+
+    return image

peps/management/commands/generate_pep_pages.py

@@ -4,9 +4,9 @@
 from django.core.management.base import NoArgsCommand
 from django.conf import settings
 
-from peps.converters import get_pep0_page, get_pep_page
+from peps.converters import get_pep0_page, get_pep_page, add_pep_image
 
-pep_number_re = re.compile(r'pep-(\d+)\.html')
+pep_number_re = re.compile(r'pep-(\d+)')
 
 
 class Command(NoArgsCommand):
@@ -23,6 +23,13 @@ class Command(NoArgsCommand):
     """
     help = "Generate PEP Page objects from rendered HTML"
 
+    def is_pep_page(self, path):
+        return path.startswith('pep-') and path.endswith('.html')
+
+    def is_image(self, path):
+        # All images are pngs
+        return path.endswith('.png')
+
     def handle_noargs(self, **options):
         verbosity = int(options['verbosity'])
 
@@ -36,11 +43,18 @@ def verbose(msg):
         verbose("Generating PEP0 index page")
         pep0_page = get_pep0_page()
 
+        image_paths = set()
+
         # Find pep pages
         for f in os.listdir(settings.PEP_REPO_PATH):
 
+            if self.is_image(f):
+                verbose("- Deferring import of image '{}'".format(f))
+                image_paths.add(f)
+                continue
+
             # Skip files we aren't looking for
-            if not f.startswith('pep-') or not f.endswith('.html'):
+            if not self.is_pep_page(f):
                 verbose("- Skipping non-PEP file '{}'".format(f))
                 continue
 
@@ -50,6 +64,18 @@ def verbose(msg):
                 pep_number = pep_match.groups(1)[0]
                 p = get_pep_page(pep_number)
             else:
-                verbose("- Skipping invalid {}'".format(f))
+                verbose("- Skipping invalid '{}'".format(f))
+
+        # Find pep images. This needs to happen afterwards, because we need
+        for img in image_paths:
+            pep_match = pep_number_re.match(img)
+            if pep_match:
+                pep_number = pep_match.groups(1)[0]
+                verbose("Generating image for PEP {} at '{}'".format(
+                    pep_number, img))
+                add_pep_image(pep_number, img)
+            else:
+                verbose("- Skipping non-PEP related image '{}'".format(img))
+
 
         verbose("== Finished")

peps/tests/fake_pep_repo/pep-3001.html

@@ -58,7 +58,8 @@ <h1><a class="toc-backref" href="#id2">Removal of obsolete modules</a></h1>
 <p>All modules marked as deprecated in 2.x versions should be removed for
 Python 3000.  The same applies to modules which are seen as obsolete today,
 but are too widely used to be deprecated or removed.  Python 3000 is the
-big occasion to get rid of them.</p>
+big occasion to get rid of them. <img alt='pep-3001-1.png' 
+src='pep-3001-1.png'/></p>
 <p>There will have to be a document listing all removed modules, together
 with information on possible substitutes or alternatives.  This infor-
 mation will also have to be provided by the python3warn.py porting

peps/tests/test_commands.py

@@ -1,11 +1,14 @@
 import os
 
+from bs4 import BeautifulSoup
+
 from django.test import TestCase
 from django.conf import settings
 from django.core.management import call_command
 from django.core.exceptions import ImproperlyConfigured
 from django.test.utils import override_settings
 
+from pages.models import Image
 
 FAKE_PEP_REPO = os.path.join(settings.BASE, 'peps/tests/fake_pep_repo/')
 
@@ -21,3 +24,10 @@ def test_generate_pep_pages(self):
     @override_settings(PEP_REPO_PATH=FAKE_PEP_REPO)
     def test_generate_pep_pages_real(self):
         call_command('generate_pep_pages')
+
+    @override_settings(PEP_REPO_PATH=FAKE_PEP_REPO)
+    def test_image_generated(self):
+        call_command('generate_pep_pages')
+        img = Image.objects.get(page__path='dev/peps/pep-3001/')
+        soup = BeautifulSoup(img.page.content.raw)
+        self.assertTrue(settings.MEDIA_URL in soup.find('img')['src'])