test_parsing: Adapt to new structure of course pages.

This uses the observation from @aruljohn from issues coursera-dl#472 and coursera-dl#474 and and
potentially fixes issues coursera-dl#472, coursera-dl#473, coursera-dl#474.

edx_dl/parsing.py

@@ -239,7 +239,21 @@ def extract_courses_from_html(self, page, BASE_URL):
         Extracts courses (Course) from the html page
         """
         soup = BeautifulSoup(page)
-        courses_soup = soup.find_all('div', 'course')
+
+        # First, try with new course structure (as of December 2017).  If
+        # that doesn't work, we fallback to an older course structure
+        # (released with version 0.1.6). If even that doesn't work, then we
+        # try with the oldest course structure (that was current before
+        # version 0.1.6).
+        #
+        # rbrito---This code is ugly.
+
+        courses_soup = soup.find_all('article', 'course')
+        if len(courses_soup) == 0:
+            courses_soup = soup.find_all('div', 'course')
+        if len(courses_soup) == 0:
+            courses_soup = soup.find_all('div', 'course audit')
+
         courses = []
 
         for course_soup in courses_soup:

test_parsing.py

@@ -121,6 +121,7 @@ def test_extract_sections(file, num_sections_expected, num_subsections_expected)
 @pytest.mark.parametrize(
     'filename,site,num_courses_expected,num_available_courses_expected', [
         ('test/html/dashboard-version-with-articles.html', 'https://courses.edx.org', 18, 14),
+        ('test/html/dashboard-version-with-divs.html', 'https://courses.edx.org', 18, 14),
     ]
 )
 def test_extract_courses_from_html(filename, site, num_courses_expected, num_available_courses_expected):