Add multi-page support to update.

JulienD · Mar 17, 2017 · a1bd02e · a1bd02e
1 parent 5ad4868
commit a1bd02e
Show file tree

Hide file tree

Showing 6 changed files with 96 additions and 13 deletions.
diff --git a/dscan/common/update_api.py b/dscan/common/update_api.py
@@ -19,6 +19,7 @@
 
 GH = 'https://github.com/'
 UW = './.update-workspace/'
+TAG_PAGES = 5
 
 def github_tags_newer(github_repo, versions_file, update_majors):
     """
@@ -36,16 +37,19 @@ def github_tags_newer(github_repo, versions_file, update_majors):
     vf = VersionsFile(versions_file)
     current_highest = vf.highest_version_major(update_majors)
 
-    tags_url = '%s%stags' % (GH, github_repo)
-    resp = requests.get(tags_url)
-    bs = BeautifulSoup(resp.text, 'lxml')
+    pages_done = 0
+    after = ""
+    while(pages_done < TAG_PAGES):
+        tags_url = '%s%stags?after=%s' % (GH, github_repo, after)
+        resp = requests.get(tags_url)
+        bs = BeautifulSoup(resp.text, 'lxml')
 
-    gh_versions = []
-    for tag in bs.find_all('span', {'class':'tag-name'}):
-        gh_versions.append(tag.text)
+        gh_versions = []
+        for tag in bs.find_all('span', {'class':'tag-name'}):
+            gh_versions.append(tag.text)
+            after = tag.text
 
-    print(gh_versions)
-    assert(False)
+        pages_done += 1
 
     newer = _newer_tags_get(current_highest, gh_versions)
 
@@ -107,7 +111,6 @@ def _newer_tags_get(current_highest, versions):
     for major in current_highest:
         highest_version = current_highest[major]
         for version in versions:
-            print(version, highest_version)
             if version.startswith(major) and version_gt(version,
                     highest_version):
                 newer.append(version)

diff --git a/dscan/tests/resources/github_response.2.html b/dscan/tests/resources/github_response.2.html
@@ -0,0 +1,16 @@
+<html>
+    <head>
+
+    </head>
+    <body>
+        <span class="tag-name">7.34</span>
+        <span class="tag-name">7.33</span>
+        <span class="tag-name">7.32</span>
+        <span class="tag-name">7.31</span>
+        <span class="tag-name">7.30</span>
+        <span class="tag-name">6.34</span>
+        <span class="tag-name">6.33</span>
+        <span class="tag-name">6.32</span>
+        <span class="tag-name">6.21</span>
+    </body>
+</html>
diff --git a/dscan/tests/resources/github_response.3.html b/dscan/tests/resources/github_response.3.html
@@ -0,0 +1,16 @@
+<html>
+    <head>
+
+    </head>
+    <body>
+        <span class="tag-name">7.34</span>
+        <span class="tag-name">7.33</span>
+        <span class="tag-name">7.32</span>
+        <span class="tag-name">7.31</span>
+        <span class="tag-name">7.30</span>
+        <span class="tag-name">6.34</span>
+        <span class="tag-name">6.33</span>
+        <span class="tag-name">6.32</span>
+        <span class="tag-name">6.11</span>
+    </body>
+</html>
diff --git a/dscan/tests/resources/github_response.4.html b/dscan/tests/resources/github_response.4.html
@@ -0,0 +1,16 @@
+<html>
+    <head>
+
+    </head>
+    <body>
+        <span class="tag-name">7.34</span>
+        <span class="tag-name">7.33</span>
+        <span class="tag-name">7.32</span>
+        <span class="tag-name">7.31</span>
+        <span class="tag-name">7.30</span>
+        <span class="tag-name">6.34</span>
+        <span class="tag-name">6.33</span>
+        <span class="tag-name">6.32</span>
+        <span class="tag-name">6.1</span>
+    </body>
+</html>
diff --git a/dscan/tests/resources/github_response.5.html b/dscan/tests/resources/github_response.5.html
@@ -0,0 +1,16 @@
+<html>
+    <head>
+
+    </head>
+    <body>
+        <span class="tag-name">7.34</span>
+        <span class="tag-name">7.33</span>
+        <span class="tag-name">7.32</span>
+        <span class="tag-name">7.31</span>
+        <span class="tag-name">7.30</span>
+        <span class="tag-name">6.34</span>
+        <span class="tag-name">6.33</span>
+        <span class="tag-name">6.32</span>
+        <span class="tag-name">5.9</span>
+    </body>
+</html>
diff --git a/dscan/tests/update_tests.py b/dscan/tests/update_tests.py
@@ -65,10 +65,17 @@ def tearDown(self):
 
     def gh_mock(self):
         # github_response.html has 7.34 & 6.34 as the latest tags.
-        gh_resp = open(dscan.PWD + 'tests/resources/github_response.html').read()
-        responses.add(responses.GET, 'https://github.com/drupal/drupal/tags', body=gh_resp)
-        responses.add(responses.GET, 'https://github.com/silverstripe/silverstripe-framework/releases')
-        pass
+        gh_resp_1 = open(dscan.PWD + 'tests/resources/github_response.html').read()
+        gh_resp_2 = open(dscan.PWD + 'tests/resources/github_response.2.html').read()
+        gh_resp_3 = open(dscan.PWD + 'tests/resources/github_response.3.html').read()
+        gh_resp_4 = open(dscan.PWD + 'tests/resources/github_response.4.html').read()
+        gh_resp_5 = open(dscan.PWD + 'tests/resources/github_response.5.html').read()
+
+        responses.add(responses.GET, 'https://github.com/drupal/drupal/tags?after=', body=gh_resp_1, match_querystring=True)
+        responses.add(responses.GET, 'https://github.com/drupal/drupal/tags?after=6.31', body=gh_resp_2, match_querystring=True)
+        responses.add(responses.GET, 'https://github.com/drupal/drupal/tags?after=6.21', body=gh_resp_3, match_querystring=True)
+        responses.add(responses.GET, 'https://github.com/drupal/drupal/tags?after=6.11', body=gh_resp_4, match_querystring=True)
+        responses.add(responses.GET, 'https://github.com/drupal/drupal/tags?after=6.1', body=gh_resp_5, match_querystring=True)
 
     def mock_update_all(self):
         self.updater.update_version = Mock(spec=self.updater.update_version)
@@ -106,6 +113,15 @@ def test_github_tag_newer(self):
             vf().highest_version_major.return_value = {'6': '6.33', '7': '7.34'}
             assert github_tags_newer('drupal/drupal/', 'not_a_real_file.xml', ['6', '7'])
 
+    def test_github_tag_newer_multi_page(self):
+        self.gh_mock()
+        with patch('dscan.common.update_api.VersionsFile') as vf:
+            vf().highest_version_major.return_value = {'6': '6.99', '7':
+                    '7.99', '5': '5.8'}
+            # should be true because of 5.9 in response mocks.
+            assert github_tags_newer('drupal/drupal/', 'not_a_real_file.xml', ['6', '7', '5'])
+
+
     def test_github_repo(self):
         with patch('dscan.common.update_api.GitRepo.__init__', return_value=None, autospec=True) as gri:
             with patch('dscan.common.update_api.GitRepo.init', autospec=True) as grii: