From dc538665e787f9fba52e57b738567f80b36a786a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 13 Mar 2023 10:57:07 -0400 Subject: [PATCH] fix github timestamp creation (#111) Signed-off-by: Alex Goodman --- src/vunnel/providers/github/parser.py | 12 +++++++++++- tests/unit/providers/github/test_github.py | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/vunnel/providers/github/parser.py b/src/vunnel/providers/github/parser.py index 1d3930c6..d27fc29c 100644 --- a/src/vunnel/providers/github/parser.py +++ b/src/vunnel/providers/github/parser.py @@ -155,8 +155,15 @@ def _parse(self, data): def get(self): # determine if a run was completed by looking for a timestamp metadata = self.db.get_metadata() + + # why rstrip(Z)? Previous code had been incorrectly adding a Z to the end of the timestamp manually + # instead of using the isoformat() method. This lead to a parsing problem with the github API (naturally). + # in the future this rstrip() should be removed when all cached data in CI is updated. self.timestamp = metadata.data.get("timestamp") - current_timestamp = f"{datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}Z" + if self.timestamp: + self.timestamp = self.timestamp.rstrip("Z") + + current_timestamp = datetime.datetime.now(tz=datetime.timezone.utc).isoformat() has_cursor = True # Process everything that was persisted first @@ -169,6 +176,9 @@ def get(self): # point in time, if a cursor is present, it will be used as well data = self._download() + if "errors" in data: + raise RuntimeError(f"Error downloading advisories: {data['errors']}") + page_info = data["data"]["securityAdvisories"]["pageInfo"] if page_info["hasNextPage"]: self.cursor = has_cursor = page_info.get("endCursor") diff --git a/tests/unit/providers/github/test_github.py b/tests/unit/providers/github/test_github.py index 626371ef..091d80fa 100644 --- a/tests/unit/providers/github/test_github.py +++ b/tests/unit/providers/github/test_github.py @@ -343,7 +343,7 @@ def test_get_commits_timestamp(self, fake_get_query, tmpdir, empty_response): metadata = database.get_metadata() timestamp = metadata.data["timestamp"] assert isinstance(timestamp, str) - assert timestamp.endswith("Z") + assert timestamp.endswith("Z") or timestamp.endswith("+00:00") def test_get_commits_timestamp_with_cursors(self, advisories, fake_get_query, tmpdir, empty_response): fake_get_query([empty_response, advisories(has_next_page=True)]) @@ -355,7 +355,7 @@ def test_get_commits_timestamp_with_cursors(self, advisories, fake_get_query, tm metadata = database.get_metadata() timestamp = metadata.data["timestamp"] assert isinstance(timestamp, str) - assert timestamp.endswith("Z") + assert timestamp.endswith("Z") or timestamp.endswith("+00:00") def test_has_next_page(self, advisories, fake_get_query, tmpdir, empty_response): fake_get_query([empty_response, advisories(has_next_page=True)])