Parse response code for start urls; handle shell script failure modes (…

…linode#1917)

ci/docs404.py

@@ -37,11 +37,15 @@ def __init__(self, *args, **kwargs):
              callback='parse_item', follow=True),
     )
 
+    def parse_start_url(self, response):
+        return self.parse_item(response)
+
     def parse_item(self, response):
         item = Docs404Item()
 
         if response.status == 404:
-            item['referer'] = response.request.headers.get('Referer')
+            ref = response.request.headers.get('Referer')
+            item['referer'] = ref if ref else 'orphaned link'
             item['status'] = response.status
             item['url'] = response.url
             return item

ci/scripts/docs404.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -euxo pipefail
 
 pip install -r ci/requirements.txt
-python ci/docs404.py && python ci/test_urls.py
+python ci/docs404.py