Add crawler detection page (duckduckgo#22)

* Add crawler detection page * Update page to match template * Add link to crawler test * Update breadcrumbs
upgradepixels · Feb 8, 2021 · 36c2b35 · 36c2b35
1 parent cdeb103
commit 36c2b35
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 0 deletions.
diff --git a/crawler/fail.json b/crawler/fail.json
diff --git a/crawler/index.html b/crawler/index.html
@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width">
+  <title>Crawler Detectability test</title>
+</head>  
+
+<body>
+<p><a href="../index.html">[Home]</a> ↣ <strong>[Crawler Detectability test]</strong></p>
+
+<p>This page tests against common ways of detecting headless browsers.</p>
+
+<p>A pass means that this looks like a normal Chrome browser, fail means it's Headless Chrome.</p>
+
+<table>
+    <tr>
+        <th>Test</th>
+        <th>Result</th>
+    </tr>
+    <tbody id="results"></tbody>
+</table>
+<template id="result-row">
+    <tr>
+        <td class="key"></td>
+        <td class="value"></td>
+    </tr>
+</template>
+<script>
+  (async () => {
+    const ul = document.getElementById('results');
+    const template = document.getElementById('result-row');
+
+    function displayResult(test, result, detectValue) {
+      const listEntry = template.content.cloneNode(true);
+      listEntry.querySelector('.key').textContent = test;
+      listEntry.querySelector('.value').textContent = result ? 'fail' : 'pass';
+      ul.appendChild(listEntry)
+      if (result) {
+        // this is used to allow the crawler to log test failures via its network log
+        fetch(`./fail.json?test=${test}&${detectValue}`);
+      }
+    }
+
+    displayResult('userAgent', /HeadlessChrome/.test(window.navigator.userAgent), window.navigator.userAgent);
+    displayResult('plugins', navigator.plugins.length == 0, navigator.plugins.length);
+    displayResult('languages', navigator.languages == "", navigator.languages);
+    displayResult('webdriver', !!navigator.webdriver, navigator.webdriver)
+    displayResult('window.chrome', !window.chrome || !window.chrome.runtime, JSON.stringify(window.chrome))
+
+    const permissionStatus = await navigator.permissions.query({ name: 'notifications' });
+    const isHeadless = Notification.permission === 'denied' && permissionStatus.state === 'prompt'
+    displayResult('Notification.permission', isHeadless, `${Notification.permission} - ${permissionStatus.state}`)
+  })()
+</script>
+</body>
+</html>
diff --git a/index.html b/index.html
@@ -49,5 +49,11 @@ <h2>Privacy Protections Tests</h2>
 	<li><a href='http://privacy-test-pages.glitch.me/privacy-protections/https-upgrades/'>HTTPS upgrades</a></li>
 </ul>
 
+<h2>Other</h2>
+
+<ul>
+	<li><a href='./crawler/'>Crawler Detectability test</a></li>
+</ul>
+
 </body>
 </html>