fixes matomo-org#5802 - adds some bot detections

caohejie · Sep 15, 2018 · 9b2c853 · 9b2c853
1 parent 08414dc
commit 9b2c853
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 42 deletions.
diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml
@@ -751,14 +751,26 @@
   user_agent: COMODO SSL Checker
   bot:
     name: Generic Bot
+- 
+  user_agent: [email protected]
+  bot:
+    name: Generic Bot
 - 
   user_agent: 'HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: [email protected])'
   bot:
     name: Generic Bot
+- 
+  user_agent: Mozilla/5.0 (compatible; +http://tweetedtimes.com)
+  bot:
+    name: Generic Bot
 - 
   user_agent: Mozilla/5.0 (compatible; Shareaholicbot/2.0; +http://www.shareaholic.com/bot)
   bot:
     name: Generic Bot
+- 
+  user_agent: Mozilla/5.0 (compatible; TrendsmapResolver/0.1)
+  bot:
+    name: Generic Bot
 - 
   user_agent: niki-bot
   bot:
@@ -1687,6 +1699,15 @@
   user_agent: Octopus 1.0.2
   bot:
     name: Octopus
+- 
+  user_agent: omgili/0.5 +http://omgili.com
+  bot:
+    name: Omgili bot
+    category: Search bot
+    url: http://www.omgili.com/Crawler.html
+    producer:
+      name: Omgili
+      url: http://www.omgili.com
 - 
   user_agent: omgilibot/0.3 +http://www.omgili.com/Crawler.html
   bot:

diff --git a/Tests/fixtures/smartphone-3.yml b/Tests/fixtures/smartphone-3.yml
@@ -5869,46 +5869,6 @@
     model: "6.1"
   os_family: Android
   browser_family: Chrome
-- 
-  user_agent: Mozilla/5.0 (Linux; Android 8.1.0; Nokia 7 plus Build/OPR1.170623.026) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.109 Mobile Safari/537.36
-  os:
-    name: Android
-    short_name: AND
-    version: "8.1.0"
-    platform: ""
-  client:
-    type: browser
-    name: Chrome Mobile
-    short_name: CM
-    version: "65.0.3325.109"
-    engine: Blink
-    engine_version: ""
-  device:
-    type: smartphone
-    brand: NK
-    model: 7 plus
-  os_family: Android
-  browser_family: Chrome
-- 
-  user_agent: Mozilla/5.0 (Linux; Android 7.1.1; TA-1004 Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36
-  os:
-    name: Android
-    short_name: AND
-    version: "7.1.1"
-    platform: ""
-  client:
-    type: browser
-    name: Chrome Mobile
-    short_name: CM
-    version: "60.0.3112.116"
-    engine: Blink
-    engine_version: ""
-  device:
-    type: smartphone
-    brand: NK
-    model: 8
-  os_family: Android
-  browser_family: Chrome
 - 
   user_agent: Nokia210/2.0 (04.12) Profile/MIDP-2.1 Configuration/CLDC-1.1 UCWEB/2.0 (Java; U; MIDP-2.0; en-US; Nokia210) U2/1.0.0 UCBrowser/9.3.0.326 U2/1.0.0 Mobile
   os:
@@ -5969,6 +5929,46 @@
     model: 500
   os_family: Symbian
   browser_family: Nokia Browser
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 8.1.0; Nokia 7 plus Build/OPR1.170623.026) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.109 Mobile Safari/537.36
+  os:
+    name: Android
+    short_name: AND
+    version: "8.1.0"
+    platform: ""
+  client:
+    type: browser
+    name: Chrome Mobile
+    short_name: CM
+    version: "65.0.3325.109"
+    engine: Blink
+    engine_version: ""
+  device:
+    type: smartphone
+    brand: NK
+    model: 7 plus
+  os_family: Android
+  browser_family: Chrome
+- 
+  user_agent: Mozilla/5.0 (Linux; Android 7.1.1; TA-1004 Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Mobile Safari/537.36
+  os:
+    name: Android
+    short_name: AND
+    version: "7.1.1"
+    platform: ""
+  client:
+    type: browser
+    name: Chrome Mobile
+    short_name: CM
+    version: "60.0.3112.116"
+    engine: Blink
+    engine_version: ""
+  device:
+    type: smartphone
+    brand: NK
+    model: 8
+  os_family: Android
+  browser_family: Chrome
 - 
   user_agent: OneBrowser/3.1 (Nokia7230/06.90)
   os:

diff --git a/regexes/bots.yml b/regexes/bots.yml
@@ -847,7 +847,7 @@
 - regex: 'Octopus [0-9]'
   name: 'Octopus'
 
-- regex: 'omgilibot'
+- regex: 'omgili(?:bot)?'
   name: 'Omgili bot'
   category: 'Search bot'
   url: 'http://www.omgili.com/Crawler.html'
@@ -1691,7 +1691,7 @@
 - regex: 'Server Density Service Monitoring.*'
   name: 'Server Density'
 
-- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?! Build)|zeal|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|Minimo|RackspaceBot)'
+- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com||tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?! Build)|zeal|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|Minimo|RackspaceBot)'
   name: 'Generic Bot'
 
 - regex: '^sentry'