Merge pull request allinurl#1874 from Titaniumtown/master

Add even more crawlers
simhaonline · Aug 12, 2020 · b5c6ddb · b5c6ddb
2 parents a5507eb + 7e70929
commit b5c6ddb
Show file tree

Hide file tree

Showing 2 changed files with 106 additions and 82 deletions.
diff --git a/config/browsers.list b/config/browsers.list
@@ -15,92 +15,104 @@
 #
 # The items below are sample crawlers, adjust as needed.
 
-Chef Client									Crawlers
-Abonti											Crawlers
-SISTRIX											Crawlers
-DotBot											Crawlers
+Chef Client								Crawlers
+Abonti									Crawlers
+SISTRIX									Crawlers
+DotBot									Crawlers
 Speedy	Spider							Crawlers
-Sosospider									Crawlers
-BPImageWalker								Crawlers
-DoCoMo											Crawlers
-GSLFbot											Crawlers
-YodaoBot										Crawlers
-AddThis											Crawlers
-Purebot											Crawlers
-CCBot												Crawlers
-findlinks										Crawlers
-ichiro											Crawlers
-Linguee Bot									Crawlers
-Gigabot											Crawlers
+Sosospider								Crawlers
+BPImageWalker							Crawlers
+DoCoMo									Crawlers
+GSLFbot									Crawlers
+YodaoBot								Crawlers
+AddThis									Crawlers
+Purebot									Crawlers
+CCBot									Crawlers
+findlinks								Crawlers
+ichiro									Crawlers
+Linguee Bot								Crawlers
+Gigabot									Crawlers
 BacklinkCrawler							Crawlers
-distilator									Crawlers
-Aboundex										Crawlers
-UnwindFetchor								Crawlers
-SBIder											Crawlers
-TestNutch										Crawlers
-DomainCrawler								Crawlers
+distilator								Crawlers
+Aboundex								Crawlers
+UnwindFetchor							Crawlers
+SBIder									Crawlers
+TestNutch								Crawlers
+DomainCrawler							Crawlers
 NextGenSearchBot						Crawlers
 SEOENGWorldBot							Crawlers
-Cityreview									Crawlers
-PagePeeker									Crawlers
-JS-Kit											Crawlers
-ScreenerBot									Crawlers
-ShowyouBot									Crawlers
-SolomonoBot									Crawlers
-Domnutch										Crawlers
-MaxPoint										Crawlers
-NCBot												Crawlers
-TosCrawler									Crawlers
-Updownerbot									Crawlers
-OpenWebSpider								Crawlers
-WordPress										Crawlers
-PEAR												Crawlers
-ZumBot											Crawlers
-YisouSpider									Crawlers
-W3C													Crawlers
-vcheck											Crawlers
+Cityreview								Crawlers
+PagePeeker								Crawlers
+JS-Kit									Crawlers
+ScreenerBot								Crawlers
+ShowyouBot								Crawlers
+SolomonoBot								Crawlers
+Domnutch								Crawlers
+MaxPoint								Crawlers
+NCBot									Crawlers
+TosCrawler								Crawlers
+Updownerbot								Crawlers
+OpenWebSpider							Crawlers
+WordPress								Crawlers
+PEAR									Crawlers
+ZumBot									Crawlers
+YisouSpider								Crawlers
+W3C										Crawlers
+vcheck									Crawlers
 PercolateCrawler						Crawlers
-NING												Crawlers
-gvfs												Crawlers
-CatchBot										Crawlers
-Combine											Crawlers
-A6-Indexer									Crawlers
-Altresium										Crawlers
-Comodo											Crawlers
-crawler4j										Crawlers
-Cricket											Crawlers
-EC2LinkFinder								Crawlers
-envolk											Crawlers
-GeoHasher										Crawlers
-HTMLParser									Crawlers
-MLBot												Crawlers
-Jaxified										Crawlers
-LinkWalker									Crawlers
-nutch												Crawlers
-PostRank										Crawlers
+NING									Crawlers
+gvfs									Crawlers
+CatchBot								Crawlers
+Combine									Crawlers
+A6-Indexer								Crawlers
+Altresium								Crawlers
+Comodo									Crawlers
+crawler4j								Crawlers
+Cricket									Crawlers
+EC2LinkFinder							Crawlers
+envolk									Crawlers
+GeoHasher								Crawlers
+HTMLParser								Crawlers
+MLBot									Crawlers
+Jaxified								Crawlers
+LinkWalker								Crawlers
+nutch									Crawlers
+PostRank								Crawlers
 keybase-proofs							Crawlers
-CommonCrawler								Crawlers
-X-CAD-SE										Crawlers
-Safeassign									Crawlers
-Nmap												Crawlers
-sqlmap											Crawlers
-Jorgee											Crawlers
-PxBroker										Crawlers
-Seekport										Crawlers
-adscanner										Crawlers
-AfD-Verbotsverfahren_JETZT!	Crawlers
-DuckDuckGo-favicons-Bot									Crawlers
-bingbot											Crawlers
-Discordbot										Crawlers
+CommonCrawler							Crawlers
+X-CAD-SE								Crawlers
+Safeassign								Crawlers
+Nmap									Crawlers
+sqlmap									Crawlers
+Jorgee									Crawlers
+PxBroker								Crawlers
+Seekport								Crawlers
+adscanner								Crawlers
+AfD-Verbotsverfahren_JETZT!             Crawlers
+DuckDuckGo-favicons-Bot					Crawlers
+bingbot									Crawlers
+Discordbot								Crawlers
+ZoominfoBot                             Crawlers
+Googlebot                               Crawlers
+DotBot                                  Crawlers
+AhrefsBot                               Crawlers
+SemrushBot                              Crawlers
+Adsbot                                  Crawlers
+BLEXBot                                 Crawlers
+NetcraftSurveyAgent                     Crawlers
+Netcraft Web Server Survey              Crawlers
+masscan                                 Crawlers
+MJ12bot                                 Crawlers
+Pandalytics                             Crawlers
 
-Vienna											Feeds
-Windows-RSS-Platform				Feeds
-newsbeuter									Feeds
-Wrangler										Feeds
-Fever												Feeds
-Tiny												Feeds
-FreshRSS										Feeds
-KrISS												Feeds
-SimplePie										Feeds
-Feedsubs										Feeds
-UniversalFeedParser					Feeds
+Vienna									Feeds
+Windows-RSS-Platform				    Feeds
+newsbeuter								Feeds
+Wrangler								Feeds
+Fever									Feeds
+Tiny									Feeds
+FreshRSS								Feeds
+KrISS									Feeds
+SimplePie								Feeds
+Feedsubs								Feeds
+UniversalFeedParser					    Feeds
diff --git a/src/browsers.c b/src/browsers.c
@@ -190,6 +190,18 @@ static const char *browsers[][2] = {
   {"DuckDuckGo-Favicons-Bot", "Crawlers"},
   {"bingbot", "Crawlers"},
   {"Discordbot", "Crawlers"},
+  {"ZoominfoBot", "Crawlers"},
+  {"Googlebot", "Crawlers"},
+  {"DotBot", "Crawlers"},
+  {"AhrefsBot", "Crawlers"},
+  {"SemrushBot", "Crawlers"},
+  {"Adsbot", "Crawlers"},
+  {"BLEXBot", "Crawlers"},
+  {"NetcraftSurveyAgent", "Crawlers"},
+  {"Netcraft Web Server Survey", "Crawlers"},
+  {"masscan", "Crawlers"},
+  {"MJ12bot", "Crawlers"},
+  {"Pandalytics", "Crawlers"},
 
   /* Podcast fetchers */
   {"Downcast", "Podcasts"},