Skip to content

Commit

Permalink
fix: Bug in naming of known robots
Browse files Browse the repository at this point in the history
  • Loading branch information
@nielsbasjes
nielsbasjes committed Dec 22, 2024
1 parent 07f21d5 commit 2f58fa9
Show file tree
Hide file tree
Showing 5 changed files with 447 additions and 433 deletions.
22 changes: 11 additions & 11 deletions analyzer/src/main/resources/UserAgents/FederatedSocialMedia.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ config:
User-Agent : 'FediDB/0.5.0; +https://fedidb.org/crawler.html'
expected:
DeviceClass : 'Robot'
DeviceName : 'Fedidb'
DeviceName : 'Fedidb Crawler'
DeviceBrand : 'Fedidb'
OperatingSystemClass : 'Cloud'
OperatingSystemName : 'Cloud'
Expand All @@ -345,17 +345,17 @@ config:
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'Fedidb'
LayoutEngineVersion : '??'
LayoutEngineVersionMajor : '??'
LayoutEngineNameVersion : 'Fedidb ??'
LayoutEngineNameVersionMajor : 'Fedidb ??'
LayoutEngineName : 'Fedidb Crawler'
LayoutEngineVersion : '0.5.0'
LayoutEngineVersionMajor : '0'
LayoutEngineNameVersion : 'Fedidb Crawler 0.5.0'
LayoutEngineNameVersionMajor : 'Fedidb Crawler 0'
AgentClass : 'Robot'
AgentName : 'Fedidb'
AgentVersion : '??'
AgentVersionMajor : '??'
AgentNameVersion : 'Fedidb ??'
AgentNameVersionMajor : 'Fedidb ??'
AgentName : 'Fedidb Crawler'
AgentVersion : '0.5.0'
AgentVersionMajor : '0'
AgentNameVersion : 'Fedidb Crawler 0.5.0'
AgentNameVersionMajor : 'Fedidb Crawler 0'
AgentInformationUrl : 'https://fedidb.org/crawler.html'


Expand Down
4 changes: 2 additions & 2 deletions analyzer/src/main/resources/UserAgents/RobotBaseRules.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,10 @@ config:
- 'OperatingSystemVersion : 11099 :"<<<null>>>"'
- 'LayoutEngineClass : 11099 :"Robot"'
- 'LayoutEngineName : 11099 :@BrandName'
- 'LayoutEngineVersion : 11099 :"<<<null>>>"'
- 'LayoutEngineVersion : 2100 :"<<<null>>>"'
- 'AgentClass : 11099 :"Robot"'
- 'AgentName : 11099 :@BrandName'
- 'AgentVersion : 11099 :"<<<null>>>"'
- 'AgentVersion : 2100 :"<<<null>>>"'


- matcher:
Expand Down
97 changes: 48 additions & 49 deletions analyzer/src/main/resources/UserAgents/Robots-Libraries-Tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ config:
User-Agent : 'Jakarta Commons-HttpClient/3.1'
expected:
DeviceClass : 'Robot'
DeviceName : 'Apache Jakarta Commons HttpClient'
DeviceName : 'Apache Jakarta Commons-HttpClient'
DeviceBrand : 'Apache'
OperatingSystemClass : 'Cloud'
OperatingSystemName : 'Cloud'
Expand Down Expand Up @@ -121,17 +121,17 @@ config:
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'AHC'
LayoutEngineName : 'AsyncHttpClient'
LayoutEngineVersion : '2.0'
LayoutEngineVersionMajor : '2'
LayoutEngineNameVersion : 'AHC 2.0'
LayoutEngineNameVersionMajor : 'AHC 2'
LayoutEngineNameVersion : 'AsyncHttpClient 2.0'
LayoutEngineNameVersionMajor : 'AsyncHttpClient 2'
AgentClass : 'Robot'
AgentName : 'AHC'
AgentName : 'AsyncHttpClient'
AgentVersion : '2.0'
AgentVersionMajor : '2'
AgentNameVersion : 'AHC 2.0'
AgentNameVersionMajor : 'AHC 2'
AgentNameVersion : 'AsyncHttpClient 2.0'
AgentNameVersionMajor : 'AsyncHttpClient 2'


# Looks like the new JDK HTTP client
Expand All @@ -141,7 +141,7 @@ config:
User-Agent : 'Java-http-client/17'
expected:
DeviceClass : 'Robot'
DeviceName : 'Oracle Java'
DeviceName : 'Oracle Java-Http-Client'
DeviceBrand : 'Oracle'
OperatingSystemClass : 'Cloud'
OperatingSystemName : 'Cloud'
Expand Down Expand Up @@ -323,17 +323,17 @@ config:
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'PostmanRuntime'
LayoutEngineName : 'Postman Runtime'
LayoutEngineVersion : '7.28.2'
LayoutEngineVersionMajor : '7'
LayoutEngineNameVersion : 'PostmanRuntime 7.28.2'
LayoutEngineNameVersionMajor : 'PostmanRuntime 7'
LayoutEngineNameVersion : 'Postman Runtime 7.28.2'
LayoutEngineNameVersionMajor : 'Postman Runtime 7'
AgentClass : 'Robot'
AgentName : 'PostmanRuntime'
AgentName : 'Postman Runtime'
AgentVersion : '7.28.2'
AgentVersionMajor : '7'
AgentNameVersion : 'PostmanRuntime 7.28.2'
AgentNameVersionMajor : 'PostmanRuntime 7'
AgentNameVersion : 'Postman Runtime 7.28.2'
AgentNameVersionMajor : 'Postman Runtime 7'


# Looks like this tool from Datadog
Expand Down Expand Up @@ -381,18 +381,17 @@ config:
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'Wink Client'
LayoutEngineName : 'Wink'
LayoutEngineVersion : 'v1.1.2'
LayoutEngineVersionMajor : 'v1'
LayoutEngineNameVersion : 'Wink Client v1.1.2'
LayoutEngineNameVersionMajor : 'Wink Client v1'
LayoutEngineNameVersion : 'Wink v1.1.2'
LayoutEngineNameVersionMajor : 'Wink v1'
AgentClass : 'Robot'
AgentName : 'Wink Client'
AgentName : 'Wink'
AgentVersion : 'v1.1.2'
AgentVersionMajor : 'v1'
AgentNameVersion : 'Wink Client v1.1.2'
AgentNameVersionMajor : 'Wink Client v1'

AgentNameVersion : 'Wink v1.1.2'
AgentNameVersionMajor : 'Wink v1'

# Promise based HTTP client for the browser and node.js
# https://github.com/axios/axios
Expand Down Expand Up @@ -468,17 +467,17 @@ config:
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'got'
LayoutEngineName : 'Got'
LayoutEngineVersion : '??'
LayoutEngineVersionMajor : '??'
LayoutEngineNameVersion : 'got ??'
LayoutEngineNameVersionMajor : 'got ??'
LayoutEngineNameVersion : 'Got ??'
LayoutEngineNameVersionMajor : 'Got ??'
AgentClass : 'Robot'
AgentName : 'GOT'
AgentName : 'Got'
AgentVersion : '??'
AgentVersionMajor : '??'
AgentNameVersion : 'GOT ??'
AgentNameVersionMajor : 'GOT ??'
AgentNameVersion : 'Got ??'
AgentNameVersionMajor : 'Got ??'
AgentInformationUrl : 'https://github.com/sindresorhus/got'


Expand All @@ -489,26 +488,26 @@ config:
User-Agent : 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'
expected:
DeviceClass : 'Robot'
DeviceName : 'Node-Fetch Node Fetch'
DeviceBrand : 'Node-Fetch'
DeviceName : 'Node Fetch'
DeviceBrand : 'Node Fetch'
OperatingSystemClass : 'Cloud'
OperatingSystemName : 'Cloud'
OperatingSystemVersion : '??'
OperatingSystemVersionMajor : '??'
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'node-fetch'
LayoutEngineName : 'Node Fetch'
LayoutEngineVersion : '1.0'
LayoutEngineVersionMajor : '1'
LayoutEngineNameVersion : 'node-fetch 1.0'
LayoutEngineNameVersionMajor : 'node-fetch 1'
LayoutEngineNameVersion : 'Node Fetch 1.0'
LayoutEngineNameVersionMajor : 'Node Fetch 1'
AgentClass : 'Robot'
AgentName : 'Node-Fetch'
AgentName : 'Node Fetch'
AgentVersion : '1.0'
AgentVersionMajor : '1'
AgentNameVersion : 'Node-Fetch 1.0'
AgentNameVersionMajor : 'Node-Fetch 1'
AgentNameVersion : 'Node Fetch 1.0'
AgentNameVersionMajor : 'Node Fetch 1'
AgentInformationUrl : 'https://github.com/bitinn/node-fetch'


Expand All @@ -528,17 +527,17 @@ config:
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'Go-http-client'
LayoutEngineName : 'Go http client'
LayoutEngineVersion : '1.1'
LayoutEngineVersionMajor : '1'
LayoutEngineNameVersion : 'Go-http-client 1.1'
LayoutEngineNameVersionMajor : 'Go-http-client 1'
LayoutEngineNameVersion : 'Go http client 1.1'
LayoutEngineNameVersionMajor : 'Go http client 1'
AgentClass : 'Robot'
AgentName : 'Go-http-client'
AgentName : 'Go http client'
AgentVersion : '1.1'
AgentVersionMajor : '1'
AgentNameVersion : 'Go-http-client 1.1'
AgentNameVersionMajor : 'Go-http-client 1'
AgentNameVersion : 'Go http client 1.1'
AgentNameVersionMajor : 'Go http client 1'


# .NET library
Expand All @@ -548,8 +547,8 @@ config:
User-Agent : 'ServiceStack .NET Client 5.90'
expected:
DeviceClass : 'Robot'
DeviceName : 'Servicestack .net Client'
DeviceBrand : 'Servicestack'
DeviceName : 'ServiceStack .net Client'
DeviceBrand : 'ServiceStack'
OperatingSystemClass : 'Cloud'
OperatingSystemName : 'Cloud'
OperatingSystemVersion : '??'
Expand Down Expand Up @@ -586,17 +585,17 @@ config:
OperatingSystemNameVersion : 'Cloud ??'
OperatingSystemNameVersionMajor : 'Cloud ??'
LayoutEngineClass : 'Robot'
LayoutEngineName : 'Xb2.NET'
LayoutEngineName : 'Xb2 Xbase++ Client'
LayoutEngineVersion : '4.0.00.1354'
LayoutEngineVersionMajor : '4'
LayoutEngineNameVersion : 'Xb2.NET 4.0.00.1354'
LayoutEngineNameVersionMajor : 'Xb2.NET 4'
LayoutEngineNameVersion : 'Xb2 Xbase++ Client 4.0.00.1354'
LayoutEngineNameVersionMajor : 'Xb2 Xbase++ Client 4'
AgentClass : 'Robot'
AgentName : 'Xb2.NET'
AgentName : 'Xb2 Xbase++ Client'
AgentVersion : '4.0.00.1354'
AgentVersionMajor : '4'
AgentNameVersion : 'Xb2.NET 4.0.00.1354'
AgentNameVersionMajor : 'Xb2.NET 4'
AgentNameVersion : 'Xb2 Xbase++ Client 4.0.00.1354'
AgentNameVersionMajor : 'Xb2 Xbase++ Client 4'


# FIXME: Actually this is most likely a Flutter app trying to retrieve data
Expand Down
Loading

0 comments on commit 2f58fa9

Please sign in to comment.