Skip to content

Commit

Permalink
Add short DOM notation
Browse files Browse the repository at this point in the history
  • Loading branch information
@AliasIO
AliasIO committed May 5, 2021
1 parent 5270da9 commit 35090b3
Show file tree
Hide file tree
Showing 8 changed files with 248 additions and 559 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ Patterns (regular expressions) are kept in [`src/technologies.json`](https://git
},
"dom": {
"#example-id": {
"exists": "",
"attributes": {
"class": "example-class"
},
Expand Down Expand Up @@ -277,7 +278,7 @@ Plus any of:
</tr>
<tr>
<td><code>dom</code></td>
<td>Object</td>
<td>String | Array | Object</td>
<td>
Uses a
<a
Expand Down
25 changes: 19 additions & 6 deletions schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,25 @@
}
},
"dom": {
"type": "object",
"additionalProperties": false,
"patternProperties": {
"^.+$": {
}
}
"oneOf": [
{
"type": "array",
"items": {
"$ref": "#/definitions/non-empty-non-blank-string"
}
},
{
"$ref": "#/definitions/non-empty-non-blank-string"
},
{
"type": "object",
"additionalProperties": false,
"patternProperties": {
"^.+$": {
}
}
}
]
},
"dns": {
"type": "object",
Expand Down
112 changes: 68 additions & 44 deletions src/drivers/npm/driver.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,30 @@ function analyzeJs(js) {
function analyzeDom(dom) {
return Array.prototype.concat.apply(
[],
dom.map(({ name, selector, text, property, attribute, value }) => {
dom.map(({ name, selector, exists, text, property, attribute, value }) => {
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)

if (text) {
return analyzeManyToMany(technology, 'dom.text', { [selector]: [text] })
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}

if (property) {
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}

if (typeof property !== 'undefined') {
return analyzeManyToMany(technology, `dom.properties.${property}`, {
[selector]: [value],
})
}

if (attribute) {
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(technology, `dom.attributes.${attribute}`, {
[selector]: [value],
})
Expand Down Expand Up @@ -424,7 +432,7 @@ class Site {

await sleep(1000)

// page.on('console', (message) => this.log(message.text()))
page.on('console', (message) => this.log(message.text())) // TODO

// Links
const links = await this.promiseTimeout(
Expand Down Expand Up @@ -560,68 +568,84 @@ class Site {
: !!value

Object.keys(dom).forEach((selector) => {
const nodes = document.querySelectorAll(selector)
let nodes = []

try {
nodes = document.querySelectorAll(selector)
} catch (error) {
// Continue
}

if (!nodes.length) {
return
}

dom[selector].forEach(({ text, properties, attributes }) => {
nodes.forEach((node) => {
if (text) {
const value = node.textContent.trim()

if (value) {
dom[selector].forEach(
({ exists, text, properties, attributes }) => {
nodes.forEach((node) => {
if (exists) {
technologies.push({
name,
selector,
text: value,
exists: '',
})
}
}

if (properties) {
Object.keys(properties).forEach((property) => {
if (
Object.prototype.hasOwnProperty.call(node, property)
) {
const value = node[property]
if (text) {
const value = node.textContent.trim()

if (value) {
technologies.push({
name,
selector,
text: value,
})
}
}

if (properties) {
Object.keys(properties).forEach((property) => {
if (
Object.prototype.hasOwnProperty.call(node, property)
) {
const value = node[property]

if (typeof value !== 'undefined') {
technologies.push({
name,
selector,
property,
value: toScalar(value),
})
}
}
})
}

if (attributes) {
Object.keys(attributes).forEach((attribute) => {
if (node.hasAttribute(attribute)) {
const value = node.getAttribute(attribute)

if (typeof value !== 'undefined') {
technologies.push({
name,
selector,
property,
attribute,
value: toScalar(value),
})
}
}
})
}

if (attributes) {
Object.keys(attributes).forEach((attribute) => {
if (node.hasAttribute(attribute)) {
const value = node.getAttribute(attribute)

technologies.push({
name,
selector,
attribute,
value: toScalar(value),
})
}
})
}
})
})
})
}
})
}
)
})

return technologies
}, [])
},
Wappalyzer.technologies
.filter(({ dom }) => dom)
.filter(({ dom }) => dom && dom.constructor === Object)
.map(({ name, dom }) => ({ name, dom }))
)
).catch(() => [])
Expand Down
1 change: 1 addition & 0 deletions src/drivers/webextension/images/icons/GraphQL.svg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
27 changes: 23 additions & 4 deletions src/drivers/webextension/js/content.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,11 @@ const Content = {
},
(response) => {
chrome.runtime.lastError
? reject(new Error(chrome.runtime.lastError.message))
? reject(
new Error(
`${chrome.runtime.lastError}: Driver.${func}(${args})`
)
)
: resolve(response)
}
)
Expand Down Expand Up @@ -164,7 +168,7 @@ const Content = {

// DOM
const dom = technologies
.filter(({ dom }) => dom)
.filter(({ dom }) => dom && dom.constructor === Object)
.map(({ name, dom }) => ({ name, dom }))
.reduce((technologies, { name, dom }) => {
const toScalar = (value) =>
Expand All @@ -173,14 +177,29 @@ const Content = {
: !!value

Object.keys(dom).forEach((selector) => {
const nodes = document.querySelectorAll(selector)
let nodes = []

try {
nodes = document.querySelectorAll(selector)
} catch (error) {
// eslint-disable-next-line no-console
console.error(error)
}

if (!nodes.length) {
return
}

dom[selector].forEach(({ text, properties, attributes }) => {
dom[selector].forEach(({ exists, text, properties, attributes }) => {
nodes.forEach((node) => {
if (exists) {
technologies.push({
name,
selector,
exists: '',
})
}

if (text) {
const value = node.textContent.trim()

Expand Down
67 changes: 40 additions & 27 deletions src/drivers/webextension/js/driver.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,35 +186,50 @@ const Driver = {
url,
Array.prototype.concat.apply(
[],
dom.map(({ name, selector, text, property, attribute, value }) => {
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)
dom.map(
(
{ name, selector, exists, text, property, attribute, value },
index
) => {
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)

if (text) {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}

if (property) {
return analyzeManyToMany(technology, `dom.properties.${property}`, {
[selector]: [value],
})
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}

if (attribute) {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.properties.${property}`,
{
[selector]: [value],
}
)
}

return []
})
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}

return []
}
)
)
)
},
Expand Down Expand Up @@ -263,8 +278,6 @@ const Driver = {
* @param {Object} request
*/
async onWebRequestComplete(request) {
console.log('xxx', request.responseHeaders)

if (request.responseHeaders) {
if (await Driver.isDisabledDomain(request.url)) {
return
Expand Down
Loading

0 comments on commit 35090b3

Please sign in to comment.