feat(pill): phase 2

.github/workflows/dir.yml

@@ -0,0 +1,18 @@
+name: dirfetcher
+on:
+  workflow_dispatch:
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: master
+      - uses: oNaiPs/secrets-to-env-action@v1
+        with:
+          secrets: ${{ toJSON(secrets) }}
+      - run: ls
+      - name: Run
+        run: |
+          npm install
+          node dirbg/server.js

.github/workflows/dir2.yml

@@ -0,0 +1,14 @@
+name: DIRARTICLE
+on:
+  workflow_dispatch:
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: master
+      - name: Run
+        run: |
+          npm install
+          node dirbg/perpage.mjs

.github/workflows/drik.yml

@@ -0,0 +1,14 @@
+name: DOREEN
+on:
+  workflow_dispatch:
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: master
+      - name: Run
+        run: |
+          npm install
+          node darik/server.js

.github/workflows/joking.yml

@@ -0,0 +1,18 @@
+name: JOKING
+on:
+  workflow_dispatch:
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: master
+      - uses: oNaiPs/secrets-to-env-action@v1
+        with:
+          secrets: ${{ toJSON(secrets) }}
+      - run: ls
+      - name: Run
+        run: |
+          npm install
+          node vicove/refactor.mjs

.gitignore

@@ -0,0 +1,3 @@
+node_modules
+/node_modules
+node_modules/*

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "git.ignoreLimitWarning": true
+}

Procfile

@@ -0,0 +1 @@
+web: node server.mjs

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 5.1.x   | :white_check_mark: |
+| 5.0.x   | :x:                |
+| 4.0.x   | :white_check_mark: |
+| < 4.0   | :x:                |
+
+## Reporting a Vulnerability
+
+Use this section to tell people how to report a vulnerability.
+
+Tell them where to go, how often they can expect to get an update on a
+reported vulnerability, what to expect if the vulnerability is accepted or
+declined, etc.

_darik.js

@@ -0,0 +1,144 @@
+const request = require('request-promise')
+
+const async = require('async')
+const sanitizeHtml = require('sanitize-html')
+const cheerio = require('cheerio')
+const levelup = require('levelup')
+const leveldown = require('leveldown')
+const { html2json } = require('html2json')
+const fetch = require('node-fetch')
+const { extend } = require('lodash')
+const md5 = require('md5')
+var db = levelup(leveldown('/tmp/dsadsad111543543'))
+
+async function doRequest (url) {
+  return new Promise((resolve, reject) => {
+    request(url, (error, res, body) => {
+      if (!error && res.statusCode == 200) {
+        resolve(body)
+      } else {
+        reject(error)
+      }
+    })
+  })
+}
+const insert = (json, callback) => {
+  fetch('https://db.rudixlab.com/v2/query', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      type: 'insert',
+      args: {
+        table: { name: 'News', schema: 'public' },
+        source: 'DB',
+        objects: [
+          {
+            ...json
+          }
+        ],
+        returning: ['id', 'uid']
+      }
+    })
+  })
+    .then(result => {
+      return result.json()
+    })
+    .then(data => {
+      console.log(data)
+      callback(data)
+    })
+}
+
+async function darik (params, callback) {
+  const htmlString = await doRequest('https://dariknews.bg/novini')
+  const $x = cheerio.load(htmlString)
+  let extended = []
+  const arrx = $x('.list-item h2 a')
+    .toArray()
+    .map(x => ({
+      title: $x(x).text(),
+      href: 'https:' + $x(x).attr('href'),
+      id: $x(x)
+        .attr('href')
+        .split('/')
+    }))
+
+  async.eachSeries(
+    arrx,
+    (i, cb) => {
+      fetch(i.href)
+        .then(res => res.text())
+        .then(body => {
+          const $ = cheerio.load(body)
+          const title = $('.article-title').text()
+          const media =
+            'https:' + $('.content .img-wrapper figure img').attr('src')
+          const tags = $('.content')
+            .text()
+            .replace(/[`~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/gi, ' ')
+            .replace(/[\r\n\t]+/g, ' ')
+            .split(' ')
+            .filter(x => x.length > 8)
+            .map(x => x.toLowerCase())
+          const clean = sanitizeHtml($('.content').html(), {
+            allowedTags: ['p', 'img'],
+            allowedAttributes: {
+              img: ['data-original', 'class', 'src']
+            }
+          }).replace(/data-original/g, 'src')
+
+          const react = html2json(clean).child.map((item, i) => {
+            return { id: i, ...item }
+          })
+
+          if (title.length > 20) {
+            extended.push({
+              title,
+              media,
+              md5: md5(title),
+              source: 'dariknews.bg',
+              react,
+              tags,
+              uid: md5(title)
+            })
+            cb()
+          } else {
+            cb()
+          }
+        })
+    },
+    err => {
+      async.eachSeries(
+        extended,
+        (i, cb) => {
+          db.get(i.uid, (e, data) => {
+            if (!data) {
+              db.put(i.uid, 'xxx', function () {
+                insert(i, () => {
+                  cb()
+                })
+              })
+            } else {
+              console.log('cached')
+
+              cb()
+            }
+          })
+        },
+        err => {
+          callback(extended)
+        }
+      )
+    }
+  )
+}
+
+if (!process.env.PORT) {
+  darik('', () => {})
+}
+module.exports = {
+  darik
+  //imgur,
+}

_dir.bg.js

@@ -0,0 +1,150 @@
+const request = require('request-promise')
+
+const async = require('async')
+const sanitizeHtml = require('sanitize-html')
+const cheerio = require('cheerio')
+const levelup = require('levelup')
+const leveldown = require('leveldown')
+const { html2json } = require('html2json')
+const fetch = require('node-fetch')
+const { extend } = require('lodash')
+const md5 = require('md5')
+var db = levelup(leveldown('/tmp/dsadsad111543541113'))
+
+async function doRequest (url) {
+  return new Promise((resolve, reject) => {
+    request(url, (error, res, body) => {
+      if (!error && res.statusCode == 200) {
+        resolve(body)
+      } else {
+        reject(error)
+      }
+    })
+  })
+}
+const insert = (json, callback) => {
+  fetch('https://db.rudixlab.com/v2/query', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      type: 'insert',
+      args: {
+        table: { name: 'News', schema: 'public' },
+        source: 'DB',
+        objects: [
+          {
+            ...json
+          }
+        ],
+        returning: ['id', 'uid']
+      }
+    })
+  })
+    .then(result => {
+      return result.json()
+    })
+    .then(data => {
+      console.log(data)
+      callback(data)
+    })
+}
+
+async function dir (params, callback) {
+  const htmlString = await doRequest('https://m.dir.bg')
+  const $x = cheerio.load(htmlString)
+  let extended = []
+  const arry = $x('.inner-wrapper .list-article a')
+    .toArray()
+    .map(x => ({
+      title: $x(x).text(),
+      href: $x(x).attr('href'),
+      uid: md5($x(x).attr('href')),
+      id: $x(x)
+        .attr('href')
+        .split('/')
+    }))
+
+  const arrx = arry.filter(
+    x =>
+      !x.href.includes('viber') &&
+      !x.href.includes('javascript') &&
+      x.href.length > 20
+  )
+
+  async.eachSeries(
+    arrx,
+    (i, cb) => {
+      fetch(i.href)
+        .then(res => res.text())
+        .then(body => {
+          const $ = cheerio.load(body)
+          const title = $('meta[property="og:title"]').attr('content')
+          const media = $('meta[property="og:image"]').attr('content')
+          const tags = $('.article-body')
+            .text()
+            .replace(/[`~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/gi, ' ')
+            .replace(/[\r\n\t]+/g, ' ')
+            .split(' ')
+            .filter(x => x.length > 8)
+            .map(x => x.toLowerCase())
+          const clean = sanitizeHtml($('.article-body').html(), {
+            allowedTags: ['p', 'img'],
+            allowedAttributes: {
+              img: ['data-original', 'class', 'src']
+            }
+          }).replace(/data-original/g, 'src')
+          const react = html2json(clean).child.map((item, i) => {
+            return { id: i, ...item }
+          })
+
+          if (title.length > 20) {
+            extended.push({
+              title,
+              media,
+              md5: md5(title),
+              uid: md5(title),
+              source: 'dir.bg',
+              tags,
+              react
+            })
+            cb()
+          } else {
+            cb()
+          }
+        })
+    },
+    err => {
+      async.each(
+        extended,
+        (i, cb) => {
+          db.get(i.uid, (e, data) => {
+            if (!data) {
+              db.put(i.uid, 'xxx', function () {
+                insert(i, () => {
+                  cb()
+                })
+              })
+            } else {
+              console.log('cached')
+
+              cb()
+            }
+          })
+        },
+        err => {
+          callback(extended)
+        }
+      )
+    }
+  )
+}
+
+if (!process.env.PORT) {
+  dir('', () => {})
+}
+module.exports = {
+  dir
+  //imgur,
+}