Merge pull request #1 from makeros/feat/custom-field-schema

Feat/custom field schema

.circleci/config.yml

@@ -34,4 +34,4 @@ jobs:
           key: v1-dependencies-{{ checksum "package.json" }}
 
       # run tests!
-      - run: npm test
+      - run: npm test

.gitignore

@@ -35,3 +35,5 @@ jspm_packages
 
 # Optional REPL history
 .node_repl_history
+
+.vscode/

README.md

@@ -45,8 +45,8 @@ node your-app.js | pino-gelf log -e
 // Enable single custom field logging
 node your-app.js | pino-gelf log -c environment
 
-// Enable multiple custom field logging
-node your-app.js | pino-gelf log -c environment,colour
+// Enable multiple custom field logging using custom schema json file
+node your-app.js | pino-gelf log -c my_schema.json
 
 // Enable local output
 node your-app.js | pino-gelf log -v
@@ -66,6 +66,7 @@ Verbose Logging|`false`
 
 ## Example
 
+### Custom fields 
 Given the Pino log message (formatted as JSON for readability):
 ```
 {
@@ -75,38 +76,43 @@ Given the Pino log message (formatted as JSON for readability):
   "level":30,
   "time":1481840140708,
   "msg":"request completed",
-  "res":{
-    "statusCode":304,
-    "header":"HTTP/1.1 304 Not Modified\r\nX-Powered-By: Express\r\nETag: W/\"d-bNNVbesNpUvKBgtMOUeYOQ\"\r\nDate: Thu, 15 Dec 2016 22:15:40 GMT\r\nConnection: keep-alive\r\n\r\n"
-  },
+  "customField":"test",
+  "res":{"statusCode":304},
   "responseTime":8,
-  "environment":"dev",
-  "colour":"black",
   "req":{
-    "id":1,
     "method":"GET",
-    "url":"/",
     "headers":{
       "host":"localhost:3000",
-      "connection":"keep-alive",
-      "if-none-match":"W/\"d-bNNVbesNpUvKBgtMOUeYOQ\"",
-      "upgrade-insecure-requests":"1",
-      "accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-      "user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14",
-      "accept-language":"en-gb",
-      "dnt":"1",
-      "accept-encoding":"gzip, deflate"
+      "user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14"}
     },
-    "remoteAddress":"::1",
-    "remotePort":52829
-  },
   "v":1
 }
 ```
 
+Given custom schema json file (my_custom_schema.json):
+```
+{
+  "title": "GELF Schema",
+  "type": "object",
+  "properties": {
+    "_status_code": {
+      "type": "integer",
+      "source": "res.statusCode"
+    },
+    "_user_agent": {
+      "type": "string",
+      "source": "req.headers.user-agent" 
+    },
+    "customField": {
+      "type": "string"
+    }
+  }
+}
+
+```
 And the usage:
 ```
-node server.js | pino-gelf log -e -c environment,colour
+node server.js | pino-gelf log -e -c my_custom_schema.json
 ```
 
 Pino GELF will send the following message to your Graylog server (formatted here as JSON for readability):
@@ -116,24 +122,12 @@ Pino GELF will send the following message to your Graylog server (formatted here
   "host":"han",
   "short_message":"request completed",
   "full_message":"request completed",
-  "timestamp":1481840588.672,
+  "timestamp":1481840140.708,
   "level":6,
   "facility":"pino-gelf-test-app",
-  "req":{
-    "id":1,
-    "method":"GET",
-    "url":"/",
-    "headers":"{\"host\":\"localhost:3000\",\"accept-encoding\":\"gzip, deflate\",\"connection\":\"keep-alive\",\"upgrade-insecure-requests\":\"1\",\"accept\":\"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14\",\"accept-language\":\"en-gb\",\"dnt\":\"1\",\"cache-control\":\"max-age=0\"}",
-    "remoteAddress":"::1",
-    "remotePort":52843
-  },
-  "res":{
-    "statusCode":200,
-    "header":"\"HTTP/1.1 200 OK\r\nX-Powered-By: Express\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: 13\r\nETag: W/\\"d-bNNVbesNpUvKBgtMOUeYOQ\\"\r\nDate: Thu, 15 Dec 2016 22:23:08 GMT\r\nConnection: keep-alive\r\n\r\n\""
-  },
-  "responseTime":9,
-  "colour":"black",
-  "environment":"dev"
+  "_status_code":304,
+  "_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14",
+  "customField":"test"
 }
 ```
 

__mocks__/custom-schema.json

@@ -0,0 +1,13 @@
+{
+  "title": "GELF Schema",
+  "type": "object",
+  "properties": {
+    "test1": {
+      "type": "integer",
+      "source": "nested.mock"
+    },
+    "test2": {
+      "type": "string"
+    }
+  }
+}

__mocks__/dgram.js

@@ -1,12 +1,12 @@
 const dgram = jest.genMockFromModule('dgram'); // eslint-disable-line
 
-function createSocket() {
+function createSocket () {
   return {
     send: () => {},
     close: () => {}
-  };
+  }
 }
 
-dgram.createSocket = createSocket;
+dgram.createSocket = createSocket
 
-module.exports = dgram;
+module.exports = dgram

__mocks__/readme-custom-schema-example.json

@@ -0,0 +1,17 @@
+{
+  "title": "GELF Schema",
+  "type": "object",
+  "properties": {
+    "_status_code": {
+      "type": "integer",
+      "source": "res.statusCode"
+    },
+    "_user_agent": {
+      "type": "string",
+      "source": "req.headers.user-agent" 
+    },
+    "customField": {
+      "type": "string"
+    }
+  }
+}

index.js

@@ -1,19 +1,12 @@
 #! /usr/bin/env node
 
-'use strict';
-
-const program = require('commander');
-const version = require('./package.json').version;
-const pinoGelf = require('./lib/pino-gelf');
-
-function parseCustomFields (val) {
-  if (val === undefined) return [];
-  if (val.includes(',')) return val.split(',');
-  return [ val ];
-}
+const program = require('commander')
+const path = require('path')
+const version = require('./package.json').version
+const pinoGelf = require('./lib/pino-gelf')
 
 program
-  .version(version);
+  .version(version)
 
 program
   .command('log')
@@ -22,20 +15,19 @@ program
   .option('-p, --port [port]', 'Graylog Port', parseInt)
   .option('-m, --max-chunk-size [maxChunkSize]', 'Graylog Input Maximum Chunk Size', parseInt)
   .option('-e, --use-express-middleware-preset')
-  .option('-c, --specify-custom-fields [keyList]', 'Comma Separated List of Custom Keys', parseCustomFields)
+  .option('-c, --specify-custom-schema [json file]', 'A JSON file with schema')
   .option('-v, --verbose', 'Output GELF to console')
   .action(function () {
     const opts = {
-      customKeys: this.specifyCustomFields || [],
+      customSchema: this.specifyCustomSchema ? require(path.resolve(this.specifyCustomSchema)) : false,
       host: this.host || '127.0.0.1',
       maxChunkSize: this.maxChunkSize || 1420,
       port: this.port || 12201,
       useExpressMiddlewarePreset: this.useExpressMiddlewarePreset || false,
       verbose: this.verbose || false
-    };
-
-    pinoGelf(opts);
-  });
+    }
+    pinoGelf(opts)
+  })
 
 program
-  .parse(process.argv);
+  .parse(process.argv)

lib/pino-gelf.js

@@ -1,12 +1,12 @@
-'use strict';
+'use strict'
 
-const fastJsonParse = require('fast-json-parse');
-const pipeline = require('./pipeline');
-const pump = require('pump');
-const split = require('split2');
+const fastJsonParse = require('fast-json-parse')
+const pipeline = require('./pipeline')
+const pump = require('pump')
+const split = require('split2')
 
 module.exports = function (opts) {
-  const pipe = pipeline(opts);
-  pump(process.stdin, split(fastJsonParse), pipe);
-  process.on('SIGINT', function () { process.exit(0); });
-};
+  const pipe = pipeline(opts)
+  pump(process.stdin, split(fastJsonParse), pipe)
+  process.on('SIGINT', function () { process.exit(0) })
+}

lib/pipeline.js

@@ -1,26 +1,23 @@
-'use strict';
-
-const through2 = require('through2');
-const transformer = require('./transformer');
-const Transport = require('./transport');
-const utils = require('./utils');
+const through2 = require('through2')
+const transformer = require('./transformer')
+const Transport = require('./transport')
+const utils = require('./utils')
 
 module.exports = function (opts) {
   return through2.obj(function (data, enc, cb) {
     if (data.value) {
-      const transform = transformer(opts);
-      const message = transform(data.value);
-
+      const transform = transformer(opts)
+      const message = transform(data.value)
       if (opts.verbose) {
-        const stringify = utils.stringify(opts);
-        const messageString = stringify(message);
-        setImmediate(function () { process.stdout.write(messageString); });
+        const stringify = utils.stringify(opts)
+        const messageString = stringify(message)
+        setImmediate(function () { process.stdout.write(messageString + '\n') })
       }
 
-      const transport = new Transport(opts);
-      transport.emit('log', message);
+      const transport = new Transport(opts)
+      transport.emit('log', message)
     }
 
-    cb();
-  });
-};
+    cb()
+  })
+}

lib/transformer/custom-gelf.js

@@ -1,7 +1,11 @@
-'use strict';
+const _ = require('lodash')
 
-const _ = require('lodash');
-
-module.exports = function (data, customKeys) {
-  return _.pick(data, customKeys);
-};
+module.exports = function (data, customSchema) {
+  // console.log(customSchema)
+  return Object.entries(customSchema.properties)
+    .reduce((acc, entry) => {
+      const sourcePath = entry[1].source ? entry[1].source : entry[0]
+      acc[entry[0]] = _.get(data, sourcePath, undefined)
+      return acc
+    }, {})
+}

lib/transformer/express-gelf.js

@@ -1,26 +1,26 @@
-'use strict';
+'use strict'
 
-const _ = require('lodash');
+const _ = require('lodash')
 
-const standardKeys = ['pid', 'hostname', 'name', 'level', 'time', 'msg', 'v'];
+const standardKeys = ['pid', 'hostname', 'name', 'level', 'time', 'msg', 'v']
 
 function filterStandardKeys (data) {
   return Object
     .keys(data)
-    .filter(function (key) { return !standardKeys.includes(key); });
+    .filter(function (key) { return !standardKeys.includes(key) })
 }
 
 module.exports = function (data) {
-  const expressKeys = filterStandardKeys(data);
-  const expressGelf = _.pick(data, expressKeys);
+  const expressKeys = filterStandardKeys(data)
+  const expressGelf = _.pick(data, expressKeys)
 
-  if(_.has(expressGelf, 'req') && _.has(expressGelf.req, 'headers')) {
-    expressGelf.req.headers = JSON.stringify(expressGelf.req.headers);
+  if (_.has(expressGelf, 'req') && _.has(expressGelf.req, 'headers')) {
+    expressGelf.req.headers = JSON.stringify(expressGelf.req.headers)
   }
 
-  if(_.has(expressGelf, 'res') && _.has(expressGelf.res, 'header')) {
-    expressGelf.res.header = JSON.stringify(expressGelf.res.header);
+  if (_.has(expressGelf, 'res') && _.has(expressGelf.res, 'header')) {
+    expressGelf.res.header = JSON.stringify(expressGelf.res.header)
   }
 
-  return expressGelf;
-};
+  return expressGelf
+}

lib/transformer/index.js

@@ -1,21 +1,29 @@
-'use strict';
+'use strict'
 
-const buildStandardGelf = require('./standard-gelf');
-const buildExpressMiddlewareGelf = require('./express-gelf');
-const buildCustomGelf = require('./custom-gelf');
+const buildStandardGelf = require('./standard-gelf')
+const buildExpressMiddlewareGelf = require('./express-gelf')
+const buildCustomGelf = require('./custom-gelf')
 
 module.exports = function (opts) {
   return function (data) {
-    const standardGelf = buildStandardGelf(data);
+    const standardGelf = buildStandardGelf(data)
 
     const expressGelf = opts.useExpressMiddlewarePreset
       ? buildExpressMiddlewareGelf(data)
-      : {};
+      : {}
 
-    const customGelf = opts.customKeys.length > 0
-      ? buildCustomGelf(data, opts.customKeys)
-      : {};
+    const customGelf = opts.customSchema
+      ? buildCustomGelf(data, opts.customSchema)
+      : {}
+      // return Object.assign({}, standardGelf, expressGelf, customGelf);
+      // console.log(standardGelf, expressGelf, customGelf)
+    const result = {
+      ...standardGelf,
+      ...expressGelf,
+      ...customGelf
+    }
+    // console.log(result)
 
-    return Object.assign({}, standardGelf, expressGelf, customGelf);
-  };
-};
+    return result
+  }
+}