fix(dao) handle non-UTF8 values (Kong#1457)

kong/dao/dao.lua

@@ -13,10 +13,13 @@
 -- @module kong.dao
 
 local Object = require "kong.vendor.classic"
+local utils = require "kong.tools.utils"
 local Errors = require "kong.dao.errors"
 local schemas_validation = require "kong.dao.schemas_validation"
 local event_types = require("kong.core.events").TYPES
 
+local RANDOM_VALUE = utils.random_string()
+
 local function check_arg(arg, arg_n, exp_type)
   if type(arg) ~= exp_type then
     local info = debug.getinfo(2)
@@ -35,6 +38,14 @@ local function check_not_empty(tbl, arg_n)
   end
 end
 
+local function check_utf8(tbl, arg_n)
+  for k, v in pairs(tbl) do
+    if not utils.validate_utf8(v) then
+      tbl[k] = RANDOM_VALUE -- Force a random string
+    end
+  end
+end
+
 -- Publishes an event, if an event handler has been specified.
 -- Currently this propagates the events cluster-wide.
 -- @param[type=string] type The event type to publish
@@ -116,6 +127,7 @@ end
 -- @treturn table err If an error occured, a table describing the issue.
 function DAO:find(tbl)
   check_arg(tbl, 1, "table")
+  check_utf8(tbl, 1)
 
   local model = self.model_mt(tbl)
   if not model:has_primary_keys() then
@@ -138,7 +150,9 @@ end
 function DAO:find_all(tbl)
   if tbl ~= nil then
     check_arg(tbl, 1, "table")
+    check_utf8(tbl, 1)
     check_not_empty(tbl, 1)
+
     local ok, err = schemas_validation.is_schema_subset(tbl, self.schema)
     if not ok then
       return nil, Errors.schema(err)

kong/tools/utils.lua

@@ -297,4 +297,30 @@ function _M.load_module_if_exists(module_name)
   end
 end
 
+local find = string.find
+local tostring = tostring
+
+-- Numbers taken from table 3-7 in www.unicode.org/versions/Unicode6.2.0/UnicodeStandard-6.2.pdf
+-- find-based solution inspired by http://notebook.kulchenko.com/programming/fixing-malformed-utf8-in-lua
+function _M.validate_utf8(val)
+  local str = tostring(val)
+  local i, len = 1, #str
+  while i <= len do
+    if     i == find(str, "[%z\1-\127]", i) then i = i + 1
+    elseif i == find(str, "[\194-\223][\123-\191]", i) then i = i + 2
+    elseif i == find(str,        "\224[\160-\191][\128-\191]", i)
+        or i == find(str, "[\225-\236][\128-\191][\128-\191]", i)
+        or i == find(str,        "\237[\128-\159][\128-\191]", i)
+        or i == find(str, "[\238-\239][\128-\191][\128-\191]", i) then i = i + 3
+    elseif i == find(str,        "\240[\144-\191][\128-\191][\128-\191]", i)
+        or i == find(str, "[\241-\243][\128-\191][\128-\191][\128-\191]", i)
+        or i == find(str,        "\244[\128-\143][\128-\191][\128-\191]", i) then i = i + 4
+    else
+      return false, i
+    end
+  end
+
+  return true
+end
+
 return _M

spec/01-unit/04-utils_spec.lua

@@ -98,6 +98,12 @@ describe("Utils", function()
   end)
 
   describe("string", function()
+    it("checks valid UTF8 values", function()
+      assert.True(utils.validate_utf8("hello"))
+      assert.True(utils.validate_utf8(123))
+      assert.True(utils.validate_utf8(true))
+      assert.False(utils.validate_utf8(string.char(105, 213, 205, 149)))
+    end)
     describe("random_string()", function()
       it("should return a random string", function()
         local first = utils.random_string()

spec/02-integration/02-dao/03-crud_spec.lua

@@ -270,6 +270,13 @@ helpers.for_each_dao(function(kong_config)
         assert.falsy(err)
         assert.same({}, rows)
       end)
+      it("handles non-string values", function()
+        local rows, err = apis:find_all {
+          request_host = string.char(105, 213, 205, 149)
+        }
+        assert.falsy(err)
+        assert.same({}, rows)
+      end)
 
       describe("errors", function()
         it("handle invalid arg", function()

spec/03-plugins/01-basic-auth/03-access_spec.lua

@@ -37,6 +37,11 @@ describe("Plugin: basic-auth (access)", function()
       password = "kong",
       consumer_id = consumer.id
     })
+     assert(helpers.dao.basicauth_credentials:insert {
+      username = "user123",
+      password = "password123",
+      consumer_id = consumer.id
+    })
   end)
   teardown(function()
     if client then client:close() end
@@ -128,6 +133,29 @@ describe("Plugin: basic-auth (access)", function()
       })
       assert.res_status(200, res)
     end)
+    it("authenticates valid credentials in Authorization", function()
+      local res = assert(client:send {
+        method = "GET",
+        path = "/status/200",
+        headers = {
+          ["Authorization"] = "Basic dXNlcjEyMzpwYXNzd29yZDEyMw==",
+          ["Host"] = "basic-auth1.com"
+        }
+      })
+      assert.res_status(200, res)
+    end)
+    it("returns 403 for valid Base64 encoding", function()
+      local res = assert(client:send {
+        method = "GET",
+        path = "/status/200",
+        headers = {
+          ["Authorization"] = "Basic adXNlcjEyMzpwYXNzd29yZDEyMw==",
+          ["Host"] = "basic-auth1.com"
+        }
+      })
+      local body = assert.res_status(403, res)
+      assert.equal([[{"message":"Invalid authentication credentials"}]], body)
+    end)
     it("authenticates valid credentials in Proxy-Authorization", function()
       local res = assert(client:send {
         method = "GET",