feat(core) implement internal dns with loadbalancing, replace dnsmasq (…

…Kong#1587)

Implements an internal DNS resolver, some new settings and a bugfix

 - uses same dns resolution in Kong proxy and Kong cli
 - bugfix: wildcarded hostnames are now properly validated
 - implements SRV record resolution
 - implements internal (weighted) roundrobin loadbalancing on the dns records
 - drops dnsmasq as a dependency
 - implements the `retries` setting (per api)
 - implements the `keepalive` setting (global)
 - now using the `balancer_by_lua` directives
 - revert back to standard `pgmoon` from the workaround `pgmoon-mashape` clone
 - exports ipv4, ipv6 and hostname verifying and normalization to the utils module

kong-0.9.4-0.rockspec

@@ -21,14 +21,15 @@ dependencies = {
   "version == 0.2",
   "lapis == 1.5.1",
   "lua-cassandra == dev-0",
-  "pgmoon-mashape == 2.0.1",
+  "pgmoon == 1.6.0",
   "luatz == 0.3",
   "lua_system_constants == 0.1.1",
   "lua-resty-iputils == 0.2.1",
   "luacrypto == 0.3.2",
   "luasyslog == 1.0.0",
   "lua_pack == 1.0.4",
-  "lua-resty-worker-events == 0.3.0"
+  "dns == 0.2.1",
+  "lua-resty-worker-events == 0.3.0",
 }
 build = {
   type = "builtin",
@@ -64,7 +65,6 @@ build = {
     ["kong.cmd.utils.serf_signals"] = "kong/cmd/utils/serf_signals.lua",
     ["kong.cmd.utils.nginx_signals"] = "kong/cmd/utils/nginx_signals.lua",
     ["kong.cmd.utils.prefix_handler"] = "kong/cmd/utils/prefix_handler.lua",
-    ["kong.cmd.utils.dnsmasq_signals"] = "kong/cmd/utils/dnsmasq_signals.lua",
 
     ["kong.api"] = "kong/api/init.lua",
     ["kong.api.api_helpers"] = "kong/api/api_helpers.lua",
@@ -76,6 +76,7 @@ build = {
     ["kong.api.routes.cache"] = "kong/api/routes/cache.lua",
     ["kong.api.routes.cluster"] = "kong/api/routes/cluster.lua",
 
+    ["kong.tools.dns"] = "kong/tools/dns.lua",
     ["kong.tools.utils"] = "kong/tools/utils.lua",
     ["kong.tools.printable"] = "kong/tools/printable.lua",
     ["kong.tools.responses"] = "kong/tools/responses.lua",
@@ -92,6 +93,7 @@ build = {
     ["kong.core.events"] = "kong/core/events.lua",
     ["kong.core.error_handlers"] = "kong/core/error_handlers.lua",
     ["kong.core.globalpatches"] = "kong/core/globalpatches.lua",
+    ["kong.core.balancer"] = "kong/core/balancer.lua",
 
     ["kong.dao.errors"] = "kong/dao/errors.lua",
     ["kong.dao.schemas_validation"] = "kong/dao/schemas_validation.lua",

kong.conf.default

@@ -109,6 +109,12 @@
                                  # to the SSL key for the `admin_listen_ssl`
                                  # address.
 
+#nginx_keepalive = 60            # Sets the maximum number of idle keepalive 
+                                 # connections to upstream servers that are 
+                                 # preserved in the cache of each worker 
+                                 # process. When this number is exceeded, the 
+                                 # least recently used connections are closed.
+
 #------------------------------------------------------------------------------
 # DATASTORE
 #------------------------------------------------------------------------------
@@ -262,18 +268,13 @@
 # DNS RESOLVER
 #------------------------------------------------------------------------------
 
-#dnsmasq = on                    # Toggles if Kong should start/stop dnsmasq,
-                                 # which can be used as the Nginx DNS resolver.
-                                 # Using dnsmasq allows Nginx to resolve
-                                 # domains defined in /etc/hosts.
-                                 # dnsmasq must be installed and available in
-                                 # your $PATH.
-
-#dnsmasq_port = 8053             # The port on which dnsmasq should listen to
-                                 # for queries.
+#dns_resolver =                  # Comma separated list of name servers, each
+                                 # entry in `ipv4[:port]` format to be used by
+                                 # Kong. If not specified the nameservers in
+                                 # the local `resolv.conf` file will be used.
+                                 # Port defaults to 53 if omitted.
 
-#dns_resolver = 8.8.8.8          # Configure a name server to be used by Nginx.
-                                 # Only valid when `dnsmasq` is disabled.
+#dns_hostsfile = /etc/hosts      # The `hosts` file to use.
 
 #------------------------------------------------------------------------------
 # DEVELOPMENT & MISCELLANEOUS

kong/cmd/compile.lua

@@ -24,7 +24,7 @@ Example usage:
  }
 
 Note:
- Third-party services such as Serf and dnsmasq need to be properly configured
+ Third-party services such as Serf need to be properly configured
  and started for Kong to be fully compatible while embedded.
 
 Options:

kong/cmd/health.lua

@@ -21,7 +21,6 @@ local function execute(args)
   local pids = {
     serf = conf.serf_pid,
     nginx = conf.nginx_pid,
-    dnsmasq = conf.dnsmasq and conf.dnsmasq_pid or nil
   }
 
   local count = 0

kong/cmd/quit.lua

@@ -1,4 +1,3 @@
-local dnsmasq_signals = require "kong.cmd.utils.dnsmasq_signals"
 local nginx_signals = require "kong.cmd.utils.nginx_signals"
 local serf_signals = require "kong.cmd.utils.serf_signals"
 local conf_loader = require "kong.conf_loader"
@@ -40,10 +39,6 @@ local function execute(args)
   local dao = assert(DAOFactory.new(conf))
   assert(serf_signals.stop(conf, dao))
 
-  if conf.dnsmasq then
-    assert(dnsmasq_signals.stop(conf))
-  end
-
   log("Kong stopped (gracefully)")
 end
 

kong/cmd/reload.lua

@@ -1,4 +1,3 @@
-local dnsmasq_signals = require "kong.cmd.utils.dnsmasq_signals"
 local prefix_handler = require "kong.cmd.utils.prefix_handler"
 local nginx_signals = require "kong.cmd.utils.nginx_signals"
 local serf_signals = require "kong.cmd.utils.serf_signals"
@@ -22,9 +21,6 @@ local function execute(args)
     prefix = args.prefix
   }))
   assert(prefix_handler.prepare_prefix(conf, args.nginx_conf))
-  if conf.dnsmasq then
-    assert(dnsmasq_signals.start(conf))
-  end
 
   local dao = assert(DAOFactory.new(conf))
   assert(serf_signals.start(conf, dao))

kong/cmd/restart.lua

@@ -16,7 +16,7 @@ end
 local lapp = [[
 Usage: kong restart [OPTIONS]
 
-Restart a Kong node (and other configured services like dnsmasq and Serf)
+Restart a Kong node (and other configured services like Serf)
 in the given prefix directory.
 
 This command is equivalent to doing both 'kong stop' and

kong/cmd/start.lua

@@ -1,4 +1,3 @@
-local dnsmasq_signals = require "kong.cmd.utils.dnsmasq_signals"
 local prefix_handler = require "kong.cmd.utils.prefix_handler"
 local nginx_signals = require "kong.cmd.utils.nginx_signals"
 local serf_signals = require "kong.cmd.utils.serf_signals"
@@ -20,24 +19,18 @@ local function execute(args)
   xpcall(function()
     assert(prefix_handler.prepare_prefix(conf, args.nginx_conf))
     assert(dao:run_migrations())
-    if conf.dnsmasq then
-      assert(dnsmasq_signals.start(conf))
-    end
     assert(serf_signals.start(conf, dao))
     assert(nginx_signals.start(conf))
     log("Kong started")
   end, function(e)
-    log.verbose("could not start Kong, stopping services")
-    pcall(nginx_signals.stop(conf))
-    pcall(serf_signals.stop(conf, dao))
-    if conf.dnsmasq then
-      pcall(dnsmasq_signals.stop(conf))
-    end
     err = e -- cannot throw from this function
-    log.verbose("stopped services")
   end)
 
   if err then
+    log.verbose("could not start Kong, stopping services")
+    pcall(nginx_signals.stop(conf))
+    pcall(serf_signals.stop(conf, dao))
+    log.verbose("stopped services")
     error(err) -- report to main error handler
   end
 end

kong/cmd/stop.lua

@@ -1,4 +1,3 @@
-local dnsmasq_signals = require "kong.cmd.utils.dnsmasq_signals"
 local nginx_signals = require "kong.cmd.utils.nginx_signals"
 local serf_signals = require "kong.cmd.utils.serf_signals"
 local conf_loader = require "kong.conf_loader"
@@ -21,9 +20,6 @@ local function execute(args)
   local dao = assert(DAOFactory.new(conf))
   assert(nginx_signals.stop(conf))
   assert(serf_signals.stop(conf, dao))
-  if conf.dnsmasq then
-    assert(dnsmasq_signals.stop(conf))
-  end
   log("Kong stopped")
 end
 

kong/cmd/utils/prefix_handler.lua

@@ -168,9 +168,6 @@ local function compile_conf(kong_config, conf_template)
     tostring = tostring
   }
 
-  if kong_config.dnsmasq then
-    compile_env["dns_resolver"] = "127.0.0.1:"..kong_config.dnsmasq_port
-  end
   if kong_config.anonymous_reports and socket.dns.toip(constants.SYSLOG.ADDRESS) then
     compile_env["syslog_reports"] = fmt("error_log syslog:server=%s:%d error;",
                                         constants.SYSLOG.ADDRESS, constants.SYSLOG.PORT)

kong/conf_loader.lua

@@ -7,16 +7,17 @@ local pl_config = require "pl.config"
 local pl_file = require "pl.file"
 local pl_path = require "pl.path"
 local tablex = require "pl.tablex"
+local utils = require "kong.tools.utils"
 local log = require "kong.cmd.utils.log"
 
+local ipv4_port_pattern = "^(%d+)%.(%d+)%.(%d+)%.(%d+):(%d+)$"
+
 local DEFAULT_PATHS = {
   "/etc/kong/kong.conf",
   "/etc/kong.conf"
 }
 
 local PREFIX_PATHS = {
-  dnsmasq_pid = {"pids", "dnsmasq.pid"}
-  ;
   serf_pid = {"pids", "serf.pid"},
   serf_log = {"logs", "serf.log"},
   serf_event = {"serf", "serf_event.sh"},
@@ -60,6 +61,7 @@ local CONF_INFERENCES = {
   cluster_listen_rpc = {typ = "string"},
   cluster_advertise = {typ = "string"},
   nginx_worker_processes = {typ = "string"},
+  nginx_keepalive = {typ = "number"},
 
   database = {enum = {"postgres", "cassandra"}},
   pg_port = {typ = "number"},
@@ -82,8 +84,7 @@ local CONF_INFERENCES = {
   cluster_profile = {enum = {"local", "lan", "wan"}},
   cluster_ttl_on_failure = {typ = "number"},
 
-  dnsmasq = {typ = "boolean"},
-  dnsmasq_port = {typ = "number"},
+  dns_resolver = {typ = "array"},
 
   ssl = {typ = "boolean"},
   admin_ssl = {typ = "boolean"},
@@ -206,13 +207,16 @@ local function check_and_infer(conf)
     end
   end
 
-  if conf.dns_resolver and conf.dnsmasq then
-    errors[#errors+1] = "must disable dnsmasq when a custom DNS resolver is specified"
-  elseif not conf.dns_resolver and not conf.dnsmasq then
-    errors[#errors+1] = "must specify a custom DNS resolver when dnsmasq is turned off"
+  if conf.dns_resolver then
+    for _, server in ipairs(conf.dns_resolver) do
+      local dns = utils.normalize_ip(server)
+      if (not dns) or (dns.type ~= "ipv4") then
+        errors[#errors+1] = "dns_resolver must be a comma separated list in the form of IPv4 or IPv4:port"
+        break -- one error is enough
+      end
+    end
   end
 
-  local ipv4_port_pattern = "^(%d+)%.(%d+)%.(%d+)%.(%d+):(%d+)$"
   if not conf.cluster_listen:match(ipv4_port_pattern) then
     errors[#errors+1] = "cluster_listen must be in the form of IPv4:port"
   end
@@ -409,6 +413,10 @@ local function load(path, custom_conf)
 
   log.verbose("prefix in use: %s", conf.prefix)
 
+  -- initialize the dns client, so the globally patched tcp.connect method
+  -- will work from here onwards.
+  assert(require("kong.tools.dns")(conf))
+
   return setmetatable(conf, nil) -- remove Map mt
 end