docs: impl the initial support for Tarantool docs

- impl DocumentationManager module for docs managing (initialization, downloading, updating etc.)
- impl parser for Tarantool *.rst documentation
  Now supported:
      * module name
      * in-module functions: name + description (too dummy)
- impl doc injection into completion items

tarantool-lsp/doc-manager.lua

@@ -0,0 +1,134 @@
+local http = require('http.client')
+local fio = require('fio')
+local log = require('tarantool-lsp.log')
+
+local parser = require('tarantool-lsp.doc-parser')
+
+local DocumentationManager = {}
+
+local function downloadDocs()
+    local DWNLD_URL = "https://github.com/tarantool/doc/archive/1.10.zip"
+    local ARCHIVE_NAME = "doc.zip"
+    local response = http.get(DWNLD_URL)
+    if response.status ~= 200 then
+        return nil, "Can't download docs. HTTP code is " .. tostring(response.status)
+    end
+
+    local filepath = fio.pathjoin(fio.tempdir(), ARCHIVE_NAME)
+    local out = fio.open(filepath, { 'O_WRONLY', 'O_CREAT' }, tonumber('777', 8))
+    out:write(response.body)
+    out:close()
+
+    return filepath
+end
+
+local function extractArchive(filepath, dest_dir)
+    local status = os.execute("unzip")
+    if status ~= 0 then
+        return nil, "Unzip isn't installed on this system"
+    end
+
+    status = os.execute(string.format("unzip %s -d %d", filepath, dest_dir))
+    if status ~= 0 then
+        return nil, "Can't unzip file"
+    end
+
+    return true
+end
+
+-- Parse only box.* namespace now
+local function parseDocs(doc_path)
+    local work_dir = fio.pathjoin(doc_path, "doc-1.10", "doc", "1.10", "book", "box")
+    local docs = fio.glob(fio.pathjoin(work_dir, "*.rst"))
+
+    local terms = {}
+    for _, doc_file in ipairs(docs) do
+        local f = fio.open(doc_file, { 'O_RDONLY' })
+        local text = f:read()
+        local ok, trace = xpcall(parser.parseDocFile, debug.traceback, text, terms)
+
+        f:close()
+        if not ok then
+            log.error("Error parse %s file. Traceback: %s. Exit...", trace, doc_file)
+            os.exit(1)
+        end
+    end
+
+    return terms
+end
+
+function DocumentationManager:init()
+    -- TODO: Check current revision of docs
+    -- local filepath, err = downloadDocs()
+    -- if err ~= nil then
+    --     return nil, err
+    -- end
+
+    local filepath = "/Users/a.barsegyan/tarantool-lsp"
+
+    -- self.doc_dir = fio.dirname(filepath)
+    -- local _, err = extractArchive(filepath, self.doc_dir)
+    -- if err ~= nil then
+    --     return nil, err
+    -- end
+
+    self.terms = parseDocs(filepath)
+
+    local unsorted = {}
+    for k, _ in pairs(self.terms) do
+        table.insert(unsorted, k)
+    end
+    table.sort(unsorted)
+    self.termsSorted = unsorted
+
+    return true
+end
+
+function DocumentationManager:getCompletions(str)
+    local similar = false
+    local completions = {}
+
+    local function deep_level(str)
+        local lvl = 0
+        for delim in str:gmatch("[.:]") do
+            lvl = lvl + 1
+        end
+
+        return lvl
+    end
+
+    local str_deep_level = deep_level(str)
+
+    local function is_completion(term)
+        local is = term:match("^" .. str)
+        if is then
+            if str_deep_level < deep_level(term) then
+                return false
+            end
+
+            return true
+        end
+
+        return false
+    end
+
+    for _, term in ipairs(self.termsSorted) do
+        if is_completion(term) then
+            if not similar then
+                similar = true
+            end
+
+            table.insert(completions, term)
+        elseif similar then
+            break
+        end
+    end
+
+    return completions
+end
+
+function DocumentationManager:get(term)
+    return self.terms[term]
+end
+
+return DocumentationManager

tarantool-lsp/doc-parser.lua

@@ -0,0 +1,189 @@
+local log = require('tarantool-lsp.log')
+
+local function ltrim(s)
+  return (s:gsub("^%s*", ""))
+end
+
+local function rtrim(s)
+  local n = #s
+  while n > 0 and s:find("^%s", n) do n = n - 1 end
+  return s:sub(1, n)
+end
+
+local completionKinds = {
+    Text = 1,
+    Method = 2,
+    Function = 3,
+    Constructor = 4,
+    Field = 5,
+    Variable = 6,
+    Class = 7,
+    Interface = 8,
+    Module = 9,
+    Property = 10,
+    Unit = 11,
+    Value = 12,
+    Enum = 13,
+    Keyword = 14,
+    Snippet = 15,
+    Color = 16,
+    File = 17,
+    Reference = 18,
+}
+
+local function parseFunction(scope, moduleName)
+    local is, ie, funcName = scope:find("^([%w.:_]+)")
+    scope = scope:match("%([^\n]*%)\n\n(.*)", ie)
+
+    local termDescription = scope:match("^(.*)\n%s*\n%s*%:%w+[%s%w%-]*%:")
+    -- Temporaly solution
+    if not termDescription then
+        termDescription = scope
+    end
+    termDescription = rtrim(ltrim(termDescription or ""))
+
+    if moduleName and not funcName:find(moduleName) then
+        funcName = moduleName .. '.' .. funcName
+    end
+
+    return { name = funcName, description = termDescription, type = completionKinds['Function'] }
+end
+
+local function parseIndex(scope)
+    local is, ie = scope:find("%+[%=]+%+[%=]+%+\n")
+    local index_rows = scope:sub(ie + 1, scope:len())
+
+    local index = {}
+    local ROW_SEPARATOR = "%+[%-]+%+[%-]+%+\n"
+    local TEXT_REGEXP = "[%w.,:()%s'`+/-]+"
+    local FUNC_REGEXP = "[%w.()]+"
+    -- local WORD_REGEXP = "[%w.,:()'`+/-]+"
+    -- local SENTENCE_REGEXP = "[" .. WORD_REGEXP .. "%s?]*"
+    local FUNC_REGEXP = "[%w._()]+"
+    local ROW_REGEXP = "[%s]*%|[%s]*%:ref%:%`(" .. FUNC_REGEXP .. ")"
+
+    local i = 1
+    while index_rows:find(ROW_REGEXP, i) do
+        local is, ie, func_name = index_rows:find(ROW_REGEXP, i)
+        local row_dump = index_rows:sub(ie + 1, index_rows:find(ROW_SEPARATOR, ie + 1))
+
+        local desc = ""
+        for desc_row in row_dump:gmatch("[^\n][%s]*%|[%s]*(" .. TEXT_REGEXP .. ")%|\n") do
+            desc = desc .. desc_row
+        end
+
+        index[func_name:gsub("%(%)", "")] = rtrim(desc):gsub("[%s]+", " ")
+        i = ie + 1
+    end
+
+    return index
+end
+
+-- Parse only functions
+local function parseDocFile(text, terms)
+    local function findNextTerm(text, pos)
+        local terms = {
+            -- Directives
+            { pattern = "%.%. module%:%:", name = "module" },
+            { pattern = "%.%. function%:%:", name = "function" },
+
+            -- Headings
+            { pattern = "[%-]+\n[%s]+Submodule%s", name = "submodule" },
+            { pattern = "[%=]+\n[%s]+Overview[%s]*\n[%=]+\n\n", name = "overview" },
+            { pattern = "[%=]+\n[%s]+Index[%s]*\n[%=]+\n\n", name = "index" }
+        }
+
+        local nextTerm
+        for _, term in ipairs(terms) do
+            local is, ie = text:find(term.pattern, pos)
+            if is then
+                if not nextTerm then
+                    nextTerm = { pos = is, term = term, e_pos = ie }
+                else
+                    if is < nextTerm.pos then
+                        nextTerm = { pos = is, term = term, e_pos = ie }
+                    end
+                end
+            end
+        end
+
+        return nextTerm
+    end
+
+    local function truncateScope(text, startPos)
+        local nextTerm = findNextTerm(text, startPos)
+        local lastPos = text:len()
+        if nextTerm then
+            lastPos = nextTerm.pos
+        end
+
+        return text:sub(startPos, lastPos - 1)
+    end
+
+    -- Scope for functions and other objects
+    local moduleName
+
+    local i = 1
+    -- local terms = {}
+    while findNextTerm(text, i) do
+        local nextTerm = findNextTerm(text, i)
+        local nextNextTerm = findNextTerm(text, nextTerm.pos + 1)
+
+        if nextTerm.term.name == "module" then
+            local is, ie, moduleName = text:find("%.%. module%:%: ([%w.:_]*)\n", nextTerm.pos)
+            local currentModule = terms[moduleName]
+            if not currentModule then
+                terms[moduleName] = { name = moduleName, type = completionKinds['Module'] }
+                currentModule = terms[moduleName]
+            end
+
+            if not currentModule.description then
+                currentModule.description = truncateScope(text, ie + 1)
+            end
+        elseif nextTerm.term.name == "function" then
+            local is, ie = text:find("%.%. function%:%:", nextTerm.pos)
+
+            -- Skip space between directive and term name
+            local scope = text:sub(ie + 2, nextNextTerm and nextNextTerm.pos or text:len())
+            local term = parseFunction(scope, moduleName)
+
+            if terms[term.name] then
+                terms[term.name].description = term.description
+            else
+                terms[term.name] = term
+            end
+        elseif nextTerm.term.name == "submodule" then
+            moduleName = text:match("%`([%w.]+)%`", nextTerm.pos) -- Remained part [%s]*\n[%-]+\n
+            terms[moduleName] = { name = moduleName, type = completionKinds['Module'] }
+        elseif nextTerm.term.name == "overview" then
+            -- TODO: Uncomment
+            -- assert(moduleName ~= nil, "Module name should be setted")
+            if moduleName then
+                local currentModule = terms[moduleName]
+                if not currentModule then
+                    terms[moduleName] = { name = moduleName }
+                    currentModule = terms[moduleName]
+                end
+                currentModule.description = truncateScope(text, nextTerm.e_pos)
+            end
+        elseif nextTerm.term.name == "index" then
+            local index = parseIndex(truncateScope(text, nextTerm.e_pos))
+            for func, desc in pairs(index) do
+                local term = terms[func]
+                if not term then
+                    -- TODO: Maybe it's not a function...
+                    terms[func] = { name = func, type = completionKinds['Function'] }
+                    term = terms[func]
+                end
+
+                term.brief = desc
+            end
+        end
+
+        i = nextTerm.pos + 1
+    end
+end
+
+return {
+    parseDocFile = parseDocFile
+}

tarantool-lsp/methods.lua

@@ -3,6 +3,7 @@ local analyze = require 'tarantool-lsp.analyze'
 local rpc     = require 'tarantool-lsp.rpc'
 local log     = require 'tarantool-lsp.log'
 local utf     = require 'tarantool-lsp.unicode'
+local docs 	  = require('tarantool-lsp.doc-manager')
 local json    = require 'json'
 local unpack  = table.unpack or unpack
 
@@ -22,6 +23,13 @@ function method_handlers.initialize(params, id)
 	log.info("Config.root = %q", Config.root)
 	analyze.load_completerc(Config.root)
 	analyze.load_luacheckrc(Config.root)
+
+	local ok, err = docs:init()
+	if err ~= nil then
+		log.info("Docs subsystem error: %s", err)
+	end
+	-- log.info("AAAA %t", docs.terms)
+
 	--ClientCapabilities = params.capabilities
 	Initialized = true
 	-- hopefully this is modest enough
@@ -606,9 +614,24 @@ method_handlers["textDocument/completion"] = function(params, id)
 	local left_part = current_line:sub(0, params.position.character)
 	local last_token = left_part:match("[%w.:_]*$")
 	if last_token then
-		local completions = console.completion_handler(last_token, 0, last_token:len()) or {}
-		-- Completion handler returns input string at the first element
-		for _, cmplt in fun.tail(completions) do
+		local raw_completions = {}
+		local ADD_COMPLETION = function(cmplt)
+			raw_completions[cmplt] = true
+		end
+
+		-- [?] Completion handler returns input string at the first element
+		local tnt_completions = console.completion_handler(last_token, 0, last_token:len()) or {}
+		local doc_completions = docs:getCompletions(last_token)
+		fun.each(ADD_COMPLETION, fun.tail(tnt_completions))
+		fun.each(ADD_COMPLETION, fun.remove_if(function(cmplt)
+			if raw_completions[cmplt .. '('] then
+				return false
+			end
+
+			return true
+		end, doc_completions))
+
+		for _, cmplt in fun.map(function(cmplt) return cmplt end, raw_completions) do
 			local showedCmplt = cmplt
 			local insertedCmplt = cmplt
 			local cmpltKind = completionKinds["Field"]
@@ -618,12 +641,14 @@ method_handlers["textDocument/completion"] = function(params, id)
 				showedCmplt = cmplt:gsub("%(", "")
 			end
 
+			local doc = docs:get(showedCmplt)
+
 			table.insert(items, {
 				label = showedCmplt,
-				kind = cmpltKind,
+				kind = doc and doc.type or cmpltKind,
 				insertText = insertedCmplt,
-				-- documentation = "box.cfg{} option",
-				detail = "detail information"
+				documentation = doc and doc.description,
+				detail = doc and doc.brief
 			})
 		end
 	end