Refactor WorkingFiles and CompletionManager

* WorkingFiles::files : vector -> unordered_map
* Add timestamp to WorkingFile

* Rename "comp-preload" thread to "preamble"
* Rename CompletionManager to SemaManager as it is used by "diag" "comp" "preamble"
* Rename clang_complete.* to sema_manager.*
* Merge SemaManager::{preloads,sessions}
* Add initialization option session.maxNum
* In DiagnosticMain, if an included file was modified, cancel the DiagTask and create a PreambleTask instead. The task sets `from_diag` so as to trigger immediate DiagTask after the preamble is built.

CMakeLists.txt

@@ -180,7 +180,6 @@ file(GLOB SOURCES src/*.cc src/*.h src/serializers/*.cc src/serializers/*.h
 target_sources(ccls PRIVATE third_party/siphash.cc)
 
 target_sources(ccls PRIVATE
-  src/clang_complete.cc
   src/clang_tu.cc
   src/config.cc
   src/filesystem.cc
@@ -197,6 +196,7 @@ target_sources(ccls PRIVATE
   src/position.cc
   src/project.cc
   src/query.cc
+  src/sema_manager.cc
   src/serializer.cc
   src/test.cc
   src/utils.cc

src/config.hh

@@ -228,6 +228,10 @@ struct Config {
     std::vector<std::string> whitelist;
   } index;
 
+  struct Session {
+    int maxNum = 10;
+  } session;
+
   struct WorkspaceSymbol {
     int caseSensitivity = 1;
     // Maximum workspace search results.
@@ -261,12 +265,13 @@ MAKE_REFLECT_STRUCT(Config::Index, blacklist, comments, initialBlacklist,
                     initialWhitelist, multiVersion, multiVersionBlacklist,
                     multiVersionWhitelist, onChange, threads, trackDependency,
                     whitelist);
+MAKE_REFLECT_STRUCT(Config::Session, maxNum);
 MAKE_REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort);
 MAKE_REFLECT_STRUCT(Config::Xref, maxNum);
 MAKE_REFLECT_STRUCT(Config, compilationDatabaseCommand,
                     compilationDatabaseDirectory, cacheDirectory, cacheFormat,
                     clang, client, codeLens, completion, diagnostics, highlight,
-                    index, workspaceSymbol, xref);
+                    index, session, workspaceSymbol, xref);
 
 extern Config *g_config;
 

src/indexer.cc

@@ -3,11 +3,11 @@
 
 #include "indexer.hh"
 
-#include "clang_complete.hh"
 #include "clang_tu.hh"
 #include "log.hh"
 #include "pipeline.hh"
 #include "platform.hh"
+#include "sema_manager.hh"
 #include "serializer.hh"
 
 #include <clang/AST/AST.h>
@@ -1219,10 +1219,11 @@ void Init() {
 }
 
 std::vector<std::unique_ptr<IndexFile>>
-Index(CompletionManager *completion, WorkingFiles *wfiles, VFS *vfs,
+Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
       const std::string &opt_wdir, const std::string &file,
       const std::vector<const char *> &args,
-      const std::vector<std::pair<std::string, std::string>> &remapped, bool &ok) {
+      const std::vector<std::pair<std::string, std::string>> &remapped,
+      bool &ok) {
   ok = true;
   auto PCH = std::make_shared<PCHContainerOperations>();
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = llvm::vfs::getRealFileSystem();
@@ -1243,7 +1244,7 @@ Index(CompletionManager *completion, WorkingFiles *wfiles, VFS *vfs,
     bool done_remap = false;
 #if 0
     std::shared_ptr<CompletionSession> session =
-      completion->TryGetSession(file, false, false);
+      manager->TryGetSession(file, false, false);
     if (session)
       if (auto preamble = session->GetPreamble()) {
         Bufs.push_back(llvm::MemoryBuffer::getMemBuffer(buf));

src/indexer.hh

@@ -312,14 +312,14 @@ struct IndexFile {
   std::string ToString();
 };
 
-struct CompletionManager;
+struct SemaManager;
 struct WorkingFiles;
 struct VFS;
 
 namespace idx {
 void Init();
 std::vector<std::unique_ptr<IndexFile>>
-Index(CompletionManager *complete, WorkingFiles *wfiles, VFS *vfs,
+Index(SemaManager *complete, WorkingFiles *wfiles, VFS *vfs,
       const std::string &opt_wdir, const std::string &file,
       const std::vector<const char *> &args,
       const std::vector<std::pair<std::string, std::string>> &remapped,

src/message_handler.hh

@@ -12,7 +12,7 @@
 #include <vector>
 
 namespace ccls {
-struct CompletionManager;
+struct SemaManager;
 struct VFS;
 struct IncludeComplete;
 struct Project;
@@ -193,7 +193,7 @@ struct ReplyOnce {
 };
 
 struct MessageHandler {
-  CompletionManager *clang_complete = nullptr;
+  SemaManager *manager = nullptr;
   DB *db = nullptr;
   IncludeComplete *include_complete = nullptr;
   Project *project = nullptr;

src/messages/ccls_call.cc

@@ -189,11 +189,10 @@ void MessageHandler::ccls_call(Reader &reader, ReplyOnce &reply) {
              param.levels);
   } else {
     QueryFile *file = FindFile(reply, param.textDocument.uri.GetPath());
-    if (!file)
+    WorkingFile *wf = file ? wfiles->GetFile(file->def->path) : nullptr;
+    if (!wf)
       return;
-    WorkingFile *working_file = wfiles->GetFileByFilename(file->def->path);
-    for (SymbolRef sym :
-         FindSymbolsAtLocation(working_file, file, param.position)) {
+    for (SymbolRef sym : FindSymbolsAtLocation(wf, file, param.position)) {
       if (sym.kind == Kind::Func) {
         result = BuildInitial(this, sym.usr, param.callee, param.callType,
                               param.qualified, param.levels);

src/messages/ccls_inheritance.cc

@@ -137,9 +137,8 @@ void Inheritance(MessageHandler *m, Param &param, ReplyOnce &reply) {
     QueryFile *file = m->FindFile(reply, param.textDocument.uri.GetPath());
     if (!file)
       return;
-    WorkingFile *wfile = m->wfiles->GetFileByFilename(file->def->path);
-
-    for (SymbolRef sym : FindSymbolsAtLocation(wfile, file, param.position))
+    WorkingFile *wf = m->wfiles->GetFile(file->def->path);
+    for (SymbolRef sym : FindSymbolsAtLocation(wf, file, param.position))
       if (sym.kind == Kind::Func || sym.kind == Kind::Type) {
         result = BuildInitial(m, sym, param.derived, param.qualified,
                               param.levels);

src/messages/ccls_member.cc

@@ -270,11 +270,10 @@ void MessageHandler::ccls_member(Reader &reader, ReplyOnce &reply) {
       result.reset();
   } else {
     QueryFile *file = FindFile(reply, param.textDocument.uri.GetPath());
-    if (!file)
+    WorkingFile *wf = file ? wfiles->GetFile(file->def->path) : nullptr;
+    if (!wf)
       return;
-    WorkingFile *wfile = wfiles->GetFileByFilename(file->def->path);
-    for (SymbolRef sym :
-         FindSymbolsAtLocation(wfile, file, param.position)) {
+    for (SymbolRef sym : FindSymbolsAtLocation(wf, file, param.position)) {
       switch (sym.kind) {
       case Kind::Func:
       case Kind::Type:

src/messages/ccls_navigate.cc

@@ -31,14 +31,13 @@ void MessageHandler::ccls_navigate(Reader &reader,
   Param param;
   Reflect(reader, param);
   QueryFile *file = FindFile(reply, param.textDocument.uri.GetPath());
-  if (!file)
+  WorkingFile *wf = file ? wfiles->GetFile(file->def->path) : nullptr;
+  if (!wf)
     return;
-
-  WorkingFile *wfile = wfiles->GetFileByFilename(file->def->path);
   Position ls_pos = param.position;
-  if (wfile && wfile->index_lines.size())
-    if (auto line = wfile->GetIndexPosFromBufferPos(ls_pos.line,
-                                                    &ls_pos.character, false))
+  if (wf->index_lines.size())
+    if (auto line =
+            wf->GetIndexPosFromBufferPos(ls_pos.line, &ls_pos.character, false))
       ls_pos.line = *line;
   Pos pos{(int16_t)ls_pos.line, (int16_t)ls_pos.character};
 
@@ -85,7 +84,7 @@ void MessageHandler::ccls_navigate(Reader &reader,
   }
   std::vector<Location> result;
   if (res)
-    if (auto ls_range = GetLsRange(wfile, *res)) {
+    if (auto ls_range = GetLsRange(wf, *res)) {
       Location &ls_loc = result.emplace_back();
       ls_loc.uri = param.textDocument.uri;
       ls_loc.range = *ls_range;

src/messages/ccls_reload.cc

@@ -1,10 +1,10 @@
 // Copyright 2017-2018 ccls Authors
 // SPDX-License-Identifier: Apache-2.0
 
-#include "clang_complete.hh"
 #include "message_handler.hh"
 #include "pipeline.hh"
 #include "project.hh"
+#include "sema_manager.hh"
 #include "working_files.hh"
 
 #include <queue>
@@ -28,7 +28,7 @@ void MessageHandler::ccls_reload(Reader &reader) {
     vfs->Clear();
     db->clear();
     project->Index(wfiles, RequestId());
-    clang_complete->FlushAllSessions();
+    manager->Clear();
     return;
   }
 }

src/messages/ccls_vars.cc

@@ -20,13 +20,12 @@ void MessageHandler::ccls_vars(Reader &reader, ReplyOnce &reply) {
   Param param;
   Reflect(reader, param);
   QueryFile *file = FindFile(reply, param.textDocument.uri.GetPath());
-  if (!file)
+  WorkingFile *wf = file ? wfiles->GetFile(file->def->path) : nullptr;
+  if (!wf)
     return;
-  WorkingFile *working_file = wfiles->GetFileByFilename(file->def->path);
 
   std::vector<Location> result;
-  for (SymbolRef sym :
-       FindSymbolsAtLocation(working_file, file, param.position)) {
+  for (SymbolRef sym : FindSymbolsAtLocation(wf, file, param.position)) {
     Usr usr = sym.usr;
     switch (sym.kind) {
     default:

src/messages/initialize.cc

@@ -1,7 +1,7 @@
 // Copyright 2017-2018 ccls Authors
 // SPDX-License-Identifier: Apache-2.0
 
-#include "clang_complete.hh"
+#include "sema_manager.hh"
 #include "filesystem.hh"
 #include "include_complete.hh"
 #include "log.hh"
@@ -229,7 +229,7 @@ void *Indexer(void *arg_) {
   delete arg;
   std::string name = "indexer" + std::to_string(idx);
   set_thread_name(name.c_str());
-  pipeline::Indexer_Main(h->clang_complete, h->vfs, h->project, h->wfiles);
+  pipeline::Indexer_Main(h->manager, h->vfs, h->project, h->wfiles);
   return nullptr;
 }
 } // namespace
@@ -325,6 +325,8 @@ void Initialize(MessageHandler *m, InitializeParam &param, ReplyOnce &reply) {
 
   LOG_S(INFO) << "dispatch initial index requests";
   m->project->Index(m->wfiles, reply.id);
+
+  m->manager->sessions.SetCapacity(g_config->session.maxNum);
 }
 
 void MessageHandler::initialize(Reader &reader, ReplyOnce &reply) {

src/messages/textDocument_code.cc

@@ -21,13 +21,12 @@ MAKE_REFLECT_STRUCT(CodeAction, title, kind, edit);
 }
 void MessageHandler::textDocument_codeAction(CodeActionParam &param,
                                              ReplyOnce &reply) {
-  WorkingFile *wf =
-      wfiles->GetFileByFilename(param.textDocument.uri.GetPath());
+  WorkingFile *wf = wfiles->GetFile(param.textDocument.uri.GetPath());
   if (!wf)
     return;
   std::vector<CodeAction> result;
   std::vector<Diagnostic> diagnostics;
-  wfiles->DoAction([&]() { diagnostics = wf->diagnostics_; });
+  wfiles->WithLock([&]() { diagnostics = wf->diagnostics; });
   for (Diagnostic &diag : diagnostics)
     if (diag.fixits_.size() &&
         (param.range.Intersects(diag.range) ||
@@ -85,17 +84,16 @@ void MessageHandler::textDocument_codeLens(TextDocumentParam &param,
   std::string path = param.textDocument.uri.GetPath();
 
   QueryFile *file = FindFile(reply, path);
-  WorkingFile *wfile =
-      file ? wfiles->GetFileByFilename(file->def->path) : nullptr;
-  if (!wfile) {
+  WorkingFile *wf = file ? wfiles->GetFile(file->def->path) : nullptr;
+  if (!wf) {
     return;
   }
 
   auto Add = [&](const char *singular, Cmd_xref show, Range range, int num,
                  bool force_display = false) {
     if (!num && !force_display)
       return;
-    std::optional<lsRange> ls_range = GetLsRange(wfile, range);
+    std::optional<lsRange> ls_range = GetLsRange(wf, range);
     if (!ls_range)
       return;
     CodeLens &code_lens = result.emplace_back();

src/messages/textDocument_completion.cc

@@ -1,12 +1,12 @@
 // Copyright 2017-2018 ccls Authors
 // SPDX-License-Identifier: Apache-2.0
 
-#include "clang_complete.hh"
 #include "fuzzy_match.hh"
 #include "include_complete.hh"
 #include "log.hh"
 #include "message_handler.hh"
 #include "pipeline.hh"
+#include "sema_manager.hh"
 #include "working_files.hh"
 
 #include <clang/Sema/CodeCompleteConsumer.h>
@@ -439,7 +439,7 @@ void MessageHandler::textDocument_completion(CompletionParam &param,
   static CompleteConsumerCache<std::vector<CompletionItem>> cache;
   CompletionList result;
   std::string path = param.textDocument.uri.GetPath();
-  WorkingFile *file = wfiles->GetFileByFilename(path);
+  WorkingFile *file = wfiles->GetFile(path);
   if (!file) {
     return;
   }
@@ -511,7 +511,7 @@ void MessageHandler::textDocument_completion(CompletionParam &param,
   }
 #endif
 
-  CompletionManager::OnComplete callback =
+  SemaManager::OnComplete callback =
       [filter, path, begin_pos, end_pos, reply,
        buffer_line](CodeCompleteConsumer *OptConsumer) {
         if (!OptConsumer)
@@ -536,11 +536,9 @@ void MessageHandler::textDocument_completion(CompletionParam &param,
     cache.WithLock([&]() { Consumer.ls_items = cache.result; });
     callback(&Consumer);
   } else {
-    clang_complete->completion_request_.PushBack(
-        std::make_unique<CompletionManager::CompletionRequest>(
-            reply.id, param.textDocument, begin_pos,
-            std::make_unique<CompletionConsumer>(CCOpts, false), CCOpts,
-            callback));
+    manager->comp_tasks.PushBack(std::make_unique<SemaManager::CompTask>(
+        reply.id, param.textDocument.uri.GetPath(), begin_pos,
+        std::make_unique<CompletionConsumer>(CCOpts, false), CCOpts, callback));
   }
 }
 } // namespace ccls

src/messages/textDocument_definition.cc

@@ -37,15 +37,15 @@ void MessageHandler::textDocument_definition(TextDocumentPositionParam &param,
                                              ReplyOnce &reply) {
   int file_id;
   QueryFile *file = FindFile(reply, param.textDocument.uri.GetPath(), &file_id);
-  if (!file)
+  WorkingFile *wf = file ? wfiles->GetFile(file->def->path) : nullptr;
+  if (!wf)
     return;
 
   std::vector<Location> result;
   Maybe<Use> on_def;
-  WorkingFile *wfile = wfiles->GetFileByFilename(file->def->path);
   Position &ls_pos = param.position;
 
-  for (SymbolRef sym : FindSymbolsAtLocation(wfile, file, ls_pos, true)) {
+  for (SymbolRef sym : FindSymbolsAtLocation(wf, file, ls_pos, true)) {
     // Special cases which are handled:
     //  - symbol has declaration but no definition (ie, pure virtual)
     //  - goto declaration while in definition of recursive type
@@ -96,7 +96,7 @@ void MessageHandler::textDocument_definition(TextDocumentPositionParam &param,
     // Find the best match of the identifier at point.
     if (!range) {
       Position position = param.position;
-      const std::string &buffer = wfile->buffer_content;
+      const std::string &buffer = wf->buffer_content;
       std::string_view query = LexIdentifierAroundPos(position, buffer);
       std::string_view short_query = query;
       {
@@ -160,7 +160,7 @@ void MessageHandler::textDocument_typeDefinition(
   QueryFile *file = FindFile(reply, param.textDocument.uri.GetPath());
   if (!file)
     return;
-  WorkingFile *working_file = wfiles->GetFileByFilename(file->def->path);
+  WorkingFile *working_file = wfiles->GetFile(file->def->path);
 
   std::vector<Location> result;
   auto Add = [&](const QueryType &type) {