First draft: replace libclang indexer with clangIndex

.clang-format

@@ -1 +1 @@
-BasedOnStyle: Chromium
+BasedOnStyle: LLVM

cmake/FindClang.cmake

@@ -69,6 +69,7 @@ set(_Clang_REQUIRED_VARS Clang_LIBRARY Clang_INCLUDE_DIR Clang_EXECUTABLE
                          LLVM_INCLUDE_DIR LLVM_BUILD_INCLUDE_DIR)
 
 _Clang_find_library(Clang_LIBRARY clang)
+_Clang_find_add_library(clangIndex)
 _Clang_find_add_library(clangFrontend)
 _Clang_find_add_library(clangParse)
 _Clang_find_add_library(clangSerialization)

src/clang_tu.cc

@@ -259,67 +259,6 @@ std::string ClangCursor::get_type_description() const {
   return ::ToString(clang_getTypeSpelling(type));
 }
 
-std::string ClangCursor::get_comments() const {
-  CXSourceRange range = clang_Cursor_getCommentRange(cx_cursor);
-  if (clang_Range_isNull(range))
-    return {};
-
-  unsigned start_column;
-  clang_getSpellingLocation(clang_getRangeStart(range), nullptr, nullptr,
-                            &start_column, nullptr);
-
-  // Get associated comment text.
-  CXString cx_raw = clang_Cursor_getRawCommentText(cx_cursor);
-  int pad = -1;
-  std::string ret;
-  for (const char* p = clang_getCString(cx_raw); *p;) {
-    // The first line starts with a comment marker, but the rest needs
-    // un-indenting.
-    unsigned skip = start_column - 1;
-    for (; skip > 0 && (*p == ' ' || *p == '\t'); p++)
-      skip--;
-    const char* q = p;
-    while (*q != '\n' && *q)
-      q++;
-    if (*q)
-      q++;
-    // A minimalist approach to skip Doxygen comment markers.
-    // See https://www.stack.nl/~dimitri/doxygen/manual/docblocks.html
-    if (pad < 0) {
-      // First line, detect the length of comment marker and put into |pad|
-      const char* begin = p;
-      while (*p == '/' || *p == '*')
-        p++;
-      if (*p == '<' || *p == '!')
-        p++;
-      if (*p == ' ')
-        p++;
-      pad = int(p - begin);
-    } else {
-      // Other lines, skip |pad| bytes
-      int prefix = pad;
-      while (prefix > 0 &&
-             (*p == ' ' || *p == '/' || *p == '*' || *p == '<' || *p == '!'))
-        prefix--, p++;
-    }
-    ret.insert(ret.end(), p, q);
-    p = q;
-  }
-  clang_disposeString(cx_raw);
-  while (ret.size() && isspace(ret.back()))
-    ret.pop_back();
-  if (EndsWith(ret, "*/")) {
-    ret.resize(ret.size() - 2);
-  } else if (EndsWith(ret, "\n/")) {
-    ret.resize(ret.size() - 2);
-  }
-  while (ret.size() && isspace(ret.back()))
-    ret.pop_back();
-  if (ret.empty())
-    return {};
-  return ret;
-}
-
 std::string ClangCursor::ToString() const {
   return ::ToString(get_kind()) + " " + get_spell_name();
 }

src/clang_tu.h

@@ -85,7 +85,6 @@ class ClangCursor {
   bool is_valid_kind() const;
 
   std::string get_type_description() const;
-  std::string get_comments() const;
 
   std::string ToString() const;
 

src/clang_utils.cc

@@ -3,6 +3,7 @@
 #include "platform.h"
 
 #include "filesystem.hh"
+using namespace clang;
 using namespace llvm;
 
 namespace {
@@ -125,6 +126,20 @@ std::string FileName(CXFile file) {
   return ret;
 }
 
+std::string FileName(const FileEntry& file) {
+  StringRef Name = file.tryGetRealPathName();
+  if (Name.empty())
+    Name = file.getName();
+  std::string ret = NormalizePath(Name);
+  // Resolve /usr/include/c++/7.3.0 symlink.
+  if (!StartsWith(ret, g_config->projectRoot)) {
+    SmallString<256> dest;
+    sys::fs::real_path(ret, dest);
+    ret = dest.str();
+  }
+  return ret;
+}
+
 std::string ToString(CXString cx_string) {
   std::string string;
   if (cx_string.data != nullptr) {

src/clang_utils.h

@@ -3,6 +3,7 @@
 #include "lsp_diagnostic.h"
 
 #include <clang-c/Index.h>
+#include <clang/Basic/FileManager.h>
 
 #include <optional>
 #include <vector>
@@ -12,6 +13,7 @@ std::optional<lsDiagnostic> BuildAndDisposeDiagnostic(CXDiagnostic diagnostic,
 
 // Returns the absolute path to |file|.
 std::string FileName(CXFile file);
+std::string FileName(const clang::FileEntry& file);
 
 std::string ToString(CXString cx_string);
 

src/file_consumer.cc

@@ -23,11 +23,6 @@ std::optional<std::string> GetFileContents(
 
 }  // namespace
 
-bool operator==(const CXFileUniqueID& a, const CXFileUniqueID& b) {
-  return a.data[0] == b.data[0] && a.data[1] == b.data[1] &&
-         a.data[2] == b.data[2];
-}
-
 FileContents::FileContents(const std::string& path, const std::string& content)
     : path(path), content(content) {
   line_offsets_.push_back(0);
@@ -98,14 +93,10 @@ FileConsumer::FileConsumer(VFS* vfs, const std::string& parse_file)
     : vfs_(vfs), parse_file_(parse_file), thread_id_(g_thread_id) {}
 
 IndexFile* FileConsumer::TryConsumeFile(
-    CXFile file,
-    bool* is_first_ownership,
+    const clang::FileEntry& File,
     std::unordered_map<std::string, FileContents>* file_contents_map) {
-  assert(is_first_ownership);
-
-  CXFileUniqueID file_id;
-  if (clang_getFileUniqueID(file, &file_id) != 0) {
-    std::string file_name = FileName(file);
+  std::string file_name = FileName(File);
+  if (!File.isValid()) {
     if (!file_name.empty()) {
       LOG_S(ERROR) << "Could not get unique file id for " << file_name
                    << " when parsing " << parse_file_;
@@ -114,33 +105,27 @@ IndexFile* FileConsumer::TryConsumeFile(
   }
 
   // Try to find cached local result.
-  auto it = local_.find(file_id);
-  if (it != local_.end()) {
-    *is_first_ownership = false;
+  unsigned UID = File.getUID();
+  auto it = local_.find(UID);
+  if (it != local_.end())
     return it->second.get();
-  }
-
-  std::string file_name = FileName(file);
 
   // We did not take the file from global. Cache that we failed so we don't try
   // again and return nullptr.
   if (!vfs_->Mark(file_name, thread_id_, 2)) {
-    local_[file_id] = nullptr;
+    local_[UID] = nullptr;
     return nullptr;
   }
 
   // Read the file contents, if we fail then we cannot index the file.
   std::optional<std::string> contents =
       GetFileContents(file_name, file_contents_map);
-  if (!contents) {
-    *is_first_ownership = false;
+  if (!contents)
     return nullptr;
-  }
 
   // Build IndexFile instance.
-  *is_first_ownership = true;
-  local_[file_id] = std::make_unique<IndexFile>(file_name, *contents);
-  return local_[file_id].get();
+  local_[UID] = std::make_unique<IndexFile>(UID, file_name, *contents);
+  return local_[UID].get();
 }
 
 std::vector<std::unique_ptr<IndexFile>> FileConsumer::TakeLocalState() {

src/file_consumer.h

@@ -5,6 +5,7 @@
 #include "utils.h"
 
 #include <clang-c/Index.h>
+#include <clang/Basic/FileManager.h>
 
 #include <functional>
 #include <mutex>
@@ -13,10 +14,6 @@
 
 struct IndexFile;
 
-// Needed for unordered_map usage below.
-MAKE_HASHABLE(CXFileUniqueID, t.data[0], t.data[1], t.data[2]);
-bool operator==(const CXFileUniqueID& a, const CXFileUniqueID& b);
-
 struct FileContents {
   FileContents() = default;
   FileContents(const std::string& path, const std::string& content);
@@ -56,24 +53,20 @@ struct VFS {
 struct FileConsumer {
   FileConsumer(VFS* vfs, const std::string& parse_file);
 
-  // Returns true if this instance owns given |file|. This will also attempt to
-  // take ownership over |file|.
-  //
   // Returns IndexFile for the file or nullptr. |is_first_ownership| is set
   // to true iff the function just took ownership over the file. Otherwise it
   // is set to false.
   //
   // note: file_contents is passed as a parameter instead of as a member
   // variable since it is large and we do not want to copy it.
-  IndexFile* TryConsumeFile(CXFile file,
-                            bool* is_first_ownership,
+  IndexFile* TryConsumeFile(const clang::FileEntry& file,
                             std::unordered_map<std::string, FileContents>* file_contents);
 
   // Returns and passes ownership of all local state.
   std::vector<std::unique_ptr<IndexFile>> TakeLocalState();
 
  private:
-  std::unordered_map<CXFileUniqueID, std::unique_ptr<IndexFile>> local_;
+  std::unordered_map<unsigned, std::unique_ptr<IndexFile>> local_;
   VFS* vfs_;
   std::string parse_file_;
   int thread_id_;