Merge pull request swiftlang#25953 from JDevlieghere/swift-overlay-vfs

[ClangImporter] Merge Swift & Clang VFS

include/swift/AST/DiagnosticsClangImporter.def

@@ -91,10 +91,6 @@ WARNING(implicit_bridging_header_imported_from_module,none,
         "is deprecated and will be removed in a later version of Swift",
         (StringRef, Identifier))
 
-WARNING(clang_vfs_overlay_is_ignored,none,
-        "ignoring '-ivfsoverlay' options provided to '-Xcc' in favor of "
-        "'-vfsoverlay'", ())
-
 #ifndef DIAG_NO_UNDEF
 # if defined(DIAG)
 #  undef DIAG

include/swift/Basic/SourceManager.h

@@ -23,10 +23,16 @@
 
 namespace swift {
 
+static inline llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>
+getRealOverlayFileSystem() {
+  return llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>(
+      new llvm::vfs::OverlayFileSystem(llvm::vfs::getRealFileSystem()));
+}
+
 /// This class manages and owns source buffers.
 class SourceManager {
   llvm::SourceMgr LLVMSourceMgr;
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> FileSystem;
   unsigned CodeCompletionBufferID = 0U;
   unsigned CodeCompletionOffset;
 
@@ -49,9 +55,9 @@ class SourceManager {
   mutable std::pair<const char *, const VirtualFile*> CachedVFile = {nullptr, nullptr};
 
 public:
-  SourceManager(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
-                    llvm::vfs::getRealFileSystem())
-    : FileSystem(FS) {}
+  SourceManager(llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> FS =
+                    getRealOverlayFileSystem())
+      : FileSystem(FS) {}
 
   llvm::SourceMgr &getLLVMSourceMgr() {
     return LLVMSourceMgr;
@@ -60,11 +66,12 @@ class SourceManager {
     return LLVMSourceMgr;
   }
 
-  void setFileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
+  void
+  setFileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> FS) {
     FileSystem = FS;
   }
 
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getFileSystem() {
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> getFileSystem() {
     return FileSystem;
   }
 
@@ -254,4 +261,3 @@ class SourceManager {
 } // end namespace swift
 
 #endif // SWIFT_BASIC_SOURCEMANAGER_H
-

lib/ClangImporter/ClangImporter.cpp

@@ -1012,14 +1012,35 @@ ClangImporter::create(ASTContext &ctx,
 
   // Set up the file manager.
   {
-    if (!ctx.SearchPathOpts.VFSOverlayFiles.empty()) {
-      // If the clang instance has overlays it means the user has provided
-      // -ivfsoverlay options and swift -vfsoverlay options.  We're going to
-      // clobber their file system with our own, so warn about it.
-      if (!instance.getHeaderSearchOpts().VFSOverlayFiles.empty()) {
-        ctx.Diags.diagnose(SourceLoc(), diag::clang_vfs_overlay_is_ignored);
-      }
+    if (instance.getHeaderSearchOpts().VFSOverlayFiles.empty()) {
       instance.setVirtualFileSystem(ctx.SourceMgr.getFileSystem());
+    } else {
+      // Initialize the clang VFS from its compiler invocation.
+      instance.createFileManager();
+
+      // Create a new overlay file system for the clang importer with the clang
+      // VFS as its root.
+      auto ClangImporterFS =
+          llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>(
+              new llvm::vfs::OverlayFileSystem(
+                  &instance.getVirtualFileSystem()));
+
+      auto SwiftFS = ctx.SourceMgr.getFileSystem();
+      auto it = SwiftFS->overlays_rbegin();
+      auto end = SwiftFS->overlays_rend();
+
+      // The Swift file system is an overlay file system with the real file
+      // system as its root. Skip the root so we query the other overlays
+      // before falling back to the real file system.
+      it++;
+
+      // Add all remaining Swift overlay file systems to the new file system.
+      while (it != end) {
+        ClangImporterFS->pushOverlay(*it);
+        it++;
+      }
+
+      instance.setVirtualFileSystem(ClangImporterFS);
     }
     instance.createFileManager();
   }

lib/Frontend/Frontend.cpp

@@ -235,9 +235,6 @@ static bool loadAndValidateVFSOverlay(
     const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &BaseFS,
     const llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> &OverlayFS,
     DiagnosticEngine &Diag) {
-  // FIXME: It should be possible to allow chained lookup of later VFS overlays
-  // through the mapping defined by earlier overlays.
-  // See rdar://problem/39440687
   auto Buffer = BaseFS->getBufferForFile(File);
   if (!Buffer) {
     Diag.diagnose(SourceLoc(), diag::cannot_open_file, File,
@@ -264,14 +261,7 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() {
     hadAnyFailure |=
         loadAndValidateVFSOverlay(File, BaseFS, OverlayFS, Diagnostics);
   }
-
-  // If we successfully loaded all the overlays, let the source manager and
-  // diagnostic engine take advantage of the overlay file system.
-  if (!hadAnyFailure &&
-      (OverlayFS->overlays_begin() != OverlayFS->overlays_end())) {
-    SourceMgr.setFileSystem(OverlayFS);
-  }
-
+  SourceMgr.setFileSystem(OverlayFS);
   return hadAnyFailure;
 }
 

lib/Frontend/ParseableInterfaceModuleLoader.cpp

@@ -273,7 +273,7 @@ static Optional<StringRef> getRelativeDepPath(StringRef DepPath,
 /// output path.
 /// \note Needs to be in the swift namespace so CompilerInvocation can see it.
 class swift::ParseableInterfaceBuilder {
-  llvm::vfs::FileSystem &fs;
+  llvm::vfs::OverlayFileSystem &fs;
   DiagnosticEngine &diags;
   const StringRef interfacePath;
   const StringRef moduleName;
@@ -765,7 +765,7 @@ class ParseableInterfaceModuleLoaderImpl {
   using AccessPathElem = std::pair<Identifier, SourceLoc>;
   friend class swift::ParseableInterfaceModuleLoader;
   ASTContext &ctx;
-  llvm::vfs::FileSystem &fs;
+  llvm::vfs::OverlayFileSystem &fs;
   DiagnosticEngine &diags;
   ModuleRebuildInfo rebuildInfo;
   const StringRef modulePath;

test/Frontend/Inputs/vfs/a-modulemap

@@ -1,3 +1,7 @@
 module VFSMappedModule {
   header "VFSMappedModule.h"
 }
+
+module YetAnotherVFSMappedModule {
+  header "YetAnotherVFSMappedModule.h"
+}

test/Frontend/Inputs/vfs/b-header

@@ -0,0 +1 @@
+#define MAJOR_SUCCESS 1

test/Frontend/Inputs/vfs/quaternary-vfsoverlay.yaml

@@ -0,0 +1,17 @@
+{
+  'version': 0,
+  'use-external-names': false,
+  'roots': [
+    {
+      'name': 'OUT_DIR', 'type': 'directory',
+      'contents': [
+        { 'name': 'YetAnotherVFSMappedModule.h', 'type': 'file',
+          'external-contents': 'INPUT_DIR/vfs/b-header'
+        },
+        { 'name': 'YetAnotherVFSMappedModule.framework/Headers/VFSMappedModule.h', 'type': 'file',
+          'external-contents': 'INPUT_DIR/vfs/b-header'
+        },
+      ]
+    },
+  ]
+}

test/Frontend/vfs.swift

@@ -2,6 +2,7 @@
 // RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/vfsoverlay.yaml > %t/overlay.yaml
 // RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/secondary-vfsoverlay.yaml > %t/secondary-overlay.yaml
 // RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/tertiary-vfsoverlay.yaml > %t/tertiary-overlay.yaml
+// RUN: sed -e "s|INPUT_DIR|%/S/Inputs|g" -e "s|OUT_DIR|%/t|g" %S/Inputs/vfs/quaternary-vfsoverlay.yaml > %t/quaternary-vfsoverlay.yaml
 
 // RUN: not %target-swift-frontend -vfsoverlay %/t/overlay.yaml -typecheck %s %/t/mapped-file.swift -serialize-diagnostics-path %/t/basic.dia 2>&1 | %FileCheck -check-prefix=BASIC_MAPPING_ERROR %s
 // RUN: c-index-test -read-diagnostics %/t/basic.dia 2>&1 | %FileCheck -check-prefix=BASIC_MAPPING_ERROR %s
@@ -20,10 +21,11 @@
 // RUN: %target-swift-frontend(mock-sdk: %clang-importer-sdk) -I %/t -DTEST_VFS_CLANG_IMPORTER -Xcc -ivfsoverlay -Xcc %/t/overlay.yaml -typecheck %/s
 
 // If we see -ivfsoverlay and -vfsoverlay, we'll clobber Clang's VFS with our own.
-// RUN: %target-swift-frontend(mock-sdk: %clang-importer-sdk) -I %/t -DTEST_VFS_CLANG_IMPORTER -vfsoverlay %/t/overlay.yaml -Xcc -ivfsoverlay -Xcc %/t/overlay.yaml -typecheck %/s  2>&1 | %FileCheck -check-prefix=WARN_VFS_CLOBBERED %s
-
-// WARN_VFS_CLOBBERED: warning: ignoring '-ivfsoverlay' options provided to '-Xcc' in favor of '-vfsoverlay'
+// RUN: %target-swift-frontend(mock-sdk: %clang-importer-sdk) -I %/t -DTEST_VFS_CLANG_IMPORTER -DTEST_VFS_CLANG_IMPORTER_MERGE -vfsoverlay %/t/overlay.yaml -Xcc -ivfsoverlay -Xcc %/t/quaternary-vfsoverlay.yaml -typecheck %/s  2>&1
 
 #if TEST_VFS_CLANG_IMPORTER
 import VFSMappedModule
+#if TEST_VFS_CLANG_IMPORTER_MERGE
+import YetAnotherVFSMappedModule
+#endif
 #endif

unittests/FrontendTool/ModuleLoadingTests.cpp

@@ -87,8 +87,10 @@ class ParseableInterfaceModuleLoaderTest : public testing::Test {
     SourceManager sourceMgr;
 
     // Create a file system that tracks how many times a file has been opened.
-    llvm::IntrusiveRefCntPtr<OpenTrackingFileSystem> fs(
-      new OpenTrackingFileSystem(sourceMgr.getFileSystem()));
+    llvm::IntrusiveRefCntPtr<OpenTrackingFileSystem> baseFS(
+        new OpenTrackingFileSystem(sourceMgr.getFileSystem()));
+    auto fs = llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>(
+        new llvm::vfs::OverlayFileSystem(baseFS));
 
     sourceMgr.setFileSystem(fs);
     PrintingDiagnosticConsumer printingConsumer;
@@ -126,7 +128,7 @@ class ParseableInterfaceModuleLoaderTest : public testing::Test {
     ASSERT_TRUE(fs->exists(cachedModulePath));
 
     // Assert that we've only opened this file once, to write it.
-    ASSERT_EQ((unsigned)1, fs->numberOfOpens(cachedModulePath));
+    ASSERT_EQ((unsigned)1, baseFS->numberOfOpens(cachedModulePath));
 
     auto bufOrErr = fs->getBufferForFile(cachedModulePath);
     ASSERT_TRUE(bufOrErr);