Skip to content

Commit

Permalink
[Caching] Preliminary simple macro support for caching
Browse files Browse the repository at this point in the history
Preliminary caching support for macro. Current implementation only
supports macros that built into the toolchain by:
* Inserting the plugin into the CASFS
* Lookup plugin via physical file system
  • Loading branch information
cachemeifyoucan committed Feb 14, 2024
1 parent bca1dd7 commit 121fc4e
Show file tree
Hide file tree
Showing 4 changed files with 243 additions and 3 deletions.
62 changes: 62 additions & 0 deletions lib/AST/ModuleDependencies.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "swift/Frontend/Frontend.h"
#include "llvm/CAS/CASProvidingFileSystem.h"
#include "llvm/CAS/CachingOnDiskFileSystem.h"
#include "llvm/Config/config.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/PrefixMapper.h"
Expand Down Expand Up @@ -461,6 +462,67 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
FS->status(LayoutFile);
}
}

// Add plugin dylibs from the toolchain only by look through the plugin search
// directory and add those within the runtime resource dir.
SmallString<256> PluginDir(Opts.RuntimeResourcePath);
if (PluginDir.empty())
return;

llvm::sys::path::append(PluginDir, "host");
// Helper function to look for plugin to include into CASFS if provided
// directory is within the toolchain plugin.
auto recordFiles = [&](StringRef Path) {
if (!Path.starts_with(PluginDir))
return;
std::error_code EC;
for (auto I = FS->dir_begin(Path, EC); I != llvm::vfs::directory_iterator();
I = I.increment(EC)) {
if (I->type() != llvm::sys::fs::file_type::regular_file)
continue;
#if defined(_WIN32)
constexpr StringRef libPrefix{};
constexpr StringRef libSuffix = ".dll";
#else
constexpr StringRef libPrefix = "lib";
constexpr StringRef libSuffix = LTDL_SHLIB_EXT;
#endif
StringRef filename = llvm::sys::path::filename(I->path());
if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix))
FS->status(I->path());
}
};
for (auto &entry : Opts.PluginSearchOpts) {
switch (entry.getKind()) {

// '-load-plugin-library <library path>'.
case PluginSearchOption::Kind::LoadPluginLibrary: {
auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
FS->status(val.LibraryPath);
break;
}

// '-load-plugin-executable <executable path>#<module name>, ...'.
case PluginSearchOption::Kind::LoadPluginExecutable: {
// We don't have executable plugin in toolchain.
break;
}

// '-plugin-path <library search path>'.
case PluginSearchOption::Kind::PluginPath: {
auto &val = entry.get<PluginSearchOption::PluginPath>();
recordFiles(val.SearchPath);
break;
}

// '-external-plugin-path <library search path>#<server path>'.
case PluginSearchOption::Kind::ExternalPluginPath: {
auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
recordFiles(val.SearchPath);
break;
}
}
}
}

void SwiftDependencyTracker::startTracking() {
Expand Down
16 changes: 13 additions & 3 deletions lib/AST/PluginLoader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "swift/Basic/SourceManager.h"
#include "swift/Parse/Lexer.h"
#include "llvm/Config/config.h"
#include "llvm/Support/VirtualFileSystem.h"

using namespace swift;

Expand Down Expand Up @@ -59,6 +60,15 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) {
return "";
}

static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
getPluginLoadingFS(ASTContext &Ctx) {
// If there is a clang include tree FS, using real file system to load plugin
// as the FS in SourceMgr doesn't support directory iterator.
if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot)
return llvm::vfs::getRealFileSystem();
return Ctx.SourceMgr.getFileSystem();
}

llvm::DenseMap<Identifier, PluginLoader::PluginEntry> &
PluginLoader::getPluginMap() {
if (PluginMap.has_value()) {
Expand Down Expand Up @@ -86,7 +96,7 @@ PluginLoader::getPluginMap() {
(void)result;
};

auto fs = Ctx.SourceMgr.getFileSystem();
auto fs = getPluginLoadingFS(Ctx);
std::error_code ec;

for (auto &entry : Ctx.SearchPathOpts.PluginSearchOpts) {
Expand Down Expand Up @@ -162,7 +172,7 @@ PluginLoader::lookupPluginByModuleName(Identifier moduleName) {

llvm::Expected<LoadedLibraryPlugin *>
PluginLoader::loadLibraryPlugin(StringRef path) {
auto fs = Ctx.SourceMgr.getFileSystem();
auto fs = getPluginLoadingFS(Ctx);
SmallString<128> resolvedPath;
if (auto err = fs->getRealPath(path, resolvedPath)) {
return llvm::createStringError(err, err.message());
Expand All @@ -186,7 +196,7 @@ PluginLoader::loadLibraryPlugin(StringRef path) {

llvm::Expected<LoadedExecutablePlugin *>
PluginLoader::loadExecutablePlugin(StringRef path) {
auto fs = Ctx.SourceMgr.getFileSystem();
auto fs = getPluginLoadingFS(Ctx);
SmallString<128> resolvedPath;
if (auto err = fs->getRealPath(path, resolvedPath)) {
return llvm::createStringError(err, err.message());
Expand Down
74 changes: 74 additions & 0 deletions test/CAS/macro_option_set.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// REQUIRES: swift_swift_parser

// RUN: %empty-directory(%t)

// RUN: %target-swift-frontend -scan-dependencies -module-name MyApp -module-cache-path %t/clang-module-cache -O \
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
// RUN: %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -plugin-path %swift-plugin-dir

// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS

// FS: SwiftMacros

// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd
// RUN: %swift_frontend_plain @%t/SwiftShims.cmd

// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json Swift > %t/Swift.cmd
// RUN: %swift_frontend_plain @%t/Swift.cmd

// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json Swift moduleCacheKey | tr -d '\n' > %t/Swift.key
// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:SwiftShims moduleCacheKey | tr -d '\n' > %t/Shims.key
// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd

// RUN: echo "[{" > %/t/map.json
// RUN: echo "\"moduleName\": \"Swift\"," >> %/t/map.json
// RUN: echo "\"modulePath\": \"Swift.swiftmodule\"," >> %/t/map.json
// RUN: echo -n "\"moduleCacheKey\": " >> %/t/map.json
// RUN: cat %t/Swift.key >> %/t/map.json
// RUN: echo "," >> %/t/map.json
// RUN: echo "\"isFramework\": false" >> %/t/map.json
// RUN: echo "}," >> %/t/map.json
// RUN: echo "{" >> %/t/map.json
// RUN: echo "\"moduleName\": \"SwiftShims\"," >> %/t/map.json
// RUN: echo "\"clangModulePath\": \"SwiftShims.pcm\"," >> %/t/map.json
// RUN: echo -n "\"clangModuleCacheKey\": " >> %/t/map.json
// RUN: cat %t/Shims.key >> %/t/map.json
// RUN: echo "," >> %/t/map.json
// RUN: echo "\"isFramework\": false" >> %/t/map.json
// RUN: echo "}]" >> %/t/map.json
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid

// RUN: %target-swift-frontend \
// RUN: -typecheck -verify -cache-compile-job -cas-path %t/cas \
// RUN: -swift-version 5 -disable-implicit-swift-modules \
// RUN: -plugin-path %swift-plugin-dir \
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
// RUN: %s @%t/MyApp.cmd

import Swift

@attached(member, names: named(RawValue), named(rawValue), named(`init`), arbitrary)
@attached(extension, conformances: OptionSet)
public macro OptionSet<RawType>() =
#externalMacro(module: "SwiftMacros", type: "OptionSetMacro")

@OptionSet<UInt8>
struct ShippingOptions {
private enum Options: Int {
case nextDay
case secondDay
case priority
case standard
}

static let express: ShippingOptions = [.nextDay, .secondDay]
static let all: ShippingOptions = [.express, .priority, .standard]
}

let options = ShippingOptions.express
assert(options.contains(.nextDay))
assert(options.contains(.secondDay))
assert(!options.contains(.standard))

94 changes: 94 additions & 0 deletions test/CAS/macro_plugin.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// REQUIRES: swift_swift_parser

/// Test loading and external library through `-load-plugin-library`
/// TODO: switch this test case to use `-external-plugin-path`.

// RUN: %empty-directory(%t)
// RUN: %empty-directory(%t/plugins)
//
//== Build the plugin library
// RUN: %host-build-swift \
// RUN: -swift-version 5 \
// RUN: -emit-library \
// RUN: -o %t/plugins/%target-library-name(MacroDefinition) \
// RUN: -module-name=MacroDefinition \
// RUN: %S/../Macros/Inputs/syntax_macro_definitions.swift \
// RUN: -g -no-toolchain-stdlib-rpath

// RUN: %target-swift-frontend -scan-dependencies -module-name MyApp -module-cache-path %t/clang-module-cache -O \
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
// RUN: %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -load-plugin-library %t/plugins/%target-library-name(MacroDefinition)

// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS

// FS: MacroDefinition

// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd
// RUN: %swift_frontend_plain @%t/SwiftShims.cmd

// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json Swift > %t/Swift.cmd
// RUN: %swift_frontend_plain @%t/Swift.cmd

// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json Swift moduleCacheKey | tr -d '\n' > %t/Swift.key
// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:SwiftShims moduleCacheKey | tr -d '\n' > %t/Shims.key
// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd

// RUN: echo "[{" > %/t/map.json
// RUN: echo "\"moduleName\": \"Swift\"," >> %/t/map.json
// RUN: echo "\"modulePath\": \"Swift.swiftmodule\"," >> %/t/map.json
// RUN: echo -n "\"moduleCacheKey\": " >> %/t/map.json
// RUN: cat %t/Swift.key >> %/t/map.json
// RUN: echo "," >> %/t/map.json
// RUN: echo "\"isFramework\": false" >> %/t/map.json
// RUN: echo "}," >> %/t/map.json
// RUN: echo "{" >> %/t/map.json
// RUN: echo "\"moduleName\": \"SwiftShims\"," >> %/t/map.json
// RUN: echo "\"clangModulePath\": \"SwiftShims.pcm\"," >> %/t/map.json
// RUN: echo -n "\"clangModuleCacheKey\": " >> %/t/map.json
// RUN: cat %t/Shims.key >> %/t/map.json
// RUN: echo "," >> %/t/map.json
// RUN: echo "\"isFramework\": false" >> %/t/map.json
// RUN: echo "}]" >> %/t/map.json
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid

// RUN: %target-swift-frontend \
// RUN: -typecheck -verify -cache-compile-job -cas-path %t/cas \
// RUN: -swift-version 5 -disable-implicit-swift-modules \
// RUN: -load-plugin-library %t/plugins/%target-library-name(MacroDefinition) \
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
// RUN: %s @%t/MyApp.cmd

@attached(extension, conformances: P, names: named(requirement))
macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro")

protocol P {
static func requirement()
}

struct Wrapped: P {
static func requirement() {
print("Wrapped.requirement")
}
}

@DelegatedConformance
struct Generic<Element> {}

// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Generic: P where Element: P {\n static func requirement() {\n Element.requirement()\n }\n}"}}

func requiresP(_ value: (some P).Type) {
value.requirement()
}

requiresP(Generic<Wrapped>.self)

struct Outer {
@DelegatedConformance
struct Nested<Element> {}
}

// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Outer.Nested: P where Element: P {\n static func requirement() {\n Element.requirement()\n }\n}"}}

requiresP(Outer.Nested<Wrapped>.self)

0 comments on commit 121fc4e

Please sign in to comment.