Skip to content

Commit

Permalink
[Remarks] Add a specialized RemarkStreamer for SIL remarks
Browse files Browse the repository at this point in the history
This allows the usage of the whole remark infrastructure developed in
LLVM, which includes a new binary format, metadata in object files, etc.

This gets rid of the YAMLTraits-based remark serialization and does the
plumbing for hooking to LLVM's main remark streamer.

For more about the idea behind LLVM's main remark streamer, see the
docs/Remarks.rst changes in https://reviews.llvm.org/D73676.

The flags are now:

* -save-optimization-record: enable remarks, defaults to YAML
* -save-optimization-record=<format>: enable remarks, use <format> for
serialization
* -save-optimization-record-passes <regex>: only serialize passes that
match <regex>.

The YAMLTraits in swift had a different `flow` setting for the debug
location, resulting in some test changes.
  • Loading branch information
francisvm committed Mar 3, 2020
1 parent e7b2850 commit e724eba
Show file tree
Hide file tree
Showing 29 changed files with 505 additions and 230 deletions.
2 changes: 2 additions & 0 deletions include/swift/AST/DiagnosticsFrontend.def
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,8 @@ NOTE(compiled_module_invalid_reason,none,
ERROR(unknown_forced_module_loading_mode,none,
"unknown value for SWIFT_FORCE_MODULE_LOADING variable: '%0'",
(StringRef))
ERROR(error_creating_remark_serializer,none,
"error while creating remark serializer: '%0'", (StringRef))

REMARK(interface_file_lock_failure,none,
"could not acquire lock file for module interface '%0'", (StringRef))
Expand Down
10 changes: 9 additions & 1 deletion include/swift/AST/SILOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include "swift/Basic/OptimizationMode.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Remarks/RemarkFormat.h"
#include <string>
#include <climits>

Expand Down Expand Up @@ -161,10 +162,17 @@ class SILOptions {
/// pipeline or after serialization.
bool StripOwnershipAfterSerialization = true;

/// The name of the file to which the backend should save YAML optimization
/// The name of the file to which the backend should save optimization
/// records.
std::string OptRecordFile;

/// The regex that filters the passes that should be saved to the optimization
/// records.
std::string OptRecordPasses;

/// The format used for serializing remarks (default: YAML)
llvm::remarks::Format OptRecordFormat = llvm::remarks::Format::YAML;

SILOptions() {}

/// Return a hash code of any components from these options that should
Expand Down
3 changes: 2 additions & 1 deletion include/swift/Basic/FileTypes.def
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ TYPE("tbd", TBD, "tbd", "")
TYPE("module-trace", ModuleTrace, "trace.json", "")

TYPE("index-data", IndexData, "", "")
TYPE("opt-record", OptRecord, "opt.yaml", "")
TYPE("yaml-opt-record", YAMLOptRecord, "opt.yaml", "")
TYPE("bitstream-opt-record",BitstreamOptRecord, "opt.bitstream", "")

// Overlay files declare wrapper modules, called "separately-imported overlays",
// that should be automatically imported when a particular module is imported.
Expand Down
3 changes: 3 additions & 0 deletions include/swift/Driver/ToolChain.h
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,9 @@ class ToolChain {
virtual void validateArguments(DiagnosticEngine &diags,
const llvm::opt::ArgList &args,
StringRef defaultTarget) const {}

llvm::Expected<file_types::ID>
remarkFileTypeFromArgs(const llvm::opt::ArgList &Args) const;
};
} // end namespace driver
} // end namespace swift
Expand Down
12 changes: 11 additions & 1 deletion include/swift/Option/Options.td
Original file line number Diff line number Diff line change
Expand Up @@ -548,10 +548,20 @@ def Rpass_missed_EQ : Joined<["-"], "Rpass-missed=">,

def save_optimization_record : Flag<["-"], "save-optimization-record">,
Flags<[FrontendOption]>, HelpText<"Generate a YAML optimization record file">;
def save_optimization_record_EQ : Joined<["-"], "save-optimization-record=">,
Flags<[FrontendOption]>,
HelpText<"Generate an optimization record file in a specific format "
"(default: YAML)">, MetaVarName<"<format>">;
def save_optimization_record_path :
Separate<["-"], "save-optimization-record-path">,
Flags<[FrontendOption, ArgumentIsPath]>,
HelpText<"Specify the file name of any generated YAML optimization record">;
HelpText<"Specify the file name of any generated optimization record">;
def save_optimization_record_passes :
Separate<["-"], "save-optimization-record-passes">,
Flags<[FrontendOption]>,
HelpText<"Only include passes which match a specified regular expression in"
"the generated optimization record "
"(by default, include all passes)">, MetaVarName<"<regex>">;

// Platform options.
def enable_app_extension : Flag<["-"], "application-extension">,
Expand Down
22 changes: 16 additions & 6 deletions include/swift/SIL/OptimizationRemark.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#define SWIFT_SIL_OPTIMIZATIONREMARKEMITTER_H

#include "swift/Basic/SourceLoc.h"
#include "swift/Demangling/Demangler.h"
#include "swift/SIL/SILBasicBlock.h"
#include "swift/SIL/SILInstruction.h"
#include "swift/SIL/SILModule.h"
Expand Down Expand Up @@ -75,23 +76,30 @@ template <typename DerivedT> class Remark {
StringRef passName;

/// Textual identifier for the remark (single-word, camel-case). Can be used
/// by external tools reading the YAML output file for optimization remarks to
/// by external tools reading the output file for optimization remarks to
/// identify the remark.
StringRef identifier;
SmallString<32> identifier;

/// Source location for the diagnostics.
SourceLoc location;

/// The function for the diagnostics.
SILFunction *function;

/// The demangled name of \p Function.
SmallString<64> demangledFunctionName;

/// Indentation used if this remarks is printed as a debug message.
unsigned indentDebugWidth = 0;

protected:
Remark(StringRef identifier, SILInstruction &i)
: identifier(identifier), location(i.getLoc().getSourceLoc()),
function(i.getParent()->getParent()) {}
: identifier((Twine("sil.") + identifier).str()),
location(i.getLoc().getSourceLoc()),
function(i.getParent()->getParent()),
demangledFunctionName(Demangle::demangleSymbolAsString(
function->getName(),
Demangle::DemangleOptions::SimplifiedUIDemangleOptions())) {}

public:
DerivedT &operator<<(StringRef s) {
Expand All @@ -112,11 +120,13 @@ template <typename DerivedT> class Remark {
StringRef getPassName() const { return passName; }
StringRef getIdentifier() const { return identifier; }
SILFunction *getFunction() const { return function; }
StringRef getDemangledFunctionName() const { return demangledFunctionName; }
SourceLoc getLocation() const { return location; }
std::string getMsg() const;
std::string getDebugMsg() const;
Remark<DerivedT> &getRemark() { return *this; }
SmallVector<Argument, 4> &getArgs() { return args; }
ArrayRef<Argument> getArgs() const { return args; }

void setPassName(StringRef name) { passName = name; }
};
Expand Down Expand Up @@ -156,7 +166,7 @@ class Emitter {
void emit(T remarkBuilder, decltype(remarkBuilder()) * = nullptr) {
using RemarkT = decltype(remarkBuilder());
// Avoid building the remark unless remarks are enabled.
if (isEnabled<RemarkT>() || module.getOptRecordStream()) {
if (isEnabled<RemarkT>() || module.getSILRemarkStreamer()) {
auto rb = remarkBuilder();
rb.setPassName(passName);
emit(rb);
Expand All @@ -171,7 +181,7 @@ class Emitter {
using RemarkT = decltype(remarkBuilder());
// Avoid building the remark unless remarks are enabled.
bool emitRemark = emitter && (emitter->isEnabled<RemarkT>() ||
emitter->module.getOptRecordStream());
emitter->module.getSILRemarkStreamer());
// Same for DEBUG.
bool shouldEmitDebug = false;
#ifndef NDEBUG
Expand Down
20 changes: 11 additions & 9 deletions include/swift/SIL/SILModule.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ class SILUndef;
class SourceFile;
class SerializedSILLoader;
class SILFunctionBuilder;
class SILRemarkStreamer;

namespace Lowering {
class SILGenModule;
Expand Down Expand Up @@ -232,13 +233,11 @@ class SILModule {
// The list of SILProperties in the module.
PropertyListType properties;

/// This is the underlying raw stream of OptRecordStream.
///
/// It is also owned by SILModule in order to keep their lifetime in sync.
std::unique_ptr<llvm::raw_ostream> OptRecordRawStream;
/// The remark output stream used to record SIL remarks to a file.
std::unique_ptr<llvm::raw_fd_ostream> silRemarkStream;

/// If non-null, the YAML file where remarks should be recorded.
std::unique_ptr<llvm::yaml::Output> OptRecordStream;
/// The remark streamer used to serialize SIL remarks to a file.
std::unique_ptr<swift::SILRemarkStreamer> silRemarkStreamer;

/// This is a cache of intrinsic Function declarations to numeric ID mappings.
llvm::DenseMap<Identifier, IntrinsicInfo> IntrinsicIDCache;
Expand Down Expand Up @@ -524,9 +523,12 @@ class SILModule {
return coverageMaps;
}

llvm::yaml::Output *getOptRecordStream() { return OptRecordStream.get(); }
void setOptRecordStream(std::unique_ptr<llvm::yaml::Output> &&Stream,
std::unique_ptr<llvm::raw_ostream> &&RawStream);
swift::SILRemarkStreamer *getSILRemarkStreamer() {
return silRemarkStreamer.get();
}
void setSILRemarkStreamer(
std::unique_ptr<llvm::raw_fd_ostream> &&remarkStream,
std::unique_ptr<swift::SILRemarkStreamer> &&remarkStreamer);

// This is currently limited to VarDecl because the visibility of global
// variables and class properties is straightforward, while the visibility of
Expand Down
104 changes: 104 additions & 0 deletions include/swift/SIL/SILRemarkStreamer.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
//===--- SILRemarkStreamer.h - Interface for streaming remarks --*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
/// \file
/// This file defines the interface used to stream SIL optimization diagnostics
/// through LLVM's RemarkStreamer interface.
//
//===----------------------------------------------------------------------===//

#ifndef SWIFT_SIL_SILREMARKSTREAMER_H
#define SWIFT_SIL_SILREMARKSTREAMER_H

#include "swift/SIL/OptimizationRemark.h"
#include "llvm/Remarks/RemarkStreamer.h"

namespace swift {

class SILRemarkStreamer {
llvm::remarks::RemarkStreamer &llvmStreamer;
// Source manager for resolving source locations.
const SourceManager &srcMgr;
/// Convert diagnostics into LLVM remark objects.
/// The lifetime of the members of the result is bound to the lifetime of
/// the SIL remarks.
template <typename RemarkT>
llvm::remarks::Remark
toLLVMRemark(const OptRemark::Remark<RemarkT> &remark) const;

public:
SILRemarkStreamer(llvm::remarks::RemarkStreamer &llvmStreamer,
const SourceManager &srcMgr)
: llvmStreamer(llvmStreamer), srcMgr(srcMgr) {}
/// Emit a remark through the streamer.
template <typename RemarkT>
void emit(const OptRemark::Remark<RemarkT> &remark);
};

std::pair<std::unique_ptr<llvm::raw_fd_ostream>,
std::unique_ptr<SILRemarkStreamer>>
createSILRemarkStreamer(SILModule &srcMgr, StringRef filename, StringRef passes,
llvm::remarks::Format format,
DiagnosticEngine &diagEngine, SourceManager &sourceMgr);

// Implementation for template member functions.

// OptRemark type -> llvm::remarks::Type
template <typename RemarkT> static llvm::remarks::Type toRemarkType() {
if (std::is_same<RemarkT, OptRemark::RemarkPassed>::value)
return llvm::remarks::Type::Passed;
if (std::is_same<RemarkT, OptRemark::RemarkMissed>::value)
return llvm::remarks::Type::Missed;
llvm_unreachable("Unknown remark type");
}

static inline Optional<llvm::remarks::RemarkLocation>
toRemarkLocation(const SourceLoc &loc, const SourceManager &srcMgr) {
if (!loc.isValid())
return None;

StringRef file = srcMgr.getDisplayNameForLoc(loc);
unsigned line, col;
std::tie(line, col) = srcMgr.getLineAndColumn(loc);
return llvm::remarks::RemarkLocation{file, line, col};
}

template <typename RemarkT>
llvm::remarks::Remark SILRemarkStreamer::toLLVMRemark(
const OptRemark::Remark<RemarkT> &optRemark) const {
llvm::remarks::Remark llvmRemark; // The result.
llvmRemark.RemarkType = toRemarkType<RemarkT>();
llvmRemark.PassName = optRemark.getPassName();
llvmRemark.RemarkName = optRemark.getIdentifier();
llvmRemark.FunctionName = optRemark.getDemangledFunctionName();
llvmRemark.Loc = toRemarkLocation(optRemark.getLocation(), srcMgr);

for (const OptRemark::Argument &arg : optRemark.getArgs()) {
llvmRemark.Args.emplace_back();
llvmRemark.Args.back().Key = arg.key;
llvmRemark.Args.back().Val = arg.val;
llvmRemark.Args.back().Loc = toRemarkLocation(arg.loc, srcMgr);
}

return llvmRemark;
}

template <typename RemarkT>
void SILRemarkStreamer::emit(const OptRemark::Remark<RemarkT> &optRemark) {
if (!llvmStreamer.matchesFilter(optRemark.getPassName()))
return;

return llvmStreamer.getSerializer().emit(toLLVMRemark(optRemark));
}

} // namespace swift
#endif
9 changes: 6 additions & 3 deletions lib/Basic/FileTypes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ bool file_types::isTextual(ID Id) {
case file_types::TY_ImportedModules:
case file_types::TY_TBD:
case file_types::TY_ModuleTrace:
case file_types::TY_OptRecord:
case file_types::TY_YAMLOptRecord:
case file_types::TY_SwiftModuleInterfaceFile:
case file_types::TY_PrivateSwiftModuleInterfaceFile:
case file_types::TY_SwiftOverlayFile:
Expand All @@ -102,6 +102,7 @@ bool file_types::isTextual(ID Id) {
case file_types::TY_Nothing:
case file_types::TY_Remapping:
case file_types::TY_IndexData:
case file_types::TY_BitstreamOptRecord:
return false;
case file_types::TY_INVALID:
llvm_unreachable("Invalid type ID.");
Expand Down Expand Up @@ -146,7 +147,8 @@ bool file_types::isAfterLLVM(ID Id) {
case file_types::TY_Remapping:
case file_types::TY_IndexData:
case file_types::TY_ModuleTrace:
case file_types::TY_OptRecord:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
case file_types::TY_SwiftModuleInterfaceFile:
case file_types::TY_PrivateSwiftModuleInterfaceFile:
return false;
Expand Down Expand Up @@ -195,7 +197,8 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
case file_types::TY_Remapping:
case file_types::TY_IndexData:
case file_types::TY_ModuleTrace:
case file_types::TY_OptRecord:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
return false;
case file_types::TY_INVALID:
llvm_unreachable("Invalid type ID.");
Expand Down
19 changes: 14 additions & 5 deletions lib/Driver/Driver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Remarks/RemarkFormat.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
Expand Down Expand Up @@ -1941,7 +1942,8 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
case file_types::TY_ImportedModules:
case file_types::TY_TBD:
case file_types::TY_ModuleTrace:
case file_types::TY_OptRecord:
case file_types::TY_YAMLOptRecord:
case file_types::TY_BitstreamOptRecord:
case file_types::TY_SwiftModuleInterfaceFile:
case file_types::TY_PrivateSwiftModuleInterfaceFile:
case file_types::TY_SwiftCrossImportDir:
Expand Down Expand Up @@ -2766,6 +2768,7 @@ Job *Driver::buildJobsForAction(Compilation &C, const JobAction *JA,
Output.get());

if (C.getArgs().hasArg(options::OPT_save_optimization_record,
options::OPT_save_optimization_record_EQ,
options::OPT_save_optimization_record_path))
chooseOptimizationRecordPath(C, workingDirectory, Buf, Output.get());

Expand Down Expand Up @@ -3202,12 +3205,18 @@ void Driver::chooseOptimizationRecordPath(Compilation &C,
CommandOutput *Output) const {
const OutputInfo &OI = C.getOutputInfo();
if (OI.CompilerMode == OutputInfo::Mode::SingleCompile) {
llvm::Expected<file_types::ID> FileType =
C.getToolChain().remarkFileTypeFromArgs(C.getArgs());
if (!FileType) {
Diags.diagnose({}, diag::error_creating_remark_serializer,
llvm::toString(FileType.takeError()));
return;
}
auto filename = *getOutputFilenameFromPathArgOrAsTopLevel(
OI, C.getArgs(), options::OPT_save_optimization_record_path,
file_types::TY_OptRecord, /*TreatAsTopLevelOutput=*/true,
workingDirectory, Buf);
OI, C.getArgs(), options::OPT_save_optimization_record_path, *FileType,
/*TreatAsTopLevelOutput=*/true, workingDirectory, Buf);

Output->setAdditionalOutputForType(file_types::TY_OptRecord, filename);
Output->setAdditionalOutputForType(*FileType, filename);
} else
// FIXME: We should use the OutputMap in this case.
Diags.diagnose({}, diag::warn_opt_remark_disabled);
Expand Down
Loading

0 comments on commit e724eba

Please sign in to comment.