diff --git a/include/swift/SIL/SILNodes.def.rej b/include/swift/SIL/SILNodes.def.rej deleted file mode 100644 index bd5c06c134680..0000000000000 --- a/include/swift/SIL/SILNodes.def.rej +++ /dev/null @@ -1,19 +0,0 @@ -*************** -*** 405,414 **** - ConversionInst, None, DoesNotRelease) - SINGLE_VALUE_INST(ConvertEscapeToNoEscapeInst, convert_escape_to_noescape, - ConversionInst, None, DoesNotRelease) -- SINGLE_VALUE_INST(ThinFunctionToPointerInst, thin_function_to_pointer, -- ConversionInst, None, DoesNotRelease) -- SINGLE_VALUE_INST(PointerToThinFunctionInst, pointer_to_thin_function, -- ConversionInst, None, DoesNotRelease) - BRIDGED_SINGLE_VALUE_INST(RefToBridgeObjectInst, ref_to_bridge_object, - ConversionInst, None, DoesNotRelease) - BRIDGED_SINGLE_VALUE_INST(BridgeObjectToRefInst, bridge_object_to_ref, ---- 405,410 ---- - ConversionInst, None, DoesNotRelease) - SINGLE_VALUE_INST(ConvertEscapeToNoEscapeInst, convert_escape_to_noescape, - ConversionInst, None, DoesNotRelease) - BRIDGED_SINGLE_VALUE_INST(RefToBridgeObjectInst, ref_to_bridge_object, - ConversionInst, None, DoesNotRelease) - BRIDGED_SINGLE_VALUE_INST(BridgeObjectToRefInst, bridge_object_to_ref, diff --git a/stdlib/public/BackDeployConcurrency/Actor.cpp b/stdlib/public/BackDeployConcurrency/Actor.cpp new file mode 100644 index 0000000000000..c14628bfd445c --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Actor.cpp @@ -0,0 +1,2002 @@ +///===--- Actor.cpp - Standard actor implementation ------------------------===/// +/// +/// This source file is part of the Swift.org open source project +/// +/// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +/// Licensed under Apache License v2.0 with Runtime Library Exception +/// +/// See https:///swift.org/LICENSE.txt for license information +/// See https:///swift.org/CONTRIBUTORS.txt for the list of Swift project authors +/// +///===----------------------------------------------------------------------===/// +/// +/// The default actor implementation for Swift actors, plus related +/// routines such as generic executor enqueuing and switching. +/// +///===----------------------------------------------------------------------===/// + +#include "ConcurrencyRuntime.h" + +#ifdef _WIN32 +// On Windows, an include below triggers an indirect include of minwindef.h +// which contains a definition of the `max` macro, generating an error in our +// use of std::max in this file. This define prevents those macros from being +// defined. +#define NOMINMAX +#endif + +#include "CompatibilityOverride.h" +#include "swift/Runtime/Atomic.h" +#include "swift/Runtime/Casting.h" +#include "swift/Runtime/Once.h" +#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/ThreadLocal.h" +#include "swift/Runtime/ThreadLocalStorage.h" +#include "swift/Basic/ListMerger.h" +#ifndef SWIFT_CONCURRENCY_BACK_DEPLOYMENT +#include "llvm/Config/config.h" +#else +// All platforms where we care about back deployment have a known +// configurations. +#define HAVE_PTHREAD_H 1 +#define SWIFT_OBJC_INTEROP 1 +#endif +#include "llvm/ADT/PointerIntPair.h" +#include "Actor.h" +#include "Task.h" +#include "TaskPrivate.h" +#include "VoucherSupport.h" + +#if SWIFT_CONCURRENCY_ENABLE_DISPATCH +#include +#endif + +#if defined(__APPLE__) +#include +#elif defined(__ANDROID__) +#include +#endif + +#if defined(__ELF__) +#include +#endif + +#if defined(__APPLE__) +#include +#elif defined(__ANDROID__) +#include +#endif + +#if defined(__ELF__) +#include +#endif + +#if HAVE_PTHREAD_H +#include + +// Only use __has_include since HAVE_PTHREAD_NP_H is not provided. +#if __has_include() +#include +#endif +#endif + +#if defined(_WIN32) +#include +#include +#include +#endif + +#if SWIFT_OBJC_INTEROP +extern "C" void *objc_autoreleasePoolPush(); +extern "C" void objc_autoreleasePoolPop(void *); +#endif + +using namespace swift; + +/// Should we yield the thread? +static bool shouldYieldThread() { + // FIXME: system scheduler integration + return false; +} + +/*****************************************************************************/ +/******************************* TASK TRACKING ******************************/ +/*****************************************************************************/ + +namespace { + +/// An extremely silly class which exists to make pointer +/// default-initialization constexpr. +template struct Pointer { + T *Value; + constexpr Pointer() : Value(nullptr) {} + constexpr Pointer(T *value) : Value(value) {} + operator T *() const { return Value; } + T *operator->() const { return Value; } +}; + +/// A class which encapsulates the information we track about +/// the current thread and active executor. +class ExecutorTrackingInfo { + /// A thread-local variable pointing to the active tracking + /// information about the current thread, if any. + /// + /// TODO: this is obviously runtime-internal and therefore not + /// reasonable to make ABI. We might want to also provide a way + /// for generated code to efficiently query the identity of the + /// current executor, in order to do a cheap comparison to avoid + /// doing all the work to suspend the task when we're already on + /// the right executor. It would make sense for that to be a + /// separate thread-local variable (or whatever is most efficient + /// on the target platform). + static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( + Pointer, ActiveInfoInThread, + SWIFT_CONCURRENCY_EXECUTOR_TRACKING_INFO_KEY); + + /// The active executor. + ExecutorRef ActiveExecutor = ExecutorRef::generic(); + + /// Whether this context allows switching. Some contexts do not; + /// for example, we do not allow switching from swift_job_run + /// unless the passed-in executor is generic. + bool AllowsSwitching = true; + + VoucherManager voucherManager; + + /// The tracking info that was active when this one was entered. + ExecutorTrackingInfo *SavedInfo; + +public: + ExecutorTrackingInfo() = default; + + ExecutorTrackingInfo(const ExecutorTrackingInfo &) = delete; + ExecutorTrackingInfo &operator=(const ExecutorTrackingInfo &) = delete; + + /// Unconditionally initialize a fresh tracking state on the + /// current state, shadowing any previous tracking state. + /// leave() must be called beforet the object goes out of scope. + void enterAndShadow(ExecutorRef currentExecutor) { + ActiveExecutor = currentExecutor; + SavedInfo = ActiveInfoInThread.get(); + ActiveInfoInThread.set(this); + } + + void swapToJob(Job *job) { voucherManager.swapToJob(job); } + + void restoreVoucher(AsyncTask *task) { voucherManager.restoreVoucher(task); } + + ExecutorRef getActiveExecutor() const { + return ActiveExecutor; + } + + void setActiveExecutor(ExecutorRef newExecutor) { + ActiveExecutor = newExecutor; + } + + + bool allowsSwitching() const { + return AllowsSwitching; + } + + /// Disallow switching in this tracking context. This should only + /// be set on a new tracking info, before any jobs are run in it. + void disallowSwitching() { + AllowsSwitching = false; + } + + static ExecutorTrackingInfo *current() { + return ActiveInfoInThread.get(); + } + + void leave() { + voucherManager.leave(); + ActiveInfoInThread.set(SavedInfo); + } +}; + +class ActiveTask { + /// A thread-local variable pointing to the active tracking + /// information about the current thread, if any. + static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL(Pointer, Value, + SWIFT_CONCURRENCY_TASK_KEY); + +public: + static void set(AsyncTask *task) { Value.set(task); } + static AsyncTask *get() { return Value.get(); } +}; + +/// Define the thread-locals. +SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( + Pointer, + ActiveTask::Value, + SWIFT_CONCURRENCY_TASK_KEY); + +SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( + Pointer, + ExecutorTrackingInfo::ActiveInfoInThread, + SWIFT_CONCURRENCY_EXECUTOR_TRACKING_INFO_KEY); + +} // end anonymous namespace + +void swift::runJobInEstablishedExecutorContext(Job *job) { + _swift_tsan_acquire(job); + +#if SWIFT_OBJC_INTEROP + auto pool = objc_autoreleasePoolPush(); +#endif + + if (auto task = dyn_cast(job)) { + // Update the active task in the current thread. + ActiveTask::set(task); + + // Update the task status to say that it's running on the + // current thread. If the task suspends somewhere, it should + // update the task status appropriately; we don't need to update + // it afterwards. + task->flagAsRunning(); + + task->runInFullyEstablishedContext(); + + assert(ActiveTask::get() == nullptr && + "active task wasn't cleared before susspending?"); + } else { + // There's no extra bookkeeping to do for simple jobs besides swapping in + // the voucher. + ExecutorTrackingInfo::current()->swapToJob(job); + job->runSimpleInFullyEstablishedContext(); + } + +#if SWIFT_OBJC_INTEROP + objc_autoreleasePoolPop(pool); +#endif + + _swift_tsan_release(job); +} + +void swift::adoptTaskVoucher(AsyncTask *task) { + ExecutorTrackingInfo::current()->swapToJob(task); +} + +void swift::restoreTaskVoucher(AsyncTask *task) { + ExecutorTrackingInfo::current()->restoreVoucher(task); +} + +SWIFT_CC(swift) +AsyncTask *swift::swift_task_getCurrent() { + return ActiveTask::get(); +} + +AsyncTask *swift::_swift_task_clearCurrent() { + auto task = ActiveTask::get(); + ActiveTask::set(nullptr); + return task; +} + +SWIFT_CC(swift) +static ExecutorRef swift_task_getCurrentExecutorImpl() { + auto currentTracking = ExecutorTrackingInfo::current(); + auto result = (currentTracking ? currentTracking->getActiveExecutor() + : ExecutorRef::generic()); + SWIFT_TASK_DEBUG_LOG("getting current executor %p", result.getIdentity()); + return result; +} + +#if defined(_WIN32) +static HANDLE __initialPthread = INVALID_HANDLE_VALUE; +#endif + +/// Determine whether we are currently executing on the main thread +/// independently of whether we know that we are on the main actor. +static bool isExecutingOnMainThread() { +#if defined(__linux__) + return syscall(SYS_gettid) == getpid(); +#elif defined(_WIN32) + if (__initialPthread == INVALID_HANDLE_VALUE) { + DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), + GetCurrentProcess(), &__initialPthread, 0, FALSE, + DUPLICATE_SAME_ACCESS); + } + + return __initialPthread == GetCurrentThread(); +#else + return pthread_main_np() == 1; +#endif +} + +JobPriority swift::swift_task_getCurrentThreadPriority() { +#if defined(__APPLE__) + return static_cast(qos_class_self()); +#else + if (isExecutingOnMainThread()) + return JobPriority::UserInitiated; + + return JobPriority::Unspecified; +#endif +} + +SWIFT_CC(swift) +static bool swift_task_isCurrentExecutorImpl(ExecutorRef executor) { + if (auto currentTracking = ExecutorTrackingInfo::current()) { + return currentTracking->getActiveExecutor() == executor; + } + + return executor.isMainExecutor() && isExecutingOnMainThread(); +} + +/// Logging level for unexpected executors: +/// 0 - no logging +/// 1 - warn on each instance +/// 2 - fatal error +static unsigned unexpectedExecutorLogLevel = 1; + +static void checkUnexpectedExecutorLogLevel(void *context) { + const char *levelStr = getenv("SWIFT_UNEXPECTED_EXECUTOR_LOG_LEVEL"); + if (!levelStr) + return; + + long level = strtol(levelStr, nullptr, 0); + if (level >= 0 && level < 3) + unexpectedExecutorLogLevel = level; +} + +SWIFT_CC(swift) +void swift::swift_task_reportUnexpectedExecutor( + const unsigned char *file, uintptr_t fileLength, bool fileIsASCII, + uintptr_t line, ExecutorRef executor) { + // Make sure we have an appropriate log level. + static swift_once_t logLevelToken; + swift_once(&logLevelToken, checkUnexpectedExecutorLogLevel, nullptr); + + bool isFatalError = false; + switch (unexpectedExecutorLogLevel) { + case 0: + return; + + case 1: + isFatalError = false; + break; + + case 2: + isFatalError = true; + break; + } + + const char *functionIsolation; + const char *whereExpected; + if (executor.isMainExecutor()) { + functionIsolation = "@MainActor function"; + whereExpected = "the main thread"; + } else { + functionIsolation = "actor-isolated function"; + whereExpected = "the same actor"; + } + + char *message; + swift_asprintf( + &message, + "%s: data race detected: %s at %.*s:%d was not called on %s\n", + isFatalError ? "error" : "warning", functionIsolation, + (int)fileLength, file, (int)line, whereExpected); + + if (_swift_shouldReportFatalErrorsToDebugger()) { + RuntimeErrorDetails details = { + .version = RuntimeErrorDetails::currentVersion, + .errorType = "actor-isolation-violation", + .currentStackDescription = "Actor-isolated function called from another thread", + .framesToSkip = 1, + }; + _swift_reportToDebugger( + isFatalError ? RuntimeErrorFlagFatal : RuntimeErrorFlagNone, message, + &details); + } + +#if defined(_WIN32) +#define STDERR_FILENO 2 + _write(STDERR_FILENO, message, strlen(message)); +#else + write(STDERR_FILENO, message, strlen(message)); +#endif +#if defined(__APPLE__) + asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", message); +#elif defined(__ANDROID__) + __android_log_print(ANDROID_LOG_FATAL, "SwiftRuntime", "%s", message); +#endif + + free(message); + + if (isFatalError) + abort(); +} + +/*****************************************************************************/ +/*********************** DEFAULT ACTOR IMPLEMENTATION ************************/ +/*****************************************************************************/ + +namespace { + +class DefaultActorImpl; + +/// A job to process a default actor. Allocated inline in the actor. +class ProcessInlineJob : public Job { +public: + ProcessInlineJob(JobPriority priority) + : Job({JobKind::DefaultActorInline, priority}, &process) {} + + SWIFT_CC(swiftasync) + static void process(Job *job); + + static bool classof(const Job *job) { + return job->Flags.getKind() == JobKind::DefaultActorInline; + } +}; + +/// A job to process a default actor that's allocated separately from +/// the actor but doesn't need the override mechanics. +class ProcessOutOfLineJob : public Job { + DefaultActorImpl *Actor; +public: + ProcessOutOfLineJob(DefaultActorImpl *actor, JobPriority priority) + : Job({JobKind::DefaultActorSeparate, priority}, &process), + Actor(actor) {} + + SWIFT_CC(swiftasync) + static void process(Job *job); + + static bool classof(const Job *job) { + return job->Flags.getKind() == JobKind::DefaultActorSeparate; + } +}; + +/// A job to process a default actor with a new priority; allocated +/// separately from the actor. +class ProcessOverrideJob; + +/// Information about the currently-running processing job. +struct RunningJobInfo { + enum KindType : uint8_t { + Inline, Override, Other + }; + KindType Kind; + JobPriority Priority; + ProcessOverrideJob *OverrideJob; + + bool wasInlineJob() const { + return Kind == Inline; + } + + static RunningJobInfo forOther(JobPriority priority) { + return {Other, priority, nullptr}; + } + static RunningJobInfo forInline(JobPriority priority) { + return {Inline, priority, nullptr}; + } + static RunningJobInfo forOverride(ProcessOverrideJob *job); + + void setAbandoned(); + void setRunning(); + bool waitForActivation(); +}; + +class JobRef { + enum : uintptr_t { + NeedsPreprocessing = 0x1, + IsOverride = 0x2, + JobMask = ~uintptr_t(NeedsPreprocessing | IsOverride) + }; + + /// A Job* that may have one of the two bits above mangled into it. + uintptr_t Value; + + JobRef(Job *job, unsigned flags) + : Value(reinterpret_cast(job) | flags) {} +public: + constexpr JobRef() : Value(0) {} + + /// Return a reference to a job that's been properly preprocessed. + static JobRef getPreprocessed(Job *job) { + /// We allow null pointers here. + return { job, 0 }; + } + + /// Return a reference to a job that hasn't been preprocesssed yet. + static JobRef getUnpreprocessed(Job *job) { + assert(job && "passing a null job"); + return { job, NeedsPreprocessing }; + } + + /// Return a reference to an override job, which needs special + /// treatment during preprocessing. + static JobRef getOverride(ProcessOverrideJob *job); + + /// Is this a null reference? + operator bool() const { return Value != 0; } + + /// Does this job need to be pre-processed before we can treat + /// the job queue as a proper queue? + bool needsPreprocessing() const { + return Value & NeedsPreprocessing; + } + + /// Is this an unpreprocessed override job? + bool isOverride() const { + return Value & IsOverride; + } + + /// Given that this is an override job, return it. + ProcessOverrideJob *getAsOverride() const { + assert(isOverride()); + return reinterpret_cast(Value & JobMask); + } + ProcessOverrideJob *getAsPreprocessedOverride() const; + + Job *getAsJob() const { + assert(!isOverride()); + return reinterpret_cast(Value & JobMask); + } + Job *getAsPreprocessedJob() const { + assert(!isOverride() && !needsPreprocessing()); + return reinterpret_cast(Value); + } + + bool operator==(JobRef other) const { + return Value == other.Value; + } + bool operator!=(JobRef other) const { + return Value != other.Value; + } +}; + +/// The default actor implementation. +/// +/// Ownership of the actor is subtle. Jobs are assumed to keep the actor +/// alive as long as they're executing on it; this allows us to avoid +/// retaining and releasing whenever threads are scheduled to run a job. +/// While jobs are enqueued on the actor, there is a conceptual shared +/// ownership of the currently-enqueued jobs which is passed around +/// between threads and processing jobs and managed using extra retains +/// and releases of the actor. The basic invariant is as follows: +/// +/// - Let R be 1 if there are jobs enqueued on the actor or if a job +/// is currently running on the actor; otherwise let R be 0. +/// - Let N be the number of active processing jobs for the actor. +/// - N >= R +/// - There are N - R extra retains of the actor. +/// +/// We can think of this as there being one "owning" processing job +/// and K "extra" jobs. If there is a processing job that is actively +/// running the actor, it is always the owning job; otherwise, any of +/// the N jobs may win the race to become the owning job. +/// +/// We then have the following ownership rules: +/// +/// - When we enqueue the first job on an actor, then R becomes 1, and +/// we must create a processing job so that N >= R. We do not need to +/// retain the actor. +/// - When we create an extra job to process an actor (e.g. because of +/// priority overrides), N increases but R remains the same. We must +/// retain the actor. +/// - When we start running an actor, our job definitively becomes the +/// owning job, but neither N nor R changes. We do not need to retain +/// the actor. +/// - When we go to start running an actor and for whatever reason we +/// don't actually do so, we are eliminating an extra processing job, +/// and so N decreases but R remains the same. We must release the +/// actor. +/// - When we are running an actor and give it up, and there are no +/// remaining jobs on it, then R becomes 0 and N decreases by 1. +/// We do not need to release the actor. +/// - When we are running an actor and give it up, and there are jobs +/// remaining on it, then R remains 1 but N is decreasing by 1. +/// We must either release the actor or create a new processing job +/// for it to maintain the balance. +class DefaultActorImpl : public HeapObject { + enum class Status { + /// The actor is not currently scheduled. Completely redundant + /// with the job list being empty. + Idle, + + /// There is currently a job scheduled to process the actor at the + /// stored max priority. + Scheduled, + + /// There is currently a thread processing the actor at the stored + /// max priority. + Running, + + /// The actor is a zombie that's been fully released but is still + /// running. We delay deallocation until its running thread gives + /// it up, which fortunately doesn't touch anything in the + /// actor except for the DefaultActorImpl. + /// + /// To coordinate between the releasing thread and the running + /// thread, we have a two-stage "latch". This is because the + /// releasing thread does not atomically decide to not deallocate. + /// Fortunately almost all of the overhead here is only incurred + /// when we actually do end up in the zombie state. + Zombie_Latching, + Zombie_ReadyForDeallocation + }; + + struct Flags : public FlagSet { + enum : size_t { + Status_offset = 0, + Status_width = 3, + + HasActiveInlineJob = 3, + + IsDistributedRemote = 4, + + MaxPriority = 8, + MaxPriority_width = JobFlags::Priority_width, + + // FIXME: add a reference to the running thread ID so that we + // can boost priorities. + }; + + /// What is the current high-level status of this actor? + FLAGSET_DEFINE_FIELD_ACCESSORS(Status_offset, Status_width, Status, + getStatus, setStatus) + + bool isAnyRunningStatus() const { + auto status = getStatus(); + return status == Status::Running || + status == Status::Zombie_Latching || + status == Status::Zombie_ReadyForDeallocation; + } + + bool isAnyZombieStatus() const { + auto status = getStatus(); + return status == Status::Zombie_Latching || + status == Status::Zombie_ReadyForDeallocation; + } + + /// Is there currently an active processing job allocated inline + /// in the actor? + FLAGSET_DEFINE_FLAG_ACCESSORS(HasActiveInlineJob, + hasActiveInlineJob, setHasActiveInlineJob) + + /// Is the actor a distributed 'remote' actor? + /// I.e. it does not have storage for user-defined properties and all + /// function call must be transformed into $distributed_ function calls. + FLAGSET_DEFINE_FLAG_ACCESSORS(IsDistributedRemote, + isDistributedRemote, setIsDistributedRemote) + + /// What is the maximum priority of jobs that are currently running + /// or enqueued on this actor? + /// + /// Note that the above isn't quite correct: we don't actually + /// lower this after we finish processing higher-priority tasks. + /// (Doing so introduces some subtleties around kicking off + /// lower-priority processing jobs.) + FLAGSET_DEFINE_FIELD_ACCESSORS(MaxPriority, MaxPriority_width, + JobPriority, + getMaxPriority, setMaxPriority) + }; + + /// This is designed to fit into two words, which can generally be + /// done lock-free on all our supported platforms. + struct alignas(2 * sizeof(void*)) State { + JobRef FirstJob; + struct Flags Flags; + }; + + swift::atomic CurrentState; + + friend class ProcessInlineJob; + union { + // When the ProcessInlineJob storage is initialized, its metadata pointer + // will point to Job's metadata. When it isn't, the metadata pointer is + // NULL. Use HeapObject to initialize the metadata pointer to NULL and allow + // it to be checked without fully initializing the ProcessInlineJob. + HeapObject JobStorageHeapObject{nullptr}; + + ProcessInlineJob JobStorage; + }; + +public: + /// Properly construct an actor, except for the heap header. + void initialize(bool isDistributedRemote = false) { + auto flags = Flags(); + flags.setIsDistributedRemote(isDistributedRemote); + new (&CurrentState) swift::atomic(State{JobRef(), flags}); + JobStorageHeapObject.metadata = nullptr; + } + + /// Properly destruct an actor, except for the heap header. + void destroy(); + + /// Properly respond to the last release of a default actor. Note + /// that the actor will have been completely torn down by the time + /// we reach this point. + void deallocate(); + + /// Add a job to this actor. + void enqueue(Job *job); + + /// Take over running this actor in the current thread, if possible. + bool tryAssumeThread(RunningJobInfo runner); + + /// Give up running this actor in the current thread. + void giveUpThread(RunningJobInfo runner); + + /// Claim the next job off the actor or give it up. + Job *claimNextJobOrGiveUp(bool actorIsOwned, RunningJobInfo runner); + + /// Check if the actor is actually a distributed *remote* actor. + /// + /// Note that a distributed *local* actor instance is the same as any other + /// ordinary default (local) actor, and no special handling is needed for them. + bool isDistributedRemote(); + +private: + void deallocateUnconditional(); + + /// Schedule an inline processing job. This can generally only be + /// done if we know nobody else is trying to do it at the same time, + /// e.g. if this thread just sucessfully transitioned the actor from + /// Idle to Scheduled. + void scheduleNonOverrideProcessJob(JobPriority priority, + bool hasActiveInlineJob); + + static DefaultActorImpl *fromInlineJob(Job *job) { + assert(isa(job)); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Winvalid-offsetof" + return reinterpret_cast( + reinterpret_cast(job) - offsetof(DefaultActorImpl, JobStorage)); +#pragma clang diagnostic pop + } + + class OverrideJobCache { + ProcessOverrideJob *Job = nullptr; + bool IsNeeded = false; +#ifndef NDEBUG + bool WasCommitted = false; +#endif + public: + OverrideJobCache() = default; + OverrideJobCache(const OverrideJobCache &) = delete; + OverrideJobCache &operator=(const OverrideJobCache &) = delete; + ~OverrideJobCache() { + assert(WasCommitted && "didn't commit override job!"); + } + + void addToState(DefaultActorImpl *actor, State &newState); + void setNotNeeded() { IsNeeded = false; } + void commit(); + }; +}; + +} /// end anonymous namespace + +static_assert(sizeof(DefaultActorImpl) <= sizeof(DefaultActor) && + alignof(DefaultActorImpl) <= alignof(DefaultActor), + "DefaultActorImpl doesn't fit in DefaultActor"); + +static DefaultActorImpl *asImpl(DefaultActor *actor) { + return reinterpret_cast(actor); +} + +static DefaultActor *asAbstract(DefaultActorImpl *actor) { + return reinterpret_cast(actor); +} + +/*****************************************************************************/ +/*********************** DEFAULT ACTOR IMPLEMENTATION ************************/ +/*****************************************************************************/ + +void DefaultActorImpl::destroy() { + auto oldState = CurrentState.load(std::memory_order_relaxed); + while (true) { + assert(!oldState.FirstJob && "actor has queued jobs at destruction"); + if (oldState.Flags.getStatus() == Status::Idle) return; + + assert(oldState.Flags.getStatus() == Status::Running && + "actor scheduled but not running at destruction"); + + // If the actor is currently running, set it to zombie status + // so that we know to deallocate it when it's given up. + auto newState = oldState; + newState.Flags.setStatus(Status::Zombie_Latching); + if (CurrentState.compare_exchange_weak(oldState, newState, + std::memory_order_relaxed, + std::memory_order_relaxed)) + return; + } +} + +void DefaultActorImpl::deallocate() { + // If we're in a zombie state waiting to latch, put the actor in the + // ready-for-deallocation state, but don't actually deallocate yet. + // Note that we should never see the actor already in the + // ready-for-deallocation state; giving up the actor while in the + // latching state will always put it in Idle state. + auto oldState = CurrentState.load(std::memory_order_relaxed); + while (oldState.Flags.getStatus() == Status::Zombie_Latching) { + auto newState = oldState; + newState.Flags.setStatus(Status::Zombie_ReadyForDeallocation); + if (CurrentState.compare_exchange_weak(oldState, newState, + std::memory_order_relaxed, + std::memory_order_relaxed)) + return; + } + + assert(oldState.Flags.getStatus() == Status::Idle); + + deallocateUnconditional(); +} + +void DefaultActorImpl::deallocateUnconditional() { + if (JobStorageHeapObject.metadata != nullptr) + JobStorage.~ProcessInlineJob(); + auto metadata = cast(this->metadata); + swift_deallocObject(this, metadata->getInstanceSize(), + metadata->getInstanceAlignMask()); +} + +/// Given that a job is enqueued normally on a default actor, get/set +/// the next job in the actor's queue. +/// +/// Note that this must not be used on the override jobs that can appear +/// in the queue; those jobs are not actually in the actor's queue (they're +/// on the global execution queues). So the actor's actual queue flows +/// through the NextJob field on those objects rather than through +/// the SchedulerPrivate fields. +static JobRef getNextJobInQueue(Job *job) { + return *reinterpret_cast(job->SchedulerPrivate); +} +static void setNextJobInQueue(Job *job, JobRef next) { + *reinterpret_cast(job->SchedulerPrivate) = next; +} + +/// Schedule a processing job that doesn't have to be an override job. +/// +/// We can either do this with inline storage or heap-allocated. +/// To ues inline storage, we need to verify that the hasActiveInlineJob +/// flag is not set in the state and then successfully set it. The +/// argument reports that this has happened correctly. +/// +/// We should only schedule a non-override processing job at all if +/// we're transferring ownership of the jobs in it; see the ownership +/// comment on DefaultActorImpl. +void DefaultActorImpl::scheduleNonOverrideProcessJob(JobPriority priority, + bool hasActiveInlineJob) { + Job *job; + if (hasActiveInlineJob) { + job = new ProcessOutOfLineJob(this, priority); + } else { + if (JobStorageHeapObject.metadata != nullptr) + JobStorage.~ProcessInlineJob(); + job = new (&JobStorage) ProcessInlineJob(priority); + } + swift_task_enqueueGlobal(job); +} + + +namespace { + +/// A job to process a specific default actor at a higher priority than +/// it was previously running at. +/// +/// When an override job is successfully registered with an actor +/// (not enqueued there), the thread processing the actor and the +/// thread processing the override job coordinate by each calling +/// one of a set of methods on the object. +class ProcessOverrideJob : public Job { + DefaultActorImpl *Actor; + + ConditionVariable::Mutex Lock; + ConditionVariable Queue; + + /// Has the actor made a decision about this job yet? + bool IsResolvedByActor = false; + + /// Has the job made a decision about itself yet? + bool IsResolvedByJob = false; + + /// Has this job been abandoned? + bool IsAbandoned = false; + +public: + /// SchedulerPrivate in an override job is used for actually scheduling + /// the job, so the actor queue goes through this instead. + /// + /// We also use this temporarily for the list of override jobs on + /// the actor that we need to wake up. + JobRef NextJob; + +public: + ProcessOverrideJob(DefaultActorImpl *actor, JobPriority priority, + JobRef nextJob) + : Job({JobKind::DefaultActorOverride, priority}, &process), + Actor(actor), NextJob(nextJob) {} + + DefaultActorImpl *getActor() const { return Actor; } + + /// Called by the job to notify the actor that the job has chosen + /// to abandon its work. This is irrevocable: the job is not going + /// to have a thread behind it. + /// + /// This may delete the job or cause it to be deleted on another thread. + void setAbandoned() { + bool shouldDelete = false; + Lock.withLock([&] { + assert(!IsResolvedByJob && "job already resolved itself"); + IsResolvedByJob = true; + IsAbandoned = true; + shouldDelete = IsResolvedByJob && IsResolvedByActor; + }); + if (shouldDelete) delete this; + } + + /// Called by the job to notify the actor that the job has successfully + /// taken over the actor and is now running it. + /// + /// This may delete the job object or cause it to be deleted on + /// another thread. + void setRunning() { + bool shouldDelete = false; + Lock.withLock([&] { + assert(!IsResolvedByJob && "job already resolved itself"); + IsResolvedByJob = true; + shouldDelete = IsResolvedByJob && IsResolvedByActor; + }); + if (shouldDelete) delete this; + } + + /// Called by the job to wait for the actor to resolve what the job + /// should do. + bool waitForActivation() { + bool isActivated = false; + Lock.withLockOrWait(Queue, [&] { + assert(!IsResolvedByJob && "job already resolved itself"); + if (IsResolvedByActor) { + isActivated = !IsAbandoned; + IsResolvedByJob = true; + return true; + } + return false; + }); + delete this; + return isActivated; + } + + /// Called by the actor to notify this job that the actor thinks it + /// should try to take over the actor. It's okay if that doesn't + /// succeed (as long as that's because some other job is going to + /// take over). + /// + /// This may delete the job or cause it to be deleted on another + /// thread. + bool wakeAndActivate() { + bool shouldDelete = false; + bool mayHaveBeenActivated = false; + Lock.withLockThenNotifyAll(Queue, [&] { + assert(!IsResolvedByActor && "actor already resolved this sjob"); + IsResolvedByActor = true; + mayHaveBeenActivated = IsResolvedByJob && !IsAbandoned; + shouldDelete = IsResolvedByJob && IsResolvedByActor; + }); + if (shouldDelete) delete this; + return mayHaveBeenActivated; + } + + /// Called by the actor to notify this job that the actor does not + /// think it should try to take over the actor. It's okay if the + /// job successfully takes over the actor anyway. + /// + /// This may delete the job or cause it to be deleted on another + /// thread. + void wakeAndAbandon() { + bool shouldDelete = false; + Lock.withLockThenNotifyAll(Queue, [&] { + assert(!IsResolvedByActor && "actor already resolved this job"); + IsResolvedByActor = true; + IsAbandoned = true; + shouldDelete = IsResolvedByJob && IsResolvedByActor; + }); + if (shouldDelete) delete this; + } + + SWIFT_CC(swiftasync) + static void process(Job *job); + + static bool classof(const Job *job) { + return job->Flags.getKind() == JobKind::DefaultActorOverride; + } +}; + +} /// end anonymous namespace + +JobRef JobRef::getOverride(ProcessOverrideJob *job) { + return JobRef(job, NeedsPreprocessing | IsOverride); +} +ProcessOverrideJob *JobRef::getAsPreprocessedOverride() const { + return cast_or_null(getAsPreprocessedJob()); +} +RunningJobInfo RunningJobInfo::forOverride(ProcessOverrideJob *job) { + return {Override, job->getPriority(), job}; +} + +/// Flag that the current processing job has been abandoned +/// and will not be running the actor. +void RunningJobInfo::setAbandoned() { + if (OverrideJob) { + OverrideJob->setAbandoned(); + OverrideJob = nullptr; + } +} + +/// Flag that the current processing job is now running the actor. +void RunningJobInfo::setRunning() { + if (OverrideJob) { + OverrideJob->setRunning(); + OverrideJob = nullptr; + } +} + +/// Try to wait for the current processing job to be activated, +/// if that's possible. It's okay to call this multiple times +/// (or to call setAbandoned/setRunning after it) as long as +/// it's all on a single value. +bool RunningJobInfo::waitForActivation() { + if (Kind == Override) { + // If we don't have an override job, it's because we've already + // waited for activation successfully. + if (!OverrideJob) return true; + + bool result = OverrideJob->waitForActivation(); + OverrideJob = nullptr; + return result; + } + return false; +} + +/// Wake all the overrides in the given list, activating the first +/// that exactly matches the target priority, if any. +static void wakeOverrides(ProcessOverrideJob *nextOverride, + Optional targetPriority) { + bool hasAlreadyActivated = false; + while (nextOverride) { + // We have to advance to the next override before we call one of + // the wake methods because they can delete the job immediately + // (and even if they don't, we'd still be racing with deletion). + auto cur = nextOverride; + nextOverride = cur->NextJob.getAsPreprocessedOverride(); + + if (hasAlreadyActivated || + !targetPriority) + cur->wakeAndAbandon(); + else + hasAlreadyActivated = cur->wakeAndActivate(); + } +} + +/// Flag that an override job is needed and create it. +void DefaultActorImpl::OverrideJobCache::addToState(DefaultActorImpl *actor, + State &newState) { + IsNeeded = true; + auto newPriority = newState.Flags.getMaxPriority(); + auto nextJob = newState.FirstJob; + if (Job) { + Job->Flags.setPriority(newPriority); + Job->NextJob = nextJob; + } else { + // Override jobs are always "extra" from the perspective of our + // ownership rules and so require a retain of the actor. We must + // do this before changing the actor state because other jobs may + // race to release the actor as soon as we change the actor state. + swift_retain(actor); + Job = new ProcessOverrideJob(actor, newPriority, nextJob); + } + newState.FirstJob = JobRef::getOverride(Job); +} + +/// Schedule the override job if we created one and still need it. +/// If we created one but didn't end up needing it (which can happen +/// with a race to override), destroy it. +void DefaultActorImpl::OverrideJobCache::commit() { +#ifndef NDEBUG + assert(!WasCommitted && "committing override job multiple timee"); + WasCommitted = true; +#endif + + if (Job) { + if (IsNeeded) { + swift_task_enqueueGlobal(Job); + } else { + swift_release(Job->getActor()); + delete Job; + } + } +} + +namespace { + +struct JobQueueTraits { + static Job *getNext(Job *job) { + return getNextJobInQueue(job).getAsPreprocessedJob(); + } + static void setNext(Job *job, Job *next) { + setNextJobInQueue(job, JobRef::getPreprocessed(next)); + } + static int compare(Job *lhs, Job *rhs) { + return descendingPriorityOrder(lhs->getPriority(), rhs->getPriority()); + } +}; + +} // end anonymous namespace + +/// Preprocess the prefix of the actor's queue that hasn't already +/// been preprocessed: +/// +/// - Split the jobs into registered overrides and actual jobs. +/// - Append the actual jobs to any already-preprocessed job list. +/// +/// The returned job should become the new root of the job queue +/// (or may be immediately dequeued, in which its successor should). +/// All of the jobs in this list are guaranteed to be non-override jobs. +static Job *preprocessQueue(JobRef first, + JobRef previousFirst, + Job *previousFirstNewJob, + ProcessOverrideJob *&overridesToWake) { + assert(previousFirst || previousFirstNewJob == nullptr); + + if (!first.needsPreprocessing()) + return first.getAsPreprocessedJob(); + + using ListMerger = swift::ListMerger; + ListMerger newJobs; + + while (first != previousFirst) { + // If we find something that doesn't need preprocessing, it must've + // been left by a previous queue-processing, which means that + // this must be our first attempt to preprocess in this processing. + // Just treat the queue from this point as a well-formed whole + // to which we need to add any new items we might've just found. + if (!first.needsPreprocessing()) { + assert(!previousFirst && !previousFirstNewJob); + previousFirstNewJob = first.getAsPreprocessedJob(); + break; + } + + // If the job is an override, add it to the list of override jobs + // that we need to wake up. Note that the list of override jobs + // flows through NextJob; we must not use getNextJobInQueue because + // that touches queue-private state, and the override job is + // not enqueued on the actor, merely registered with it. + if (first.isOverride()) { + auto overrideJob = first.getAsOverride(); + first = overrideJob->NextJob; + overrideJob->NextJob = JobRef::getPreprocessed(overridesToWake); + overridesToWake = overrideJob; + continue; + } + + // If the job isn't an override, add it to the front of the list of + // jobs we're building up. Note that this reverses the order of + // jobs; since enqueue() always adds jobs to the front, reversing + // the order effectively makes the actor queue FIFO, which is what + // we want. + auto job = first.getAsJob(); + first = getNextJobInQueue(job); + newJobs.insertAtFront(job); + } + + // If there are jobs already in the queue, put the new jobs at the end. + auto firstNewJob = newJobs.release(); + if (!firstNewJob) { + firstNewJob = previousFirstNewJob; + } else if (previousFirstNewJob) { + // Merge the jobs we just processed into the existing job list. + ListMerger merge(previousFirstNewJob); + merge.merge(firstNewJob); + firstNewJob = merge.release(); + } + + return firstNewJob; +} + +void DefaultActorImpl::giveUpThread(RunningJobInfo runner) { + SWIFT_TASK_DEBUG_LOG("giving up thread for actor %p", this); + auto oldState = CurrentState.load(std::memory_order_acquire); + assert(oldState.Flags.isAnyRunningStatus()); + + ProcessOverrideJob *overridesToWake = nullptr; + auto firstNewJob = preprocessQueue(oldState.FirstJob, JobRef(), nullptr, + overridesToWake); + + _swift_tsan_release(this); + while (true) { + // In Zombie_ReadyForDeallocation state, nothing else should + // be touching the atomic, and there's no point updating it. + if (oldState.Flags.getStatus() == Status::Zombie_ReadyForDeallocation) { + wakeOverrides(overridesToWake, oldState.Flags.getMaxPriority()); + deallocateUnconditional(); + return; + } + + // In Zombie_Latching state, we should try to update to Idle; + // if we beat the releasing thread, it'll deallocate. + // In Running state, we may need to schedule a processing job. + + State newState = oldState; + newState.FirstJob = JobRef::getPreprocessed(firstNewJob); + if (firstNewJob) { + assert(oldState.Flags.getStatus() == Status::Running); + newState.Flags.setStatus(Status::Scheduled); + } else { + newState.Flags.setStatus(Status::Idle); + } + + // If the runner was an inline job, it's no longer active. + if (runner.wasInlineJob()) { + newState.Flags.setHasActiveInlineJob(false); + } + + bool hasMoreJobs = (bool) newState.FirstJob; + bool hasOverrideAtNewPriority = false; + bool hasActiveInlineJob = newState.Flags.hasActiveInlineJob(); + bool needsNewProcessJob = hasMoreJobs && !hasOverrideAtNewPriority; + + // If we want to create a new inline job below, be sure to claim that + // in the new state. + if (needsNewProcessJob && !hasActiveInlineJob) { + newState.Flags.setHasActiveInlineJob(true); + } + + auto firstPreprocessed = oldState.FirstJob; + if (!CurrentState.compare_exchange_weak(oldState, newState, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_acquire)) { + // Preprocess any new queue items. + firstNewJob = preprocessQueue(oldState.FirstJob, + firstPreprocessed, + firstNewJob, + overridesToWake); + + // Try again. + continue; + } + +#define LOG_STATE_TRANSITION \ + SWIFT_TASK_DEBUG_LOG("actor %p transitioned from %zx to %zx (%s)\n", this, \ + oldState.Flags.getOpaqueValue(), \ + newState.Flags.getOpaqueValue(), __FUNCTION__) + LOG_STATE_TRANSITION; + + // The priority of the remaining work. + auto newPriority = newState.Flags.getMaxPriority(); + + // Process the override commands we found. + wakeOverrides(overridesToWake, newPriority); + + // This is the actor's owning job; per the ownership rules (see + // the comment on DefaultActorImpl), if there are remaining + // jobs, we need to balance out our ownership one way or another. + // We also, of course, need to ensure that there's a thread that's + // actually going to process the actor. + if (hasMoreJobs) { + // If we know that there's an override job at the new priority, + // we can let it become the owning job. We just need to release. + if (hasOverrideAtNewPriority) { + swift_release(this); + + // Otherwies, enqueue a job that will try to take over running + // with the new priority. This also ensures that there's a job + // at that priority which will actually take over the actor. + } else { + scheduleNonOverrideProcessJob(newPriority, hasActiveInlineJob); + } + } + + return; + } +} + +/// Claim the next job on the actor or give it up forever. +/// +/// The running thread doesn't need to already own the actor to do this. +/// It does need to be participating correctly in the ownership +/// scheme as a "processing job"; see the comment on DefaultActorImpl. +Job *DefaultActorImpl::claimNextJobOrGiveUp(bool actorIsOwned, + RunningJobInfo runner) { + auto oldState = CurrentState.load(std::memory_order_acquire); + + // The status had better be Running unless we're trying to acquire + // our first job. + assert(oldState.Flags.isAnyRunningStatus() || !actorIsOwned); + + // If we don't yet own the actor, we need to try to claim the actor + // first; we cannot safely access the queue memory yet because other + // threads may concurrently be trying to do this. + if (!actorIsOwned) { + // We really shouldn't ever be in a state where we're trying to take + // over a non-running actor if the actor is in a zombie state. + assert(!oldState.Flags.isAnyZombieStatus()); + + while (true) { + // A helper function when the only change we need to try is to + // update for an inline runner. + auto tryUpdateForInlineRunner = [&]{ + if (!runner.wasInlineJob()) return true; + + auto newState = oldState; + newState.Flags.setHasActiveInlineJob(false); + auto success = CurrentState.compare_exchange_weak(oldState, newState, + /*success*/ std::memory_order_relaxed, + /*failure*/ std::memory_order_acquire); + if (success) LOG_STATE_TRANSITION; + return success; + }; + + // If the actor is out of work, or its priority doesn't match our + // priority, don't try to take over the actor. + if (!oldState.FirstJob) { + + // The only change we need here is inline-runner bookkeeping. + if (!tryUpdateForInlineRunner()) + continue; + + // We're eliminating a processing thread; balance ownership. + swift_release(this); + runner.setAbandoned(); + return nullptr; + } + + // If the actor is currently running, we'd need to wait for + // it to stop. We can do this if we're an override job; + // otherwise we need to exit. + if (oldState.Flags.getStatus() == Status::Running) { + if (!runner.waitForActivation()) { + // The only change we need here is inline-runner bookkeeping. + if (!tryUpdateForInlineRunner()) + continue; + + swift_release(this); + return nullptr; + } + + // Fall through into the compare-exchange below, but anticipate + // that the actor is now Scheduled instead of Running. + oldState.Flags.setStatus(Status::Scheduled); + } + + // Try to set the state as Running. + assert(oldState.Flags.getStatus() == Status::Scheduled); + auto newState = oldState; + newState.Flags.setStatus(Status::Running); + + // Also do our inline-runner bookkeeping. + if (runner.wasInlineJob()) + newState.Flags.setHasActiveInlineJob(false); + + if (!CurrentState.compare_exchange_weak(oldState, newState, + /*success*/ std::memory_order_relaxed, + /*failure*/ std::memory_order_acquire)) + continue; + LOG_STATE_TRANSITION; + _swift_tsan_acquire(this); + + // If that succeeded, we can proceed to the main body. + oldState = newState; + runner.setRunning(); + break; + } + } + + assert(oldState.Flags.isAnyRunningStatus()); + + // We should have taken care of the inline-job bookkeeping now. + assert(!oldState.Flags.hasActiveInlineJob() || !runner.wasInlineJob()); + + // Okay, now it's safe to look at queue state. + // Preprocess any queue items at the front of the queue. + ProcessOverrideJob *overridesToWake = nullptr; + auto newFirstJob = preprocessQueue(oldState.FirstJob, JobRef(), + nullptr, overridesToWake); + + Optional remainingJobPriority; + _swift_tsan_release(this); + while (true) { + // In Zombie_ReadyForDeallocation state, nothing else should + // be touching the atomic, and there's no point updating it. + if (oldState.Flags.getStatus() == Status::Zombie_ReadyForDeallocation) { + wakeOverrides(overridesToWake, oldState.Flags.getMaxPriority()); + deallocateUnconditional(); + return nullptr; + } + + State newState = oldState; + + // If the priority we're currently running with is adqeuate for + // all the remaining jobs, try to dequeue something. + // FIXME: should this be an exact match in priority instead of + // potentially running jobs with too high a priority? + Job *jobToRun; + if (newFirstJob) { + jobToRun = newFirstJob; + newState.FirstJob = getNextJobInQueue(newFirstJob); + newState.Flags.setStatus(Status::Running); + + // Otherwise, we should give up the thread. + } else { + jobToRun = nullptr; + newState.FirstJob = JobRef::getPreprocessed(newFirstJob); + newState.Flags.setStatus(newFirstJob ? Status::Scheduled + : Status::Idle); + } + + // Try to update the queue. The changes we've made to the queue + // structure need to be made visible even if we aren't dequeuing + // anything. + auto firstPreprocessed = oldState.FirstJob; + if (!CurrentState.compare_exchange_weak(oldState, newState, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_acquire)) { + // Preprocess any new queue items, which will have been formed + // into a linked list leading to the last head we observed. + // (The fact that that job may not be the head anymore doesn't + // matter; we're looking for an exact match with that.) + newFirstJob = preprocessQueue(oldState.FirstJob, + firstPreprocessed, + newFirstJob, + overridesToWake); + + // Loop to retry updating the state. + continue; + } + LOG_STATE_TRANSITION; + + // We successfully updated the state. + + // If we're giving up the thread with jobs remaining, we need + // to release the actor, and we should wake overrides with the + // right priority. + Optional remainingJobPriority; + if (!jobToRun && newFirstJob) { + remainingJobPriority = newState.Flags.getMaxPriority(); + } + + // Wake the overrides. + wakeOverrides(overridesToWake, remainingJobPriority); + + // Per the ownership rules (see the comment on DefaultActorImpl), + // release the actor if we're giving up the thread with jobs + // remaining. We intentionally do this after wakeOverrides to + // try to get the overrides running a little faster. + if (remainingJobPriority) + swift_release(this); + + return jobToRun; + } +} + +SWIFT_CC(swift) +static void swift_job_runImpl(Job *job, ExecutorRef executor) { + ExecutorTrackingInfo trackingInfo; + + // swift_job_run is a primary entrypoint for executors telling us to + // run jobs. Actor executors won't expect us to switch off them + // during this operation. But do allow switching if the executor + // is generic. + if (!executor.isGeneric()) trackingInfo.disallowSwitching(); + + trackingInfo.enterAndShadow(executor); + + SWIFT_TASK_DEBUG_LOG("%s(%p)", __func__, job); + runJobInEstablishedExecutorContext(job); + + trackingInfo.leave(); + + // Give up the current executor if this is a switching context + // (which, remember, only happens if we started out on a generic + // executor) and we've switched to a default actor. + auto currentExecutor = trackingInfo.getActiveExecutor(); + if (trackingInfo.allowsSwitching() && currentExecutor.isDefaultActor()) { + // Use an underestimated priority; if this means we create an + // extra processing job in some cases, that's probably okay. + auto runner = RunningJobInfo::forOther(JobPriority(0)); + asImpl(currentExecutor.getDefaultActor())->giveUpThread(runner); + } +} + +/// The primary function for processing an actor on a thread. Start +/// processing the given default actor as the active default actor on +/// the current thread, and keep processing whatever actor we're +/// running when code returns back to us until we're not processing +/// any actors anymore. +/// +/// \param currentActor is expected to be passed in as retained to ensure that +/// the actor lives for the duration of job execution. +/// Note that this may conflict with the retain/release +/// design in the DefaultActorImpl, but it does fix bugs! +SWIFT_CC(swiftasync) +static void processDefaultActor(DefaultActorImpl *currentActor, + RunningJobInfo runner) { + SWIFT_TASK_DEBUG_LOG("processDefaultActor %p", currentActor); + DefaultActorImpl *actor = currentActor; + + // If we actually have work to do, we'll need to set up tracking info. + // Optimistically assume that we will; the alternative (an override job + // took over the actor first) is rare. + ExecutorTrackingInfo trackingInfo; + trackingInfo.enterAndShadow( + ExecutorRef::forDefaultActor(asAbstract(currentActor))); + + // Remember whether we've already taken over the actor. + bool threadIsRunningActor = false; + while (true) { + assert(currentActor); + + // Immediately check if we've been asked to yield the thread. + if (shouldYieldThread()) + break; + + // Try to claim another job from the current actor, taking it over + // if we haven't already. + auto job = currentActor->claimNextJobOrGiveUp(threadIsRunningActor, + runner); + + SWIFT_TASK_DEBUG_LOG("processDefaultActor %p claimed job %p", currentActor, + job); + + // If we failed to claim a job, we have nothing to do. + if (!job) { + // We also gave up the actor as part of failing to claim it. + // Make sure we don't try to give up the actor again. + currentActor = nullptr; + break; + } + + // This thread now owns the current actor. + threadIsRunningActor = true; + + // Run the job. + runJobInEstablishedExecutorContext(job); + + // The current actor may have changed after the job. + // If it's become nil, or not a default actor, we have nothing to do. + auto currentExecutor = trackingInfo.getActiveExecutor(); + + SWIFT_TASK_DEBUG_LOG("processDefaultActor %p current executor now %p", + currentActor, currentExecutor.getIdentity()); + + if (!currentExecutor.isDefaultActor()) { + // The job already gave up the thread for us. + // Make sure we don't try to give up the actor again. + currentActor = nullptr; + break; + } + currentActor = asImpl(currentExecutor.getDefaultActor()); + } + + // Leave the tracking info. + trackingInfo.leave(); + + // If we still have an active actor, we should give it up. + if (threadIsRunningActor && currentActor) { + currentActor->giveUpThread(runner); + } + + swift_release(actor); +} + +SWIFT_CC(swiftasync) +void ProcessInlineJob::process(Job *job) { + DefaultActorImpl *actor = DefaultActorImpl::fromInlineJob(job); + + // Pull the priority out of the job before we do anything that might + // invalidate it. + auto targetPriority = job->getPriority(); + auto runner = RunningJobInfo::forInline(targetPriority); + + swift_retain(actor); + return processDefaultActor(actor, runner); // 'return' forces tail call +} + +SWIFT_CC(swiftasync) +void ProcessOutOfLineJob::process(Job *job) { + auto self = cast(job); + DefaultActorImpl *actor = self->Actor; + + // Pull the priority out of the job before we do anything that might + // invalidate it. + auto targetPriority = job->getPriority(); + auto runner = RunningJobInfo::forOther(targetPriority); + + delete self; + + swift_retain(actor); + return processDefaultActor(actor, runner); // 'return' forces tail call +} + +SWIFT_CC(swiftasync) +void ProcessOverrideJob::process(Job *job) { + auto self = cast(job); + + // Pull the actor and priority out of the job. + auto actor = self->Actor; + auto runner = RunningJobInfo::forOverride(self); + + swift_retain(actor); + return processDefaultActor(actor, runner); // 'return' forces tail call +} + +void DefaultActorImpl::enqueue(Job *job) { + auto oldState = CurrentState.load(std::memory_order_relaxed); + + OverrideJobCache overrideJob; + + while (true) { + assert(!oldState.Flags.isAnyZombieStatus() && + "enqueuing work on a zombie actor"); + auto newState = oldState; + + // Put the job at the front of the job list (which will get + // reversed during preprocessing). + setNextJobInQueue(job, oldState.FirstJob); + newState.FirstJob = JobRef::getUnpreprocessed(job); + + auto oldStatus = oldState.Flags.getStatus(); + bool wasIdle = oldStatus == Status::Idle; + + // Update the priority: the priority of the job we're adding + // if the actor was idle, or the max if not. Only the running + // thread can decrease the actor's priority once it's non-idle. + // (But note that the job we enqueue can still observe a + // lowered priority.) + auto oldPriority = oldState.Flags.getMaxPriority(); + auto newPriority = + wasIdle ? job->getPriority() + : std::max(oldPriority, job->getPriority()); + newState.Flags.setMaxPriority(newPriority); + + // If we need an override job, create it (if necessary) and + // register it with the queue. + bool needsOverride = false; + if (needsOverride) { + overrideJob.addToState(this, newState); + } else { + overrideJob.setNotNeeded(); + } + + // If we don't need an override job, then we might be able to + // create an inline job; flag that. + bool hasActiveInlineJob = newState.Flags.hasActiveInlineJob(); + if (wasIdle && !hasActiveInlineJob) + newState.Flags.setHasActiveInlineJob(true); + + // Make sure the status is at least Scheduled. We'll actually + // schedule the job below, if we succeed at this. + if (wasIdle) { + newState.Flags.setStatus(Status::Scheduled); + } + + // Try the compare-exchange, and try again if it fails. + if (!CurrentState.compare_exchange_weak(oldState, newState, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_relaxed)) + continue; + LOG_STATE_TRANSITION; + + // Okay, we successfully updated the status. Schedule a job to + // process the actor if necessary. + + // Commit the override job if we created one. + overrideJob.commit(); + + // If the actor is currently idle, schedule it using the + // invasive job. + if (wasIdle) { + assert(!needsOverride); + scheduleNonOverrideProcessJob(newPriority, hasActiveInlineJob); + } + + return; + } +} + +bool DefaultActorImpl::tryAssumeThread(RunningJobInfo runner) { + // We have to load-acquire in order to properly order accesses to + // the actor's state for the new task. + auto oldState = CurrentState.load(std::memory_order_acquire); + + // If the actor is currently idle, try to mark it as running. + while (oldState.Flags.getStatus() == Status::Idle) { + assert(!oldState.FirstJob); + auto newState = oldState; + newState.Flags.setStatus(Status::Running); + newState.Flags.setMaxPriority(runner.Priority); + + if (CurrentState.compare_exchange_weak(oldState, newState, + /*success*/ std::memory_order_relaxed, + /*failure*/ std::memory_order_acquire)) { + LOG_STATE_TRANSITION; + _swift_tsan_acquire(this); + return true; + } + } + + assert(!oldState.Flags.isAnyZombieStatus() && + "trying to assume a zombie actor"); + + return false; +} + +void swift::swift_defaultActor_initialize(DefaultActor *_actor) { + asImpl(_actor)->initialize(); +} + +void swift::swift_defaultActor_destroy(DefaultActor *_actor) { + asImpl(_actor)->destroy(); +} + +void swift::swift_defaultActor_enqueue(Job *job, DefaultActor *_actor) { + asImpl(_actor)->enqueue(job); +} + +void swift::swift_defaultActor_deallocate(DefaultActor *_actor) { + asImpl(_actor)->deallocate(); +} + +static bool isDefaultActorClass(const ClassMetadata *metadata) { + assert(metadata->isTypeMetadata()); + while (true) { + // Trust the class descriptor if it says it's a default actor. + if (metadata->getDescription()->isDefaultActor()) + return true; + + // Go to the superclass. + metadata = metadata->Superclass; + + // If we run out of Swift classes, it's not a default actor. + if (!metadata || !metadata->isTypeMetadata()) return false; + } +} + +void swift::swift_defaultActor_deallocateResilient(HeapObject *actor) { + auto metadata = cast(actor->metadata); + if (isDefaultActorClass(metadata)) + return swift_defaultActor_deallocate(static_cast(actor)); + + swift_deallocObject(actor, metadata->getInstanceSize(), + metadata->getInstanceAlignMask()); +} + +/// FIXME: only exists for the quick-and-dirty MainActor implementation. +namespace swift { + Metadata* MainActorMetadata = nullptr; +} + +/*****************************************************************************/ +/****************************** ACTOR SWITCHING ******************************/ +/*****************************************************************************/ + +/// Can the current executor give up its thread? +static bool canGiveUpThreadForSwitch(ExecutorTrackingInfo *trackingInfo, + ExecutorRef currentExecutor) { + assert(trackingInfo || currentExecutor.isGeneric()); + + // Some contexts don't allow switching at all. + if (trackingInfo && !trackingInfo->allowsSwitching()) + return false; + + // We can certainly "give up" a generic executor to try to run + // a task for an actor. + if (currentExecutor.isGeneric()) + return true; + + // If the current executor is a default actor, we know how to make + // it give up its thread. + if (currentExecutor.isDefaultActor()) + return true; + + return false; +} + +/// Tell the current executor to give up its thread, given that it +/// returned true from canGiveUpThreadForSwitch(). +/// +/// Note that we don't update DefaultActorProcessingFrame here; we'll +/// do that in runOnAssumedThread. +static void giveUpThreadForSwitch(ExecutorRef currentExecutor, + RunningJobInfo runner) { + if (currentExecutor.isGeneric()) + return; + + asImpl(currentExecutor.getDefaultActor())->giveUpThread(runner); +} + +/// Try to assume control of the current thread for the given executor +/// in order to run the given job. +/// +/// This doesn't actually run the job yet. +/// +/// Note that we don't update DefaultActorProcessingFrame here; we'll +/// do that in runOnAssumedThread. +static bool tryAssumeThreadForSwitch(ExecutorRef newExecutor, + RunningJobInfo runner) { + // If the new executor is generic, we don't need to do anything. + if (newExecutor.isGeneric()) { + return true; + } + + // If the new executor is a default actor, ask it to assume the thread. + if (newExecutor.isDefaultActor()) { + return asImpl(newExecutor.getDefaultActor())->tryAssumeThread(runner); + } + + return false; +} + +/// Given that we've assumed control of an executor on this thread, +/// continue to run the given task on it. +SWIFT_CC(swiftasync) +static void runOnAssumedThread(AsyncTask *task, ExecutorRef executor, + ExecutorTrackingInfo *oldTracking, + RunningJobInfo runner) { + // Note that this doesn't change the active task and so doesn't + // need to either update ActiveTask or flagAsRunning/flagAsSuspended. + + // If there's alreaady tracking info set up, just change the executor + // there and tail-call the task. We don't want these frames to + // potentially accumulate linearly. + if (oldTracking) { + oldTracking->setActiveExecutor(executor); + + return task->runInFullyEstablishedContext(); // 'return' forces tail call + } + + // Otherwise, set up tracking info. + ExecutorTrackingInfo trackingInfo; + trackingInfo.enterAndShadow(executor); + + // Run the new task. + task->runInFullyEstablishedContext(); + + // Leave the tracking frame, and give up the current actor if + // we have one. + // + // In principle, we could execute more tasks from the actor here, but + // that's probably not a reasonable thing to do in an assumed context + // rather than a dedicated actor-processing job. + executor = trackingInfo.getActiveExecutor(); + trackingInfo.leave(); + + SWIFT_TASK_DEBUG_LOG("leaving assumed thread, current executor is %p", + executor.getIdentity()); + + if (executor.isDefaultActor()) + asImpl(executor.getDefaultActor())->giveUpThread(runner); +} + +SWIFT_CC(swiftasync) +static void swift_task_switchImpl(SWIFT_ASYNC_CONTEXT AsyncContext *resumeContext, + TaskContinuationFunction *resumeFunction, + ExecutorRef newExecutor) { + auto trackingInfo = ExecutorTrackingInfo::current(); + auto currentExecutor = + (trackingInfo ? trackingInfo->getActiveExecutor() + : ExecutorRef::generic()); + SWIFT_TASK_DEBUG_LOG("trying to switch from executor %p to %p", + currentExecutor.getIdentity(), + newExecutor.getIdentity()); + + // If the current executor is compatible with running the new executor, + // we can just immediately continue running with the resume function + // we were passed in. + if (!currentExecutor.mustSwitchToRun(newExecutor)) { + return resumeFunction(resumeContext); // 'return' forces tail call + } + + auto task = swift_task_getCurrent(); + assert(task && "no current task!"); + + // Park the task for simplicity instead of trying to thread the + // initial resumption information into everything below. + task->ResumeContext = resumeContext; + task->ResumeTask = resumeFunction; + + // Okay, we semantically need to switch. + auto runner = RunningJobInfo::forOther(task->getPriority()); + + // If the current executor can give up its thread, and the new executor + // can take over a thread, try to do so; but don't do this if we've + // been asked to yield the thread. + if (canGiveUpThreadForSwitch(trackingInfo, currentExecutor) && + !shouldYieldThread() && + tryAssumeThreadForSwitch(newExecutor, runner)) { + SWIFT_TASK_DEBUG_LOG( + "switch succeeded, task %p assumed thread for executor %p", task, + newExecutor.getIdentity()); + giveUpThreadForSwitch(currentExecutor, runner); + // 'return' forces tail call + return runOnAssumedThread(task, newExecutor, trackingInfo, runner); + } + + // Otherwise, just asynchronously enqueue the task on the given + // executor. + SWIFT_TASK_DEBUG_LOG("switch failed, task %p enqueued on executor %p", task, + newExecutor.getIdentity()); + task->flagAsSuspended(); + _swift_task_clearCurrent(); + swift_task_enqueue(task, newExecutor); +} + +/*****************************************************************************/ +/************************* GENERIC ACTOR INTERFACES **************************/ +/*****************************************************************************/ + +// Implemented in Swift to avoid some annoying hard-coding about +// SerialExecutor's protocol witness table. We could inline this +// with effort, though. +extern "C" SWIFT_CC(swift) +void _swift_task_enqueueOnExecutor(Job *job, HeapObject *executor, + const Metadata *selfType, + const SerialExecutorWitnessTable *wtable); + +SWIFT_CC(swift) +static void swift_task_enqueueImpl(Job *job, ExecutorRef executor) { + SWIFT_TASK_DEBUG_LOG("enqueue job %p on executor %p", job, + executor.getIdentity()); + + assert(job && "no job provided"); + + _swift_tsan_release(job); + + if (executor.isGeneric()) + return swift_task_enqueueGlobal(job); + + if (executor.isDefaultActor()) + return asImpl(executor.getDefaultActor())->enqueue(job); + + auto wtable = executor.getSerialExecutorWitnessTable(); + auto executorObject = executor.getIdentity(); + auto executorType = swift_getObjectType(executorObject); + _swift_task_enqueueOnExecutor(job, executorObject, executorType, wtable); +} + +#define OVERRIDE_ACTOR COMPATIBILITY_OVERRIDE +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH + + +/*****************************************************************************/ +/***************************** DISTRIBUTED ACTOR *****************************/ +/*****************************************************************************/ + +OpaqueValue* +swift::swift_distributedActor_remote_initialize(const Metadata *actorType) { + auto *classMetadata = actorType->getClassObject(); + + // TODO(distributed): make this allocation smaller + // ==== Allocate the memory for the remote instance + HeapObject *alloc = swift_allocObject(classMetadata, + classMetadata->getInstanceSize(), + classMetadata->getInstanceAlignMask()); + + // TODO: remove this memset eventually, today we only do this to not have + // to modify the destructor logic, as releasing zeroes is no-op + memset(alloc + 1, 0, classMetadata->getInstanceSize() - sizeof(HeapObject)); + + // TODO(distributed): a remote one does not have to have the "real" + // default actor body, e.g. we don't need an executor at all; so + // we can allocate more efficiently and only share the flags/status field + // between the both memory representations + // --- Currently we ride on the DefaultActorImpl to reuse the memory layout + // of the flags etc. So initialize the default actor into the allocation. + auto actor = asImpl(reinterpret_cast(alloc)); + actor->initialize(/*remote*/true); + assert(actor->isDistributedRemote()); + + return reinterpret_cast(actor); +} + +bool swift::swift_distributed_actor_is_remote(DefaultActor *_actor) { + return asImpl(_actor)->isDistributedRemote(); +} + +bool DefaultActorImpl::isDistributedRemote() { + auto state = CurrentState.load(std::memory_order_relaxed); + return state.Flags.isDistributedRemote(); +} diff --git a/stdlib/public/BackDeployConcurrency/Actor.h b/stdlib/public/BackDeployConcurrency/Actor.h new file mode 100644 index 0000000000000..913b67d5efd2d --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Actor.h @@ -0,0 +1,45 @@ +//===--- Actor.h - ABI structures for actors --------------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing actors. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_ACTOR_H +#define SWIFT_ABI_ACTOR_H + +#include "swift/ABI/HeapObject.h" +#include "swift/ABI/MetadataValues.h" +#include "ConditionVariable.h" + +namespace swift { + +/// The default actor implementation. This is the layout of both +/// the DefaultActor and NSDefaultActor classes. +class alignas(Alignment_DefaultActor) DefaultActor : public HeapObject { +public: + // These constructors do not initialize the actor instance, and the + // destructor does not destroy the actor instance; you must call + // swift_defaultActor_{initialize,destroy} yourself. + constexpr DefaultActor(const HeapMetadata *metadata) + : HeapObject(metadata), PrivateData{} {} + + constexpr DefaultActor(const HeapMetadata *metadata, + InlineRefCounts::Immortal_t immortal) + : HeapObject(metadata, immortal), PrivateData{} {} + + void *PrivateData[NumWords_DefaultActor]; +}; + +} // end namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/Actor.swift b/stdlib/public/BackDeployConcurrency/Actor.swift new file mode 100644 index 0000000000000..788b5ad908e88 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Actor.swift @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +/// Common protocol to which all actors conform. +/// +/// The `Actor` protocol generalizes over all actor types. Actor types +/// implicitly conform to this protocol. +@available(SwiftStdlib 5.1, *) +public protocol Actor: AnyObject, Sendable { + + /// Retrieve the executor for this actor as an optimized, unowned + /// reference. + /// + /// This property must always evaluate to the same executor for a + /// given actor instance, and holding on to the actor must keep the + /// executor alive. + /// + /// This property will be implicitly accessed when work needs to be + /// scheduled onto this actor. These accesses may be merged, + /// eliminated, and rearranged with other work, and they may even + /// be introduced when not strictly required. Visible side effects + /// are therefore strongly discouraged within this property. + nonisolated var unownedExecutor: UnownedSerialExecutor { get } +} + +/// Called to initialize the default actor instance in an actor. +/// The implementation will call this within the actor's initializer. +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_defaultActor_initialize") +public func _defaultActorInitialize(_ actor: AnyObject) + +/// Called to destroy the default actor instance in an actor. +/// The implementation will call this within the actor's deinit. +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_defaultActor_destroy") +public func _defaultActorDestroy(_ actor: AnyObject) + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_enqueueMainExecutor") +@usableFromInline +internal func _enqueueOnMain(_ job: UnownedJob) diff --git a/stdlib/public/BackDeployConcurrency/AsyncCompactMapSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncCompactMapSequence.swift new file mode 100644 index 0000000000000..72889e9b52c81 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncCompactMapSequence.swift @@ -0,0 +1,131 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that maps the given closure over the + /// asynchronous sequence’s elements, omitting results that don't return a + /// value. + /// + /// Use the `compactMap(_:)` method to transform every element received from + /// a base asynchronous sequence, while also discarding any `nil` results + /// from the closure. Typically, you use this to transform from one type of + /// element to another. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `5`. The closure provided to the `compactMap(_:)` + /// method takes each `Int` and looks up a corresponding `String` from a + /// `romanNumeralDict` dictionary. Because there is no key for `4`, the closure + /// returns `nil` in this case, which `compactMap(_:)` omits from the + /// transformed asynchronous sequence. + /// + /// let romanNumeralDict: [Int : String] = + /// [1: "I", 2: "II", 3: "III", 5: "V"] + /// + /// let stream = Counter(howHigh: 5) + /// .compactMap { romanNumeralDict[$0] } + /// for await numeral in stream { + /// print("\(numeral) ", terminator: " ") + /// } + /// // Prints: I II III V + /// + /// - Parameter transform: A mapping closure. `transform` accepts an element + /// of this sequence as its parameter and returns a transformed value of the + /// same or of a different type. + /// - Returns: An asynchronous sequence that contains, in order, the + /// non-`nil` elements produced by the `transform` closure. + @inlinable + public __consuming func compactMap( + _ transform: @escaping (Element) async -> ElementOfResult? + ) -> AsyncCompactMapSequence { + return AsyncCompactMapSequence(self, transform: transform) + } +} + +/// An asynchronous sequence that maps a given closure over the asynchronous +/// sequence’s elements, omitting results that don't return a value. +@available(SwiftStdlib 5.1, *) +public struct AsyncCompactMapSequence { + @usableFromInline + let base: Base + + @usableFromInline + let transform: (Base.Element) async -> ElementOfResult? + + @usableFromInline + init( + _ base: Base, + transform: @escaping (Base.Element) async -> ElementOfResult? + ) { + self.base = base + self.transform = transform + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncCompactMapSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The compact map sequence produces whatever type of element its + /// transforming closure produces. + public typealias Element = ElementOfResult + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the compact map sequence. + public struct Iterator: AsyncIteratorProtocol { + public typealias Element = ElementOfResult + + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let transform: (Base.Element) async -> ElementOfResult? + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + transform: @escaping (Base.Element) async -> ElementOfResult? + ) { + self.baseIterator = baseIterator + self.transform = transform + } + + /// Produces the next element in the compact map sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns `nil`. Otherwise, `next()` calls the + /// transforming closure on the received element, returning it if the + /// transform returns a non-`nil` value. If the transform returns `nil`, + /// this method continues to wait for further elements until it gets one + /// that transforms to a non-`nil` value. + @inlinable + public mutating func next() async rethrows -> ElementOfResult? { + while true { + guard let element = try await baseIterator.next() else { + return nil + } + + if let transformed = await transform(element) { + return transformed + } + } + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), transform: transform) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncDropFirstSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncDropFirstSequence.swift new file mode 100644 index 0000000000000..67095a1b16589 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncDropFirstSequence.swift @@ -0,0 +1,137 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Omits a specified number of elements from the base asynchronous sequence, + /// then passes through all remaining elements. + /// + /// Use `dropFirst(_:)` when you want to drop the first *n* elements from the + /// base sequence and pass through the remaining elements. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `dropFirst(_:)` method causes the modified + /// sequence to ignore the values `0` through `4`, and instead emit `5` through `10`: + /// + /// for await number in Counter(howHigh: 10).dropFirst(3) { + /// print("\(number) ", terminator: " ") + /// } + /// // prints "4 5 6 7 8 9 10" + /// + /// If the number of elements to drop exceeds the number of elements in the + /// sequence, the result is an empty sequence. + /// + /// - Parameter count: The number of elements to drop from the beginning of + /// the sequence. `count` must be greater than or equal to zero. + /// - Returns: An asynchronous sequence that drops the first `count` + /// elements from the base sequence. + @inlinable + public __consuming func dropFirst( + _ count: Int = 1 + ) -> AsyncDropFirstSequence { + precondition(count >= 0, + "Can't drop a negative number of elements from an async sequence") + return AsyncDropFirstSequence(self, dropping: count) + } +} + +/// An asynchronous sequence which omits a specified number of elements from the +/// base asynchronous sequence, then passes through all remaining elements. +@available(SwiftStdlib 5.1, *) +public struct AsyncDropFirstSequence { + @usableFromInline + let base: Base + + @usableFromInline + let count: Int + + @usableFromInline + init(_ base: Base, dropping count: Int) { + self.base = base + self.count = count + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncDropFirstSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The drop-first sequence produces whatever type of element its base + /// iterator produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the drop-first sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + var count: Int + + @usableFromInline + init(_ baseIterator: Base.AsyncIterator, count: Int) { + self.baseIterator = baseIterator + self.count = count + } + + /// Produces the next element in the drop-first sequence. + /// + /// Until reaching the number of elements to drop, this iterator calls + /// `next()` on its base iterator and discards the result. If the base + /// iterator returns `nil`, indicating the end of the sequence, this + /// iterator returns `nil`. After reaching the number of elements to + /// drop, this iterator passes along the result of calling `next()` on + /// the base iterator. + @inlinable + public mutating func next() async rethrows -> Base.Element? { + var remainingToDrop = count + while remainingToDrop > 0 { + guard try await baseIterator.next() != nil else { + count = 0 + return nil + } + remainingToDrop -= 1 + } + count = 0 + return try await baseIterator.next() + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), count: count) + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncDropFirstSequence { + /// Omits a specified number of elements from the base asynchronous sequence, + /// then passes through all remaining elements. + /// + /// When you call `dropFirst(_:)` on an asynchronous sequence that is already + /// an `AsyncDropFirstSequence`, the returned sequence simply adds the new + /// drop count to the current drop count. + @inlinable + public __consuming func dropFirst( + _ count: Int = 1 + ) -> AsyncDropFirstSequence { + // If this is already a AsyncDropFirstSequence, we can just sum the current + // drop count and additional drop count. + precondition(count >= 0, + "Can't drop a negative number of elements from an async sequence") + return AsyncDropFirstSequence(base, dropping: self.count + count) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncDropWhileSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncDropWhileSequence.swift new file mode 100644 index 0000000000000..e82fca9968b95 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncDropWhileSequence.swift @@ -0,0 +1,129 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Omits elements from the base asynchronous sequence until a given closure + /// returns false, after which it passes through all remaining elements. + /// + /// Use `drop(while:)` to omit elements from an asynchronous sequence until + /// the element received meets a condition you specify. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `drop(while:)` method causes the modified + /// sequence to ignore received values until it encounters one that is + /// divisible by `3`: + /// + /// let stream = Counter(howHigh: 10) + /// .drop { $0 % 3 != 0 } + /// for await number in stream { + /// print("\(number) ", terminator: " ") + /// } + /// // prints "3 4 5 6 7 8 9 10" + /// + /// After the predicate returns `false`, the sequence never executes it again, + /// and from then on the sequence passes through elements from its underlying + /// sequence as-is. + /// + /// - Parameter predicate: A closure that takes an element as a parameter and + /// returns a Boolean value indicating whether to drop the element from the + /// modified sequence. + /// - Returns: An asynchronous sequence that skips over values from the + /// base sequence until the provided closure returns `false`. + @inlinable + public __consuming func drop( + while predicate: @escaping (Element) async -> Bool + ) -> AsyncDropWhileSequence { + AsyncDropWhileSequence(self, predicate: predicate) + } +} + +/// An asynchronous sequence which omits elements from the base sequence until a +/// given closure returns false, after which it passes through all remaining +/// elements. +@available(SwiftStdlib 5.1, *) +public struct AsyncDropWhileSequence { + @usableFromInline + let base: Base + + @usableFromInline + let predicate: (Base.Element) async -> Bool + + @usableFromInline + init( + _ base: Base, + predicate: @escaping (Base.Element) async -> Bool + ) { + self.base = base + self.predicate = predicate + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncDropWhileSequence: AsyncSequence { + + /// The type of element produced by this asynchronous sequence. + /// + /// The drop-while sequence produces whatever type of element its base + /// sequence produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the drop-while sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + var predicate: ((Base.Element) async -> Bool)? + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + predicate: @escaping (Base.Element) async -> Bool + ) { + self.baseIterator = baseIterator + self.predicate = predicate + } + + /// Produces the next element in the drop-while sequence. + /// + /// This iterator calls `next()` on its base iterator and evaluates the + /// result with the `predicate` closure. As long as the predicate returns + /// `true`, this method returns `nil`. After the predicate returns `false`, + /// for a value received from the base iterator, this method returns that + /// value. After that, the iterator returns values received from its + /// base iterator as-is, and never executes the predicate closure again. + @inlinable + public mutating func next() async rethrows -> Base.Element? { + while let predicate = self.predicate { + guard let element = try await baseIterator.next() else { + return nil + } + if await predicate(element) == false { + self.predicate = nil + return element + } + } + return try await baseIterator.next() + } + } + + /// Creates an instance of the drop-while sequence iterator. + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), predicate: predicate) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncFilterSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncFilterSequence.swift new file mode 100644 index 0000000000000..d7dc0d8505cfd --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncFilterSequence.swift @@ -0,0 +1,115 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that contains, in order, the elements of + /// the base sequence that satisfy the given predicate. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `filter(_:)` method returns `true` for even + /// values and `false` for odd values, thereby filtering out the odd values: + /// + /// let stream = Counter(howHigh: 10) + /// .filter { $0 % 2 == 0 } + /// for await number in stream { + /// print("\(number) ", terminator: " ") + /// } + /// // Prints: 2 4 6 8 10 + /// + /// - Parameter isIncluded: A closure that takes an element of the + /// asynchronous sequence as its argument and returns a Boolean value + /// that indicates whether to include the element in the filtered sequence. + /// - Returns: An asynchronous sequence that contains, in order, the elements + /// of the base sequence that satisfy the given predicate. + @inlinable + public __consuming func filter( + _ isIncluded: @escaping (Element) async -> Bool + ) -> AsyncFilterSequence { + return AsyncFilterSequence(self, isIncluded: isIncluded) + } +} + +/// An asynchronous sequence that contains, in order, the elements of +/// the base sequence that satisfy a given predicate. +@available(SwiftStdlib 5.1, *) +public struct AsyncFilterSequence { + @usableFromInline + let base: Base + + @usableFromInline + let isIncluded: (Element) async -> Bool + + @usableFromInline + init( + _ base: Base, + isIncluded: @escaping (Base.Element) async -> Bool + ) { + self.base = base + self.isIncluded = isIncluded + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncFilterSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The filter sequence produces whatever type of element its base + /// sequence produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the filter sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let isIncluded: (Base.Element) async -> Bool + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + isIncluded: @escaping (Base.Element) async -> Bool + ) { + self.baseIterator = baseIterator + self.isIncluded = isIncluded + } + + /// Produces the next element in the filter sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns nil. Otherwise, `next()` evaluates the + /// result with the `predicate` closure. If the closure returns `true`, + /// `next()` returns the received element; otherwise it awaits the next + /// element from the base iterator. + @inlinable + public mutating func next() async rethrows -> Base.Element? { + while true { + guard let element = try await baseIterator.next() else { + return nil + } + if await isIncluded(element) { + return element + } + } + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), isIncluded: isIncluded) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncFlatMapSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncFlatMapSequence.swift new file mode 100644 index 0000000000000..65fcf95b7392b --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncFlatMapSequence.swift @@ -0,0 +1,156 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that concatenates the results of calling + /// the given transformation with each element of this sequence. + /// + /// Use this method to receive a single-level asynchronous sequence when your + /// transformation produces an asynchronous sequence for each element. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `5`. The transforming closure takes the received `Int` + /// and returns a new `Counter` that counts that high. For example, when the + /// transform receives `3` from the base sequence, it creates a new `Counter` + /// that produces the values `1`, `2`, and `3`. The `flatMap(_:)` method + /// "flattens" the resulting sequence-of-sequences into a single + /// `AsyncSequence`. + /// + /// let stream = Counter(howHigh: 5) + /// .flatMap { Counter(howHigh: $0) } + /// for await number in stream { + /// print("\(number)", terminator: " ") + /// } + /// // Prints: 1 1 2 1 2 3 1 2 3 4 1 2 3 4 5 + /// + /// - Parameter transform: A mapping closure. `transform` accepts an element + /// of this sequence as its parameter and returns an `AsyncSequence`. + /// - Returns: A single, flattened asynchronous sequence that contains all + /// elements in all the asychronous sequences produced by `transform`. + @inlinable + public __consuming func flatMap( + _ transform: @escaping (Element) async -> SegmentOfResult + ) -> AsyncFlatMapSequence { + return AsyncFlatMapSequence(self, transform: transform) + } +} + +/// An asynchronous sequence that concatenates the results of calling a given +/// transformation with each element of this sequence. +@available(SwiftStdlib 5.1, *) +public struct AsyncFlatMapSequence { + @usableFromInline + let base: Base + + @usableFromInline + let transform: (Base.Element) async -> SegmentOfResult + + @usableFromInline + init( + _ base: Base, + transform: @escaping (Base.Element) async -> SegmentOfResult + ) { + self.base = base + self.transform = transform + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncFlatMapSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The flat map sequence produces the type of element in the asynchronous + /// sequence produced by the `transform` closure. + public typealias Element = SegmentOfResult.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the flat map sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let transform: (Base.Element) async -> SegmentOfResult + + @usableFromInline + var currentIterator: SegmentOfResult.AsyncIterator? + + @usableFromInline + var finished = false + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + transform: @escaping (Base.Element) async -> SegmentOfResult + ) { + self.baseIterator = baseIterator + self.transform = transform + } + + /// Produces the next element in the flat map sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns `nil`. Otherwise, `next()` calls the + /// transforming closure on the received element, takes the resulting + /// asynchronous sequence, and creates an asynchronous iterator from it. + /// `next()` then consumes values from this iterator until it terminates. + /// At this point, `next()` is ready to receive the next value from the base + /// sequence. + @inlinable + public mutating func next() async rethrows -> SegmentOfResult.Element? { + while !finished { + if var iterator = currentIterator { + do { + guard let element = try await iterator.next() else { + currentIterator = nil + continue + } + // restore the iterator since we just mutated it with next + currentIterator = iterator + return element + } catch { + finished = true + throw error + } + } else { + guard let item = try await baseIterator.next() else { + finished = true + return nil + } + do { + let segment = await transform(item) + var iterator = segment.makeAsyncIterator() + guard let element = try await iterator.next() else { + currentIterator = nil + continue + } + currentIterator = iterator + return element + } catch { + finished = true + throw error + } + } + } + return nil + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), transform: transform) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncIteratorProtocol.swift b/stdlib/public/BackDeployConcurrency/AsyncIteratorProtocol.swift new file mode 100644 index 0000000000000..9c7076721e13f --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncIteratorProtocol.swift @@ -0,0 +1,97 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// A type that asynchronously supplies the values of a sequence one at a +/// time. +/// +/// The `AsyncIteratorProtocol` defines the type returned by the +/// `makeAsyncIterator()` method of the `AsyncSequence` protocol. In short, +/// the iterator is what produces the asynchronous sequence's values. The +/// protocol defines a single asynchronous method, `next()`, which either +/// produces the next element of the sequence, or returns `nil` to signal +/// the end of the sequence. +/// +/// To implement your own `AsyncSequence`, implement a wrapped type that +/// conforms to `AsyncIteratorProtocol`. The following example shows a `Counter` +/// type that uses an inner iterator to monotonically generate `Int` values +/// until reaching a `howHigh` value. While this example isn't itself +/// asychronous, it shows the shape of a custom sequence and iterator, and how +/// to use it as if it were asynchronous: +/// +/// struct Counter : AsyncSequence { +/// typealias Element = Int +/// let howHigh: Int +/// +/// struct AsyncIterator : AsyncIteratorProtocol { +/// let howHigh: Int +/// var current = 1 +/// mutating func next() async -> Int? { +/// // A genuinely asychronous implementation uses the `Task` +/// // API to check for cancellation here and return early. +/// guard current <= howHigh else { +/// return nil +/// } +/// +/// let result = current +/// current += 1 +/// return result +/// } +/// } +/// +/// func makeAsyncIterator() -> AsyncIterator { +/// return AsyncIterator(howHigh: howHigh) +/// } +/// } +/// +/// At the call site, this looks like: +/// +/// for await i in Counter(howHigh: 10) { +/// print(i, terminator: " ") +/// } +/// // Prints: 1 2 3 4 5 6 7 8 9 10 +/// +/// ### End of Iteration +/// +/// The iterator returns `nil` to indicate the end of the sequence. After +/// returning `nil` (or throwing an error) from `next()`, the iterator enters +/// a terminal state, and all future calls to `next()` must return `nil`. +/// +/// ### Cancellation +/// +/// Types conforming to `AsyncIteratorProtocol` should use the cancellation +/// primitives provided by Swift's `Task` API. The iterator can choose how to +/// handle and respond to cancellation, including: +/// +/// - Checking the `isCancelled` value of the current `Task` inside `next()` +/// and returning `nil` to terminate the sequence. +/// - Calling `checkCancellation()` on the `Task`, which throws a +/// `CancellationError`. +/// - Implementing `next()` with a +/// `withTaskCancellationHandler(handler:operation:)` invocation to +/// immediately react to cancellation. +/// +/// If the iterator needs to clean up on cancellation, it can do so after +/// checking for cancellation as described above, or in `deinit` if it's +/// a reference type. +@available(SwiftStdlib 5.1, *) +@rethrows +public protocol AsyncIteratorProtocol { + associatedtype Element + /// Asynchronously advances to the next element and returns it, or ends the + /// sequence if there is no next element. + /// + /// - Returns: The next element, if it exists, or `nil` to signal the end of + /// the sequence. + mutating func next() async throws -> Element? +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncLet.cpp b/stdlib/public/BackDeployConcurrency/AsyncLet.cpp new file mode 100644 index 0000000000000..048ed25c16a51 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncLet.cpp @@ -0,0 +1,551 @@ +//===--- AsyncLet.h - async let object management -00------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Object management routines for asynchronous task objects. +// +//===----------------------------------------------------------------------===// + +#include "CompatibilityOverride.h" +#include "ConcurrencyRuntime.h" +#include "swift/ABI/Metadata.h" +#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/HeapObject.h" +#include "llvm/ADT/PointerIntPair.h" +#include "AsyncLet.h" +#include "Debug.h" +#include "Task.h" +#include "TaskOptions.h" +#include "TaskPrivate.h" + +#if !defined(_WIN32) +#include +#endif + +using namespace swift; + +namespace { +class alignas(Alignment_AsyncLet) AsyncLetImpl: public ChildTaskStatusRecord { +public: + // This is where we could define a Status or other types important for async-let + +private: + // Flags stored in the low bits of the task pointer. + enum { + HasResult = 1 << 0, + DidAllocateFromParentTask = 1 << 1, + }; + + /// The task that was kicked off to initialize this `async let`, + /// and flags. + llvm::PointerIntPair taskAndFlags; + + /// Reserved space for a future_wait context frame, used during suspensions + /// on the child task future. + std::aligned_storage::type futureWaitContextStorage; + + friend class ::swift::AsyncTask; + +public: + explicit AsyncLetImpl(AsyncTask* task) + : ChildTaskStatusRecord(task), + taskAndFlags(task, 0) { + assert(task->hasChildFragment() && "async let task must be a child task."); + } + + /// Returns the task record representing this async let task. + /// The record is stored in the parent task, and should be removed when the + /// async let goes out of scope. + ChildTaskStatusRecord *getTaskRecord() { + return reinterpret_cast(this); + } + + AsyncTask *getTask() const { + return taskAndFlags.getPointer(); + } + + bool hasResultInBuffer() const { + return taskAndFlags.getInt() & HasResult; + } + + void setHasResultInBuffer(bool value = true) { + if (value) + taskAndFlags.setInt(taskAndFlags.getInt() | HasResult); + else + taskAndFlags.setInt(taskAndFlags.getInt() & ~HasResult); + } + + bool didAllocateFromParentTask() const { + return taskAndFlags.getInt() & DidAllocateFromParentTask; + } + + void setDidAllocateFromParentTask(bool value = true) { + if (value) + taskAndFlags.setInt(taskAndFlags.getInt() | DidAllocateFromParentTask); + else + taskAndFlags.setInt(taskAndFlags.getInt() & ~DidAllocateFromParentTask); + } + + // The compiler preallocates a large fixed space for the `async let`, with the + // intent that most of it be used for the child task context. The next two + // methods return the address and size of that space. + + /// Return a pointer to the unused space within the async let block. + void *getPreallocatedSpace() { + return (void*)(this + 1); + } + + /// Return the size of the unused space within the async let block. + static constexpr size_t getSizeOfPreallocatedSpace() { + return sizeof(AsyncLet) - sizeof(AsyncLetImpl); + } + + TaskFutureWaitAsyncContext *getFutureContext() { + return reinterpret_cast(&futureWaitContextStorage); + } +}; // end AsyncLetImpl + +} // end anonymous namespace + + +/******************************************************************************/ +/************************* ASYNC LET IMPLEMENTATION ***************************/ +/******************************************************************************/ + +static_assert(sizeof(AsyncLetImpl) <= sizeof(AsyncLet) && + alignof(AsyncLetImpl) <= alignof(AsyncLet), + "AsyncLetImpl doesn't fit in AsyncLet"); + +static AsyncLetImpl *asImpl(AsyncLet *alet) { + return reinterpret_cast(alet); +} + +static AsyncLetImpl *asImpl(const AsyncLet *alet) { + return reinterpret_cast( + const_cast(alet)); +} + +void swift::asyncLet_addImpl(AsyncTask *task, AsyncLet *asyncLet, + bool didAllocateInParentTask) { + AsyncLetImpl *impl = new (asyncLet) AsyncLetImpl(task); + impl->setDidAllocateFromParentTask(didAllocateInParentTask); + + auto record = impl->getTaskRecord(); + assert(impl == record && "the async-let IS the task record"); + + // ok, now that the group actually is initialized: attach it to the task + swift_task_addStatusRecord(record); +} + +// ============================================================================= +// ==== start ------------------------------------------------------------------ + +SWIFT_CC(swift) +void swift::swift_asyncLet_start(AsyncLet *alet, + TaskOptionRecord *options, + const Metadata *futureResultType, + void *closureEntryPoint, + HeapObject *closureContext) { + auto flags = TaskCreateFlags(); + flags.setEnqueueJob(true); + + AsyncLetTaskOptionRecord asyncLetOptionRecord(alet); + asyncLetOptionRecord.Parent = options; + + swift_task_create( + flags.getOpaqueValue(), + &asyncLetOptionRecord, + futureResultType, + closureEntryPoint, closureContext); +} + +SWIFT_CC(swift) +void swift::swift_asyncLet_begin(AsyncLet *alet, + TaskOptionRecord *options, + const Metadata *futureResultType, + void *closureEntryPoint, + HeapObject *closureContext, + void *resultBuffer) { + SWIFT_TASK_DEBUG_LOG("creating async let buffer of type %s at %p", + swift_getTypeName(futureResultType, true).data, + resultBuffer); + + auto flags = TaskCreateFlags(); + flags.setEnqueueJob(true); + + AsyncLetWithBufferTaskOptionRecord asyncLetOptionRecord(alet, resultBuffer); + asyncLetOptionRecord.Parent = options; + + swift_task_create( + flags.getOpaqueValue(), + &asyncLetOptionRecord, + futureResultType, + closureEntryPoint, closureContext); +} + +// ============================================================================= +// ==== wait ------------------------------------------------------------------- + +SWIFT_CC(swiftasync) +static void swift_asyncLet_waitImpl( + OpaqueValue *result, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, TaskContinuationFunction *resumeFunction, + AsyncContext *callContext) { + auto task = alet->getTask(); + swift_task_future_wait(result, callerContext, task, resumeFunction, + callContext); +} + +SWIFT_CC(swiftasync) +static void swift_asyncLet_wait_throwingImpl( + OpaqueValue *result, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + ThrowingTaskFutureWaitContinuationFunction *resumeFunction, + AsyncContext * callContext) { + auto task = alet->getTask(); + swift_task_future_wait_throwing(result, callerContext, task, resumeFunction, + callerContext); +} + +// ============================================================================= +// ==== get ------------------------------------------------------------------- + +SWIFT_CC(swiftasync) +static void swift_asyncLet_getImpl(SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + void *resultBuffer, + TaskContinuationFunction *resumeFunction, + AsyncContext *callContext) { + // Don't need to do anything if the result buffer is already populated. + if (asImpl(alet)->hasResultInBuffer()) { + return resumeFunction(callerContext); + } + + // Mark the async let as having its result populated. + // The only task that can ask this of the async let is the same parent task + // that's currently executing, so we can set it now and tail-call future_wait, + // since by the time we can call back it will be populated. + asImpl(alet)->setHasResultInBuffer(); + swift_task_future_wait(reinterpret_cast(resultBuffer), + callerContext, alet->getTask(), + resumeFunction, callContext); +} + +struct AsyncLetContinuationContext: AsyncContext { + AsyncLet *alet; + OpaqueValue *resultBuffer; +}; + +static_assert(sizeof(AsyncLetContinuationContext) <= sizeof(TaskFutureWaitAsyncContext), + "compiler provides the same amount of context space to each"); + +SWIFT_CC(swiftasync) +static void _asyncLet_get_throwing_continuation( + SWIFT_ASYNC_CONTEXT AsyncContext *callContext, + SWIFT_CONTEXT void *error) { + auto continuationContext = static_cast(callContext); + auto alet = continuationContext->alet; + + // If the future completed successfully, its result is now in the async let + // buffer. + if (!error) { + asImpl(alet)->setHasResultInBuffer(); + } + + // Continue the caller's execution. + auto throwingResume + = reinterpret_cast(callContext->ResumeParent); + return throwingResume(callContext->Parent, error); +} + +SWIFT_CC(swiftasync) +static void swift_asyncLet_get_throwingImpl( + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + void *resultBuffer, + ThrowingTaskFutureWaitContinuationFunction *resumeFunction, + AsyncContext *callContext) { + // Don't need to do anything if the result buffer is already populated. + if (asImpl(alet)->hasResultInBuffer()) { + return resumeFunction(callerContext, nullptr); + } + + auto aletContext = static_cast(callContext); + aletContext->ResumeParent + = reinterpret_cast(resumeFunction); + aletContext->Parent = callerContext; + aletContext->alet = alet; + auto futureContext = asImpl(alet)->getFutureContext(); + + // Unlike the non-throwing variant, whether we end up with a result depends + // on the success of the task. If we raise an error, then the result buffer + // will not be populated. Save the async let binding so we can fetch it + // after completion. + return swift_task_future_wait_throwing( + reinterpret_cast(resultBuffer), + aletContext, alet->getTask(), + _asyncLet_get_throwing_continuation, + futureContext); +} + +// ============================================================================= +// ==== end -------------------------------------------------------------------- + +SWIFT_CC(swift) +static void swift_asyncLet_endImpl(AsyncLet *alet) { + auto task = alet->getTask(); + + // Cancel the task as we exit the scope + swift_task_cancel(task); + + // Remove the child record from the parent task + auto record = asImpl(alet)->getTaskRecord(); + swift_task_removeStatusRecord(record); + + // TODO: we need to implicitly await either before the end or here somehow. + + // and finally, release the task and free the async-let + AsyncTask *parent = swift_task_getCurrent(); + assert(parent && "async-let must have a parent task"); + + SWIFT_TASK_DEBUG_LOG("async let end of task %p, parent: %p", task, parent); + _swift_task_dealloc_specific(parent, task); +} + +// ============================================================================= +// ==== finish ----------------------------------------------------------------- + +SWIFT_CC(swiftasync) +// FIXME: noinline to work around an LLVM bug where the outliner breaks +// musttail. +SWIFT_NOINLINE +static void asyncLet_finish_after_task_completion(SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + TaskContinuationFunction *resumeFunction, + AsyncContext *callContext, + SWIFT_CONTEXT void *error) { + auto task = alet->getTask(); + + // Remove the child record from the parent task + auto record = asImpl(alet)->getTaskRecord(); + swift_task_removeStatusRecord(record); + + // and finally, release the task and destroy the async-let + assert(swift_task_getCurrent() && "async-let must have a parent task"); + + SWIFT_TASK_DEBUG_LOG("async let end of task %p, parent: %p", task, + swift_task_getCurrent()); + // Destruct the task. + task->~AsyncTask(); + // Deallocate it out of the parent, if it was allocated there. + if (alet->didAllocateFromParentTask()) { + swift_task_dealloc(task); + } + + return reinterpret_cast(resumeFunction) + (callerContext, error); +} + + +SWIFT_CC(swiftasync) +static void _asyncLet_finish_continuation( + SWIFT_ASYNC_CONTEXT AsyncContext *callContext, + SWIFT_CONTEXT void *error) { + // Retrieve the async let pointer from the context. + auto continuationContext + = reinterpret_cast(callContext); + auto alet = continuationContext->alet; + auto resultBuffer = continuationContext->resultBuffer; + + // Destroy the error, or the result that was stored to the buffer. + if (error) { + swift_errorRelease((SwiftError*)error); + } else { + alet->getTask()->futureFragment()->getResultType()->vw_destroy(resultBuffer); + } + + // Clean up the async let now that the task has finished. + return asyncLet_finish_after_task_completion(callContext->Parent, + alet, + callContext->ResumeParent, + callContext, + nullptr); +} + +SWIFT_CC(swiftasync) +static void swift_asyncLet_finishImpl(SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + void *resultBuffer, + TaskContinuationFunction *resumeFunction, + AsyncContext *callContext) { + auto task = alet->getTask(); + + // If the result buffer is already populated, then we just need to destroy + // the value in it and then clean up the task. + if (asImpl(alet)->hasResultInBuffer()) { + task->futureFragment()->getResultType()->vw_destroy( + reinterpret_cast(resultBuffer)); + return asyncLet_finish_after_task_completion(callerContext, + alet, + resumeFunction, + callContext, + nullptr); + } + // Otherwise, cancel the task and let it finish first. + swift_task_cancel(task); + + // Save the async let pointer in the context so we can clean it up once the + // future completes. + auto aletContext = static_cast(callContext); + aletContext->Parent = callerContext; + aletContext->ResumeParent = resumeFunction; + aletContext->alet = alet; + aletContext->resultBuffer = reinterpret_cast(resultBuffer); + auto futureContext = asImpl(alet)->getFutureContext(); + + // TODO: It would be nice if we could await the future without having to + // provide a buffer to store the value to, since we're going to dispose of + // it anyway. + return swift_task_future_wait_throwing( + reinterpret_cast(resultBuffer), + callContext, alet->getTask(), + _asyncLet_finish_continuation, + futureContext); +} + +// ============================================================================= +// ==== consume ---------------------------------------------------------------- + +SWIFT_CC(swiftasync) +static void _asyncLet_consume_continuation( + SWIFT_ASYNC_CONTEXT AsyncContext *callContext) { + // Retrieve the async let pointer from the context. + auto continuationContext + = reinterpret_cast(callContext); + auto alet = continuationContext->alet; + + // Clean up the async let now that the task has finished. + return asyncLet_finish_after_task_completion(callContext->Parent, alet, + callContext->ResumeParent, + callContext, + nullptr); +} + +SWIFT_CC(swiftasync) +static void swift_asyncLet_consumeImpl(SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + void *resultBuffer, + TaskContinuationFunction *resumeFunction, + AsyncContext *callContext) { + // If the result buffer is already populated, then we just need to clean up + // the task. + if (asImpl(alet)->hasResultInBuffer()) { + return asyncLet_finish_after_task_completion(callerContext, + alet, + resumeFunction, + callContext, + nullptr); + } + + // Save the async let pointer in the context so we can clean it up once the + // future completes. + auto aletContext = static_cast(callContext); + aletContext->Parent = callerContext; + aletContext->ResumeParent = resumeFunction; + aletContext->alet = alet; + auto futureContext = asImpl(alet)->getFutureContext(); + + // Await completion of the task. We'll destroy the task afterward. + return swift_task_future_wait( + reinterpret_cast(resultBuffer), + callContext, alet->getTask(), + _asyncLet_consume_continuation, + futureContext); +} + +SWIFT_CC(swiftasync) +static void _asyncLet_consume_throwing_continuation( + SWIFT_ASYNC_CONTEXT AsyncContext *callContext, + SWIFT_CONTEXT void *error) { + // Get the async let pointer so we can destroy the task. + auto continuationContext = static_cast(callContext); + auto alet = continuationContext->alet; + + return asyncLet_finish_after_task_completion(callContext->Parent, + alet, + callContext->ResumeParent, + callContext, + error); +} + +SWIFT_CC(swiftasync) +static void swift_asyncLet_consume_throwingImpl( + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + void *resultBuffer, + ThrowingTaskFutureWaitContinuationFunction *resumeFunction, + AsyncContext *callContext) { + // If the result buffer is already populated, we just need to clean up the + // task. + if (asImpl(alet)->hasResultInBuffer()) { + return asyncLet_finish_after_task_completion(callerContext, + alet, + reinterpret_cast(resumeFunction), + callContext, + nullptr); + } + + auto aletContext = static_cast(callContext); + aletContext->ResumeParent + = reinterpret_cast(resumeFunction); + aletContext->Parent = callerContext; + aletContext->alet = alet; + auto futureContext = asImpl(alet)->getFutureContext(); + + // Unlike the non-throwing variant, whether we end up with a result depends + // on the success of the task. If we raise an error, then the result buffer + // will not be populated. Save the async let binding so we can fetch it + // after completion. + return swift_task_future_wait_throwing( + reinterpret_cast(resultBuffer), + aletContext, alet->getTask(), + _asyncLet_consume_throwing_continuation, + futureContext); +} + +// ============================================================================= +// ==== AsyncLet Implementation ------------------------------------------------ + +AsyncTask* AsyncLet::getTask() const { + return asImpl(this)->getTask(); +} + +void *AsyncLet::getPreallocatedSpace() { + return asImpl(this)->getPreallocatedSpace(); +} + +size_t AsyncLet::getSizeOfPreallocatedSpace() { + return AsyncLetImpl::getSizeOfPreallocatedSpace(); +} + +bool AsyncLet::didAllocateFromParentTask() { + return asImpl(this)->didAllocateFromParentTask(); +} + +void AsyncLet::setDidAllocateFromParentTask(bool value) { + return asImpl(this)->setDidAllocateFromParentTask(value); +} + +// ============================================================================= + +#define OVERRIDE_ASYNC_LET COMPATIBILITY_OVERRIDE +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH diff --git a/stdlib/public/BackDeployConcurrency/AsyncLet.h b/stdlib/public/BackDeployConcurrency/AsyncLet.h new file mode 100644 index 0000000000000..4e77ba6867698 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncLet.h @@ -0,0 +1,66 @@ +//===--- AsyncLet.h - ABI structures for async let -00-----------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing task groups. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_TASK_ASYNC_LET_H +#define SWIFT_ABI_TASK_ASYNC_LET_H + +#include "swift/ABI/HeapObject.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/Config.h" +#include "swift/Basic/RelativePointer.h" +#include "swift/Basic/STLExtras.h" +#include "Task.h" + +namespace swift { + +/// Represents an in-flight `async let`, i.e. the Task that is computing the +/// result of the async let, along with the awaited status and other metadata. +class alignas(Alignment_AsyncLet) AsyncLet { +public: + // These constructors do not initialize the AsyncLet instance, and the + // destructor does not destroy the AsyncLet instance; you must call + // swift_asyncLet_{start,end} yourself. + constexpr AsyncLet() + : PrivateData{} {} + + void *PrivateData[NumWords_AsyncLet]; + + // TODO: we could offer a "was awaited on" check here + + /// Returns the child task that is associated with this async let. + /// The tasks completion is used to fulfil the value represented by this async let. + AsyncTask *getTask() const; + + // The compiler preallocates a large fixed space for the `async let`, with the + // intent that most of it be used for the child task context. The next two + // methods return the address and size of that space. + + /// Return a pointer to the unused space within the async let block. + void *getPreallocatedSpace(); + + /// Return the size of the unused space within the async let block. + static size_t getSizeOfPreallocatedSpace(); + + /// Was the task allocated out of the parent's allocator? + bool didAllocateFromParentTask(); + + /// Flag that the task was allocated from the parent's allocator. + void setDidAllocateFromParentTask(bool value = true); +}; + +} // end namespace swift + +#endif // SWIFT_ABI_TASK_ASYNC_LET_H diff --git a/stdlib/public/BackDeployConcurrency/AsyncLet.swift b/stdlib/public/BackDeployConcurrency/AsyncLet.swift new file mode 100644 index 0000000000000..f4e8551e5559c --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncLet.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +// ==== Async Let ------------------------------------------------------------- +// Only has internal / builtin functions as it is not really accessible directly + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_asyncLet_start") +public func _asyncLetStart( + asyncLet: Builtin.RawPointer, + options: Builtin.RawPointer?, + operation: @Sendable () async throws -> T +) + +/// DEPRECATED. use _asyncLet_get instead +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_asyncLet_wait") +public func _asyncLetGet(asyncLet: Builtin.RawPointer) async -> T + +/// DEPRECATED. use _asyncLet_get_throwing instead +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_asyncLet_wait_throwing") +public func _asyncLetGetThrowing(asyncLet: Builtin.RawPointer) async throws -> T + +/// DEPRECATED. use _asyncLet_finish instead +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_asyncLet_end") +public func _asyncLetEnd( + asyncLet: Builtin.RawPointer // TODO: should this take __owned? +) + +/// Wait if necessary and then project the result value of an async let +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_asyncLet_get") +public func _asyncLet_get(_ asyncLet: Builtin.RawPointer, _ resultBuffer: Builtin.RawPointer) async -> Builtin.RawPointer + +/// Wait if necessary and then project the result value of an async let that throws +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_asyncLet_get_throwing") +public func _asyncLet_get_throwing(_ asyncLet: Builtin.RawPointer, _ resultBuffer: Builtin.RawPointer) async throws -> Builtin.RawPointer + +/// Wait if necessary and then tear down the async let task +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_asyncLet_finish") +public func _asyncLet_finish(_ asyncLet: Builtin.RawPointer, _ resultBuffer: Builtin.RawPointer) async + +@_silgen_name("swift_asyncLet_extractTask") +func _asyncLetExtractTask( + of asyncLet: Builtin.RawPointer +) -> Builtin.NativeObject diff --git a/stdlib/public/BackDeployConcurrency/AsyncMapSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncMapSequence.swift new file mode 100644 index 0000000000000..57aa8918c0efe --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncMapSequence.swift @@ -0,0 +1,119 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that maps the given closure over the + /// asynchronous sequence’s elements. + /// + /// Use the `map(_:)` method to transform every element received from a base + /// asynchronous sequence. Typically, you use this to transform from one type + /// of element to another. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `5`. The closure provided to the `map(_:)` method + /// takes each `Int` and looks up a corresponding `String` from a + /// `romanNumeralDict` dictionary. This means the outer `for await in` loop + /// iterates over `String` instances instead of the underlying `Int` values + /// that `Counter` produces: + /// + /// let romanNumeralDict: [Int: String] = + /// [1: "I", 2: "II", 3: "III", 5: "V"] + /// + /// let stream = Counter(howHigh: 5) + /// .map { romanNumeralDict[$0] ?? "(unknown)" } + /// for await numeral in stream { + /// print("\(numeral) ", terminator: " ") + /// } + /// // Prints: I II III (unknown) V + /// + /// - Parameter transform: A mapping closure. `transform` accepts an element + /// of this sequence as its parameter and returns a transformed value of the + /// same or of a different type. + /// - Returns: An asynchronous sequence that contains, in order, the elements + /// produced by the `transform` closure. + @inlinable + public __consuming func map( + _ transform: @escaping (Element) async -> Transformed + ) -> AsyncMapSequence { + return AsyncMapSequence(self, transform: transform) + } +} + +/// An asynchronous sequence that maps the given closure over the asynchronous +/// sequence’s elements. +@available(SwiftStdlib 5.1, *) +public struct AsyncMapSequence { + @usableFromInline + let base: Base + + @usableFromInline + let transform: (Base.Element) async -> Transformed + + @usableFromInline + init( + _ base: Base, + transform: @escaping (Base.Element) async -> Transformed + ) { + self.base = base + self.transform = transform + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncMapSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The map sequence produces whatever type of element its transforming + /// closure produces. + public typealias Element = Transformed + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the map sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let transform: (Base.Element) async -> Transformed + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + transform: @escaping (Base.Element) async -> Transformed + ) { + self.baseIterator = baseIterator + self.transform = transform + } + + /// Produces the next element in the map sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns `nil`. Otherwise, `next()` returns the result of + /// calling the transforming closure on the received element. + @inlinable + public mutating func next() async rethrows -> Transformed? { + guard let element = try await baseIterator.next() else { + return nil + } + return await transform(element) + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), transform: transform) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncPrefixSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncPrefixSequence.swift new file mode 100644 index 0000000000000..4cbcb3fa4cccc --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncPrefixSequence.swift @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Returns an asynchronous sequence, up to the specified maximum length, + /// containing the initial elements of the base asynchronous sequence. + /// + /// Use `prefix(_:)` to reduce the number of elements produced by the + /// asynchronous sequence. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `prefix(_:)` method causes the modified + /// sequence to pass through the first six values, then end. + /// + /// for await number in Counter(howHigh: 10).prefix(6) { + /// print("\(number) ") + /// } + /// // prints "1 2 3 4 5 6" + /// + /// If the count passed to `prefix(_:)` exceeds the number of elements in the + /// base sequence, the result contains all of the elements in the sequence. + /// + /// - Parameter count: The maximum number of elements to return. The value of + /// `count` must be greater than or equal to zero. + /// - Returns: An asynchronous sequence starting at the beginning of the + /// base sequence with at most `count` elements. + @inlinable + public __consuming func prefix( + _ count: Int + ) -> AsyncPrefixSequence { + precondition(count >= 0, + "Can't prefix a negative number of elements from an async sequence") + return AsyncPrefixSequence(self, count: count) + } +} + +/// An asynchronous sequence, up to a specified maximum length, +/// containing the initial elements of a base asynchronous sequence. +@available(SwiftStdlib 5.1, *) +public struct AsyncPrefixSequence { + @usableFromInline + let base: Base + + @usableFromInline + let count: Int + + @usableFromInline + init(_ base: Base, count: Int) { + self.base = base + self.count = count + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncPrefixSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The prefix sequence produces whatever type of element its base iterator + /// produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the prefix sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + var remaining: Int + + @usableFromInline + init(_ baseIterator: Base.AsyncIterator, count: Int) { + self.baseIterator = baseIterator + self.remaining = count + } + + /// Produces the next element in the prefix sequence. + /// + /// Until reaching the number of elements to include, this iterator calls + /// `next()` on its base iterator and passes through the result. After + /// reaching the maximum number of elements, subsequent calls to `next()` + /// return `nil`. + @inlinable + public mutating func next() async rethrows -> Base.Element? { + if remaining != 0 { + remaining &-= 1 + return try await baseIterator.next() + } else { + return nil + } + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), count: count) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncPrefixWhileSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncPrefixWhileSequence.swift new file mode 100644 index 0000000000000..8ec772ec5b874 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncPrefixWhileSequence.swift @@ -0,0 +1,122 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Returns an asynchronous sequence, containing the initial, consecutive + /// elements of the base sequence that satisfy the given predicate. + /// + /// Use `prefix(while:)` to produce values while elements from the base + /// sequence meet a condition you specify. The modified sequence ends when + /// the predicate closure returns `false`. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `prefix(while:)` method causes the modified + /// sequence to pass along values so long as they aren’t divisible by `2` and + /// `3`. Upon reaching `6`, the sequence ends: + /// + /// let stream = Counter(howHigh: 10) + /// .prefix { $0 % 2 != 0 || $0 % 3 != 0 } + /// for try await number in stream { + /// print("\(number) ", terminator: " ") + /// } + /// // prints "1 2 3 4 5" + /// + /// - Parameter predicate: A closure that takes an element as a parameter and + /// returns a Boolean value indicating whether the element should be + /// included in the modified sequence. + /// - Returns: An asynchronous sequence of the initial, consecutive + /// elements that satisfy `predicate`. + @inlinable + public __consuming func prefix( + while predicate: @escaping (Element) async -> Bool + ) rethrows -> AsyncPrefixWhileSequence { + return AsyncPrefixWhileSequence(self, predicate: predicate) + } +} + +/// An asynchronous sequence, containing the initial, consecutive +/// elements of the base sequence that satisfy a given predicate. +@available(SwiftStdlib 5.1, *) +public struct AsyncPrefixWhileSequence { + @usableFromInline + let base: Base + + @usableFromInline + let predicate: (Base.Element) async -> Bool + + @usableFromInline + init( + _ base: Base, + predicate: @escaping (Base.Element) async -> Bool + ) { + self.base = base + self.predicate = predicate + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncPrefixWhileSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The prefix-while sequence produces whatever type of element its base + /// iterator produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the prefix-while sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var predicateHasFailed = false + + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let predicate: (Base.Element) async -> Bool + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + predicate: @escaping (Base.Element) async -> Bool + ) { + self.baseIterator = baseIterator + self.predicate = predicate + } + + /// Produces the next element in the prefix-while sequence. + /// + /// If the predicate hasn't yet failed, this method gets the next element + /// from the base sequence and calls the predicate with it. If this call + /// succeeds, this method passes along the element. Otherwise, it returns + /// `nil`, ending the sequence. + @inlinable + public mutating func next() async rethrows -> Base.Element? { + if !predicateHasFailed, let nextElement = try await baseIterator.next() { + if await predicate(nextElement) { + return nextElement + } else { + predicateHasFailed = true + } + } + return nil + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), predicate: predicate) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncSequence.swift new file mode 100644 index 0000000000000..d6e607430cc49 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncSequence.swift @@ -0,0 +1,481 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// A type that provides asynchronous, sequential, iterated access to its +/// elements. +/// +/// An `AsyncSequence` resembles the `Sequence` type --- offering a list of +/// values you can step through one at a time --- and adds asynchronicity. An +/// `AsyncSequence` may have all, some, or none of its values available when +/// you first use it. Instead, you use `await` to receive values as they become +/// available. +/// +/// As with `Sequence`, you typically iterate through an `AsyncSequence` with a +/// `for await`-`in` loop. However, because the caller must potentially wait for values, +/// you use the `await` keyword. The following example shows how to iterate +/// over `Counter`, a custom `AsyncSequence` that produces `Int` values from +/// `1` up to a `howHigh` value: +/// +/// for await i in Counter(howHigh: 10) { +/// print(i, terminator: " ") +/// } +/// // Prints: 1 2 3 4 5 6 7 8 9 10 +/// +/// An `AsyncSequence` doesn't generate or contain the values; it just defines +/// how you access them. Along with defining the type of values as an associated +/// type called `Element`, the `AsyncSequence` defines a `makeAsyncIterator()` +/// method. This returns an instance of type `AsyncIterator`. Like the standard +/// `IteratorProtocol`, the `AsyncIteratorProtocol` defines a single `next()` +/// method to produce elements. The difference is that the `AsyncIterator` +/// defines its `next()` method as `async`, which requires a caller to wait for +/// the next value with the `await` keyword. +/// +/// `AsyncSequence` also defines methods for processing the elements you +/// receive, modeled on the operations provided by the basic `Sequence` in the +/// standard library. There are two categories of methods: those that return a +/// single value, and those that return another `AsyncSequence`. +/// +/// Single-value methods eliminate the need for a `for await`-`in` loop, and instead +/// let you make a single `await` call. For example, the `contains(_:)` method +/// returns a Boolean value that indicates if a given value exists in the +/// `AsyncSequence`. Given the `Counter` sequence from the previous example, +/// you can test for the existence of a sequence member with a one-line call: +/// +/// let found = await Counter(howHigh: 10).contains(5) // true +/// +/// Methods that return another `AsyncSequence` return a type specific to the +/// method's semantics. For example, the `.map(_:)` method returns a +/// `AsyncMapSequence` (or a `AsyncThrowingMapSequence`, if the closure you +/// provide to the `map(_:)` method can throw an error). These returned +/// sequences don't eagerly await the next member of the sequence, which allows +/// the caller to decide when to start work. Typically, you'll iterate over +/// these sequences with `for await`-`in`, like the base `AsyncSequence` you started +/// with. In the following example, the `map(_:)` method transforms each `Int` +/// received from a `Counter` sequence into a `String`: +/// +/// let stream = Counter(howHigh: 10) +/// .map { $0 % 2 == 0 ? "Even" : "Odd" } +/// for await s in stream { +/// print(s, terminator: " ") +/// } +/// // Prints: Odd Even Odd Even Odd Even Odd Even Odd Even +/// +@available(SwiftStdlib 5.1, *) +@rethrows +public protocol AsyncSequence { + /// The type of asynchronous iterator that produces elements of this + /// asynchronous sequence. + associatedtype AsyncIterator: AsyncIteratorProtocol where AsyncIterator.Element == Element + /// The type of element produced by this asynchronous sequence. + associatedtype Element + /// Creates the asynchronous iterator that produces elements of this + /// asynchronous sequence. + /// + /// - Returns: An instance of the `AsyncIterator` type used to produce + /// elements of the asynchronous sequence. + __consuming func makeAsyncIterator() -> AsyncIterator +} + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Returns the result of combining the elements of the asynchronous sequence + /// using the given closure. + /// + /// Use the `reduce(_:_:)` method to produce a single value from the elements of + /// an entire sequence. For example, you can use this method on an sequence of + /// numbers to find their sum or product. + /// + /// The `nextPartialResult` closure executes sequentially with an accumulating + /// value initialized to `initialResult` and each element of the sequence. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `4`. The `reduce(_:_:)` method sums the values + /// received from the asynchronous sequence. + /// + /// let sum = await Counter(howHigh: 4) + /// .reduce(0) { + /// $0 + $1 + /// } + /// print(sum) + /// // Prints: 10 + /// + /// + /// - Parameters: + /// - initialResult: The value to use as the initial accumulating value. + /// The `nextPartialResult` closure receives `initialResult` the first + /// time the closure runs. + /// - nextPartialResult: A closure that combines an accumulating value and + /// an element of the asynchronous sequence into a new accumulating value, + /// for use in the next call of the `nextPartialResult` closure or + /// returned to the caller. + /// - Returns: The final accumulated value. If the sequence has no elements, + /// the result is `initialResult`. + @inlinable + public func reduce( + _ initialResult: Result, + _ nextPartialResult: + (_ partialResult: Result, Element) async throws -> Result + ) async rethrows -> Result { + var accumulator = initialResult + var iterator = makeAsyncIterator() + while let element = try await iterator.next() { + accumulator = try await nextPartialResult(accumulator, element) + } + return accumulator + } + + /// Returns the result of combining the elements of the asynchronous sequence + /// using the given closure, given a mutable initial value. + /// + /// Use the `reduce(into:_:)` method to produce a single value from the + /// elements of an entire sequence. For example, you can use this method on a + /// sequence of numbers to find their sum or product. + /// + /// The `nextPartialResult` closure executes sequentially with an accumulating + /// value initialized to `initialResult` and each element of the sequence. + /// + /// Prefer this method over `reduce(_:_:)` for efficiency when the result is + /// a copy-on-write type, for example an `Array` or `Dictionary`. + /// + /// - Parameters: + /// - initialResult: The value to use as the initial accumulating value. + /// The `nextPartialResult` closure receives `initialResult` the first + /// time the closure executes. + /// - nextPartialResult: A closure that combines an accumulating value and + /// an element of the asynchronous sequence into a new accumulating value, + /// for use in the next call of the `nextPartialResult` closure or + /// returned to the caller. + /// - Returns: The final accumulated value. If the sequence has no elements, + /// the result is `initialResult`. + @inlinable + public func reduce( + into initialResult: __owned Result, + _ updateAccumulatingResult: + (_ partialResult: inout Result, Element) async throws -> Void + ) async rethrows -> Result { + var accumulator = initialResult + var iterator = makeAsyncIterator() + while let element = try await iterator.next() { + try await updateAccumulatingResult(&accumulator, element) + } + return accumulator + } +} + +@available(SwiftStdlib 5.1, *) +@inlinable +@inline(__always) +func _contains( + _ self: Source, + where predicate: (Source.Element) async throws -> Bool +) async rethrows -> Bool { + for try await element in self { + if try await predicate(element) { + return true + } + } + return false +} + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Returns a Boolean value that indicates whether the asynchronous sequence + /// contains an element that satisfies the given predicate. + /// + /// You can use the predicate to check for an element of a type that doesn’t + /// conform to the `Equatable` protocol, or to find an element that satisfies + /// a general condition. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `contains(where:)` method checks to see + /// whether the sequence produces a value divisible by `3`: + /// + /// let containsDivisibleByThree = await Counter(howHigh: 10) + /// .contains { $0 % 3 == 0 } + /// print(containsDivisibleByThree) + /// // Prints: true + /// + /// The predicate executes each time the asynchronous sequence produces an + /// element, until either the predicate finds a match or the sequence ends. + /// + /// - Parameter predicate: A closure that takes an element of the asynchronous + /// sequence as its argument and returns a Boolean value that indicates + /// whether the passed element represents a match. + /// - Returns: `true` if the sequence contains an element that satisfies + /// predicate; otherwise, `false`. + @inlinable + public func contains( + where predicate: (Element) async throws -> Bool + ) async rethrows -> Bool { + return try await _contains(self, where: predicate) + } + + /// Returns a Boolean value that indicates whether all elements produced by the + /// asynchronous sequence satisfies the given predicate. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `allSatisfy(_:)` method checks to see whether + /// all elements produced by the sequence are less than `10`. + /// + /// let allLessThanTen = await Counter(howHigh: 10) + /// .allSatisfy { $0 < 10 } + /// print(allLessThanTen) + /// // Prints: false + /// + /// The predicate executes each time the asynchronous sequence produces an + /// element, until either the predicate returns `false` or the sequence ends. + /// + /// If the asynchronous sequence is empty, this method returns `true`. + /// + /// - Parameter predicate: A closure that takes an element of the asynchronous + /// sequence as its argument and returns a Boolean value that indicates + /// whether the passed element satisfies a condition. + /// - Returns: `true` if the sequence contains only elements that satisfy + /// `predicate`; otherwise, `false`. + @inlinable + public func allSatisfy( + _ predicate: (Element) async throws -> Bool + ) async rethrows -> Bool { + return try await !contains { try await !predicate($0) } + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence where Element: Equatable { + /// Returns a Boolean value that indicates whether the asynchronous sequence + /// contains the given element. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `contains(_:)` method checks to see whether + /// the sequence produces the value `5`: + /// + /// let containsFive = await Counter(howHigh: 10) + /// .contains(5) + /// print(containsFive) + /// // Prints: true + /// + /// - Parameter search: The element to find in the asynchronous sequence. + /// - Returns: `true` if the method found the element in the asynchronous + /// sequence; otherwise, `false`. + @inlinable + public func contains(_ search: Element) async rethrows -> Bool { + for try await element in self { + if element == search { + return true + } + } + return false + } +} + +@available(SwiftStdlib 5.1, *) +@inlinable +@inline(__always) +func _first( + _ self: Source, + where predicate: (Source.Element) async throws -> Bool +) async rethrows -> Source.Element? { + for try await element in self { + if try await predicate(element) { + return element + } + } + return nil +} + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Returns the first element of the sequence that satisfies the given + /// predicate. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `first(where:)` method returns the first + /// member of the sequence that's evenly divisible by both `2` and `3`. + /// + /// let divisibleBy2And3 = await Counter(howHigh: 10) + /// .first { $0 % 2 == 0 && $0 % 3 == 0 } + /// print(divisibleBy2And3 ?? "none") + /// // Prints: 6 + /// + /// The predicate executes each time the asynchronous sequence produces an + /// element, until either the predicate finds a match or the sequence ends. + /// + /// - Parameter predicate: A closure that takes an element of the asynchronous + /// sequence as its argument and returns a Boolean value that indicates + /// whether the element is a match. + /// - Returns: The first element of the sequence that satisfies `predicate`, + /// or `nil` if there is no element that satisfies `predicate`. + @inlinable + public func first( + where predicate: (Element) async throws -> Bool + ) async rethrows -> Element? { + return try await _first(self, where: predicate) + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Returns the minimum element in the asynchronous sequence, using the given + /// predicate as the comparison between elements. + /// + /// Use this method when the asynchronous sequence's values don't conform + /// to `Comparable`, or when you want to apply a custom ordering to the + /// sequence. + /// + /// The predicate must be a *strict weak ordering* over the elements. That is, + /// for any elements `a`, `b`, and `c`, the following conditions must hold: + /// + /// - `areInIncreasingOrder(a, a)` is always `false`. (Irreflexivity) + /// - If `areInIncreasingOrder(a, b)` and `areInIncreasingOrder(b, c)` are + /// both `true`, then `areInIncreasingOrder(a, c)` is also + /// `true`. (Transitive comparability) + /// - Two elements are *incomparable* if neither is ordered before the other + /// according to the predicate. If `a` and `b` are incomparable, and `b` + /// and `c` are incomparable, then `a` and `c` are also incomparable. + /// (Transitive incomparability) + /// + /// The following example uses an enumeration of playing cards ranks, `Rank`, + /// which ranges from `ace` (low) to `king` (high). An asynchronous sequence + /// called `RankCounter` produces all elements of the array. The predicate + /// provided to the `min(by:)` method sorts ranks based on their `rawValue`: + /// + /// enum Rank: Int { + /// case ace = 1, two, three, four, five, six, seven, eight, nine, ten, jack, queen, king + /// } + /// + /// let min = await RankCounter() + /// .min { $0.rawValue < $1.rawValue } + /// print(min ?? "none") + /// // Prints: ace + /// + /// - Parameter areInIncreasingOrder: A predicate that returns `true` if its + /// first argument should be ordered before its second argument; otherwise, + /// `false`. + /// - Returns: The sequence’s minimum element, according to + /// `areInIncreasingOrder`. If the sequence has no elements, returns `nil`. + @inlinable + @warn_unqualified_access + public func min( + by areInIncreasingOrder: (Element, Element) async throws -> Bool + ) async rethrows -> Element? { + var it = makeAsyncIterator() + guard var result = try await it.next() else { + return nil + } + while let e = try await it.next() { + if try await areInIncreasingOrder(e, result) { + result = e + } + } + return result + } + + /// Returns the maximum element in the asynchronous sequence, using the given + /// predicate as the comparison between elements. + /// + /// Use this method when the asynchronous sequence's values don't conform + /// to `Comparable`, or when you want to apply a custom ordering to the + /// sequence. + /// + /// The predicate must be a *strict weak ordering* over the elements. That is, + /// for any elements `a`, `b`, and `c`, the following conditions must hold: + /// + /// - `areInIncreasingOrder(a, a)` is always `false`. (Irreflexivity) + /// - If `areInIncreasingOrder(a, b)` and `areInIncreasingOrder(b, c)` are + /// both `true`, then `areInIncreasingOrder(a, c)` is also + /// `true`. (Transitive comparability) + /// - Two elements are *incomparable* if neither is ordered before the other + /// according to the predicate. If `a` and `b` are incomparable, and `b` + /// and `c` are incomparable, then `a` and `c` are also incomparable. + /// (Transitive incomparability) + /// + /// The following example uses an enumeration of playing cards ranks, `Rank`, + /// which ranges from `ace` (low) to `king` (high). An asynchronous sequence + /// called `RankCounter` produces all elements of the array. The predicate + /// provided to the `max(by:)` method sorts ranks based on their `rawValue`: + /// + /// enum Rank: Int { + /// case ace = 1, two, three, four, five, six, seven, eight, nine, ten, jack, queen, king + /// } + /// + /// let max = await RankCounter() + /// .max { $0.rawValue < $1.rawValue } + /// print(max ?? "none") + /// // Prints: king + /// + /// - Parameter areInIncreasingOrder: A predicate that returns `true` if its + /// first argument should be ordered before its second argument; otherwise, + /// `false`. + /// - Returns: The sequence’s minimum element, according to + /// `areInIncreasingOrder`. If the sequence has no elements, returns `nil`. + @inlinable + @warn_unqualified_access + public func max( + by areInIncreasingOrder: (Element, Element) async throws -> Bool + ) async rethrows -> Element? { + var it = makeAsyncIterator() + guard var result = try await it.next() else { + return nil + } + while let e = try await it.next() { + if try await areInIncreasingOrder(result, e) { + result = e + } + } + return result + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence where Element: Comparable { + /// Returns the minimum element in an asynchronous sequence of comparable + /// elements. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `min()` method returns the minimum value + /// of the sequence. + /// + /// let min = await Counter(howHigh: 10) + /// .min() + /// print(min ?? "none") + /// // Prints: 1 + /// + /// - Returns: The sequence’s minimum element. If the sequence has no + /// elements, returns `nil`. + @inlinable + @warn_unqualified_access + public func min() async rethrows -> Element? { + return try await self.min(by: <) + } + + /// Returns the maximum element in an asynchronous sequence of comparable + /// elements. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `max()` method returns the max value + /// of the sequence. + /// + /// let max = await Counter(howHigh: 10) + /// .max() + /// print(max ?? "none") + /// // Prints: 10 + /// + /// - Returns: The sequence’s maximum element. If the sequence has no + /// elements, returns `nil`. + @inlinable + @warn_unqualified_access + public func max() async rethrows -> Element? { + return try await self.max(by: <) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncStream.cpp b/stdlib/public/BackDeployConcurrency/AsyncStream.cpp new file mode 100644 index 0000000000000..14b7caf0ad463 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncStream.cpp @@ -0,0 +1,39 @@ +//===--- AsyncStream.cpp - Multi-resume locking interface -----------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Runtime/Mutex.h" + +namespace swift { +// return the size in words for the given mutex primitive +extern "C" +size_t _swift_async_stream_lock_size() { + size_t words = sizeof(MutexHandle) / sizeof(void *); + if (words < 1) { return 1; } + return words; +} + +extern "C" +void _swift_async_stream_lock_init(MutexHandle &lock) { + MutexPlatformHelper::init(lock); +} + +extern "C" +void _swift_async_stream_lock_lock(MutexHandle &lock) { + MutexPlatformHelper::lock(lock); +} + +extern "C" +void _swift_async_stream_lock_unlock(MutexHandle &lock) { + MutexPlatformHelper::unlock(lock); +} + +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncStream.swift b/stdlib/public/BackDeployConcurrency/AsyncStream.swift new file mode 100644 index 0000000000000..d09db98297266 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncStream.swift @@ -0,0 +1,428 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020-2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// An asynchronous sequence generated from a closure that calls a continuation +/// to produce new elements. +/// +/// `AsyncStream` conforms to `AsyncSequence`, providing a convenient way to +/// create an asynchronous sequence without manually implementing an +/// asynchronous iterator. In particular, an asynchronous stream is well-suited +/// to adapt callback- or delegation-based APIs to participate with +/// `async`-`await`. +/// +/// You initialize an `AsyncStream` with a closure that receives an +/// `AsyncStream.Continuation`. Produce elements in this closure, then provide +/// them to the stream by calling the continuation's `yield(_:)` method. When +/// there are no further elements to produce, call the continuation's +/// `finish()` method. This causes the sequence iterator to produce a `nil`, +/// which terminates the sequence. The continuation conforms to `Sendable`, which permits +/// calling it from concurrent contexts external to the iteration of the +/// `AsyncStream`. +/// +/// An arbitrary source of elements can produce elements faster than they are +/// consumed by a caller iterating over them. Because of this, `AsyncStream` +/// defines a buffering behavior, allowing the stream to buffer a specific +/// number of oldest or newest elements. By default, the buffer limit is +/// `Int.max`, which means the value is unbounded. +/// +/// ### Adapting Existing Code to Use Streams +/// +/// To adapt existing callback code to use `async`-`await`, use the callbacks +/// to provide values to the stream, by using the continuation's `yield(_:)` +/// method. +/// +/// Consider a hypothetical `QuakeMonitor` type that provides callers with +/// `Quake` instances every time it detects an earthquake. To receive callbacks, +/// callers set a custom closure as the value of the monitor's +/// `quakeHandler` property, which the monitor calls back as necessary. +/// +/// class QuakeMonitor { +/// var quakeHandler: ((Quake) -> Void)? +/// +/// func startMonitoring() {…} +/// func stopMonitoring() {…} +/// } +/// +/// To adapt this to use `async`-`await`, extend the `QuakeMonitor` to add a +/// `quakes` property, of type `AsyncStream`. In the getter for this +/// property, return an `AsyncStream`, whose `build` closure -- called at +/// runtime to create the stream -- uses the continuation to perform the +/// following steps: +/// +/// 1. Creates a `QuakeMonitor` instance. +/// 2. Sets the monitor's `quakeHandler` property to a closure that receives +/// each `Quake` instance and forwards it to the stream by calling the +/// continuation's `yield(_:)` method. +/// 3. Sets the continuation's `onTermination` property to a closure that +/// calls `stopMonitoring()` on the monitor. +/// 4. Calls `startMonitoring` on the `QuakeMonitor`. +/// +/// extension QuakeMonitor { +/// +/// static var quakes: AsyncStream { +/// AsyncStream { continuation in +/// let monitor = QuakeMonitor() +/// monitor.quakeHandler = { quake in +/// continuation.yield(quake) +/// } +/// continuation.onTermination = { @Sendable _ in +/// monitor.stopMonitoring() +/// } +/// monitor.startMonitoring() +/// } +/// } +/// } +/// +/// Because the stream is an `AsyncSequence`, the call point can use the +/// `for`-`await`-`in` syntax to process each `Quake` instance as the stream +/// produces it: +/// +/// for await quake in QuakeMonitor.quakes { +/// print ("Quake: \(quake.date)") +/// } +/// print ("Stream finished.") +/// +@available(SwiftStdlib 5.1, *) +public struct AsyncStream { + /// A mechanism to interface between synchronous code and an asynchronous + /// stream. + /// + /// The closure you provide to the `AsyncStream` in + /// `init(_:bufferingPolicy:_:)` receives an instance of this type when + /// invoked. Use this continuation to provide elements to the stream by + /// calling one of the `yield` methods, then terminate the stream normally by + /// calling the `finish()` method. + /// + /// - Note: Unlike other continuations in Swift, `AsyncStream.Continuation` + /// supports escaping. + public struct Continuation: Sendable { + /// A type that indicates how the stream terminated. + /// + /// The `onTermination` closure receives an instance of this type. + public enum Termination { + + /// The stream finished as a result of calling the continuation's + /// `finish` method. + case finished + + /// The stream finished as a result of cancellation. + case cancelled + } + + /// A type that indicates the result of yielding a value to a client, by + /// way of the continuation. + /// + /// The various `yield` methods of `AsyncStream.Continuation` return this + /// type to indicate the success or failure of yielding an element to the + /// continuation. + public enum YieldResult { + + /// The stream successfully enqueued the element. + /// + /// This value represents the successful enqueueing of an element, whether + /// the stream buffers the element or delivers it immediately to a pending + /// call to `next()`. The associated value `remaining` is a hint that + /// indicates the number of remaining slots in the buffer at the time of + /// the `yield` call. + /// + /// - Note: From a thread safety point of view, `remaining` is a lower bound + /// on the number of remaining slots. This is because a subsequent call + /// that uses the `remaining` value could race on the consumption of + /// values from the stream. + case enqueued(remaining: Int) + + /// The stream didn't enqueue the element because the buffer was full. + /// + /// The associated element for this case is the element dropped by the stream. + case dropped(Element) + + /// The stream didn't enqueue the element because the stream was in a + /// terminal state. + /// + /// This indicates the stream terminated prior to calling `yield`, either + /// because the stream finished normally or through cancellation. + case terminated + } + + /// A strategy that handles exhaustion of a buffer’s capacity. + public enum BufferingPolicy { + /// Continue to add to the buffer, without imposing a limit on the number + /// of buffered elements. + case unbounded + + /// When the buffer is full, discard the newly received element. + /// + /// This strategy enforces keeping at most the specified number of oldest + /// values. + case bufferingOldest(Int) + + /// When the buffer is full, discard the oldest element in the buffer. + /// + /// This strategy enforces keeping at most the specified number of newest + /// values. + case bufferingNewest(Int) + } + + let storage: _Storage + + /// Resume the task awaiting the next iteration point by having it return + /// nomally from its suspension point with a given element. + /// + /// - Parameter value: The value to yield from the continuation. + /// - Returns: A `YieldResult` that indicates the success or failure of the + /// yield operation. + /// + /// If nothing is awaiting the next value, this method attempts to buffer the + /// result's element. + /// + /// This can be called more than once and returns to the caller immediately + /// without blocking for any awaiting consumption from the iteration. + @discardableResult + public func yield(_ value: __owned Element) -> YieldResult { + storage.yield(value) + } + + /// Resume the task awaiting the next iteration point by having it return + /// nil, which signifies the end of the iteration. + /// + /// Calling this function more than once has no effect. After calling + /// finish, the stream enters a terminal state and doesn't produces any additional + /// elements. + public func finish() { + storage.finish() + } + + /// A callback to invoke when canceling iteration of an asynchronous + /// stream. + /// + /// If an `onTermination` callback is set, using task cancellation to + /// terminate iteration of an `AsyncStream` results in a call to this + /// callback. + /// + /// Canceling an active iteration invokes the `onTermination` callback + /// first, then resumes by yielding `nil`. This means that you can perform + /// needed cleanup in the cancellation handler. After reaching a terminal + /// state as a result of cancellation, the `AsyncStream` sets the callback + /// to `nil`. + public var onTermination: (@Sendable (Termination) -> Void)? { + get { + return storage.getOnTermination() + } + nonmutating set { + storage.setOnTermination(newValue) + } + } + } + + final class _Context { + let storage: _Storage? + let produce: () async -> Element? + + init(storage: _Storage? = nil, produce: @escaping () async -> Element?) { + self.storage = storage + self.produce = produce + } + + deinit { + storage?.cancel() + } + } + + let context: _Context + + + /// Constructs an asynchronous stream for an element type, using the + /// specified buffering policy and element-producing closure. + /// + /// - Parameters: + /// - elementType: The type of element the `AsyncStream` produces. + /// - bufferingPolicy: A `Continuation.BufferingPolicy` value to + /// set the stream's buffering behavior. By default, the stream buffers an + /// unlimited number of elements. You can also set the policy to buffer a + /// specified number of oldest or newest elements. + /// - build: A custom closure that yields values to the + /// `AsyncStream`. This closure receives an `AsyncStream.Continuation` + /// instance that it uses to provide elements to the stream and terminate the + /// stream when finished. + /// + /// The `AsyncStream.Continuation` received by the `build` closure is + /// appropriate for use in concurrent contexts. It is thread safe to send and + /// finish; all calls to the continuation are serialized. However, calling + /// this from multiple concurrent contexts could result in out-of-order + /// delivery. + /// + /// The following example shows an `AsyncStream` created with this + /// initializer that produces 100 random numbers on a one-second interval, + /// calling `yield(_:)` to deliver each element to the awaiting call point. + /// When the `for` loop exits and the stream finishes by calling the + /// continuation's `finish()` method. + /// + /// let stream = AsyncStream(Int.self, + /// bufferingPolicy: .bufferingNewest(5)) { continuation in + /// Task.detached { + /// for _ in 0..<100 { + /// await Task.sleep(1 * 1_000_000_000) + /// continuation.yield(Int.random(in: 1...10)) + /// } + /// continuation.finish() + /// } + /// } + /// + /// // Call point: + /// for await random in stream { + /// print ("\(random)") + /// } + /// + public init( + _ elementType: Element.Type = Element.self, + bufferingPolicy limit: Continuation.BufferingPolicy = .unbounded, + _ build: (Continuation) -> Void + ) { + let storage: _Storage = .create(limit: limit) + context = _Context(storage: storage, produce: storage.next) + build(Continuation(storage: storage)) + } + + + /// Constructs an asynchronous stream from a given element-producing + /// closure, with an optional closure to handle cancellation. + /// + /// - Parameters: + /// - produce: A closure that asynchronously produces elements for the + /// stream. + /// - onCancel: A closure to execute when canceling the stream's task. + /// + /// Use this convenience initializer when you have an asychronous function + /// that can produce elements for the stream, and don't want to invoke + /// a continuation manually. This initializer "unfolds" your closure into + /// an asynchronous stream. The created stream handles conformance + /// to the `AsyncSequence` protocol automatically, including termination + /// (either by cancellation or by returning `nil` from the closure to finish + /// iteration). + /// + /// The following example shows an `AsyncStream` created with this + /// initializer that produces random numbers on a one-second interval. This + /// example uses the Swift multiple trailing closure syntax, which omits + /// the `unfolding` parameter label. + /// + /// let stream = AsyncStream { + /// await Task.sleep(1 * 1_000_000_000) + /// return Int.random(in: 1...10) + /// } + /// onCancel: { @Sendable () in print ("Canceled.") } + /// ) + /// + /// // Call point: + /// for await random in stream { + /// print ("\(random)") + /// } + /// + /// + public init( + unfolding produce: @escaping () async -> Element?, + onCancel: (@Sendable () -> Void)? = nil + ) { + let storage: _AsyncStreamCriticalStorage Element?>> + = .create(produce) + context = _Context { + return await withTaskCancellationHandler { + guard let result = await storage.value?() else { + storage.value = nil + return nil + } + return result + } onCancel: { + storage.value = nil + onCancel?() + } + } + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncStream: AsyncSequence { + /// The asynchronous iterator for iterating an asynchronous stream. + /// + /// This type doesn't conform to `Sendable`. Don't use it from multiple + /// concurrent contexts. It is a programmer error to invoke `next()` from a + /// concurrent context that contends with another such call, which + /// results in a call to `fatalError()`. + public struct Iterator: AsyncIteratorProtocol { + let context: _Context + + /// The next value from the asynchronous stream. + /// + /// When `next()` returns `nil`, this signifies the end of the + /// `AsyncStream`. + /// + /// It is a programmer error to invoke `next()` from a + /// concurrent context that contends with another such call, which + /// results in a call to `fatalError()`. + /// + /// If you cancel the task this iterator is running in while `next()` is + /// awaiting a value, the `AsyncStream` terminates. In this case, `next()` + /// might return `nil` immediately, or return `nil` on subsequent calls. + public mutating func next() async -> Element? { + await context.produce() + } + } + + /// Creates the asynchronous iterator that produces elements of this + /// asynchronous sequence. + public func makeAsyncIterator() -> Iterator { + return Iterator(context: context) + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncStream.Continuation { + /// Resume the task awaiting the next iteration point by having it return + /// normally from its suspension point with a given result's success value. + /// + /// - Parameter result: A result to yield from the continuation. + /// - Returns: A `YieldResult` that indicates the success or failure of the + /// yield operation. + /// + /// If nothing is awaiting the next value, the method attempts to buffer the + /// result's element. + /// + /// If you call this method repeatedly, each call returns immediately, without + /// blocking for any awaiting consumption from the iteration. + @discardableResult + public func yield( + with result: Result + ) -> YieldResult { + switch result { + case .success(let val): + return storage.yield(val) + } + } + + /// Resume the task awaiting the next iteration point by having it return + /// normally from its suspension point. + /// + /// - Returns: A `YieldResult` that indicates the success or failure of the + /// yield operation. + /// + /// Use this method with `AsyncStream` instances whose `Element` type is + /// `Void`. In this case, the `yield()` call unblocks the awaiting + /// iteration; there is no value to return. + /// + /// If you call this method repeatedly, each call returns immediately, without + /// blocking for any awaiting consumption from the iteration. + @discardableResult + public func yield() -> YieldResult where Element == Void { + return storage.yield(()) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncStreamBuffer.swift b/stdlib/public/BackDeployConcurrency/AsyncStreamBuffer.swift new file mode 100644 index 0000000000000..41aed33a72637 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncStreamBuffer.swift @@ -0,0 +1,597 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020-2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +#if ASYNC_STREAM_STANDALONE +@_exported import _Concurrency +import Darwin + +func _lockWordCount() -> Int { + let sz = + MemoryLayout.size / MemoryLayout.size + return max(sz, 1) +} + +func _lockInit(_ ptr: UnsafeRawPointer) { + UnsafeMutableRawPointer(mutating: ptr) + .assumingMemoryBound(to: os_unfair_lock.self) + .initialize(to: os_unfair_lock()) +} + +func _lock(_ ptr: UnsafeRawPointer) { + os_unfair_lock_lock(UnsafeMutableRawPointer(mutating: ptr) + .assumingMemoryBound(to: os_unfair_lock.self)) +} + +func _unlock(_ ptr: UnsafeRawPointer) { + os_unfair_lock_unlock(UnsafeMutableRawPointer(mutating: ptr) + .assumingMemoryBound(to: os_unfair_lock.self)) +} +#else +@_silgen_name("_swift_async_stream_lock_size") +func _lockWordCount() -> Int + +@_silgen_name("_swift_async_stream_lock_init") +func _lockInit(_ ptr: UnsafeRawPointer) + +@_silgen_name("_swift_async_stream_lock_lock") +func _lock(_ ptr: UnsafeRawPointer) + +@_silgen_name("_swift_async_stream_lock_unlock") +func _unlock(_ ptr: UnsafeRawPointer) +#endif + +@available(SwiftStdlib 5.1, *) +extension AsyncStream { + internal final class _Storage: @unchecked Sendable { + typealias TerminationHandler = @Sendable (Continuation.Termination) -> Void + + struct State { + var continuation: UnsafeContinuation? + var pending = _Deque() + let limit: Continuation.BufferingPolicy + var onTermination: TerminationHandler? + var terminal: Bool = false + + init(limit: Continuation.BufferingPolicy) { + self.limit = limit + } + } + // Stored as a singular structured assignment for initialization + var state: State + + private init(_doNotCallMe: ()) { + fatalError("Storage must be initialized by create") + } + + deinit { + state.onTermination?(.cancelled) + } + + private func lock() { + let ptr = + UnsafeRawPointer(Builtin.projectTailElems(self, UnsafeRawPointer.self)) + _lock(ptr) + } + + private func unlock() { + let ptr = + UnsafeRawPointer(Builtin.projectTailElems(self, UnsafeRawPointer.self)) + _unlock(ptr) + } + + func getOnTermination() -> TerminationHandler? { + lock() + let handler = state.onTermination + unlock() + return handler + } + + func setOnTermination(_ newValue: TerminationHandler?) { + lock() + withExtendedLifetime(state.onTermination) { + state.onTermination = newValue + unlock() + } + } + + func cancel() { + lock() + // swap out the handler before we invoke it to prevent double cancel + let handler = state.onTermination + state.onTermination = nil + unlock() + + // handler must be invoked before yielding nil for termination + handler?(.cancelled) + + finish() + } + + func yield(_ value: __owned Element) -> Continuation.YieldResult { + var result: Continuation.YieldResult + lock() + let limit = state.limit + let count = state.pending.count + if let continuation = state.continuation { + if count > 0 { + if !state.terminal { + switch limit { + case .unbounded: + state.pending.append(value) + result = .enqueued(remaining: .max) + case .bufferingOldest(let limit): + if count < limit { + state.pending.append(value) + result = .enqueued(remaining: limit - (count + 1)) + } else { + result = .dropped(value) + } + case .bufferingNewest(let limit): + if count < limit { + state.pending.append(value) + result = .enqueued(remaining: limit - (count + 1)) + } else if count > 0 { + result = .dropped(state.pending.removeFirst()) + state.pending.append(value) + } else { + result = .dropped(value) + } + } + } else { + result = .terminated + } + state.continuation = nil + let toSend = state.pending.removeFirst() + unlock() + continuation.resume(returning: toSend) + } else if state.terminal { + state.continuation = nil + result = .terminated + unlock() + continuation.resume(returning: nil) + } else { + state.continuation = nil + switch limit { + case .unbounded: + result = .enqueued(remaining: .max) + case .bufferingNewest(let limit): + result = .enqueued(remaining: limit) + case .bufferingOldest(let limit): + result = .enqueued(remaining: limit) + } + + unlock() + continuation.resume(returning: value) + } + } else { + if !state.terminal { + switch limit { + case .unbounded: + result = .enqueued(remaining: .max) + state.pending.append(value) + case .bufferingOldest(let limit): + if count < limit { + result = .enqueued(remaining: limit - (count + 1)) + state.pending.append(value) + } else { + result = .dropped(value) + } + case .bufferingNewest(let limit): + if count < limit { + state.pending.append(value) + result = .enqueued(remaining: limit - (count + 1)) + } else if count > 0 { + result = .dropped(state.pending.removeFirst()) + state.pending.append(value) + } else { + result = .dropped(value) + } + } + } else { + result = .terminated + } + unlock() + } + return result + } + + func finish() { + lock() + let handler = state.onTermination + state.onTermination = nil + state.terminal = true + + if let continuation = state.continuation { + if state.pending.count > 0 { + state.continuation = nil + let toSend = state.pending.removeFirst() + unlock() + handler?(.finished) + continuation.resume(returning: toSend) + } else if state.terminal { + state.continuation = nil + unlock() + handler?(.finished) + continuation.resume(returning: nil) + } else { + unlock() + handler?(.finished) + } + } else { + unlock() + handler?(.finished) + } + } + + func next(_ continuation: UnsafeContinuation) { + lock() + if state.continuation == nil { + if state.pending.count > 0 { + let toSend = state.pending.removeFirst() + unlock() + continuation.resume(returning: toSend) + } else if state.terminal { + unlock() + continuation.resume(returning: nil) + } else { + state.continuation = continuation + unlock() + } + } else { + unlock() + fatalError("attempt to await next() on more than one task") + } + } + + func next() async -> Element? { + await withTaskCancellationHandler { [cancel] in + cancel() + } operation: { + await withUnsafeContinuation { + next($0) + } + } + } + + static func create(limit: Continuation.BufferingPolicy) -> _Storage { + let minimumCapacity = _lockWordCount() + let storage = Builtin.allocWithTailElems_1( + _Storage.self, + minimumCapacity._builtinWordValue, + UnsafeRawPointer.self + ) + + let state = + UnsafeMutablePointer(Builtin.addressof(&storage.state)) + state.initialize(to: State(limit: limit)) + let ptr = UnsafeRawPointer( + Builtin.projectTailElems(storage, UnsafeRawPointer.self)) + _lockInit(ptr) + return storage + } + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingStream { + internal final class _Storage: @unchecked Sendable { + typealias TerminationHandler = @Sendable (Continuation.Termination) -> Void + enum Terminal { + case finished + case failed(Failure) + } + + struct State { + var continuation: UnsafeContinuation? + var pending = _Deque() + let limit: Continuation.BufferingPolicy + var onTermination: TerminationHandler? + var terminal: Terminal? + + init(limit: Continuation.BufferingPolicy) { + self.limit = limit + } + } + // Stored as a singular structured assignment for initialization + var state: State + + private init(_doNotCallMe: ()) { + fatalError("Storage must be initialized by create") + } + + deinit { + state.onTermination?(.cancelled) + } + + private func lock() { + let ptr = + UnsafeRawPointer(Builtin.projectTailElems(self, UnsafeRawPointer.self)) + _lock(ptr) + } + + private func unlock() { + let ptr = + UnsafeRawPointer(Builtin.projectTailElems(self, UnsafeRawPointer.self)) + _unlock(ptr) + } + + func getOnTermination() -> TerminationHandler? { + lock() + let handler = state.onTermination + unlock() + return handler + } + + func setOnTermination(_ newValue: TerminationHandler?) { + lock() + withExtendedLifetime(state.onTermination) { + state.onTermination = newValue + unlock() + } + } + + func cancel() { + lock() + // swap out the handler before we invoke it to prevent double cancel + let handler = state.onTermination + state.onTermination = nil + unlock() + + // handler must be invoked before yielding nil for termination + handler?(.cancelled) + + finish() + } + + func yield(_ value: __owned Element) -> Continuation.YieldResult { + var result: Continuation.YieldResult + lock() + let limit = state.limit + let count = state.pending.count + if let continuation = state.continuation { + if count > 0 { + if state.terminal == nil { + switch limit { + case .unbounded: + result = .enqueued(remaining: .max) + state.pending.append(value) + case .bufferingOldest(let limit): + if count < limit { + result = .enqueued(remaining: limit - (count + 1)) + state.pending.append(value) + } else { + result = .dropped(value) + } + case .bufferingNewest(let limit): + if count < limit { + state.pending.append(value) + result = .enqueued(remaining: limit - (count + 1)) + } else if count > 0 { + result = .dropped(state.pending.removeFirst()) + state.pending.append(value) + } else { + result = .dropped(value) + } + } + } else { + result = .terminated + } + state.continuation = nil + let toSend = state.pending.removeFirst() + unlock() + continuation.resume(returning: toSend) + } else if let terminal = state.terminal { + result = .terminated + state.continuation = nil + state.terminal = .finished + unlock() + switch terminal { + case .finished: + continuation.resume(returning: nil) + case .failed(let error): + continuation.resume(throwing: error) + } + } else { + switch limit { + case .unbounded: + result = .enqueued(remaining: .max) + case .bufferingOldest(let limit): + result = .enqueued(remaining: limit) + case .bufferingNewest(let limit): + result = .enqueued(remaining: limit) + } + + state.continuation = nil + unlock() + continuation.resume(returning: value) + } + } else { + if state.terminal == nil { + switch limit { + case .unbounded: + result = .enqueued(remaining: .max) + state.pending.append(value) + case .bufferingOldest(let limit): + if count < limit { + result = .enqueued(remaining: limit - (count + 1)) + state.pending.append(value) + } else { + result = .dropped(value) + } + case .bufferingNewest(let limit): + if count < limit { + state.pending.append(value) + result = .enqueued(remaining: limit - (count + 1)) + } else if count > 0 { + result = .dropped(state.pending.removeFirst()) + state.pending.append(value) + } else { + result = .dropped(value) + } + } + } else { + result = .terminated + } + unlock() + } + return result + } + + func finish(throwing error: __owned Failure? = nil) { + lock() + let handler = state.onTermination + state.onTermination = nil + if state.terminal == nil { + if let failure = error { + state.terminal = .failed(failure) + } else { + state.terminal = .finished + } + } + + if let continuation = state.continuation { + if state.pending.count > 0 { + state.continuation = nil + let toSend = state.pending.removeFirst() + unlock() + handler?(.finished(error)) + continuation.resume(returning: toSend) + } else if let terminal = state.terminal { + state.continuation = nil + unlock() + handler?(.finished(error)) + switch terminal { + case .finished: + continuation.resume(returning: nil) + case .failed(let error): + continuation.resume(throwing: error) + } + } else { + unlock() + handler?(.finished(error)) + } + } else { + unlock() + handler?(.finished(error)) + } + } + + func next(_ continuation: UnsafeContinuation) { + lock() + if state.continuation == nil { + if state.pending.count > 0 { + let toSend = state.pending.removeFirst() + unlock() + continuation.resume(returning: toSend) + } else if let terminal = state.terminal { + state.terminal = .finished + unlock() + switch terminal { + case .finished: + continuation.resume(returning: nil) + case .failed(let error): + continuation.resume(throwing: error) + } + } else { + state.continuation = continuation + unlock() + } + } else { + unlock() + fatalError("attempt to await next() on more than one task") + } + } + + func next() async throws -> Element? { + try await withTaskCancellationHandler { [cancel] in + cancel() + } operation: { + try await withUnsafeThrowingContinuation { + next($0) + } + } + } + + static func create(limit: Continuation.BufferingPolicy) -> _Storage { + let minimumCapacity = _lockWordCount() + let storage = Builtin.allocWithTailElems_1( + _Storage.self, + minimumCapacity._builtinWordValue, + UnsafeRawPointer.self + ) + + let state = + UnsafeMutablePointer(Builtin.addressof(&storage.state)) + state.initialize(to: State(limit: limit)) + let ptr = UnsafeRawPointer( + Builtin.projectTailElems(storage, UnsafeRawPointer.self)) + _lockInit(ptr) + return storage + } + } +} + +// this is used to store closures; which are two words +final class _AsyncStreamCriticalStorage: @unchecked Sendable { + var _value: Contents + private init(_doNotCallMe: ()) { + fatalError("_AsyncStreamCriticalStorage must be initialized by create") + } + + private func lock() { + let ptr = + UnsafeRawPointer(Builtin.projectTailElems(self, UnsafeRawPointer.self)) + _lock(ptr) + } + + private func unlock() { + let ptr = + UnsafeRawPointer(Builtin.projectTailElems(self, UnsafeRawPointer.self)) + _unlock(ptr) + } + + var value: Contents { + get { + lock() + let contents = _value + unlock() + return contents + } + + set { + lock() + withExtendedLifetime(_value) { + _value = newValue + unlock() + } + } + } + + static func create(_ initial: Contents) -> _AsyncStreamCriticalStorage { + let minimumCapacity = _lockWordCount() + let storage = Builtin.allocWithTailElems_1( + _AsyncStreamCriticalStorage.self, + minimumCapacity._builtinWordValue, + UnsafeRawPointer.self + ) + + let state = + UnsafeMutablePointer(Builtin.addressof(&storage._value)) + state.initialize(to: initial) + let ptr = UnsafeRawPointer( + Builtin.projectTailElems(storage, UnsafeRawPointer.self)) + _lockInit(ptr) + return storage + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncThrowingCompactMapSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncThrowingCompactMapSequence.swift new file mode 100644 index 0000000000000..53972fd6265c4 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncThrowingCompactMapSequence.swift @@ -0,0 +1,153 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that maps an error-throwing closure over + /// the base sequence’s elements, omitting results that don't return a value. + /// + /// Use the `compactMap(_:)` method to transform every element received from + /// a base asynchronous sequence, while also discarding any `nil` results + /// from the closure. Typically, you use this to transform from one type of + /// element to another. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `5`. The closure provided to the `compactMap(_:)` + /// method takes each `Int` and looks up a corresponding `String` from a + /// `romanNumeralDict` dictionary. Since there is no key for `4`, the closure + /// returns `nil` in this case, which `compactMap(_:)` omits from the + /// transformed asynchronous sequence. When the value is `5`, the closure + /// throws `MyError`, terminating the sequence. + /// + /// let romanNumeralDict: [Int : String] = + /// [1: "I", 2: "II", 3: "III", 5: "V"] + /// + /// do { + /// let stream = Counter(howHigh: 5) + /// .compactMap { (value) throws -> String? in + /// if value == 5 { + /// throw MyError() + /// } + /// return romanNumeralDict[value] + /// } + /// for try await numeral in stream { + /// print("\(numeral) ", terminator: " ") + /// } + /// } catch { + /// print("Error: \(error)") + /// } + /// // Prints: I II III Error: MyError() + /// + /// - Parameter transform: An error-throwing mapping closure. `transform` + /// accepts an element of this sequence as its parameter and returns a + /// transformed value of the same or of a different type. If `transform` + /// throws an error, the sequence ends. + /// - Returns: An asynchronous sequence that contains, in order, the + /// non-`nil` elements produced by the `transform` closure. The sequence + /// ends either when the base sequence ends or when `transform` throws an + /// error. + @inlinable + public __consuming func compactMap( + _ transform: @escaping (Element) async throws -> ElementOfResult? + ) -> AsyncThrowingCompactMapSequence { + return AsyncThrowingCompactMapSequence(self, transform: transform) + } +} + +/// An asynchronous sequence that maps an error-throwing closure over the base +/// sequence’s elements, omitting results that don't return a value. +@available(SwiftStdlib 5.1, *) +public struct AsyncThrowingCompactMapSequence { + @usableFromInline + let base: Base + + @usableFromInline + let transform: (Base.Element) async throws -> ElementOfResult? + + @usableFromInline + init( + _ base: Base, + transform: @escaping (Base.Element) async throws -> ElementOfResult? + ) { + self.base = base + self.transform = transform + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingCompactMapSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The compact map sequence produces whatever type of element its + /// transforming closure produces. + public typealias Element = ElementOfResult + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the compact map sequence. + public struct Iterator: AsyncIteratorProtocol { + public typealias Element = ElementOfResult + + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let transform: (Base.Element) async throws -> ElementOfResult? + + @usableFromInline + var finished = false + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + transform: @escaping (Base.Element) async throws -> ElementOfResult? + ) { + self.baseIterator = baseIterator + self.transform = transform + } + + /// Produces the next element in the compact map sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns `nil`. Otherwise, `next()` calls the + /// transforming closure on the received element, returning it if the + /// transform returns a non-`nil` value. If the transform returns `nil`, + /// this method continues to wait for further elements until it gets one + /// that transforms to a non-`nil` value. If calling the closure throws an + /// error, the sequence ends and `next()` rethrows the error. + @inlinable + public mutating func next() async throws -> ElementOfResult? { + while !finished { + guard let element = try await baseIterator.next() else { + finished = true + return nil + } + do { + if let transformed = try await transform(element) { + return transformed + } + } catch { + finished = true + throw error + } + } + return nil + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), transform: transform) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncThrowingDropWhileSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncThrowingDropWhileSequence.swift new file mode 100644 index 0000000000000..9051e5187e7f0 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncThrowingDropWhileSequence.swift @@ -0,0 +1,156 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Omits elements from the base sequence until a given error-throwing closure + /// returns false, after which it passes through all remaining elements. + /// + /// Use `drop(while:)` to omit elements from an asynchronous sequence until + /// the element received meets a condition you specify. If the closure you + /// provide throws an error, the sequence produces no elements and throws + /// the error instead. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The predicate passed to the `drop(while:)` + /// method throws an error if it encounters an even number, and otherwise + /// returns `true` while it receives elements less than `5`. Because the + /// predicate throws when it receives `2` from the base sequence, this example + /// throws without ever printing anything. + /// + /// do { + /// let stream = Counter(howHigh: 10) + /// .drop { + /// if $0 % 2 == 0 { + /// throw EvenError() + /// } + /// return $0 < 5 + /// } + /// for try await number in stream { + /// print("\(number) ") + /// } + /// } catch { + /// print ("\(error)") + /// } + /// // Prints: EvenError() + /// + /// After the predicate returns `false`, the sequence never executes it again, + /// and from then on the sequence passes through elements from its underlying + /// sequence. A predicate that throws an error also never executes again. + /// + /// - Parameter predicate: An error-throwing closure that takes an element as + /// a parameter and returns a Boolean value indicating whether to drop the + /// element from the modified sequence. + /// - Returns: An asynchronous sequence that skips over values until the + /// provided closure returns `false` or throws an error. + @inlinable + public __consuming func drop( + while predicate: @escaping (Element) async throws -> Bool + ) -> AsyncThrowingDropWhileSequence { + AsyncThrowingDropWhileSequence(self, predicate: predicate) + } +} + +/// An asynchronous sequence which omits elements from the base sequence until a +/// given error-throwing closure returns false, after which it passes through +/// all remaining elements. +@available(SwiftStdlib 5.1, *) +public struct AsyncThrowingDropWhileSequence { + @usableFromInline + let base: Base + + @usableFromInline + let predicate: (Base.Element) async throws -> Bool + + @usableFromInline + init( + _ base: Base, + predicate: @escaping (Base.Element) async throws -> Bool + ) { + self.base = base + self.predicate = predicate + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingDropWhileSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The drop-while sequence produces whatever type of element its base + /// sequence produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the drop-while sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let predicate: (Base.Element) async throws -> Bool + + @usableFromInline + var finished = false + + @usableFromInline + var doneDropping = false + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + predicate: @escaping (Base.Element) async throws -> Bool + ) { + self.baseIterator = baseIterator + self.predicate = predicate + } + + /// Produces the next element in the drop-while sequence. + /// + /// This iterator calls `next()` on its base iterator and evaluates the + /// result with the `predicate` closure. As long as the predicate returns + /// `true`, this method returns `nil`. After the predicate returns `false`, + /// for a value received from the base iterator, this method returns that + /// value. After that, the iterator returns values received from its + /// base iterator as-is, and never executes the predicate closure again. + /// If calling the closure throws an error, the sequence ends and `next()` + /// rethrows the error. + @inlinable + public mutating func next() async throws -> Base.Element? { + while !finished && !doneDropping { + guard let element = try await baseIterator.next() else { + return nil + } + do { + if try await predicate(element) == false { + doneDropping = true + return element + } + } catch { + finished = true + throw error + } + } + guard !finished else { + return nil + } + return try await baseIterator.next() + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), predicate: predicate) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncThrowingFilterSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncThrowingFilterSequence.swift new file mode 100644 index 0000000000000..b12458aa15a58 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncThrowingFilterSequence.swift @@ -0,0 +1,138 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that contains, in order, the elements of + /// the base sequence that satisfy the given error-throwing predicate. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `filter(_:)` method returns `true` for even + /// values and `false` for odd values, thereby filtering out the odd values, + /// but also throws an error for values divisible by 5: + /// + /// do { + /// let stream = Counter(howHigh: 10) + /// .filter { + /// if $0 % 5 == 0 { + /// throw MyError() + /// } + /// return $0 % 2 == 0 + /// } + /// for try await number in stream { + /// print("\(number) ", terminator: " ") + /// } + /// } catch { + /// print("Error: \(error)") + /// } + /// // Prints: 2 4 Error: MyError() + /// + /// - Parameter isIncluded: An error-throwing closure that takes an element + /// of the asynchronous sequence as its argument and returns a Boolean value + /// that indicates whether to include the element in the filtered sequence. + /// - Returns: An asynchronous sequence that contains, in order, the elements + /// of the base sequence that satisfy the given predicate. If the predicate + /// throws an error, the sequence contains only values produced prior to + /// the error. + @inlinable + public __consuming func filter( + _ isIncluded: @escaping (Element) async throws -> Bool + ) -> AsyncThrowingFilterSequence { + return AsyncThrowingFilterSequence(self, isIncluded: isIncluded) + } +} + +/// An asynchronous sequence that contains, in order, the elements of +/// the base sequence that satisfy the given error-throwing predicate. +@available(SwiftStdlib 5.1, *) +public struct AsyncThrowingFilterSequence { + @usableFromInline + let base: Base + + @usableFromInline + let isIncluded: (Element) async throws -> Bool + + @usableFromInline + init( + _ base: Base, + isIncluded: @escaping (Base.Element) async throws -> Bool + ) { + self.base = base + self.isIncluded = isIncluded + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingFilterSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The filter sequence produces whatever type of element its base + /// sequence produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the filter sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let isIncluded: (Base.Element) async throws -> Bool + + @usableFromInline + var finished = false + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + isIncluded: @escaping (Base.Element) async throws -> Bool + ) { + self.baseIterator = baseIterator + self.isIncluded = isIncluded + } + + /// Produces the next element in the filter sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns nil. Otherwise, `next()` evaluates the + /// result with the `predicate` closure. If the closure returns `true`, + /// `next()` returns the received element; otherwise it awaits the next + /// element from the base iterator. If calling the closure throws an error, + /// the sequence ends and `next()` rethrows the error. + @inlinable + public mutating func next() async throws -> Base.Element? { + while !finished { + guard let element = try await baseIterator.next() else { + return nil + } + do { + if try await isIncluded(element) { + return element + } + } catch { + finished = true + throw error + } + } + + return nil + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), isIncluded: isIncluded) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncThrowingFlatMapSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncThrowingFlatMapSequence.swift new file mode 100644 index 0000000000000..1b3bb8e7bd6d0 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncThrowingFlatMapSequence.swift @@ -0,0 +1,171 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that concatenates the results of calling + /// the given error-throwing transformation with each element of this + /// sequence. + /// + /// Use this method to receive a single-level asynchronous sequence when your + /// transformation produces an asynchronous sequence for each element. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `5`. The transforming closure takes the received `Int` + /// and returns a new `Counter` that counts that high. For example, when the + /// transform receives `3` from the base sequence, it creates a new `Counter` + /// that produces the values `1`, `2`, and `3`. The `flatMap(_:)` method + /// "flattens" the resulting sequence-of-sequences into a single + /// `AsyncSequence`. However, when the closure receives `4`, it throws an + /// error, terminating the sequence. + /// + /// do { + /// let stream = Counter(howHigh: 5) + /// .flatMap { (value) -> Counter in + /// if value == 4 { + /// throw MyError() + /// } + /// return Counter(howHigh: value) + /// } + /// for try await number in stream { + /// print ("\(number)", terminator: " ") + /// } + /// } catch { + /// print(error) + /// } + /// // Prints: 1 1 2 1 2 3 MyError() + /// + /// - Parameter transform: An error-throwing mapping closure. `transform` + /// accepts an element of this sequence as its parameter and returns an + /// `AsyncSequence`. If `transform` throws an error, the sequence ends. + /// - Returns: A single, flattened asynchronous sequence that contains all + /// elements in all the asychronous sequences produced by `transform`. The + /// sequence ends either when the the last sequence created from the last + /// element from base sequence ends, or when `transform` throws an error. + @inlinable + public __consuming func flatMap( + _ transform: @escaping (Element) async throws -> SegmentOfResult + ) -> AsyncThrowingFlatMapSequence { + return AsyncThrowingFlatMapSequence(self, transform: transform) + } +} + +/// An asynchronous sequence that concatenates the results of calling a given +/// error-throwing transformation with each element of this sequence. +@available(SwiftStdlib 5.1, *) +public struct AsyncThrowingFlatMapSequence { + @usableFromInline + let base: Base + + @usableFromInline + let transform: (Base.Element) async throws -> SegmentOfResult + + @usableFromInline + init( + _ base: Base, + transform: @escaping (Base.Element) async throws -> SegmentOfResult + ) { + self.base = base + self.transform = transform + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingFlatMapSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The flat map sequence produces the type of element in the asynchronous + /// sequence produced by the `transform` closure. + public typealias Element = SegmentOfResult.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the flat map sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let transform: (Base.Element) async throws -> SegmentOfResult + + @usableFromInline + var currentIterator: SegmentOfResult.AsyncIterator? + + @usableFromInline + var finished = false + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + transform: @escaping (Base.Element) async throws -> SegmentOfResult + ) { + self.baseIterator = baseIterator + self.transform = transform + } + + /// Produces the next element in the flat map sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns `nil`. Otherwise, `next()` calls the + /// transforming closure on the received element, takes the resulting + /// asynchronous sequence, and creates an asynchronous iterator from it. + /// `next()` then consumes values from this iterator until it terminates. + /// At this point, `next()` is ready to receive the next value from the base + /// sequence. If `transform` throws an error, the sequence terminates. + @inlinable + public mutating func next() async throws -> SegmentOfResult.Element? { + while !finished { + if var iterator = currentIterator { + do { + guard let element = try await iterator.next() else { + currentIterator = nil + continue + } + // restore the iterator since we just mutated it with next + currentIterator = iterator + return element + } catch { + finished = true + throw error + } + } else { + guard let item = try await baseIterator.next() else { + return nil + } + let segment: SegmentOfResult + do { + segment = try await transform(item) + var iterator = segment.makeAsyncIterator() + guard let element = try await iterator.next() else { + currentIterator = nil + continue + } + currentIterator = iterator + return element + } catch { + finished = true + currentIterator = nil + throw error + } + } + } + return nil + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), transform: transform) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncThrowingMapSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncThrowingMapSequence.swift new file mode 100644 index 0000000000000..22fb1f497ed0e --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncThrowingMapSequence.swift @@ -0,0 +1,142 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Creates an asynchronous sequence that maps the given error-throwing + /// closure over the asynchronous sequence’s elements. + /// + /// Use the `map(_:)` method to transform every element received from a base + /// asynchronous sequence. Typically, you use this to transform from one type + /// of element to another. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `5`. The closure provided to the `map(_:)` method + /// takes each `Int` and looks up a corresponding `String` from a + /// `romanNumeralDict` dictionary. This means the outer `for await in` loop + /// iterates over `String` instances instead of the underlying `Int` values + /// that `Counter` produces. Also, the dictionary doesn't provide a key for + /// `4`, and the closure throws an error for any key it can't look up, so + /// receiving this value from `Counter` ends the modified sequence with an + /// error. + /// + /// let romanNumeralDict: [Int : String] = + /// [1: "I", 2: "II", 3: "III", 5: "V"] + /// + /// do { + /// let stream = Counter(howHigh: 5) + /// .map { (value) throws -> String in + /// guard let roman = romanNumeralDict[value] else { + /// throw MyError() + /// } + /// return roman + /// } + /// for try await numeral in stream { + /// print("\(numeral) ", terminator: " ") + /// } + /// } catch { + /// print ("Error: \(error)") + /// } + /// // Prints: I II III Error: MyError() + /// + /// - Parameter transform: A mapping closure. `transform` accepts an element + /// of this sequence as its parameter and returns a transformed value of the + /// same or of a different type. `transform` can also throw an error, which + /// ends the transformed sequence. + /// - Returns: An asynchronous sequence that contains, in order, the elements + /// produced by the `transform` closure. + @inlinable + public __consuming func map( + _ transform: @escaping (Element) async throws -> Transformed + ) -> AsyncThrowingMapSequence { + return AsyncThrowingMapSequence(self, transform: transform) + } +} + +/// An asynchronous sequence that maps the given error-throwing closure over the +/// asynchronous sequence’s elements. +@available(SwiftStdlib 5.1, *) +public struct AsyncThrowingMapSequence { + @usableFromInline + let base: Base + + @usableFromInline + let transform: (Base.Element) async throws -> Transformed + + @usableFromInline + init( + _ base: Base, + transform: @escaping (Base.Element) async throws -> Transformed + ) { + self.base = base + self.transform = transform + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingMapSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The map sequence produces whatever type of element its the transforming + /// closure produces. + public typealias Element = Transformed + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the map sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let transform: (Base.Element) async throws -> Transformed + + @usableFromInline + var finished = false + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + transform: @escaping (Base.Element) async throws -> Transformed + ) { + self.baseIterator = baseIterator + self.transform = transform + } + + /// Produces the next element in the map sequence. + /// + /// This iterator calls `next()` on its base iterator; if this call returns + /// `nil`, `next()` returns nil. Otherwise, `next()` returns the result of + /// calling the transforming closure on the received element. If calling + /// the closure throws an error, the sequence ends and `next()` rethrows + /// the error. + @inlinable + public mutating func next() async throws -> Transformed? { + guard !finished, let element = try await baseIterator.next() else { + return nil + } + do { + return try await transform(element) + } catch { + finished = true + throw error + } + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), transform: transform) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncThrowingPrefixWhileSequence.swift b/stdlib/public/BackDeployConcurrency/AsyncThrowingPrefixWhileSequence.swift new file mode 100644 index 0000000000000..3037f9c7fd0c5 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncThrowingPrefixWhileSequence.swift @@ -0,0 +1,141 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +extension AsyncSequence { + /// Returns an asynchronous sequence, containing the initial, consecutive + /// elements of the base sequence that satisfy the given error-throwing + /// predicate. + /// + /// Use `prefix(while:)` to produce values while elements from the base + /// sequence meet a condition you specify. The modified sequence ends when + /// the predicate closure returns `false` or throws an error. + /// + /// In this example, an asynchronous sequence called `Counter` produces `Int` + /// values from `1` to `10`. The `prefix(_:)` method causes the modified + /// sequence to pass through values less than `8`, but throws an + /// error when it receives a value that's divisible by `5`: + /// + /// do { + /// let stream = try Counter(howHigh: 10) + /// .prefix { + /// if $0 % 5 == 0 { + /// throw MyError() + /// } + /// return $0 < 8 + /// } + /// for try await number in stream { + /// print("\(number) ", terminator: " ") + /// } + /// } catch { + /// print("Error: \(error)") + /// } + /// // Prints: 1 2 3 4 Error: MyError() + /// + /// - Parameter predicate: A error-throwing closure that takes an element of + /// the asynchronous sequence as its argument and returns a Boolean value + /// that indicates whether to include the element in the modified sequence. + /// - Returns: An asynchronous sequence that contains, in order, the elements + /// of the base sequence that satisfy the given predicate. If the predicate + /// throws an error, the sequence contains only values produced prior to + /// the error. + @inlinable + public __consuming func prefix( + while predicate: @escaping (Element) async throws -> Bool + ) rethrows -> AsyncThrowingPrefixWhileSequence { + return AsyncThrowingPrefixWhileSequence(self, predicate: predicate) + } +} + +/// An asynchronous sequence, containing the initial, consecutive +/// elements of the base sequence that satisfy the given error-throwing +/// predicate. +@available(SwiftStdlib 5.1, *) +public struct AsyncThrowingPrefixWhileSequence { + @usableFromInline + let base: Base + + @usableFromInline + let predicate: (Base.Element) async throws -> Bool + + @usableFromInline + init( + _ base: Base, + predicate: @escaping (Base.Element) async throws -> Bool + ) { + self.base = base + self.predicate = predicate + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingPrefixWhileSequence: AsyncSequence { + /// The type of element produced by this asynchronous sequence. + /// + /// The prefix-while sequence produces whatever type of element its base + /// iterator produces. + public typealias Element = Base.Element + /// The type of iterator that produces elements of the sequence. + public typealias AsyncIterator = Iterator + + /// The iterator that produces elements of the prefix-while sequence. + public struct Iterator: AsyncIteratorProtocol { + @usableFromInline + var predicateHasFailed = false + + @usableFromInline + var baseIterator: Base.AsyncIterator + + @usableFromInline + let predicate: (Base.Element) async throws -> Bool + + @usableFromInline + init( + _ baseIterator: Base.AsyncIterator, + predicate: @escaping (Base.Element) async throws -> Bool + ) { + self.baseIterator = baseIterator + self.predicate = predicate + } + + /// Produces the next element in the prefix-while sequence. + /// + /// If the predicate hasn't failed yet, this method gets the next element + /// from the base sequence and calls the predicate with it. If this call + /// succeeds, this method passes along the element. Otherwise, it returns + /// `nil`, ending the sequence. If calling the predicate closure throws an + /// error, the sequence ends and `next()` rethrows the error. + @inlinable + public mutating func next() async throws -> Base.Element? { + if !predicateHasFailed, let nextElement = try await baseIterator.next() { + do { + if try await predicate(nextElement) { + return nextElement + } else { + predicateHasFailed = true + } + } catch { + predicateHasFailed = true + throw error + } + } + return nil + } + } + + @inlinable + public __consuming func makeAsyncIterator() -> Iterator { + return Iterator(base.makeAsyncIterator(), predicate: predicate) + } +} diff --git a/stdlib/public/BackDeployConcurrency/AsyncThrowingStream.swift b/stdlib/public/BackDeployConcurrency/AsyncThrowingStream.swift new file mode 100644 index 0000000000000..0a7f0826d1668 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/AsyncThrowingStream.swift @@ -0,0 +1,471 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020-2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// An asynchronous sequence generated from an error-throwing closure that +/// calls a continuation to produce new elements. +/// +/// `AsyncThrowingStream` conforms to `AsyncSequence`, providing a convenient +/// way to create an asynchronous sequence without manually implementing an +/// asynchronous iterator. In particular, an asynchronous stream is well-suited +/// to adapt callback- or delegation-based APIs to participate with +/// `async`-`await`. +/// +/// In contrast to `AsyncStream`, this type can throw an error from the awaited +/// `next()`, which terminates the stream with the thrown error. +/// +/// You initialize an `AsyncThrowingStream` with a closure that receives an +/// `AsyncThrowingStream.Continuation`. Produce elements in this closure, then +/// provide them to the stream by calling the continuation's `yield(_:)` method. +/// When there are no further elements to produce, call the continuation's +/// `finish()` method. This causes the sequence iterator to produce a `nil`, +/// which terminates the sequence. If an error occurs, call the continuation's +/// `finish(throwing:)` method, which causes the iterator's `next()` method to +/// throw the error to the awaiting call point. The continuation is `Sendable`, +/// which permits calling it from concurrent contexts external to the iteration +/// of the `AsyncThrowingStream`. +/// +/// An arbitrary source of elements can produce elements faster than they are +/// consumed by a caller iterating over them. Because of this, `AsyncThrowingStream` +/// defines a buffering behavior, allowing the stream to buffer a specific +/// number of oldest or newest elements. By default, the buffer limit is +/// `Int.max`, which means it's unbounded. +/// +/// ### Adapting Existing Code to Use Streams +/// +/// To adapt existing callback code to use `async`-`await`, use the callbacks +/// to provide values to the stream, by using the continuation's `yield(_:)` +/// method. +/// +/// Consider a hypothetical `QuakeMonitor` type that provides callers with +/// `Quake` instances every time it detects an earthquake. To receive callbacks, +/// callers set a custom closure as the value of the monitor's +/// `quakeHandler` property, which the monitor calls back as necessary. Callers +/// can also set an `errorHandler` to receive asychronous error notifications, +/// such as the monitor service suddenly becoming unavailable. +/// +/// class QuakeMonitor { +/// var quakeHandler: ((Quake) -> Void)? +/// var errorHandler: ((Error) -> Void)? +/// +/// func startMonitoring() {…} +/// func stopMonitoring() {…} +/// } +/// +/// To adapt this to use `async`-`await`, extend the `QuakeMonitor` to add a +/// `quakes` property, of type `AsyncThrowingStream`. In the getter for +/// this property, return an `AsyncThrowingStream`, whose `build` closure -- +/// called at runtime to create the stream -- uses the continuation to +/// perform the following steps: +/// +/// 1. Creates a `QuakeMonitor` instance. +/// 2. Sets the monitor's `quakeHandler` property to a closure that receives +/// each `Quake` instance and forwards it to the stream by calling the +/// continuation's `yield(_:)` method. +/// 3. Sets the monitor's `errorHandler` property to a closure that receives +/// any error from the monitor and forwards it to the stream by calling the +/// continuation's `finish(throwing:)` method. This causes the stream's +/// iterator to throw the error and terminate the stream. +/// 4. Sets the continuation's `onTermination` property to a closure that +/// calls `stopMonitoring()` on the monitor. +/// 5. Calls `startMonitoring` on the `QuakeMonitor`. +/// +/// extension QuakeMonitor { +/// +/// static var throwingQuakes: AsyncThrowingStream { +/// AsyncThrowingStream { continuation in +/// let monitor = QuakeMonitor() +/// monitor.quakeHandler = { quake in +/// continuation.yield(quake) +/// } +/// monitor.errorHandler = { error in +/// continuation.finish(throwing: error) +/// } +/// continuation.onTermination = { @Sendable _ in +/// monitor.stopMonitoring() +/// } +/// monitor.startMonitoring() +/// } +/// } +/// } +/// +/// +/// Because the stream is an `AsyncSequence`, the call point uses the +/// `for`-`await`-`in` syntax to process each `Quake` instance as produced by the stream: +/// +/// do { +/// for try await quake in quakeStream { +/// print ("Quake: \(quake.date)") +/// } +/// print ("Stream done.") +/// } catch { +/// print ("Error: \(error)") +/// } +/// +@available(SwiftStdlib 5.1, *) +public struct AsyncThrowingStream { + /// A mechanism to interface between synchronous code and an asynchronous + /// stream. + /// + /// The closure you provide to the `AsyncThrowingStream` in + /// `init(_:bufferingPolicy:_:)` receives an instance of this type when + /// invoked. Use this continuation to provide elements to the stream by + /// calling one of the `yield` methods, then terminate the stream normally by + /// calling the `finish()` method. You can also use the continuation's + /// `finish(throwing:)` method to terminate the stream by throwing an error. + /// + /// - Note: Unlike other continuations in Swift, + /// `AsyncThrowingStream.Continuation` supports escaping. + public struct Continuation: Sendable { + /// A type that indicates how the stream terminated. + /// + /// The `onTermination` closure receives an instance of this type. + public enum Termination { + + /// The stream finished as a result of calling the continuation's + /// `finish` method. + /// + /// The associated `Failure` value provides the error that terminated + /// the stream. If no error occurred, this value is `nil`. + case finished(Failure?) + + /// The stream finished as a result of cancellation. + case cancelled + } + + /// A type that indicates the result of yielding a value to a client, by + /// way of the continuation. + /// + /// The various `yield` methods of `AsyncThrowingStream.Continuation` return + /// this type to indicate the success or failure of yielding an element to + /// the continuation. + public enum YieldResult { + + /// The stream successfully enqueued the element. + /// + /// This value represents the successful enqueueing of an element, whether + /// the stream buffers the element or delivers it immediately to a pending + /// call to `next()`. The associated value `remaining` is a hint that + /// indicates the number of remaining slots in the buffer at the time of + /// the `yield` call. + /// + /// - Note: From a thread safety perspective, `remaining` is a lower bound + /// on the number of remaining slots. This is because a subsequent call + /// that uses the `remaining` value could race on the consumption of + /// values from the stream. + case enqueued(remaining: Int) + + /// The stream didn't enqueue the element because the buffer was full. + /// + /// The associated element for this case is the element that the stream + /// dropped. + case dropped(Element) + + /// The stream didn't enqueue the element because the stream was in a + /// terminal state. + /// + /// This indicates the stream terminated prior to calling `yield`, either + /// because the stream finished normally or through cancellation, or + /// it threw an error. + case terminated + } + + /// A strategy that handles exhaustion of a buffer’s capacity. + public enum BufferingPolicy { + /// Continue to add to the buffer, treating its capacity as infinite. + case unbounded + + /// When the buffer is full, discard the newly received element. + /// + /// This strategy enforces keeping the specified amount of oldest values. + case bufferingOldest(Int) + + /// When the buffer is full, discard the oldest element in the buffer. + /// + /// This strategy enforces keeping the specified amount of newest values. + case bufferingNewest(Int) + } + + let storage: _Storage + + /// Resume the task awaiting the next iteration point by having it return + /// nomally from its suspension point with a given element. + /// + /// - Parameter value: The value to yield from the continuation. + /// - Returns: A `YieldResult` that indicates the success or failure of the + /// yield operation. + /// + /// If nothing is awaiting the next value, the method attempts to buffer the + /// result's element. + /// + /// This can be called more than once and returns to the caller immediately + /// without blocking for any awaiting consumption from the iteration. + @discardableResult + public func yield(_ value: __owned Element) -> YieldResult { + storage.yield(value) + } + + /// Resume the task awaiting the next iteration point by having it return + /// nil, which signifies the end of the iteration. + /// + /// - Parameter error: The error to throw, or `nil`, to finish normally. + /// + /// Calling this function more than once has no effect. After calling + /// finish, the stream enters a terminal state and doesn't produce any additional + /// elements. + public func finish(throwing error: __owned Failure? = nil) { + storage.finish(throwing: error) + } + + /// A callback to invoke when canceling iteration of an asynchronous + /// stream. + /// + /// If an `onTermination` callback is set, using task cancellation to + /// terminate iteration of an `AsyncThrowingStream` results in a call to this + /// callback. + /// + /// Canceling an active iteration invokes the `onTermination` callback + /// first, and then resumes by yielding `nil` or throwing an error from the + /// iterator. This means that you can perform needed cleanup in the + /// cancellation handler. After reaching a terminal state, the + /// `AsyncThrowingStream` disposes of the callback. + public var onTermination: (@Sendable (Termination) -> Void)? { + get { + return storage.getOnTermination() + } + nonmutating set { + storage.setOnTermination(newValue) + } + } + } + + final class _Context { + let storage: _Storage? + let produce: () async throws -> Element? + + init(storage: _Storage? = nil, produce: @escaping () async throws -> Element?) { + self.storage = storage + self.produce = produce + } + + deinit { + storage?.cancel() + } + } + + let context: _Context + + /// Constructs an asynchronous stream for an element type, using the + /// specified buffering policy and element-producing closure. + /// + /// - Parameters: + /// - elementType: The type of element the `AsyncThrowingStream` + /// produces. + /// - limit: The maximum number of elements to + /// hold in the buffer. By default, this value is unlimited. Use a + /// `Continuation.BufferingPolicy` to buffer a specified number of oldest + /// or newest elements. + /// - build: A custom closure that yields values to the + /// `AsyncThrowingStream`. This closure receives an + /// `AsyncThrowingStream.Continuation` instance that it uses to provide + /// elements to the stream and terminate the stream when finished. + /// + /// The `AsyncStream.Continuation` received by the `build` closure is + /// appopriate for use in concurrent contexts. It is thread safe to send and + /// finish; all calls are to the continuation are serialized. However, calling + /// this from multiple concurrent contexts could result in out-of-order + /// delivery. + /// + /// The following example shows an `AsyncStream` created with this + /// initializer that produces 100 random numbers on a one-second interval, + /// calling `yield(_:)` to deliver each element to the awaiting call point. + /// When the `for` loop exits and the stream finishes by calling the + /// continuation's `finish()` method. If the random number is divisble by 5 + /// with no remainder, the stream throws a `MyRandomNumberError`. + /// + /// let stream = AsyncThrowingStream(Int.self, + /// bufferingPolicy: .bufferingNewest(5)) { continuation in + /// Task.detached { + /// for _ in 0..<100 { + /// await Task.sleep(1 * 1_000_000_000) + /// let random = Int.random(in: 1...10) + /// if (random % 5 == 0) { + /// continuation.finish(throwing: MyRandomNumberError()) + /// return + /// } else { + /// continuation.yield(random) + /// } + /// } + /// continuation.finish() + /// } + /// } + /// + /// // Call point: + /// do { + /// for try await random in stream { + /// print ("\(random)") + /// } + /// } catch { + /// print ("\(error)") + /// } + /// + public init( + _ elementType: Element.Type = Element.self, + bufferingPolicy limit: Continuation.BufferingPolicy = .unbounded, + _ build: (Continuation) -> Void + ) where Failure == Error { + let storage: _Storage = .create(limit: limit) + context = _Context(storage: storage, produce: storage.next) + build(Continuation(storage: storage)) + } + + /// Constructs an asynchronous throwing stream from a given element-producing + /// closure. + /// + /// - Parameters: + /// - produce: A closure that asynchronously produces elements for the + /// stream. + /// + /// Use this convenience initializer when you have an asychronous function + /// that can produce elements for the stream, and don't want to invoke + /// a continuation manually. This initializer "unfolds" your closure into + /// a full-blown asynchronous stream. The created stream handles adherence to + /// the `AsyncSequence` protocol automatically. To terminate the stream with + /// an error, throw the error from your closure. + /// + /// The following example shows an `AsyncThrowingStream` created with this + /// initializer that produces random numbers on a one-second interval. If the + /// random number is divisble by 5 with no remainder, the stream throws a + /// `MyRandomNumberError`. + /// + /// let stream = AsyncThrowingStream { + /// await Task.sleep(1 * 1_000_000_000) + /// let random = Int.random(in: 1...10) + /// if (random % 5 == 0) { + /// throw MyRandomNumberError() + /// } + /// return random + /// } + /// + /// // Call point: + /// do { + /// for try await random in stream { + /// print ("\(random)") + /// } + /// } catch { + /// print ("\(error)") + /// } + /// + public init( + unfolding produce: @escaping () async throws -> Element? + ) where Failure == Error { + let storage: _AsyncStreamCriticalStorage Element?>> + = .create(produce) + context = _Context { + return try await withTaskCancellationHandler { + guard let result = try await storage.value?() else { + storage.value = nil + return nil + } + return result + } onCancel: { + storage.value = nil + } + } + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingStream: AsyncSequence { + /// The asynchronous iterator for iterating an asynchronous stream. + /// + /// This type is not `Sendable`. Don't use it from multiple + /// concurrent contexts. It is a programmer error to invoke `next()` from a + /// concurrent context that contends with another such call, which + /// results in a call to `fatalError()`. + public struct Iterator: AsyncIteratorProtocol { + let context: _Context + + /// The next value from the asynchronous stream. + /// + /// When `next()` returns `nil`, this signifies the end of the + /// `AsyncThrowingStream`. + /// + /// It is a programmer error to invoke `next()` from a concurrent context + /// that contends with another such call, which results in a call to + /// `fatalError()`. + /// + /// If you cancel the task this iterator is running in while `next()` is + /// awaiting a value, the `AsyncThrowingStream` terminates. In this case, + /// `next()` may return `nil` immediately, or else return `nil` on + /// subsequent calls. + public mutating func next() async throws -> Element? { + return try await context.produce() + } + } + + /// Creates the asynchronous iterator that produces elements of this + /// asynchronous sequence. + public func makeAsyncIterator() -> Iterator { + return Iterator(context: context) + } +} + +@available(SwiftStdlib 5.1, *) +extension AsyncThrowingStream.Continuation { + /// Resume the task awaiting the next iteration point by having it return + /// normally or throw, based on a given result. + /// + /// - Parameter result: A result to yield from the continuation. In the + /// `.success(_:)` case, this returns the associated value from the + /// iterator's `next()` method. If the result is the `failure(_:)` case, + /// this call terminates the stream with the result's error, by calling + /// `finish(throwing:)`. + /// - Returns: A `YieldResult` that indicates the success or failure of the + /// yield operation. + /// + /// If nothing is awaiting the next value and the result is success, this call + /// attempts to buffer the result's element. + /// + /// If you call this method repeatedly, each call returns immediately, without + /// blocking for any awaiting consumption from the iteration. + @discardableResult + public func yield( + with result: Result + ) -> YieldResult where Failure == Error { + switch result { + case .success(let val): + return storage.yield(val) + case .failure(let err): + storage.finish(throwing: err) + return .terminated + } + } + + /// Resume the task awaiting the next iteration point by having it return + /// nomally from its suspension point. + /// + /// - Returns: A `YieldResult` that indicates the success or failure of the + /// yield operation. + /// + /// Use this method with `AsyncThrowingStream` instances whose `Element` + /// type is `Void`. In this case, the `yield()` call unblocks the + /// awaiting iteration; there is no value to return. + /// + /// If you call this method repeatedly, each call returns immediately, + /// without blocking for any awaiting consumption from the iteration. + @discardableResult + public func yield() -> YieldResult where Element == Void { + storage.yield(()) + } +} diff --git a/stdlib/public/BackDeployConcurrency/CMakeLists.txt b/stdlib/public/BackDeployConcurrency/CMakeLists.txt index af5e50e30d333..5da354361110d 100644 --- a/stdlib/public/BackDeployConcurrency/CMakeLists.txt +++ b/stdlib/public/BackDeployConcurrency/CMakeLists.txt @@ -12,7 +12,7 @@ cmake_minimum_required(VERSION 3.19.6) -# This is always build standalone +# This is always built standalone include("${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/modules/StandaloneOverlay.cmake") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules") set(SWIFT_STDLIB_STABLE_ABI TRUE) @@ -53,16 +53,86 @@ set(swift_concurrency_options DARWIN_INSTALL_NAME_DIR "@rpath" LINK_FLAGS -lobjc) set(swift_concurrency_extra_sources - "../BackDeployConcurrency/Exclusivity.cpp" - "../BackDeployConcurrency/Metadata.cpp" + "Exclusivity.cpp" + "Metadata.cpp" "../stubs/SwiftNativeNSObject.mm") set(swift_concurrency_async_fp_mode "never") -set(LLVM_OPTIONAL_SOURCES - Clock.cpp - Clock.swift - ContinuousClock.swift - SuspendingClock.swift - TaskSleepDuration.swift) +set(SWIFT_RUNTIME_CONCURRENCY_C_FLAGS) +set(SWIFT_RUNTIME_CONCURRENCY_SWIFT_FLAGS) -add_subdirectory(../Concurrency stdlib/public/BackDeployConcurrency) +# Don't emit extended frame info on platforms other than darwin, system +# backtracer and system debugger are unlikely to support it. +list(APPEND SWIFT_RUNTIME_CONCURRENCY_C_FLAGS + "-fswift-async-fp=${swift_concurrency_async_fp_mode}") +list(APPEND SWIFT_RUNTIME_CONCURRENCY_SWIFT_FLAGS + "-Xfrontend" + "-swift-async-frame-pointer=${swift_concurrency_async_fp_mode}") + +list(APPEND SWIFT_RUNTIME_CONCURRENCY_C_FLAGS + "-D__STDC_WANT_LIB_EXT1__=1") + +add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB + CompatibilityOverride.cpp + Actor.cpp + Actor.swift + AsyncLet.cpp + AsyncLet.swift + CheckedContinuation.swift + GlobalExecutor.cpp + Errors.swift + Error.cpp + Executor.swift + AsyncCompactMapSequence.swift + AsyncDropFirstSequence.swift + AsyncDropWhileSequence.swift + AsyncFilterSequence.swift + AsyncFlatMapSequence.swift + AsyncIteratorProtocol.swift + AsyncMapSequence.swift + AsyncPrefixSequence.swift + AsyncPrefixWhileSequence.swift + AsyncSequence.swift + AsyncThrowingCompactMapSequence.swift + AsyncThrowingDropWhileSequence.swift + AsyncThrowingFilterSequence.swift + AsyncThrowingFlatMapSequence.swift + AsyncThrowingMapSequence.swift + AsyncThrowingPrefixWhileSequence.swift + GlobalActor.swift + MainActor.swift + PartialAsyncTask.swift + SourceCompatibilityShims.swift + Task.cpp + Task.swift + TaskCancellation.swift + TaskAlloc.cpp + TaskStatus.cpp + TaskGroup.cpp + TaskGroup.swift + TaskLocal.cpp + TaskLocal.swift + TaskSleep.swift + ThreadSanitizer.cpp + Mutex.cpp + AsyncStreamBuffer.swift + AsyncStream.swift + AsyncThrowingStream.swift + AsyncStream.cpp + Deque.swift + ${swift_concurrency_extra_sources} + ../Concurrency/linker-support/magic-symbols-for-install-name.c + + LINK_LIBRARIES ${swift_concurrency_link_libraries} + + C_COMPILE_FLAGS + -Dswift_Concurrency_EXPORTS ${SWIFT_RUNTIME_CONCURRENCY_C_FLAGS} + -I${SWIFT_SOURCE_DIR}/stdlib/include + SWIFT_COMPILE_FLAGS + ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} + -parse-stdlib + -Xfrontend -enable-experimental-concurrency + ${SWIFT_RUNTIME_CONCURRENCY_SWIFT_FLAGS} + ${swift_concurrency_options} + INSTALL_IN_COMPONENT ${swift_concurrency_install_component} +) diff --git a/stdlib/public/BackDeployConcurrency/CheckedContinuation.swift b/stdlib/public/BackDeployConcurrency/CheckedContinuation.swift new file mode 100644 index 0000000000000..aadc8cfa39ff8 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/CheckedContinuation.swift @@ -0,0 +1,298 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_continuation_logFailedCheck") +internal func logFailedCheck(_ message: UnsafeRawPointer) + +/// Implementation class that holds the `UnsafeContinuation` instance for +/// a `CheckedContinuation`. +@available(SwiftStdlib 5.1, *) +internal final class CheckedContinuationCanary: @unchecked Sendable { + // The instance state is stored in tail-allocated raw memory, so that + // we can atomically check the continuation state. + + private init() { fatalError("must use create") } + + private static func _create(continuation: UnsafeRawPointer, function: String) + -> Self { + let instance = Builtin.allocWithTailElems_1(self, + 1._builtinWordValue, + (UnsafeRawPointer?, String).self) + + instance._continuationPtr.initialize(to: continuation) + instance._functionPtr.initialize(to: function) + return instance + } + + private var _continuationPtr: UnsafeMutablePointer { + return UnsafeMutablePointer( + Builtin.projectTailElems(self, (UnsafeRawPointer?, String).self)) + } + private var _functionPtr: UnsafeMutablePointer { + let tailPtr = UnsafeMutableRawPointer( + Builtin.projectTailElems(self, (UnsafeRawPointer?, String).self)) + + let functionPtr = tailPtr + + MemoryLayout<(UnsafeRawPointer?, String)>.offset(of: \(UnsafeRawPointer?, String).1)! + + return functionPtr.assumingMemoryBound(to: String.self) + } + + internal static func create(continuation: UnsafeContinuation, + function: String) -> Self { + return _create( + continuation: unsafeBitCast(continuation, to: UnsafeRawPointer.self), + function: function) + } + + internal var function: String { + return _functionPtr.pointee + } + + // Take the continuation away from the container, or return nil if it's + // already been taken. + internal func takeContinuation() -> UnsafeContinuation? { + // Atomically exchange the current continuation value with a null pointer. + let rawContinuationPtr = unsafeBitCast(_continuationPtr, + to: Builtin.RawPointer.self) + let rawOld = Builtin.atomicrmw_xchg_seqcst_Word(rawContinuationPtr, + 0._builtinWordValue) + + return unsafeBitCast(rawOld, to: UnsafeContinuation?.self) + } + + deinit { + _functionPtr.deinitialize(count: 1) + // Log if the continuation was never consumed before the instance was + // destructed. + if _continuationPtr.pointee != nil { + logFailedCheck("SWIFT TASK CONTINUATION MISUSE: \(function) leaked its continuation!\n") + } + } +} + +/// A mechanism to interface +/// between synchronous and asynchronous code, +/// logging correctness violations. +/// +/// A *continuation* is an opaque representation of program state. +/// To create a continuation in asynchronous code, +/// call the `withUnsafeContinuation(function:_:)` or +/// `withUnsafeThrowingContinuation(function:_:)` function. +/// To resume the asynchronous task, +/// call the `resume(returning:)`, +/// `resume(throwing:)`, +/// `resume(with:)`, +/// or `resume()` method. +/// +/// - Important: You must call a resume method exactly once +/// on every execution path throughout the program. +/// +/// Resuming from a continuation more than once is undefined behavior. +/// Never resuming leaves the task in a suspended state indefinitely, +/// and leaks any associated resources. +/// `CheckedContinuation` logs a message +/// if either of these invariants is violated. +/// +/// `CheckedContinuation` performs runtime checks +/// for missing or multiple resume operations. +/// `UnsafeContinuation` avoids enforcing these invariants at runtime +/// because it aims to be a low-overhead mechanism +/// for interfacing Swift tasks with +/// event loops, delegate methods, callbacks, +/// and other non-`async` scheduling mechanisms. +/// However, during development, the ability to verify that the +/// invariants are being upheld in testing is important. +/// Because both types have the same interface, +/// you can replace one with the other in most circumstances, +/// without making other changes. +@available(SwiftStdlib 5.1, *) +public struct CheckedContinuation { + private let canary: CheckedContinuationCanary + + /// Creates a checked continuation from an unsafe continuation. + /// + /// Instead of calling this initializer, + /// most code calls the `withCheckedContinuation(function:_:)` or + /// `withCheckedThrowingContinuation(function:_:)` function instead. + /// You only need to initialize + /// your own `CheckedContinuation` if you already have an + /// `UnsafeContinuation` you want to impose checking on. + /// + /// - Parameters: + /// - continuation: An instance of `UnsafeContinuation` + /// that hasn't yet been resumed. + /// After passing the unsafe continuation to this initializer, + /// don't use it outside of this object. + /// - function: A string identifying the declaration that is the notional + /// source for the continuation, used to identify the continuation in + /// runtime diagnostics related to misuse of this continuation. + public init(continuation: UnsafeContinuation, function: String = #function) { + canary = CheckedContinuationCanary.create( + continuation: continuation, + function: function) + } + + /// Resume the task awaiting the continuation by having it return normally + /// from its suspension point. + /// + /// - Parameter value: The value to return from the continuation. + /// + /// A continuation must be resumed exactly once. If the continuation has + /// already been resumed through this object, then the attempt to resume + /// the continuation will trap. + /// + /// After `resume` enqueues the task, control immediately returns to + /// the caller. The task continues executing when its executor is + /// able to reschedule it. + public func resume(returning value: __owned T) { + if let c: UnsafeContinuation = canary.takeContinuation() { + c.resume(returning: value) + } else { + fatalError("SWIFT TASK CONTINUATION MISUSE: \(canary.function) tried to resume its continuation more than once, returning \(value)!\n") + } + } + + /// Resume the task awaiting the continuation by having it throw an error + /// from its suspension point. + /// + /// - Parameter error: The error to throw from the continuation. + /// + /// A continuation must be resumed exactly once. If the continuation has + /// already been resumed through this object, then the attempt to resume + /// the continuation will trap. + /// + /// After `resume` enqueues the task, control immediately returns to + /// the caller. The task continues executing when its executor is + /// able to reschedule it. + public func resume(throwing error: __owned E) { + if let c: UnsafeContinuation = canary.takeContinuation() { + c.resume(throwing: error) + } else { + fatalError("SWIFT TASK CONTINUATION MISUSE: \(canary.function) tried to resume its continuation more than once, throwing \(error)!\n") + } + } +} + +@available(SwiftStdlib 5.1, *) +extension CheckedContinuation: Sendable where T: Sendable { } + +@available(SwiftStdlib 5.1, *) +extension CheckedContinuation { + /// Resume the task awaiting the continuation by having it either + /// return normally or throw an error based on the state of the given + /// `Result` value. + /// + /// - Parameter result: A value to either return or throw from the + /// continuation. + /// + /// A continuation must be resumed exactly once. If the continuation has + /// already been resumed through this object, then the attempt to resume + /// the continuation will trap. + /// + /// After `resume` enqueues the task, control immediately returns to + /// the caller. The task continues executing when its executor is + /// able to reschedule it. + @_alwaysEmitIntoClient + public func resume(with result: Result) where E == Error { + switch result { + case .success(let val): + self.resume(returning: val) + case .failure(let err): + self.resume(throwing: err) + } + } + + /// Resume the task awaiting the continuation by having it either + /// return normally or throw an error based on the state of the given + /// `Result` value. + /// + /// - Parameter result: A value to either return or throw from the + /// continuation. + /// + /// A continuation must be resumed exactly once. If the continuation has + /// already been resumed through this object, then the attempt to resume + /// the continuation will trap. + /// + /// After `resume` enqueues the task, control immediately returns to + /// the caller. The task continues executing when its executor is + /// able to reschedule it. + @_alwaysEmitIntoClient + public func resume(with result: Result) { + switch result { + case .success(let val): + self.resume(returning: val) + case .failure(let err): + self.resume(throwing: err) + } + } + + /// Resume the task awaiting the continuation by having it return normally + /// from its suspension point. + /// + /// A continuation must be resumed exactly once. If the continuation has + /// already been resumed through this object, then the attempt to resume + /// the continuation will trap. + /// + /// After `resume` enqueues the task, control immediately returns to + /// the caller. The task continues executing when its executor is + /// able to reschedule it. + @_alwaysEmitIntoClient + public func resume() where T == Void { + self.resume(returning: ()) + } +} + +/// Suspends the current task, +/// then calls the given closure with a checked continuation for the current task. +/// +/// - Parameters: +/// - function: A string identifying the declaration that is the notional +/// source for the continuation, used to identify the continuation in +/// runtime diagnostics related to misuse of this continuation. +/// - body: A closure that takes a `CheckedContinuation` parameter. +/// You must resume the continuation exactly once. +@available(SwiftStdlib 5.1, *) +public func withCheckedContinuation( + function: String = #function, + _ body: (CheckedContinuation) -> Void +) async -> T { + return await withUnsafeContinuation { + body(CheckedContinuation(continuation: $0, function: function)) + } +} + +/// Suspends the current task, +/// then calls the given closure with a checked throwing continuation for the current task. +/// +/// - Parameters: +/// - function: A string identifying the declaration that is the notional +/// source for the continuation, used to identify the continuation in +/// runtime diagnostics related to misuse of this continuation. +/// - body: A closure that takes an `UnsafeContinuation` parameter. +/// You must resume the continuation exactly once. +/// +/// If `resume(throwing:)` is called on the continuation, +/// this function throws that error. +@available(SwiftStdlib 5.1, *) +public func withCheckedThrowingContinuation( + function: String = #function, + _ body: (CheckedContinuation) -> Void +) async throws -> T { + return try await withUnsafeThrowingContinuation { + body(CheckedContinuation(continuation: $0, function: function)) + } +} + diff --git a/stdlib/public/BackDeployConcurrency/CompatibilityOverride.cpp b/stdlib/public/BackDeployConcurrency/CompatibilityOverride.cpp new file mode 100644 index 0000000000000..10952a6b27054 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/CompatibilityOverride.cpp @@ -0,0 +1,94 @@ +//===--- CompatibiltyOverride.cpp - Back-deploying compatibility fies ---s-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Support back-deploying compatibility fixes for newer apps running on older runtimes. +// +//===----------------------------------------------------------------------===// + +#include "CompatibilityOverride.h" + +#ifndef SWIFT_RUNTIME_NO_COMPATIBILITY_OVERRIDES + +#include "../runtime/ImageInspection.h" +#include "swift/Runtime/Once.h" +#include +#include +#include +#include + +using namespace swift; + +/// The definition of the contents of the override section. +/// +/// The runtime looks in the main executable (not any libraries!) for a +/// __swift54_hooks section and uses the hooks defined therein. This struct +/// defines the layout of that section. These hooks allow extending +/// runtime functionality when running apps built with a more recent +/// compiler. If additional hooks are needed, they may be added at the +/// end, but once ABI stability hits, existing ones must not be removed +/// or rearranged. The version number at the beginning can be used to +/// indicate the presence of added functions. Until we do so, the +/// version must be set to 0. +struct OverrideSection { + uintptr_t version; + +#define OVERRIDE(name, ret, attrs, ccAttrs, namespace, typedArgs, namedArgs) \ + Override_ ## name name; +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH +}; + +static_assert(std::is_pod::value, + "OverrideSection has a set layout and must be POD."); + +// We only support mach-o for overrides, so the implementation of lookupSection +// can be mach-o specific. +#if __POINTER_WIDTH__ == 64 +using mach_header_platform = mach_header_64; +#else +using mach_header_platform = mach_header; +#endif + +extern "C" mach_header_platform *_NSGetMachExecuteHeader(); +static void *lookupSection(const char *segment, const char *section, + size_t *outSize) { + unsigned long size; + auto *executableHeader = _NSGetMachExecuteHeader(); + uint8_t *data = getsectiondata(executableHeader, segment, section, &size); + if (outSize != nullptr && data != nullptr) + *outSize = size; + return static_cast(data); +} + +static OverrideSection *getOverrideSectionPtr() { + static OverrideSection *OverrideSectionPtr; + static swift_once_t Predicate; + swift_once(&Predicate, [](void *) { + size_t Size; + OverrideSectionPtr = static_cast( + lookupSection("__DATA", COMPATIBILITY_OVERRIDE_SECTION_NAME, &Size)); + if (Size < sizeof(OverrideSection)) + OverrideSectionPtr = nullptr; + }, nullptr); + + return OverrideSectionPtr; +} + +#define OVERRIDE(name, ret, attrs, ccAttrs, namespace, typedArgs, namedArgs) \ + Override_ ## name swift::getOverride_ ## name() { \ + auto *Section = getOverrideSectionPtr(); \ + if (Section == nullptr) \ + return nullptr; \ + return Section->name; \ + } +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH + +#endif // #ifndef SWIFT_RUNTIME_NO_COMPATIBILITY_OVERRIDES diff --git a/stdlib/public/BackDeployConcurrency/CompatibilityOverride.h b/stdlib/public/BackDeployConcurrency/CompatibilityOverride.h new file mode 100644 index 0000000000000..49243cf6245c6 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/CompatibilityOverride.h @@ -0,0 +1,136 @@ +//===--- CompatibiltyOverride.h - Back-deploying compatibility fixes --*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Support back-deploying compatibility fixes for newer apps running on older runtimes. +// +//===----------------------------------------------------------------------===// + +#ifndef COMPATIBILITY_OVERRIDE_H +#define COMPATIBILITY_OVERRIDE_H + +#include "../runtime/Private.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/Metadata.h" +#include "swift/Runtime/Once.h" +#include + +namespace swift { + +// Macro utilities. +#define COMPATIBILITY_UNPAREN(...) __VA_ARGS__ +#define COMPATIBILITY_CONCAT2(x, y) x##y +#define COMPATIBILITY_CONCAT(x, y) COMPATIBILITY_CONCAT2(x, y) + +// This ridiculous construct will remove the parentheses from the argument and +// add a trailing comma, or will produce nothing when passed no argument. For +// example: +// COMPATIBILITY_UNPAREN_WITH_COMMA((1, 2, 3)) -> 1, 2, 3, +// COMPATIBILITY_UNPAREN_WITH_COMMA((4)) -> 4, +// COMPATIBILITY_UNPAREN_WITH_COMMA() -> +#define COMPATIBILITY_UNPAREN_WITH_COMMA(x) \ + COMPATIBILITY_CONCAT(COMPATIBILITY_UNPAREN_ADD_TRAILING_COMMA_, \ + COMPATIBILITY_UNPAREN_WITH_COMMA2 x) +#define COMPATIBILITY_UNPAREN_WITH_COMMA2(...) PARAMS(__VA_ARGS__) +#define COMPATIBILITY_UNPAREN_ADD_TRAILING_COMMA_PARAMS(...) __VA_ARGS__, +#define COMPATIBILITY_UNPAREN_ADD_TRAILING_COMMA_COMPATIBILITY_UNPAREN_WITH_COMMA2 + +// This ridiculous construct will preserve the parentheses around the argument, +// or will produce an empty pair of parentheses when passed no argument. For +// example: +// COMPATIBILITY_PAREN((1, 2, 3)) -> (1, 2, 3) +// COMPATIBILITY_PAREN((4)) -> (4) +// COMPATIBILITY_PAREN() -> () +#define COMPATIBILITY_PAREN(x) \ + COMPATIBILITY_CONCAT(COMPATIBILITY_PAREN_, COMPATIBILITY_PAREN2 x) +#define COMPATIBILITY_PAREN2(...) PARAMS(__VA_ARGS__) +#define COMPATIBILITY_PAREN_PARAMS(...) (__VA_ARGS__) +#define COMPATIBILITY_PAREN_COMPATIBILITY_PAREN2 () + +// Include path computation. Code that includes this file can write +// `#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH` to include the appropriate +// .def file for the current library. +#define COMPATIBILITY_OVERRIDE_INCLUDE_PATH_swiftRuntime \ + "CompatibilityOverrideRuntime.def" +#define COMPATIBILITY_OVERRIDE_INCLUDE_PATH_swift_Concurrency \ + "CompatibilityOverrideConcurrency.def" + +#define COMPATIBILITY_OVERRIDE_INCLUDE_PATH \ + COMPATIBILITY_CONCAT(COMPATIBILITY_OVERRIDE_INCLUDE_PATH_, \ + SWIFT_TARGET_LIBRARY_NAME) + +// Compatibility overrides are only supported on Darwin. +#ifndef SWIFT_RUNTIME_NO_COMPATIBILITY_OVERRIDES +#if !(defined(__APPLE__) && defined(__MACH__)) +#define SWIFT_RUNTIME_NO_COMPATIBILITY_OVERRIDES +#endif +#endif + +#ifdef SWIFT_RUNTIME_NO_COMPATIBILITY_OVERRIDES + +// Call directly through to the original implementation when we don't support +// overrides. +#define COMPATIBILITY_OVERRIDE(name, ret, attrs, ccAttrs, namespace, \ + typedArgs, namedArgs) \ + attrs ccAttrs ret namespace swift_##name COMPATIBILITY_PAREN(typedArgs) { \ + return swift_##name##Impl COMPATIBILITY_PAREN(namedArgs); \ + } + +#else // #ifdef SWIFT_RUNTIME_NO_COMPATIBILITY_OVERRIDES + +// Override section name computation. `COMPATIBILITY_OVERRIDE_SECTION_NAME` will +// resolve to string literal containing the appropriate section name for the +// current library. +#define COMPATIBILITY_OVERRIDE_SECTION_NAME_swiftRuntime "__swift56_hooks" +#define COMPATIBILITY_OVERRIDE_SECTION_NAME_swift_Concurrency "__s_async_hook" + +#define COMPATIBILITY_OVERRIDE_SECTION_NAME \ + COMPATIBILITY_CONCAT(COMPATIBILITY_OVERRIDE_SECTION_NAME_, \ + SWIFT_TARGET_LIBRARY_NAME) + +// Create typedefs for function pointers to call the original implementation. +#define OVERRIDE(name, ret, attrs, ccAttrs, namespace, typedArgs, namedArgs) \ + ccAttrs typedef ret(*Original_##name) COMPATIBILITY_PAREN(typedArgs); +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH + +// Create typedefs for override function pointers. +#define OVERRIDE(name, ret, attrs, ccAttrs, namespace, typedArgs, namedArgs) \ + ccAttrs typedef ret (*Override_##name)(COMPATIBILITY_UNPAREN_WITH_COMMA( \ + typedArgs) Original_##name originalImpl); +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH + +// Create declarations for getOverride functions. +#define OVERRIDE(name, ret, attrs, ccAttrs, namespace, typedArgs, namedArgs) \ + Override_ ## name getOverride_ ## name(); +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH + +/// Used to define an override point. The override point #defines the appropriate +/// OVERRIDE macro from CompatibilityOverride.def to this macro, then includes +/// the file to generate the override points. The original implementation of the +/// functionality must be available as swift_funcNameHereImpl. +#define COMPATIBILITY_OVERRIDE(name, ret, attrs, ccAttrs, namespace, \ + typedArgs, namedArgs) \ + attrs ccAttrs ret namespace swift_##name COMPATIBILITY_PAREN(typedArgs) { \ + static Override_##name Override; \ + static swift_once_t Predicate; \ + swift_once( \ + &Predicate, [](void *) { Override = getOverride_##name(); }, nullptr); \ + if (Override != nullptr) \ + return Override(COMPATIBILITY_UNPAREN_WITH_COMMA(namedArgs) \ + swift_##name##Impl); \ + return swift_##name##Impl COMPATIBILITY_PAREN(namedArgs); \ + } + +#endif // #else SWIFT_RUNTIME_NO_COMPATIBILITY_OVERRIDES + +} /* end namespace swift */ + +#endif /* COMPATIBILITY_OVERRIDE_H */ diff --git a/stdlib/public/BackDeployConcurrency/CompatibilityOverrideConcurrency.def b/stdlib/public/BackDeployConcurrency/CompatibilityOverrideConcurrency.def new file mode 100644 index 0000000000000..185b9cb41b59b --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/CompatibilityOverrideConcurrency.def @@ -0,0 +1,357 @@ +//===--- CompatibilityOverridesConcurrency.def - Overrides Info -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file defines x-macros used for metaprogramming with the set of +// compatibility override functions. +// +//===----------------------------------------------------------------------===// + +/// #define OVERRIDE(name, ret, attrs, ccAttrs, namespace, typedArgs, namedArgs) +/// Provides information about an overridable function. +/// - name is the name of the function, without any leading swift_ or +/// namespace. +/// - ret is the return type of the function. +/// - attrs is the attributes, if any, applied to the function definition. +/// - ccAttrs is the calling convention attributes, if any, applied to the +/// function definition and corresponding typedefs +/// - namespace is the namespace, if any, the function is in, including a +/// trailing :: +/// - typedArgs is the argument list, including types, surrounded by +/// parentheses +/// - namedArgs is the list of argument names, with no types, surrounded by +/// parentheses +/// +/// The entries are organized by group. A user may define OVERRIDE to get all +/// entries, or define one or more of the more specific OVERRIDE_* variants to +/// get only those entries. + +// NOTE: this file is used to build the definition of OverrideSection in +// CompatibilityOverride.cpp, which is part of the ABI. Moving or removing +// entries in this file will break the ABI. Additional entries can be added to +// the end. ABI breaks or version-specific changes can be accommodated by +// changing the name of the override section in that file. + +#ifdef OVERRIDE +# define OVERRIDE_ACTOR OVERRIDE +# define OVERRIDE_TASK OVERRIDE +# define OVERRIDE_ASYNC_LET OVERRIDE +# define OVERRIDE_TASK_GROUP OVERRIDE +# define OVERRIDE_TASK_LOCAL OVERRIDE +# define OVERRIDE_TASK_STATUS OVERRIDE +#else +# ifndef OVERRIDE_ACTOR +# define OVERRIDE_ACTOR(...) +# endif +# ifndef OVERRIDE_TASK +# define OVERRIDE_TASK(...) +# endif +# ifndef OVERRIDE_ASYNC_LET +# define OVERRIDE_ASYNC_LET(...) +# endif +# ifndef OVERRIDE_TASK_GROUP +# define OVERRIDE_TASK_GROUP(...) +# endif +# ifndef OVERRIDE_TASK_LOCAL +# define OVERRIDE_TASK_LOCAL(...) +# endif +# ifndef OVERRIDE_TASK_STATUS +# define OVERRIDE_TASK_STATUS(...) +# endif +#endif + +OVERRIDE_ACTOR(task_enqueue, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (class Job *job, ExecutorRef executor), + (job, executor)) + +OVERRIDE_ACTOR(job_run, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (class Job *job, ExecutorRef executor), + (job, executor)) + +OVERRIDE_ACTOR(task_getCurrentExecutor, ExecutorRef, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, ,) + +OVERRIDE_ACTOR(task_isCurrentExecutor, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (ExecutorRef executor), (executor)) + +OVERRIDE_ACTOR(task_switch, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, (SWIFT_ASYNC_CONTEXT AsyncContext *resumeToContext, + TaskContinuationFunction *resumeFunction, ExecutorRef newExecutor), + (resumeToContext, resumeFunction, newExecutor)) + +OVERRIDE_TASK(task_create_common, AsyncTaskAndContext, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + (size_t taskCreateFlags, + TaskOptionRecord *options, + const Metadata *futureResultType, + FutureAsyncSignature::FunctionType *function, + void *closureContext, + size_t initialContextSize), + (taskCreateFlags, options, futureResultType, function, + closureContext, initialContextSize)) + +OVERRIDE_TASK(task_future_wait, void, SWIFT_EXPORT_FROM(swift_Concurrency), + SWIFT_CC(swiftasync), swift::, + (OpaqueValue *result, + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, AsyncTask *task, + TaskContinuationFunction *resumeFunction, + AsyncContext *callContext), + (result, callerContext, task, resumeFunction, callContext)) + +OVERRIDE_TASK(task_future_wait_throwing, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (OpaqueValue *result, + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, AsyncTask *task, + ThrowingTaskFutureWaitContinuationFunction *resumeFunction, + AsyncContext *callContext), + (result, callerContext, task, resumeFunction, callContext)) + +OVERRIDE_TASK(continuation_resume, void, SWIFT_EXPORT_FROM(swift_Concurrency), + SWIFT_CC(swift), swift::, + (AsyncTask *continuation), + (continuation)) + +OVERRIDE_TASK(continuation_throwingResume, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + (AsyncTask *continuation), + (continuation)) + +OVERRIDE_TASK(continuation_throwingResumeWithError, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + (AsyncTask *continuation, SwiftError *error), + (continuation, error)) + +OVERRIDE_TASK(task_addCancellationHandler, + CancellationNotificationStatusRecord *, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + (CancellationNotificationStatusRecord::FunctionType handler, + void *context), + (handler, context)) + +OVERRIDE_TASK(task_removeCancellationHandler, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + (CancellationNotificationStatusRecord *record), (record)) + +OVERRIDE_TASK(task_createNullaryContinuationJob, NullaryContinuationJob *, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + (size_t priority, + AsyncTask *continuation), (priority, continuation)) + +OVERRIDE_TASK(task_asyncMainDrainQueue, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + , ) + +OVERRIDE_TASK(task_suspend, AsyncTask *, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, ,) + +OVERRIDE_TASK(continuation_init, AsyncTask *, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (ContinuationAsyncContext *context, + AsyncContinuationFlags flags), + (context, flags)) + +OVERRIDE_TASK(continuation_await, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, (ContinuationAsyncContext *context), + (context)) + +OVERRIDE_ASYNC_LET(asyncLet_wait, void, SWIFT_EXPORT_FROM(swift_Concurrency), + SWIFT_CC(swiftasync), swift::, + (OpaqueValue *result, + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, TaskContinuationFunction *resumeFn, + AsyncContext *callContext), + (result, callerContext, alet, resumeFn, callContext)) + +OVERRIDE_ASYNC_LET(asyncLet_wait_throwing, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (OpaqueValue *result, + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, + ThrowingTaskFutureWaitContinuationFunction *resume, + AsyncContext *callContext), + (result, callerContext, alet, resume, callContext)) + +OVERRIDE_ASYNC_LET(asyncLet_end, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (AsyncLet *alet), (alet)) + +OVERRIDE_ASYNC_LET(asyncLet_get, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, void *resultBuffer, + TaskContinuationFunction *resumeFn, + AsyncContext *callContext), + (callerContext, alet, resultBuffer, resumeFn, callContext)) + +OVERRIDE_ASYNC_LET(asyncLet_get_throwing, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, void *resultBuffer, + ThrowingTaskFutureWaitContinuationFunction *resumeFn, + AsyncContext *callContext), + (callerContext, alet, resultBuffer, resumeFn, callContext)) + +OVERRIDE_ASYNC_LET(asyncLet_consume, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, void *resultBuffer, + TaskContinuationFunction *resumeFn, + AsyncContext *callContext), + (callerContext, alet, resultBuffer, resumeFn, callContext)) + +OVERRIDE_ASYNC_LET(asyncLet_consume_throwing, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, void *resultBuffer, + ThrowingTaskFutureWaitContinuationFunction *resumeFn, + AsyncContext *callContext), + (callerContext, alet, resultBuffer, resumeFn, callContext)) + +OVERRIDE_ASYNC_LET(asyncLet_finish, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncLet *alet, void *resultBuffer, + TaskContinuationFunction *resumeFn, + AsyncContext *callContext), + (callerContext, alet, resultBuffer, resumeFn, callContext)) + +OVERRIDE_TASK_GROUP(taskGroup_initialize, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group, const Metadata *T), (group, T)) + +OVERRIDE_TASK_STATUS(taskGroup_attachChild, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group, AsyncTask *child), + (group, child)) + +OVERRIDE_TASK_GROUP(taskGroup_destroy, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group), (group)) + +OVERRIDE_TASK_GROUP(taskGroup_wait_next_throwing, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swiftasync), + swift::, + (OpaqueValue *resultPointer, + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + TaskGroup *_group, + ThrowingTaskFutureWaitContinuationFunction *resumeFn, + AsyncContext *callContext), + (resultPointer, callerContext, _group, resumeFn, + callContext)) + +OVERRIDE_TASK_GROUP(taskGroup_isEmpty, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group), (group)) + +OVERRIDE_TASK_GROUP(taskGroup_isCancelled, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group), (group)) + +OVERRIDE_TASK_GROUP(taskGroup_cancelAll, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group), (group)) + +OVERRIDE_TASK_GROUP(taskGroup_addPending, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group, bool unconditionally), + (group, unconditionally)) + + +OVERRIDE_TASK_LOCAL(task_reportIllegalTaskLocalBindingWithinWithTaskGroup, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), swift::, + (const unsigned char *file, uintptr_t fileLength, + bool fileIsASCII, uintptr_t line), + (file, fileLength, fileIsASCII, line)) + +OVERRIDE_TASK_LOCAL(task_localValuePush, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, + (const HeapObject *key, OpaqueValue *value, + const Metadata *valueType), + (key, value, valueType)) + +OVERRIDE_TASK_LOCAL(task_localValueGet, OpaqueValue *, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, + (const HeapObject *key), + (key)) + +OVERRIDE_TASK_LOCAL(task_localValuePop, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, ,) + +OVERRIDE_TASK_LOCAL(task_localsCopyTo, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, + (AsyncTask *target), + (target)) + +OVERRIDE_TASK_STATUS(task_addStatusRecord, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskStatusRecord *newRecord), (newRecord)) + +OVERRIDE_TASK_STATUS(task_tryAddStatusRecord, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskStatusRecord *newRecord), (newRecord)) + +OVERRIDE_TASK_STATUS(task_removeStatusRecord, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskStatusRecord *record), (record)) + +OVERRIDE_TASK_STATUS(task_hasTaskGroupStatusRecord, bool, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, , ) + +OVERRIDE_TASK_STATUS(task_attachChild, ChildTaskStatusRecord *, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (AsyncTask *child), (child)) + +OVERRIDE_TASK_STATUS(task_detachChild, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (ChildTaskStatusRecord *record), (record)) + +OVERRIDE_TASK_STATUS(task_cancel, void, SWIFT_EXPORT_FROM(swift_Concurrency), + SWIFT_CC(swift), swift::, (AsyncTask *task), (task)) + +OVERRIDE_TASK_STATUS(task_cancel_group_child_tasks, void, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (TaskGroup *group), (group)) + +OVERRIDE_TASK_STATUS(task_escalate, JobPriority, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (AsyncTask *task, JobPriority newPriority), + (task, newPriority)) + +OVERRIDE_TASK_STATUS(task_getNearestDeadline, NearestTaskDeadline, + SWIFT_EXPORT_FROM(swift_Concurrency), SWIFT_CC(swift), + swift::, (AsyncTask *task), (task)) + +#undef OVERRIDE +#undef OVERRIDE_ACTOR +#undef OVERRIDE_TASK +#undef OVERRIDE_ASYNC_LET +#undef OVERRIDE_TASK_GROUP +#undef OVERRIDE_TASK_LOCAL +#undef OVERRIDE_TASK_STATUS diff --git a/stdlib/public/BackDeployConcurrency/ConcurrencyRuntime.h b/stdlib/public/BackDeployConcurrency/ConcurrencyRuntime.h new file mode 100644 index 0000000000000..ba6b78fa053a7 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/ConcurrencyRuntime.h @@ -0,0 +1,855 @@ +//===--- Concurrency.h - Runtime interface for concurrency ------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// The runtime interface for concurrency. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_RUNTIME_CONCURRENCY_H +#define SWIFT_RUNTIME_CONCURRENCY_H + +#include "Task.h" +#include "TaskGroup.h" +#include "AsyncLet.h" +#include "TaskStatus.h" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" + +// Does the runtime use a cooperative global executor? +#if defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) +#define SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR 1 +#else +#define SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR 0 +#endif + +// Does the runtime integrate with libdispatch? +#ifndef SWIFT_CONCURRENCY_ENABLE_DISPATCH +#if SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR +#define SWIFT_CONCURRENCY_ENABLE_DISPATCH 0 +#else +#define SWIFT_CONCURRENCY_ENABLE_DISPATCH 1 +#endif +#endif + +namespace swift { +class DefaultActor; +class TaskOptionRecord; + +struct SwiftError; + +struct AsyncTaskAndContext { + AsyncTask *Task; + AsyncContext *InitialContext; +}; + +/// Create a task object. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +AsyncTaskAndContext swift_task_create( + size_t taskCreateFlags, + TaskOptionRecord *options, + const Metadata *futureResultType, + void *closureEntry, HeapObject *closureContext); + +/// Caution: not all future-initializing functions actually throw, so +/// this signature may be incorrect. +using FutureAsyncSignature = + AsyncSignature; + +/// Create a task object. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +AsyncTaskAndContext swift_task_create_common( + size_t taskCreateFlags, + TaskOptionRecord *options, + const Metadata *futureResultType, + FutureAsyncSignature::FunctionType *function, void *closureContext, + size_t initialContextSize); + +/// Allocate memory in a task. +/// +/// This must be called synchronously with the task. +/// +/// All allocations will be rounded to a multiple of MAX_ALIGNMENT. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void *swift_task_alloc(size_t size); + +/// Deallocate memory in a task. +/// +/// The pointer provided must be the last pointer allocated on +/// this task that has not yet been deallocated; that is, memory +/// must be allocated and deallocated in a strict stack discipline. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_dealloc(void *ptr); + +/// Cancel a task and all of its child tasks. +/// +/// This can be called from any thread. +/// +/// This has no effect if the task is already cancelled. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_cancel(AsyncTask *task); + +/// Cancel all the child tasks that belong to the `group`. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_cancel_group_child_tasks(TaskGroup *group); + +/// Escalate the priority of a task and all of its child tasks. +/// +/// This can be called from any thread. +/// +/// This has no effect if the task already has at least the given priority. +/// Returns the priority of the task. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +JobPriority +swift_task_escalate(AsyncTask *task, JobPriority newPriority); + +// TODO: "async let wait" and "async let destroy" would be expressed +// similar to like TaskFutureWait; + +/// Wait for a non-throwing future task to complete. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_task_future_wait(on task: _owned Builtin.NativeObject) async +/// -> Success +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_task_future_wait(OpaqueValue *, + SWIFT_ASYNC_CONTEXT AsyncContext *, AsyncTask *, + TaskContinuationFunction *, + AsyncContext *); + +/// Wait for a potentially-throwing future task to complete. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_task_future_wait_throwing(on task: _owned Builtin.NativeObject) +/// async throws -> Success +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_task_future_wait_throwing( + OpaqueValue *, + SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncTask *, + ThrowingTaskFutureWaitContinuationFunction *, + AsyncContext *); + +/// Wait for a readyQueue of a Channel to become non empty. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_taskGroup_wait_next_throwing( +/// waitingTask: Builtin.NativeObject, // current task +/// group: Builtin.RawPointer +/// ) async -> T +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) +SWIFT_CC(swiftasync) +void swift_taskGroup_wait_next_throwing( + OpaqueValue *resultPointer, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + TaskGroup *group, ThrowingTaskFutureWaitContinuationFunction *resumeFn, + AsyncContext *callContext); + +/// Initialize a `TaskGroup` in the passed `group` memory location. +/// The caller is responsible for retaining and managing the group's lifecycle. +/// +/// Its Swift signature is +/// +/// \code +/// func swift_taskGroup_initialize(group: Builtin.RawPointer) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_taskGroup_initialize(TaskGroup *group, const Metadata *T); + +/// Attach a child task to the parent task's task group record. +/// +/// This function MUST be called from the AsyncTask running the task group. +/// +/// Since the group (or rather, its record) is inserted in the parent task at +/// creation we do not need the parent task here, the group already is attached +/// to it. +/// Its Swift signature is +/// +/// \code +/// func swift_taskGroup_attachChild( +/// group: Builtin.RawPointer, +/// child: Builtin.NativeObject +/// ) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_taskGroup_attachChild(TaskGroup *group, AsyncTask *child); + +/// Its Swift signature is +/// +/// This function MUST be called from the AsyncTask running the task group. +/// +/// \code +/// func swift_taskGroup_destroy(_ group: Builtin.RawPointer) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_taskGroup_destroy(TaskGroup *group); + +/// Before starting a task group child task, inform the group that there is one +/// more 'pending' child to account for. +/// +/// This function SHOULD be called from the AsyncTask running the task group, +/// however is generally thread-safe as it only only works with the group status. +/// +/// Its Swift signature is +/// +/// \code +/// func swift_taskGroup_addPending( +/// group: Builtin.RawPointer, +/// unconditionally: Bool +/// ) -> Bool +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_taskGroup_addPending(TaskGroup *group, bool unconditionally); + +/// Cancel all tasks in the group. +/// This also prevents new tasks from being added. +/// +/// This can be called from any thread. +/// +/// Its Swift signature is +/// +/// \code +/// func swift_taskGroup_cancelAll(group: Builtin.RawPointer) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_taskGroup_cancelAll(TaskGroup *group); + +/// Check ONLY if the group was explicitly cancelled, e.g. by `cancelAll`. +/// +/// This check DOES NOT take into account the task in which the group is running +/// being cancelled or not. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_taskGroup_isCancelled(group: Builtin.RawPointer) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_taskGroup_isCancelled(TaskGroup *group); + +/// Check the readyQueue of a task group, return true if it has no pending tasks. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_taskGroup_isEmpty( +/// _ group: Builtin.RawPointer +/// ) -> Bool +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_taskGroup_isEmpty(TaskGroup *group); + +/// DEPRECATED. swift_asyncLet_begin is used instead. +/// Its Swift signature is +/// +/// \code +/// func swift_asyncLet_start( +/// asyncLet: Builtin.RawPointer, +/// options: Builtin.RawPointer?, +/// operation: __owned @Sendable () async throws -> T +/// ) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_asyncLet_start(AsyncLet *alet, + TaskOptionRecord *options, + const Metadata *futureResultType, + void *closureEntryPoint, HeapObject *closureContext); + +/// Begin an async let child task. +/// Its Swift signature is +/// +/// \code +/// func swift_asyncLet_start( +/// asyncLet: Builtin.RawPointer, +/// options: Builtin.RawPointer?, +/// operation: __owned @Sendable () async throws -> T, +/// resultBuffer: Builtin.RawPointer +/// ) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_asyncLet_begin(AsyncLet *alet, + TaskOptionRecord *options, + const Metadata *futureResultType, + void *closureEntryPoint, HeapObject *closureContext, + void *resultBuffer); + +/// This matches the ABI of a closure `(Builtin.RawPointer) async -> T` +using AsyncLetWaitSignature = + SWIFT_CC(swiftasync) + void(OpaqueValue *, + SWIFT_ASYNC_CONTEXT AsyncContext *, AsyncTask *, Metadata *); + +/// DEPRECATED. swift_asyncLet_get is used instead. +/// Wait for a non-throwing async-let to complete. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_asyncLet_wait( +/// _ asyncLet: _owned Builtin.RawPointer +/// ) async -> Success +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_asyncLet_wait(OpaqueValue *, + SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncLet *, TaskContinuationFunction *, + AsyncContext *); + +/// DEPRECATED. swift_asyncLet_get_throwing is used instead. +/// Wait for a potentially-throwing async-let to complete. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_asyncLet_wait_throwing( +/// _ asyncLet: _owned Builtin.RawPointer +/// ) async throws -> Success +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_asyncLet_wait_throwing(OpaqueValue *, + SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncLet *, + ThrowingTaskFutureWaitContinuationFunction *, + AsyncContext *); + +/// DEPRECATED. swift_asyncLet_finish is used instead. +/// Its Swift signature is +/// +/// \code +/// func swift_asyncLet_end(_ alet: Builtin.RawPointer) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_asyncLet_end(AsyncLet *alet); + +/// Get the value of a non-throwing async-let, awaiting the result if necessary. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_asyncLet_get( +/// _ asyncLet: Builtin.RawPointer, +/// _ resultBuffer: Builtin.RawPointer +/// ) async +/// \endcode +/// +/// \c result points at the variable storage for the binding. It is +/// uninitialized until the first call to \c swift_asyncLet_get or +/// \c swift_asyncLet_get_throwing. That first call initializes the storage +/// with the result of the child task. Subsequent calls do nothing and leave +/// the value in place. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_asyncLet_get(SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncLet *, + void *, + TaskContinuationFunction *, + AsyncContext *); + +/// Get the value of a throwing async-let, awaiting the result if necessary. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_asyncLet_get_throwing( +/// _ asyncLet: Builtin.RawPointer, +/// _ resultBuffer: Builtin.RawPointer +/// ) async throws +/// \endcode +/// +/// \c result points at the variable storage for the binding. It is +/// uninitialized until the first call to \c swift_asyncLet_get or +/// \c swift_asyncLet_get_throwing. That first call initializes the storage +/// with the result of the child task. Subsequent calls do nothing and leave +/// the value in place. A pointer to the storage inside the child task is +/// returned if the task completes successfully, otherwise the error from the +/// child task is thrown. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_asyncLet_get_throwing(SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncLet *, + void *, + ThrowingTaskFutureWaitContinuationFunction *, + AsyncContext *); + +/// Exit the scope of an async-let binding. If the task is still running, it +/// is cancelled, and we await its completion; otherwise, we destroy the +/// value in the variable storage. +/// +/// Its Swift signature is +/// +/// \code +/// func swift_asyncLet_finish(_ asyncLet: Builtin.RawPointer, +/// _ resultBuffer: Builtin.RawPointer) async +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_asyncLet_finish(SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncLet *, + void *, + TaskContinuationFunction *, + AsyncContext *); + +/// Get the value of a non-throwing async-let, awaiting the result if necessary, +/// and then destroy the child task. The result buffer is left initialized after +/// returning. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_asyncLet_get( +/// _ asyncLet: Builtin.RawPointer, +/// _ resultBuffer: Builtin.RawPointer +/// ) async +/// \endcode +/// +/// \c result points at the variable storage for the binding. It is +/// uninitialized until the first call to \c swift_asyncLet_get or +/// \c swift_asyncLet_get_throwing. The child task will be invalidated after +/// this call, so the `async let` can not be gotten or finished afterward. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_asyncLet_consume(SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncLet *, + void *, + TaskContinuationFunction *, + AsyncContext *); + +/// Get the value of a throwing async-let, awaiting the result if necessary, +/// and then destroy the child task. The result buffer is left initialized after +/// returning. +/// +/// This can be called from any thread. Its Swift signature is +/// +/// \code +/// func swift_asyncLet_get_throwing( +/// _ asyncLet: Builtin.RawPointer, +/// _ resultBuffer: Builtin.RawPointer +/// ) async throws +/// \endcode +/// +/// \c result points at the variable storage for the binding. It is +/// uninitialized until the first call to \c swift_asyncLet_get or +/// \c swift_asyncLet_get_throwing. That first call initializes the storage +/// with the result of the child task. Subsequent calls do nothing and leave +/// the value in place. The child task will be invalidated after +/// this call, so the `async let` can not be gotten or finished afterward. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_asyncLet_consume_throwing(SWIFT_ASYNC_CONTEXT AsyncContext *, + AsyncLet *, + void *, + ThrowingTaskFutureWaitContinuationFunction *, + AsyncContext *); + +/// Returns true if the currently executing AsyncTask has a +/// 'TaskGroupTaskStatusRecord' present. +/// +/// This can be called from any thread. +/// +/// Its Swift signature is +/// +/// \code +/// func swift_taskGroup_hasTaskGroupRecord() +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_taskGroup_hasTaskGroupRecord(); + +/// Add a status record to a task. The record should not be +/// modified while it is registered with a task. +/// +/// This must be called synchronously with the task. +/// +/// If the task is already cancelled, returns `false` but still adds +/// the status record. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_task_addStatusRecord(TaskStatusRecord *record); + +/// Add a status record to a task if the task has not already +/// been cancelled. The record should not be modified while it is +/// registered with a task. +/// +/// This must be called synchronously with the task. +/// +/// If the task is already cancelled, returns `false` and does not +/// add the status record. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_task_tryAddStatusRecord(TaskStatusRecord *record); + +/// Remove a status record from a task. After this call returns, +/// the record's memory can be freely modified or deallocated. +/// +/// This must be called synchronously with the task. The record must +/// be registered with the task or else this may crash. +/// +/// The given record need not be the last record added to +/// the task, but the operation may be less efficient if not. +/// +/// Returns false if the task has been cancelled. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_task_removeStatusRecord(TaskStatusRecord *record); + +/// Signifies whether the current task is in the middle of executing the +/// operation block of a `with(Throwing)TaskGroup(...) { }`. +/// +/// Task local values must use un-structured allocation for values bound in this +/// scope, as they may be referred to by `group.spawn`-ed tasks and therefore +/// out-life the scope of a task-local value binding. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_task_hasTaskGroupStatusRecord(); + +/// Attach a child task to its parent task and return the newly created +/// `ChildTaskStatusRecord`. +/// +/// The record must be removed with by the parent invoking +/// `swift_task_detachChild` when the child has completed. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +ChildTaskStatusRecord* swift_task_attachChild(AsyncTask *child); + +/// Remove a child task from the parent tracking it. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_detachChild(ChildTaskStatusRecord *record); + +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +size_t swift_task_getJobFlags(AsyncTask* task); + +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_task_isCancelled(AsyncTask* task); + +/// Create and add an cancellation record to the task. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +CancellationNotificationStatusRecord* +swift_task_addCancellationHandler( + CancellationNotificationStatusRecord::FunctionType handler, + void *handlerContext); + +/// Remove the passed cancellation record from the task. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_removeCancellationHandler( + CancellationNotificationStatusRecord *record); + +/// Create a NullaryContinuationJob from a continuation. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +NullaryContinuationJob* +swift_task_createNullaryContinuationJob( + size_t priority, + AsyncTask *continuation); + +/// Report error about attempting to bind a task-local value from an illegal context. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_reportIllegalTaskLocalBindingWithinWithTaskGroup( + const unsigned char *file, uintptr_t fileLength, + bool fileIsASCII, uintptr_t line); + +/// Get a task local value from either the current task, or fallback task-local +/// storage. +/// +/// Its Swift signature is +/// +/// \code +/// func _taskLocalValueGet( +/// keyType: Any.Type /*Key.Type*/ +/// ) -> UnsafeMutableRawPointer? where Key: TaskLocalKey +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +OpaqueValue* +swift_task_localValueGet(const HeapObject *key); + +/// Bind a task local key to a value in the context of either the current +/// AsyncTask if present, or in the thread-local fallback context if no task +/// available. +/// +/// Its Swift signature is +/// +/// \code +/// public func _taskLocalValuePush( +/// keyType: Any.Type/*Key.Type*/, +/// value: __owned Value +/// ) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_localValuePush(const HeapObject *key, + /* +1 */ OpaqueValue *value, + const Metadata *valueType); + +/// Pop a single task local binding from the binding stack of the current task, +/// or the fallback thread-local storage if no task is available. +/// +/// This operation must be paired up with a preceding "push" operation, as otherwise +/// it may attempt to "pop" off an empty value stuck which will lead to a crash. +/// +/// The Swift surface API ensures proper pairing of push and pop operations. +/// +/// Its Swift signature is +/// +/// \code +/// public func _taskLocalValuePop() +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_localValuePop(); + +/// Copy all task locals from the current context to the target task. +/// +/// Its Swift signature is +/// +/// \code +/// func _taskLocalValueGet(AsyncTask* task) +/// \endcode +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_localsCopyTo(AsyncTask* target); + +/// This should have the same representation as an enum like this: +/// enum NearestTaskDeadline { +/// case none +/// case alreadyCancelled +/// case active(TaskDeadline) +/// } +/// TODO: decide what this interface should really be. +struct NearestTaskDeadline { + enum Kind : uint8_t { + None, + AlreadyCancelled, + Active + }; + + TaskDeadline Value; + Kind ValueKind; +}; + +/// Returns the nearest deadline that's been registered with this task. +/// +/// This must be called synchronously with the task. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +NearestTaskDeadline +swift_task_getNearestDeadline(AsyncTask *task); + +/// Switch the current task to a new executor if we aren't already +/// running on a compatible executor. +/// +/// The resumption function pointer and continuation should be set +/// appropriately in the task. +/// +/// Generally the compiler should inline a fast-path compatible-executor +/// check to avoid doing the suspension work. This function should +/// generally be tail-called, as it may continue executing the task +/// synchronously if possible. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_task_switch(SWIFT_ASYNC_CONTEXT AsyncContext *resumeToContext, + TaskContinuationFunction *resumeFunction, + ExecutorRef newExecutor); + +/// Enqueue the given job to run asynchronously on the given executor. +/// +/// The resumption function pointer and continuation should be set +/// appropriately in the task. +/// +/// Generally you should call swift_task_switch to switch execution +/// synchronously when possible. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_enqueue(Job *job, ExecutorRef executor); + +/// Enqueue the given job to run asynchronously on the global +/// execution pool. +/// +/// The resumption function pointer and continuation should be set +/// appropriately in the task. +/// +/// Generally you should call swift_task_switch to switch execution +/// synchronously when possible. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_enqueueGlobal(Job *job); + +/// A count in nanoseconds. +using JobDelay = unsigned long long; + +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_enqueueGlobalWithDelay(JobDelay delay, Job *job); + +/// Enqueue the given job on the main executor. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_enqueueMainExecutor(Job *job); + +#if SWIFT_CONCURRENCY_ENABLE_DISPATCH + +/// Enqueue the given job on the main executor. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_enqueueOnDispatchQueue(Job *job, HeapObject *queue); + +#endif + +/// A hook to take over global enqueuing. +typedef SWIFT_CC(swift) void (*swift_task_enqueueGlobal_original)(Job *job); +SWIFT_EXPORT_FROM(swift_Concurrency) +SWIFT_CC(swift) void (*swift_task_enqueueGlobal_hook)( + Job *job, swift_task_enqueueGlobal_original original); + +/// A hook to take over global enqueuing with delay. +typedef SWIFT_CC(swift) void (*swift_task_enqueueGlobalWithDelay_original)( + unsigned long long delay, Job *job); +SWIFT_EXPORT_FROM(swift_Concurrency) +SWIFT_CC(swift) void (*swift_task_enqueueGlobalWithDelay_hook)( + unsigned long long delay, Job *job, + swift_task_enqueueGlobalWithDelay_original original); + +/// A hook to take over main executor enqueueing. +typedef SWIFT_CC(swift) void (*swift_task_enqueueMainExecutor_original)( + Job *job); +SWIFT_EXPORT_FROM(swift_Concurrency) +SWIFT_CC(swift) void (*swift_task_enqueueMainExecutor_hook)( + Job *job, swift_task_enqueueMainExecutor_original original); + +/// Initialize the runtime storage for a default actor. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_defaultActor_initialize(DefaultActor *actor); + +/// Destroy the runtime storage for a default actor. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_defaultActor_destroy(DefaultActor *actor); + +/// Deallocate an instance of a default actor. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_defaultActor_deallocate(DefaultActor *actor); + +/// Deallocate an instance of what might be a default actor. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_defaultActor_deallocateResilient(HeapObject *actor); + +/// Initialize the runtime storage for a distributed remote actor. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +OpaqueValue* +swift_distributedActor_remote_initialize(const Metadata *actorType); + +/// Enqueue a job on the default actor implementation. +/// +/// The job must be ready to run. Notably, if it's a task, that +/// means that the resumption function and context should have been +/// set appropriately. +/// +/// Jobs are assumed to be "self-consuming": once it starts running, +/// the job memory is invalidated and the executor should not access it +/// again. +/// +/// Jobs are generally expected to keep the actor alive during their +/// execution. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_defaultActor_enqueue(Job *job, DefaultActor *actor); + +/// Check if the actor is a distributed 'remote' actor instance. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_distributed_actor_is_remote(DefaultActor *actor); + +/// Do a primitive suspension of the current task, as if part of +/// a continuation, although this does not provide any of the +/// higher-level continuation semantics. The current task is returned; +/// its ResumeFunction and ResumeContext will need to be initialized, +/// and then it will need to be enqueued or run as a job later. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +AsyncTask *swift_task_suspend(); + +/// Prepare a continuation in the current task. +/// +/// The caller should initialize the Parent, ResumeParent, +/// and NormalResult fields. This function will initialize the other +/// fields with appropriate defaults; the caller may then overwrite +/// them if desired. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +AsyncTask *swift_continuation_init(ContinuationAsyncContext *context, + AsyncContinuationFlags flags); + +/// Await an initialized continuation. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync) +void swift_continuation_await(ContinuationAsyncContext *continuationContext); + +/// Resume a task from a non-throwing continuation, given a normal +/// result which has already been stored into the continuation. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_continuation_resume(AsyncTask *continuation); + +/// Resume a task from a potentially-throwing continuation, given a +/// normal result which has already been stored into the continuation. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_continuation_throwingResume(AsyncTask *continuation); + +/// Resume a task from a potentially-throwing continuation by throwing +/// an error. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_continuation_throwingResumeWithError(AsyncTask *continuation, + /* +1 */ SwiftError *error); + +/// SPI helper to log a misuse of a `CheckedContinuation` to the appropriate places in the OS. +extern "C" SWIFT_CC(swift) +void swift_continuation_logFailedCheck(const char *message); + +/// Drain the queue +/// If the binary links CoreFoundation, uses CFRunLoopRun +/// Otherwise it uses dispatchMain. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_asyncMainDrainQueue [[noreturn]](); + +/// Establish that the current thread is running as the given +/// executor, then run a job. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_job_run(Job *job, ExecutorRef executor); + +/// Return the current thread's active task reference. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +AsyncTask *swift_task_getCurrent(void); + +/// Return the current thread's active executor reference. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +ExecutorRef swift_task_getCurrentExecutor(void); + +/// Return the main-actor executor reference. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +ExecutorRef swift_task_getMainExecutor(void); + +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +bool swift_task_isCurrentExecutor(ExecutorRef executor); + +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_reportUnexpectedExecutor( + const unsigned char *file, uintptr_t fileLength, bool fileIsASCII, + uintptr_t line, ExecutorRef executor); + +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +JobPriority swift_task_getCurrentThreadPriority(void); + +#if SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR + +/// Donate this thread to the global executor until either the +/// given condition returns true or we've run out of cooperative +/// tasks to run. +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift) +void swift_task_donateThreadToGlobalExecutorUntil(bool (*condition)(void*), + void *context); + +#endif + +#ifdef __APPLE__ +/// A magic symbol whose address is the mask to apply to a frame pointer to +/// signal that it is an async frame. Do not try to read the actual value of +/// this global, it will crash. +/// +/// On ARM64_32, the address is only 32 bits, and therefore this value covers +/// the top 32 bits of the in-memory frame pointer. On other 32-bit platforms, +/// the bit is not used and the address is always 0. +SWIFT_EXPORT_FROM(swift_Concurrency) +struct { char c; } swift_async_extendedFramePointerFlags; +#endif + +} + +#pragma clang diagnostic pop + +#endif diff --git a/stdlib/public/BackDeployConcurrency/ConditionVariable.h b/stdlib/public/BackDeployConcurrency/ConditionVariable.h new file mode 100644 index 0000000000000..acc5360d6531f --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/ConditionVariable.h @@ -0,0 +1,305 @@ +//===--- ConditionVariable.h - A condition variable -------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// ConditionVariable abstraction for use in Swift back-deployed concurrency +// runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_RUNTIME_CONDITION_VARIABLE_H +#define SWIFT_RUNTIME_CONDITION_VARIABLE_H + +#include "swift/Runtime/Mutex.h" +#include +#include + +#if __has_include() +#include +#endif + +namespace swift { + +#define SWIFT_CONDITION_SUPPORTS_CONSTEXPR 1 + +typedef pthread_cond_t ConditionHandle; +typedef pthread_mutex_t ConditionMutexHandle; + +/// PThread low-level implementation that supports ConditionVariable +/// found in Mutex.h +/// +/// See ConditionVariable +struct ConditionPlatformHelper { +#if SWIFT_CONDITION_SUPPORTS_CONSTEXPR + static constexpr +#else + static +#endif + ConditionHandle + staticInit() { + return PTHREAD_COND_INITIALIZER; + }; + static void init(ConditionHandle &condition); + static void destroy(ConditionHandle &condition); + static void notifyOne(ConditionHandle &condition); + static void notifyAll(ConditionHandle &condition); + static void wait(ConditionHandle &condition, ConditionMutexHandle &mutex); +}; + +/// A stack based object that notifies one thread waiting on a condition +/// variable on destruction. +template +class ScopedNotifyOneT { + ScopedNotifyOneT() = delete; + ScopedNotifyOneT(const ScopedNotifyOneT &) = delete; + ScopedNotifyOneT &operator=(const ScopedNotifyOneT &) = delete; + + ConditionVariable &Condition; +public: + explicit ScopedNotifyOneT(ConditionVariable &c) : Condition(c) {} + + ~ScopedNotifyOneT() { + Condition.notifyOne(); + } +}; + +/// A stack based object that notifies all threads waiting on a condition +/// variable on destruction. +template +class ScopedNotifyAllT { + ScopedNotifyAllT() = delete; + ScopedNotifyAllT(const ScopedNotifyAllT &) = delete; + ScopedNotifyAllT &operator=(const ScopedNotifyAllT &) = delete; + + ConditionVariable &Condition; +public: + explicit ScopedNotifyAllT(ConditionVariable &c) : Condition(c) {} + + ~ScopedNotifyAllT() { + Condition.notifyAll(); + } +}; + +/// A ConditionVariable that works with Mutex to allow -- as an example -- +/// multi-threaded producers and consumers to signal each other in a safe way. +class ConditionVariable { + friend class ConditionMutex; + friend class StaticConditionVariable; + + ConditionVariable(const ConditionVariable &) = delete; + ConditionVariable &operator=(const ConditionVariable &) = delete; + ConditionVariable(ConditionVariable &&) = delete; + ConditionVariable &operator=(ConditionVariable &&) = delete; + +public: + ConditionVariable() { ConditionPlatformHelper::init(Handle); } + ~ConditionVariable() { ConditionPlatformHelper::destroy(Handle); } + + /// Notifies one waiter (if any exists) that the condition has been met. + /// + /// Note: To avoid missed notification it is best hold the related mutex + // lock when calling notifyOne. + void notifyOne() { ConditionPlatformHelper::notifyOne(Handle); } + + /// Notifies all waiters (if any exists) that the condition has been met. + /// + /// Note: To avoid missed notification it is best hold the related mutex + // lock when calling notifyAll. + void notifyAll() { ConditionPlatformHelper::notifyAll(Handle); } + +private: + ConditionHandle Handle; + +public: + /// A Mutex object that also supports ConditionVariables. + /// + /// This is NOT a recursive mutex. + class Mutex { + + Mutex(const Mutex &) = delete; + Mutex &operator=(const Mutex &) = delete; + Mutex(Mutex &&) = delete; + Mutex &operator=(Mutex &&) = delete; + + public: + /// Constructs a non-recursive mutex. + /// + /// If `checked` is true the mutex will attempt to check for misuse and + /// fatalError when detected. If `checked` is false (the default) the + /// mutex will make little to no effort to check for misuse (more + /// efficient). + explicit Mutex(bool checked = false); + ~Mutex(); + + /// The lock() method has the following properties: + /// - Behaves as an atomic operation. + /// - Blocks the calling thread until exclusive ownership of the mutex + /// can be obtained. + /// - Prior m.unlock() operations on the same mutex synchronize-with + /// this lock operation. + /// - The behavior is undefined if the calling thread already owns + /// the mutex (likely a deadlock). + /// - Does not throw exceptions but will halt on error (fatalError). + void lock(); + + /// The unlock() method has the following properties: + /// - Behaves as an atomic operation. + /// - Releases the calling thread's ownership of the mutex and + /// synchronizes-with the subsequent successful lock operations on + /// the same object. + /// - The behavior is undefined if the calling thread does not own + /// the mutex. + /// - Does not throw exceptions but will halt on error (fatalError). + void unlock(); + + /// The try_lock() method has the following properties: + /// - Behaves as an atomic operation. + /// - Attempts to obtain exclusive ownership of the mutex for the calling + /// thread without blocking. If ownership is not obtained, returns + /// immediately. The function is allowed to spuriously fail and return + /// even if the mutex is not currently owned by another thread. + /// - If try_lock() succeeds, prior unlock() operations on the same object + /// synchronize-with this operation. lock() does not synchronize with a + /// failed try_lock() + /// - The behavior is undefined if the calling thread already owns + /// the mutex (likely a deadlock)? + /// - Does not throw exceptions but will halt on error (fatalError). + bool try_lock(); + + /// Releases lock, waits on supplied condition, and relocks before + /// returning. + /// + /// Precondition: Mutex held by this thread, undefined otherwise. + void wait(ConditionVariable &condition); + + /// Acquires lock before calling the supplied critical section and releases + /// lock on return from critical section. + /// + /// This call can block while waiting for the lock to become available. + /// + /// For example the following mutates value while holding the mutex lock. + /// + /// ``` + /// mutex.lock([&value] { value++; }); + /// ``` + /// + /// Precondition: Mutex not held by this thread, undefined otherwise. + template + auto withLock(CriticalSection &&criticalSection) + -> decltype(std::forward(criticalSection)()) { + ScopedLock guard(*this); + return std::forward(criticalSection)(); + } + + /// Acquires lock before calling the supplied critical section. If critical + /// section returns `false` then it will wait on the supplied condition and + /// call the critical section again when wait returns (after acquiring + /// lock). If critical section returns `true` (done) it will no longer wait, + /// it will release the lock and return (lockOrWait returns to caller). + /// + /// This call can block while waiting for the lock to become available. + /// + /// For example the following will loop waiting on the condition until + /// `value > 0`. It will then "consume" that value and stop looping. + /// ...all while being correctly protected by mutex. + /// + /// ``` + /// mutex.withLockOrWait(condition, [&value] { + /// if (value > 0) { + /// value--; + /// return true; + /// } + /// return false; + /// }); + /// ``` + /// + /// Precondition: Mutex not held by this thread, undefined otherwise. + template + void withLockOrWait(ConditionVariable &condition, + CriticalSection &&criticalSection) { + withLock([&] { + while (!criticalSection()) { + wait(condition); + } + }); + } + + /// Acquires lock before calling the supplied critical section and on return + /// from critical section it notifies one waiter of supplied condition and + /// then releases the lock. + /// + /// This call can block while waiting for the lock to become available. + /// + /// For example the following mutates value while holding the mutex lock and + /// then notifies one condition waiter about this change. + /// + /// ``` + /// mutex.withLockThenNotifyOne(condition, [&value] { value++; }); + /// ``` + /// + /// Precondition: Mutex not held by this thread, undefined otherwise. + template + auto withLockThenNotifyOne(ConditionVariable &condition, + CriticalSection &&criticalSection) + -> decltype(std::forward(criticalSection)()) { + return withLock([&] { + ScopedNotifyOne guard(condition); + return std::forward(criticalSection)(); + }); + } + + /// Acquires lock before calling the supplied critical section and on return + /// from critical section it notifies all waiters of supplied condition and + /// then releases the lock. + /// + /// This call can block while waiting for the lock to become available. + /// + /// For example the following mutates value while holding the mutex lock and + /// then notifies all condition waiters about this change. + /// + /// ``` + /// mutex.withLockThenNotifyAll(condition, [&value] { value++; }); + /// ``` + /// + /// Precondition: Mutex not held by this thread, undefined otherwise. + template + auto withLockThenNotifyAll(ConditionVariable &condition, + CriticalSection &&criticalSection) + -> decltype(std::forward(criticalSection)()) { + return withLock([&] { + ScopedNotifyAll guard(condition); + return std::forward(criticalSection)(); + }); + } + + /// A stack based object that locks the supplied mutex on construction + /// and unlocks it on destruction. + /// + /// Precondition: Mutex unlocked by this thread, undefined otherwise. + typedef ScopedLockT ScopedLock; + + /// A stack based object that unlocks the supplied mutex on construction + /// and relocks it on destruction. + /// + /// Precondition: Mutex locked by this thread, undefined otherwise. + typedef ScopedLockT ScopedUnlock; + + private: + ConditionMutexHandle Handle; + }; + + using ScopedNotifyOne = ScopedNotifyOneT; + using ScopedNotifyAll = ScopedNotifyAllT; +}; + +} + +#endif diff --git a/stdlib/public/BackDeployConcurrency/CooperativeGlobalExecutor.inc b/stdlib/public/BackDeployConcurrency/CooperativeGlobalExecutor.inc new file mode 100644 index 0000000000000..a8867ecee8fe8 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/CooperativeGlobalExecutor.inc @@ -0,0 +1,197 @@ +///===--- CooperativeGlobalExecutor.inc ---------------------*- C++ -*--===/// +/// +/// This source file is part of the Swift.org open source project +/// +/// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +/// Licensed under Apache License v2.0 with Runtime Library Exception +/// +/// See https:///swift.org/LICENSE.txt for license information +/// See https:///swift.org/CONTRIBUTORS.txt for the list of Swift project authors +/// +///===------------------------------------------------------------------===/// +/// +/// The implementation of the cooperative global executor. +/// +/// This file is included into GlobalExecutor.cpp only when +/// the cooperative global executor is enabled. It is expected to +/// declare the following functions: +/// swift_task_enqueueGlobalImpl +/// swift_task_enqueueGlobalWithDelayImpl +/// swift_task_enqueueMainExecutorImpl +/// as well as any cooperative-executor-specific functions in the runtime. +/// +///===------------------------------------------------------------------===/// + +#include +#include +#include "swift/Basic/ListMerger.h" + +namespace { + +struct JobQueueTraits { + static Job *&storage(Job *cur) { + return reinterpret_cast(cur->SchedulerPrivate[0]); + } + + static Job *getNext(Job *job) { + return storage(job); + } + static void setNext(Job *job, Job *next) { + storage(job) = next; + } + static int compare(Job *lhs, Job *rhs) { + return descendingPriorityOrder(lhs->getPriority(), rhs->getPriority()); + } +}; +using JobQueueMerger = ListMerger; + +using JobDeadline = std::chrono::time_point; + +template +struct JobDeadlineStorage; + +/// Specialization for when JobDeadline fits in SchedulerPrivate. +template <> +struct JobDeadlineStorage { + static JobDeadline &storage(Job *job) { + return reinterpret_cast(job->SchedulerPrivate[1]); + } + static JobDeadline get(Job *job) { + return storage(job); + } + static void set(Job *job, JobDeadline deadline) { + new(static_cast(&storage(job))) JobDeadline(deadline); + } + static void destroy(Job *job) { + storage(job).~JobDeadline(); + } +}; + +/// Specialization for when JobDeadline doesn't fit in SchedulerPrivate. +template <> +struct JobDeadlineStorage { + static JobDeadline *&storage(Job *job) { + return reinterpret_cast(job->SchedulerPrivate[1]); + } + static JobDeadline get(Job *job) { + return *storage(job); + } + static void set(Job *job, JobDeadline deadline) { + storage(job) = new JobDeadline(deadline); + } + static void destroy(Job *job) { + delete storage(job); + } +}; + +} // end anonymous namespace + +static Job *JobQueue = nullptr; +static Job *DelayedJobQueue = nullptr; + +/// Insert a job into the cooperative global queue. +SWIFT_CC(swift) +static void swift_task_enqueueGlobalImpl(Job *job) { + assert(job && "no job provided"); + + JobQueueMerger merger(JobQueue); + merger.insert(job); + JobQueue = merger.release(); +} + +/// Enqueues a task on the main executor. +SWIFT_CC(swift) +static void swift_task_enqueueMainExecutorImpl(Job *job) { + // The cooperative executor does not distinguish between the main + // queue and the global queue. + swift_task_enqueueGlobalImpl(job); +} + +/// Insert a job into the cooperative global queue with a delay. +SWIFT_CC(swift) +static void swift_task_enqueueGlobalWithDelayImpl(JobDelay delay, + Job *newJob) { + assert(newJob && "no job provided"); + + auto deadline = std::chrono::steady_clock::now() + + std::chrono::duration_cast( + std::chrono::nanoseconds(delay)); + JobDeadlineStorage<>::set(newJob, deadline); + + Job **position = &DelayedJobQueue; + while (auto cur = *position) { + // If we find a job with a later deadline, insert here. + // Note that we maintain FIFO order. + if (deadline < JobDeadlineStorage<>::get(cur)) { + JobQueueTraits::setNext(newJob, cur); + *position = newJob; + return; + } + + // Otherwise, keep advancing through the queue. + position = &JobQueueTraits::storage(cur); + } + JobQueueTraits::setNext(newJob, nullptr); + *position = newJob; +} + +/// Recognize jobs in the delayed-jobs queue that are ready to execute +/// and move them to the primary queue. +static void recognizeReadyDelayedJobs() { + // Process all the delayed jobs. + auto nextDelayedJob = DelayedJobQueue; + if (!nextDelayedJob) return; + + auto now = std::chrono::steady_clock::now(); + JobQueueMerger readyJobs(JobQueue); + + // Pull jobs off of the delayed-jobs queue whose deadline has been + // reached, and add them to the ready queue. + while (nextDelayedJob && + JobDeadlineStorage<>::get(nextDelayedJob) <= now) { + // Destroy the storage of the deadline in the job. + JobDeadlineStorage<>::destroy(nextDelayedJob); + + auto next = JobQueueTraits::getNext(nextDelayedJob); + readyJobs.insert(nextDelayedJob); + nextDelayedJob = next; + } + + JobQueue = readyJobs.release(); + DelayedJobQueue = nextDelayedJob; +} + +/// Claim the next job from the cooperative global queue. +static Job *claimNextFromCooperativeGlobalQueue() { + while (true) { + // Move any delayed jobs that are now ready into the primary queue. + recognizeReadyDelayedJobs(); + + // If there's a job in the primary queue, run it. + if (auto job = JobQueue) { + JobQueue = JobQueueTraits::getNext(job); + return job; + } + + // If there are only delayed jobs left, sleep until the next deadline. + // TODO: should the donator have some say in this? + if (auto delayedJob = DelayedJobQueue) { + auto deadline = JobDeadlineStorage<>::get(delayedJob); + std::this_thread::sleep_until(deadline); + continue; + } + + return nullptr; + } +} + +void swift:: +swift_task_donateThreadToGlobalExecutorUntil(bool (*condition)(void *), + void *conditionContext) { + while (!condition(conditionContext)) { + auto job = claimNextFromCooperativeGlobalQueue(); + if (!job) return; + swift_job_run(job, ExecutorRef::generic()); + } +} diff --git a/stdlib/public/BackDeployConcurrency/Debug.h b/stdlib/public/BackDeployConcurrency/Debug.h new file mode 100644 index 0000000000000..1d8a686757f46 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Debug.h @@ -0,0 +1,41 @@ +//===--- Debug.h - Swift Concurrency debug helpers --------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Debugging and inspection support. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_CONCURRENCY_DEBUG_H +#define SWIFT_CONCURRENCY_DEBUG_H + +#include "swift/Runtime/Config.h" + +namespace swift { + +// Dispatch knows about these symbol names. Don't change them without consulting +// dispatch. + +/// The metadata pointer used for job objects. +SWIFT_EXPORT_FROM(swift_Concurrency) +const void *const _swift_concurrency_debug_jobMetadata; + +/// The metadata pointer used for async task objects. +SWIFT_EXPORT_FROM(swift_Concurrency) +const void *const _swift_concurrency_debug_asyncTaskMetadata; + +/// A fake metadata pointer placed at the start of async task slab allocations. +SWIFT_EXPORT_FROM(swift_Concurrency) +const void *const _swift_concurrency_debug_asyncTaskSlabMetadata; + +} // namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/Deque.swift b/stdlib/public/BackDeployConcurrency/Deque.swift new file mode 100644 index 0000000000000..580d832ffadcf --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Deque.swift @@ -0,0 +1,425 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020-2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +@available(SwiftStdlib 5.1, *) +struct _Deque { + internal struct _UnsafeHandle { + let _header: UnsafeMutablePointer<_Storage._Header> + let _elements: UnsafeMutablePointer? + + init( + header: UnsafeMutablePointer<_Storage._Header>, + elements: UnsafeMutablePointer?, + isMutable: Bool + ) { + self._header = header + self._elements = elements + } + + var header: _Storage._Header { + _header.pointee + } + + var capacity: Int { + _header.pointee.capacity + } + + var count: Int { + get { _header.pointee.count } + nonmutating set { _header.pointee.count = newValue } + } + + internal func slot(after slot: Int) -> Int { + _internalInvariant(slot < capacity) + let position = slot + 1 + if position >= capacity { + return 0 + } + return position + } + + + internal func slot(_ slot: Int, offsetBy delta: Int) -> Int { + _internalInvariant(slot <= capacity) + let position = slot + delta + if delta >= 0 { + if position >= capacity { return position - capacity } + } else { + if position < 0 { return position + capacity } + } + return position + } + + internal var endSlot: Int { + slot(startSlot, offsetBy: count) + } + + internal func uncheckedAppend(_ element: Element) { + _internalInvariant(count < capacity) + ptr(at: endSlot).initialize(to: element) + count += 1 + } + + internal func uncheckedRemoveFirst() -> Element { + _internalInvariant(count > 0) + let result = ptr(at: startSlot).move() + startSlot = slot(after: startSlot) + count -= 1 + return result + } + + internal func uncheckedRemoveFirstIfPresent() -> Element? { + if count > 0 { + let result = ptr(at: startSlot).move() + startSlot = slot(after: startSlot) + count -= 1 + return result + } else { + return nil + } + } + + struct _UnsafeWrappedBuffer { + internal let first: UnsafeBufferPointer + + internal let second: UnsafeBufferPointer? + + internal init( + _ first: UnsafeBufferPointer, + _ second: UnsafeBufferPointer? = nil + ) { + self.first = first + self.second = second + _internalInvariant(first.count > 0 || second == nil) + } + + internal init( + start: UnsafePointer, + count: Int + ) { + self.init(UnsafeBufferPointer(start: start, count: count)) + } + + internal init( + first start1: UnsafePointer, + count count1: Int, + second start2: UnsafePointer, + count count2: Int + ) { + self.init(UnsafeBufferPointer(start: start1, count: count1), + UnsafeBufferPointer(start: start2, count: count2)) + } + + internal var count: Int { first.count + (second?.count ?? 0) } + } + + internal struct _UnsafeMutableWrappedBuffer { + internal let first: UnsafeMutableBufferPointer + + internal let second: UnsafeMutableBufferPointer? + + internal init( + _ first: UnsafeMutableBufferPointer, + _ second: UnsafeMutableBufferPointer? = nil + ) { + self.first = first + self.second = second?.count == 0 ? nil : second + _internalInvariant(first.count > 0 || second == nil) + } + + internal init( + start: UnsafeMutablePointer, + count: Int + ) { + self.init(UnsafeMutableBufferPointer(start: start, count: count)) + } + + internal init( + first start1: UnsafeMutablePointer, + count count1: Int, + second start2: UnsafeMutablePointer, + count count2: Int + ) { + self.init(UnsafeMutableBufferPointer(start: start1, count: count1), + UnsafeMutableBufferPointer(start: start2, count: count2)) + } + + internal init(mutating buffer: _UnsafeWrappedBuffer) { + self.init(.init(mutating: buffer.first), + buffer.second.map { .init(mutating: $0) }) + } + } + + internal func segments() -> _UnsafeWrappedBuffer { + let wrap = capacity - startSlot + if count <= wrap { + return .init(start: ptr(at: startSlot), count: count) + } + return .init(first: ptr(at: startSlot), count: wrap, + second: ptr(at: .zero), count: count - wrap) + } + + internal func mutableSegments() -> _UnsafeMutableWrappedBuffer { + return .init(mutating: segments()) + } + + var startSlot: Int { + get { _header.pointee.startSlot } + nonmutating set { _header.pointee.startSlot = newValue } + } + + func ptr(at slot: Int) -> UnsafeMutablePointer { + _internalInvariant(slot >= 0 && slot <= capacity) + return _elements! + slot + } + + @discardableResult + func initialize( + at start: Int, + from source: UnsafeBufferPointer + ) -> Int { + _internalInvariant(start + source.count <= capacity) + guard source.count > 0 else { return start } + ptr(at: start).initialize(from: source.baseAddress!, count: source.count) + return start + source.count + } + + @discardableResult + func moveInitialize( + at start: Int, + from source: UnsafeMutableBufferPointer + ) -> Int { + _internalInvariant(start + source.count <= capacity) + guard source.count > 0 else { return start } + ptr(at: start).moveInitialize(from: source.baseAddress!, count: source.count) + return start + source.count + } + + internal func copyElements() -> _Storage { + let object = _Storage._DequeBuffer.create( + minimumCapacity: capacity, + makingHeaderWith: { _ in header }) + let result = _Storage(_buffer: ManagedBufferPointer(unsafeBufferObject: object)) + guard self.count > 0 else { return result } + result.update { target in + let source = self.segments() + target.initialize(at: startSlot, from: source.first) + if let second = source.second { + target.initialize(at: 0, from: second) + } + } + return result + } + + internal func moveElements(minimumCapacity: Int) -> _Storage { + let count = self.count + _internalInvariant(minimumCapacity >= count) + let object = _Storage._DequeBuffer.create( + minimumCapacity: minimumCapacity, + makingHeaderWith: { +#if os(OpenBSD) + let capacity = minimumCapacity +#else + let capacity = $0.capacity +#endif + return _Storage._Header( + capacity: capacity, + count: count, + startSlot: .zero) + }) + let result = _Storage(_buffer: ManagedBufferPointer(unsafeBufferObject: object)) + guard count > 0 else { return result } + result.update { target in + let source = self.mutableSegments() + let next = target.moveInitialize(at: .zero, from: source.first) + if let second = source.second { + target.moveInitialize(at: next, from: second) + } + } + self.count = 0 + return result + } + } + + enum _Storage { + internal struct _Header { + var capacity: Int + + var count: Int + + var startSlot: Int + + init(capacity: Int, count: Int, startSlot: Int) { + self.capacity = capacity + self.count = count + self.startSlot = startSlot + } + } + + internal typealias _Buffer = ManagedBufferPointer<_Header, Element> + + case empty + case buffer(_Buffer) + + internal class _DequeBuffer: ManagedBuffer<_Header, Element> { + deinit { + self.withUnsafeMutablePointers { header, elements in + let capacity = header.pointee.capacity + let count = header.pointee.count + let startSlot = header.pointee.startSlot + + if startSlot + count <= capacity { + (elements + startSlot).deinitialize(count: count) + } else { + let firstRegion = capacity - startSlot + (elements + startSlot).deinitialize(count: firstRegion) + elements.deinitialize(count: count - firstRegion) + } + } + } + } + + internal init(_buffer: _Buffer) { + self = .buffer(_buffer) + } + + internal init() { + self = .empty + } + + internal init(_ object: _DequeBuffer) { + self.init(_buffer: _Buffer(unsafeBufferObject: object)) + } + + internal var capacity: Int { + switch self { + case .empty: return 0 + case .buffer(let buffer): + return buffer.withUnsafeMutablePointerToHeader { $0.pointee.capacity } + } + + } + + internal mutating func ensure( + minimumCapacity: Int + ) { + if _slowPath(capacity < minimumCapacity) { + _ensure(minimumCapacity: minimumCapacity) + } + } + + internal static var growthFactor: Double { 1.5 } + + internal func _growCapacity( + to minimumCapacity: Int + ) -> Int { + return Swift.max(Int((Self.growthFactor * Double(capacity)).rounded(.up)), + minimumCapacity) + } + + internal mutating func _ensure( + minimumCapacity: Int + ) { + if capacity >= minimumCapacity { + self = self.read { $0.copyElements() } + } else { + let minimumCapacity = _growCapacity(to: minimumCapacity) + self = self.update { source in + source.moveElements(minimumCapacity: minimumCapacity) + } + } + } + + internal var count: Int { + switch self { + case .empty: return 0 + case .buffer(let buffer): + return buffer.withUnsafeMutablePointerToHeader { $0.pointee.count } + } + + } + + internal func read(_ body: (_UnsafeHandle) throws -> R) rethrows -> R { + switch self { + case .empty: + var header = _Header(capacity: 0, count: 0, startSlot: 0) + return try withUnsafeMutablePointer(to: &header) { headerPtr in + return try body(_UnsafeHandle(header: headerPtr, elements: nil, isMutable: false)) + } + case .buffer(let buffer): + return try buffer.withUnsafeMutablePointers { header, elements in + let handle = _UnsafeHandle(header: header, + elements: elements, + isMutable: false) + return try body(handle) + } + } + + } + + internal func update(_ body: (_UnsafeHandle) throws -> R) rethrows -> R { + switch self { + case .empty: + var header = _Header(capacity: 0, count: 0, startSlot: 0) + return try withUnsafeMutablePointer(to: &header) { headerPtr in + return try body(_UnsafeHandle(header: headerPtr, elements: nil, isMutable: false)) + } + case .buffer(let buffer): + return try buffer.withUnsafeMutablePointers { header, elements in + let handle = _UnsafeHandle(header: header, + elements: elements, + isMutable: true) + return try body(handle) + } + } + } + } + + + internal var _storage: _Storage + + init() { + _storage = _Storage() + } + + var count: Int { _storage.count } + + mutating func append(_ newElement: Element) { + _storage.ensure(minimumCapacity: _storage.count + 1) + _storage.update { + $0.uncheckedAppend(newElement) + } + } + + @discardableResult + mutating func removeFirst() -> Element { + return _storage.update { $0.uncheckedRemoveFirst() } + } + + @discardableResult + mutating func removeFirstIfPresent() -> Element? { + return _storage.update { $0.uncheckedRemoveFirstIfPresent() } + } +} + +@_alwaysEmitIntoClient @_transparent +internal func _internalInvariant( + _ condition: @autoclosure () -> Bool, + _ message: @autoclosure () -> String = String(), + file: StaticString = #file, line: UInt = #line +) { + #if INTERNAL_CHECKS_ENABLED + assert(condition(), message(), file: file, line: line) + #endif +} diff --git a/stdlib/public/BackDeployConcurrency/DispatchGlobalExecutor.inc b/stdlib/public/BackDeployConcurrency/DispatchGlobalExecutor.inc new file mode 100644 index 0000000000000..acbe281a6a2b1 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/DispatchGlobalExecutor.inc @@ -0,0 +1,245 @@ +///===--- DispatchGlobalExecutor.inc ------------------------*- C++ -*--===/// +/// +/// This source file is part of the Swift.org open source project +/// +/// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +/// Licensed under Apache License v2.0 with Runtime Library Exception +/// +/// See https:///swift.org/LICENSE.txt for license information +/// See https:///swift.org/CONTRIBUTORS.txt for the list of Swift project authors +/// +///===------------------------------------------------------------------===/// +/// +/// The implementation of the global executor when using Dispatch. +/// +/// This file is included into GlobalExecutor.cpp only when Dispatch +/// integration is enabled. It is expected to define the following +/// functions: +/// swift_task_enqueueGlobalImpl +/// swift_task_enqueueGlobalWithDelayImpl +/// swift_task_enqueueMainExecutorImpl +/// as well as any Dispatch-specific functions for the runtime. +/// +///===------------------------------------------------------------------===/// + +#if SWIFT_CONCURRENCY_ENABLE_DISPATCH +#include + +#if !defined(_WIN32) +#include +#endif + +#endif + +// Ensure that Job's layout is compatible with what Dispatch expects. +// Note: MinimalDispatchObjectHeader just has the fields we care about, it is +// not complete and should not be used for anything other than these asserts. +struct MinimalDispatchObjectHeader { + const void *VTable; + int Opaque0; + int Opaque1; + void *Linkage; +}; +static_assert( + offsetof(Job, metadata) == offsetof(MinimalDispatchObjectHeader, VTable), + "Job Metadata field must match location of Dispatch VTable field."); +static_assert(offsetof(Job, SchedulerPrivate[Job::DispatchLinkageIndex]) == + offsetof(MinimalDispatchObjectHeader, Linkage), + "Dispatch Linkage field must match Job " + "SchedulerPrivate[DispatchLinkageIndex]."); + +/// The function passed to dispatch_async_f to execute a job. +static void __swift_run_job(void *_job) { + Job *job = (Job*) _job; + auto metadata = + reinterpret_cast(job->metadata); + metadata->VTableInvoke(job, nullptr, 0); +} + +/// The type of a function pointer for enqueueing a Job object onto a dispatch +/// queue. +typedef void (*dispatchEnqueueFuncType)(dispatch_queue_t queue, void *obj, + dispatch_qos_class_t qos); + +/// Initialize dispatchEnqueueFunc and then call through to the proper +/// implementation. +static void initializeDispatchEnqueueFunc(dispatch_queue_t queue, void *obj, + dispatch_qos_class_t qos); + +/// A function pointer to the function used to enqueue a Job onto a dispatch +/// queue. Initially set to initializeDispatchEnqueueFunc, so that the first +/// call will initialize it. initializeDispatchEnqueueFunc sets it to point +/// either to dispatch_async_swift_job when it's available, otherwise to +/// dispatchEnqueueDispatchAsync. +static std::atomic dispatchEnqueueFunc{ + initializeDispatchEnqueueFunc}; + +/// A small adapter that dispatches a Job onto a queue using dispatch_async_f. +static void dispatchEnqueueDispatchAsync(dispatch_queue_t queue, void *obj, + dispatch_qos_class_t qos) { + dispatch_async_f(queue, obj, __swift_run_job); +} + +static void initializeDispatchEnqueueFunc(dispatch_queue_t queue, void *obj, + dispatch_qos_class_t qos) { + dispatchEnqueueFuncType func = nullptr; + + // Always fall back to plain dispatch_async_f on Windows for now, and + // also for back-deployed concurrency. +#if !defined(_WIN32) && !defined(SWIFT_CONCURRENCY_BACK_DEPLOYMENT) + if (runtime::environment::concurrencyEnableJobDispatchIntegration()) + func = reinterpret_cast( + dlsym(RTLD_NEXT, "dispatch_async_swift_job")); +#endif + + if (!func) + func = dispatchEnqueueDispatchAsync; + + dispatchEnqueueFunc.store(func, std::memory_order_relaxed); + + func(queue, obj, qos); +} + +/// Enqueue a Job onto a dispatch queue using dispatchEnqueueFunc. +static void dispatchEnqueue(dispatch_queue_t queue, Job *job, + dispatch_qos_class_t qos, void *executorQueue) { + job->SchedulerPrivate[Job::DispatchQueueIndex] = executorQueue; + dispatchEnqueueFunc.load(std::memory_order_relaxed)(queue, job, qos); +} + +static constexpr size_t globalQueueCacheCount = + static_cast(JobPriority::UserInteractive) + 1; +static std::atomic globalQueueCache[globalQueueCacheCount]; + +#if defined(SWIFT_CONCURRENCY_BACK_DEPLOYMENT) || !defined(__APPLE__) +extern "C" void dispatch_queue_set_width(dispatch_queue_t dq, long width); +#endif + +static dispatch_queue_t getGlobalQueue(JobPriority priority) { + size_t numericPriority = static_cast(priority); + if (numericPriority >= globalQueueCacheCount) + swift_Concurrency_fatalError(0, "invalid job priority %#zx"); + +#ifdef SWIFT_CONCURRENCY_BACK_DEPLOYMENT + std::memory_order loadOrder = std::memory_order_acquire; +#else + std::memory_order loadOrder = std::memory_order_relaxed; +#endif + + auto *ptr = &globalQueueCache[numericPriority]; + auto queue = ptr->load(loadOrder); + if (SWIFT_LIKELY(queue)) + return queue; + +#if defined(SWIFT_CONCURRENCY_BACK_DEPLOYMENT) || !defined(__APPLE__) + const int DISPATCH_QUEUE_WIDTH_MAX_LOGICAL_CPUS = -3; + + // Create a new cooperative concurrent queue and swap it in. + dispatch_queue_attr_t newQueueAttr = dispatch_queue_attr_make_with_qos_class( + DISPATCH_QUEUE_CONCURRENT, (dispatch_qos_class_t)priority, 0); + dispatch_queue_t newQueue = dispatch_queue_create( + "Swift global concurrent queue", newQueueAttr); + dispatch_queue_set_width(newQueue, DISPATCH_QUEUE_WIDTH_MAX_LOGICAL_CPUS); + + if (!ptr->compare_exchange_strong(queue, newQueue, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_acquire)) { + dispatch_release(newQueue); + return queue; + } + + return newQueue; +#else + // If we don't have a queue cached for this priority, cache it now. This may + // race with other threads doing this at the same time for this priority, but + // that's OK, they'll all end up writing the same value. + queue = dispatch_get_global_queue((dispatch_qos_class_t)priority, + /*flags*/ 0); + + // Unconditionally store it back in the cache. If we raced with another + // thread, we'll just overwrite the entry with the same value. + ptr->store(queue, std::memory_order_relaxed); +#endif + + return queue; +} + +SWIFT_CC(swift) +static void swift_task_enqueueGlobalImpl(Job *job) { + assert(job && "no job provided"); + + // We really want four things from the global execution service: + // - Enqueuing work should have minimal runtime and memory overhead. + // - Adding work should never result in an "explosion" where many + // more threads are created than the available cores. + // - Jobs should run on threads with an appropriate priority. + // - Thread priorities should temporarily elevatable to avoid + // priority inversions. + // + // Of these, the first two are the most important. Many programs + // do not rely on high-usage priority scheduling, and many priority + // inversions can be avoided at a higher level (albeit with some + // performance cost, e.g. by creating higher-priority tasks to run + // critical sections that contend with high-priority work). In + // contrast, if the async feature adds too much overhead, or if + // heavy use of it leads to thread explosions and memory exhaustion, + // programmers will have no choice but to stop using it. So if + // goals are in conflict, it's best to focus on core properties over + // priority-inversion avoidance. + + // We currently use Dispatch for our thread pool on all platforms. + // Dispatch currently backs its serial queues with a global + // concurrent queue that is prone to thread explosions when a flood + // of jobs are added to it. That problem does not apply equally + // to the global concurrent queues returned by dispatch_get_global_queue, + // which are not strictly CPU-limited but are at least much more + // cautious about adding new threads. We cannot safely elevate + // the priorities of work added to this queue using Dispatch's public + // API, but as discussed above, that is less important than avoiding + // performance problems. + JobPriority priority = job->getPriority(); + + auto queue = getGlobalQueue(priority); + + dispatchEnqueue(queue, job, (dispatch_qos_class_t)priority, + DISPATCH_QUEUE_GLOBAL_EXECUTOR); +} + + +SWIFT_CC(swift) +static void swift_task_enqueueGlobalWithDelayImpl(JobDelay delay, + Job *job) { + assert(job && "no job provided"); + + dispatch_function_t dispatchFunction = &__swift_run_job; + void *dispatchContext = job; + + JobPriority priority = job->getPriority(); + + auto queue = getGlobalQueue(priority); + + job->SchedulerPrivate[Job::DispatchQueueIndex] = + DISPATCH_QUEUE_GLOBAL_EXECUTOR; + + dispatch_time_t when = dispatch_time(DISPATCH_TIME_NOW, delay); + dispatch_after_f(when, queue, dispatchContext, dispatchFunction); +} + +SWIFT_CC(swift) +static void swift_task_enqueueMainExecutorImpl(Job *job) { + assert(job && "no job provided"); + + JobPriority priority = job->getPriority(); + + // This is an inline function that compiles down to a pointer to a global. + auto mainQueue = dispatch_get_main_queue(); + + dispatchEnqueue(mainQueue, job, (dispatch_qos_class_t)priority, mainQueue); +} + +void swift::swift_task_enqueueOnDispatchQueue(Job *job, + HeapObject *_queue) { + JobPriority priority = job->getPriority(); + auto queue = reinterpret_cast(_queue); + dispatchEnqueue(queue, job, (dispatch_qos_class_t)priority, queue); +} diff --git a/stdlib/public/BackDeployConcurrency/Error.cpp b/stdlib/public/BackDeployConcurrency/Error.cpp new file mode 100644 index 0000000000000..2fbe4b0865c0b --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Error.cpp @@ -0,0 +1,19 @@ +//===--- Error.cpp - Error handling support code --------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "Error.h" + +// swift::fatalError is not exported from libswiftCore and not shared, so define another +// internal function instead. +SWIFT_NORETURN void swift::swift_Concurrency_fatalError(uint32_t flags, const char *format, ...) { + abort(); +} diff --git a/stdlib/public/BackDeployConcurrency/Error.h b/stdlib/public/BackDeployConcurrency/Error.h new file mode 100644 index 0000000000000..1e0fd3ffd339c --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Error.h @@ -0,0 +1,30 @@ +//===--- Error.h - Swift Concurrency error helpers --------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Error handling support. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_CONCURRENCY_ERRORS_H +#define SWIFT_CONCURRENCY_ERRORS_H + +#include "../SwiftShims/Visibility.h" +#include +#include + +namespace swift { + +SWIFT_NORETURN void swift_Concurrency_fatalError(uint32_t flags, const char *format, ...); + +} // namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/Errors.swift b/stdlib/public/BackDeployConcurrency/Errors.swift new file mode 100644 index 0000000000000..50ab35b09b936 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Errors.swift @@ -0,0 +1,20 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_deletedAsyncMethodError") +public func swift_deletedAsyncMethodError() async { + fatalError("Fatal error: Call of deleted method") +} diff --git a/stdlib/public/BackDeployConcurrency/Executor.h b/stdlib/public/BackDeployConcurrency/Executor.h new file mode 100644 index 0000000000000..238b0e64acfb6 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Executor.h @@ -0,0 +1,220 @@ +//===--- Executor.h - ABI structures for executors --------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing executors. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_EXECUTOR_H +#define SWIFT_ABI_EXECUTOR_H + +#include +#include "swift/ABI/HeapObject.h" +#include "swift/Runtime/Casting.h" +#include "Actor.h" + +namespace swift { +class AsyncContext; +class AsyncTask; +class DefaultActor; +class Job; +class SerialExecutorWitnessTable; + +/// An unmanaged reference to an executor. +/// +/// This type corresponds to the type Optional in +/// Swift. The representation of nil in Optional +/// aligns with what this type calls the generic executor, so the +/// notional subtype of this type which is never generic corresponds +/// to the type Builtin.Executor. +/// +/// An executor reference is divided into two pieces: +/// +/// - The identity, which is just a (potentially ObjC) object +/// reference; when this is null, the reference is generic. +/// Equality of executor references is based solely on equality +/// of identity. +/// +/// - The implementation, which is an optional reference to a +/// witness table for the SerialExecutor protocol. When this +/// is null, but the identity is non-null, the reference is to +/// a default actor. The low bits of the implementation pointer +/// are reserved for the use of marking interesting properties +/// about the executor's implementation. The runtime masks these +/// bits off before accessing the witness table, so setting them +/// in the future should back-deploy as long as the witness table +/// reference is still present. +class ExecutorRef { + HeapObject *Identity; // Not necessarily Swift reference-countable + uintptr_t Implementation; + + // We future-proof the ABI here by masking the low bits off the + // implementation pointer before using it as a witness table. + enum: uintptr_t { + WitnessTableMask = ~uintptr_t(alignof(void*) - 1) + }; + + constexpr ExecutorRef(HeapObject *identity, uintptr_t implementation) + : Identity(identity), Implementation(implementation) {} + +public: + /// A generic execution environment. When running in a generic + /// environment, it's presumed to be okay to switch synchronously + /// to an actor. As an executor request, this represents a request + /// to drop whatever the current actor is. + constexpr static ExecutorRef generic() { + return ExecutorRef(nullptr, 0); + } + + /// Given a pointer to a default actor, return an executor reference + /// for it. + static ExecutorRef forDefaultActor(DefaultActor *actor) { + assert(actor); + return ExecutorRef(actor, 0); + } + + /// Given a pointer to a serial executor and its SerialExecutor + /// conformance, return an executor reference for it. + static ExecutorRef forOrdinary(HeapObject *identity, + const SerialExecutorWitnessTable *witnessTable) { + assert(identity); + assert(witnessTable); + return ExecutorRef(identity, reinterpret_cast(witnessTable)); + } + + HeapObject *getIdentity() const { + return Identity; + } + + /// Is this the generic executor reference? + bool isGeneric() const { + return Identity == 0; + } + + /// Is this a default-actor executor reference? + bool isDefaultActor() const { + return !isGeneric() && Implementation == 0; + } + DefaultActor *getDefaultActor() const { + assert(isDefaultActor()); + return reinterpret_cast(Identity); + } + + const SerialExecutorWitnessTable *getSerialExecutorWitnessTable() const { + assert(!isGeneric() && !isDefaultActor()); + auto table = Implementation & WitnessTableMask; + return reinterpret_cast(table); + } + + /// Do we have to do any work to start running as the requested + /// executor? + bool mustSwitchToRun(ExecutorRef newExecutor) const { + return Identity != newExecutor.Identity; + } + + /// Is this executor the main executor? + bool isMainExecutor() const; + + bool operator==(ExecutorRef other) const { + return Identity == other.Identity; + } + bool operator!=(ExecutorRef other) const { + return !(*this == other); + } +}; + +using JobInvokeFunction = + SWIFT_CC(swiftasync) + void (Job *); + +using TaskContinuationFunction = + SWIFT_CC(swiftasync) + void (SWIFT_ASYNC_CONTEXT AsyncContext *); + +using ThrowingTaskFutureWaitContinuationFunction = + SWIFT_CC(swiftasync) + void (SWIFT_ASYNC_CONTEXT AsyncContext *, SWIFT_CONTEXT void *); + + +template +class AsyncFunctionPointer; +template +struct AsyncFunctionTypeImpl; + +/// The abstract signature for an asynchronous function. +template +struct AsyncSignature; + +template +struct AsyncSignature { + bool hasDirectResult = !std::is_same::value; + using DirectResultType = DirectResultTy; + + bool hasErrorResult = HasErrorResult; + + using FunctionPointer = AsyncFunctionPointer; + using FunctionType = typename AsyncFunctionTypeImpl::type; +}; + +/// A signature for a thin async function that takes no arguments +/// and returns no results. +using ThinNullaryAsyncSignature = + AsyncSignature; + +/// A signature for a thick async function that takes no formal +/// arguments and returns no results. +using ThickNullaryAsyncSignature = + AsyncSignature; + +/// A class which can be used to statically query whether a type +/// is a specialization of AsyncSignature. +template +struct IsAsyncSignature { + static const bool value = false; +}; +template +struct IsAsyncSignature> { + static const bool value = true; +}; + +template +struct AsyncFunctionTypeImpl { + static_assert(IsAsyncSignature::value, + "template argument is not an AsyncSignature"); + + // TODO: expand and include the arguments in the parameters. + using type = TaskContinuationFunction; +}; + +template +using AsyncFunctionType = typename AsyncFunctionTypeImpl::type; + +/// A "function pointer" for an async function. +/// +/// Eventually, this will always be signed with the data key +/// using a type-specific discriminator. +template +class AsyncFunctionPointer { +public: + /// The function to run. + RelativeDirectPointer, + /*nullable*/ false, + int32_t> Function; + + /// The expected size of the context. + uint32_t ExpectedContextSize; +}; + +} + +#endif diff --git a/stdlib/public/BackDeployConcurrency/Executor.swift b/stdlib/public/BackDeployConcurrency/Executor.swift new file mode 100644 index 0000000000000..35862d5bf223f --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Executor.swift @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// A service that can execute jobs. +@available(SwiftStdlib 5.1, *) +public protocol Executor: AnyObject, Sendable { + func enqueue(_ job: UnownedJob) +} + +/// A service that executes jobs. +@available(SwiftStdlib 5.1, *) +public protocol SerialExecutor: Executor { + // This requirement is repeated here as a non-override so that we + // get a redundant witness-table entry for it. This allows us to + // avoid drilling down to the base conformance just for the basic + // work-scheduling operation. + @_nonoverride + func enqueue(_ job: UnownedJob) + + /// Convert this executor value to the optimized form of borrowed + /// executor references. + func asUnownedSerialExecutor() -> UnownedSerialExecutor +} + +/// An unowned reference to a serial executor (a `SerialExecutor` +/// value). +/// +/// This is an optimized type used internally by the core scheduling +/// operations. It is an unowned reference to avoid unnecessary +/// reference-counting work even when working with actors abstractly. +/// Generally there are extra constraints imposed on core operations +/// in order to allow this. For example, keeping an actor alive must +/// also keep the actor's associated executor alive; if they are +/// different objects, the executor must be referenced strongly by the +/// actor. +@available(SwiftStdlib 5.1, *) +@frozen +public struct UnownedSerialExecutor: Sendable { + #if compiler(>=5.5) && $BuiltinExecutor + @usableFromInline + internal var executor: Builtin.Executor + #endif + + @inlinable + internal init(_ executor: Builtin.Executor) { + #if compiler(>=5.5) && $BuiltinExecutor + self.executor = executor + #endif + } + + @inlinable + public init(ordinary executor: __shared E) { + #if compiler(>=5.5) && $BuiltinBuildExecutor + self.executor = Builtin.buildOrdinarySerialExecutorRef(executor) + #else + fatalError("Swift compiler is incompatible with this SDK version") + #endif + } +} + +// Used by the concurrency runtime +@available(SwiftStdlib 5.1, *) +@_silgen_name("_swift_task_enqueueOnExecutor") +internal func _enqueueOnExecutor(job: UnownedJob, executor: E) +where E: SerialExecutor { + executor.enqueue(job) +} + +@available(SwiftStdlib 5.1, *) +@_transparent +public // COMPILER_INTRINSIC +func _checkExpectedExecutor(_filenameStart: Builtin.RawPointer, + _filenameLength: Builtin.Word, + _filenameIsASCII: Builtin.Int1, + _line: Builtin.Word, + _executor: Builtin.Executor) { + if _taskIsCurrentExecutor(_executor) { + return + } + + _reportUnexpectedExecutor( + _filenameStart, _filenameLength, _filenameIsASCII, _line, _executor) +} + +#if !SWIFT_STDLIB_SINGLE_THREADED_RUNTIME +// This must take a DispatchQueueShim, not something like AnyObject, +// or else SILGen will emit a retain/release in unoptimized builds, +// which won't work because DispatchQueues aren't actually +// Swift-retainable. +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_enqueueOnDispatchQueue") +internal func _enqueueOnDispatchQueue(_ job: UnownedJob, + queue: DispatchQueueShim) + +/// Used by the runtime solely for the witness table it produces. +/// FIXME: figure out some way to achieve that which doesn't generate +/// all the other metadata +/// +/// Expected to work for any primitive dispatch queue; note that this +/// means a dispatch_queue_t, which is not the same as DispatchQueue +/// on platforms where that is an instance of a wrapper class. +@available(SwiftStdlib 5.1, *) +internal final class DispatchQueueShim: @unchecked Sendable, SerialExecutor { + func enqueue(_ job: UnownedJob) { + _enqueueOnDispatchQueue(job, queue: self) + } + + func asUnownedSerialExecutor() -> UnownedSerialExecutor { + return UnownedSerialExecutor(ordinary: self) + } +} +#endif diff --git a/stdlib/public/BackDeployConcurrency/GlobalActor.swift b/stdlib/public/BackDeployConcurrency/GlobalActor.swift new file mode 100644 index 0000000000000..72f0657a6de28 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/GlobalActor.swift @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// A type that represents a globally-unique actor that can be used to isolate +/// various declarations anywhere in the program. +/// +/// A type that conforms to the `GlobalActor` protocol and is marked with +/// the `@globalActor` attribute can be used as a custom attribute. Such types +/// are called global actor types, and can be applied to any declaration to +/// specify that such types are isolated to that global actor type. When using +/// such a declaration from another actor (or from nonisolated code), +/// synchronization is performed through the shared actor instance to ensure +/// mutually-exclusive access to the declaration. +@available(SwiftStdlib 5.1, *) +public protocol GlobalActor { + /// The type of the shared actor instance that will be used to provide + /// mutually-exclusive access to declarations annotated with the given global + /// actor type. + associatedtype ActorType: Actor + + /// The shared actor instance that will be used to provide mutually-exclusive + /// access to declarations annotated with the given global actor type. + /// + /// The value of this property must always evaluate to the same actor + /// instance. + static var shared: ActorType { get } + + /// The shared executor instance that will be used to provide + /// mutually-exclusive access for the global actor. + /// + /// The value of this property must be equivalent to `shared.unownedExecutor`. + static var sharedUnownedExecutor: UnownedSerialExecutor { get } +} + +@available(SwiftStdlib 5.1, *) +extension GlobalActor { + public static var sharedUnownedExecutor: UnownedSerialExecutor { + shared.unownedExecutor + } +} + diff --git a/stdlib/public/BackDeployConcurrency/GlobalExecutor.cpp b/stdlib/public/BackDeployConcurrency/GlobalExecutor.cpp new file mode 100644 index 0000000000000..b380e01191637 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/GlobalExecutor.cpp @@ -0,0 +1,131 @@ +///===--- GlobalExecutor.cpp - Global concurrent executor ------------------===/// +/// +/// This source file is part of the Swift.org open source project +/// +/// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +/// Licensed under Apache License v2.0 with Runtime Library Exception +/// +/// See https:///swift.org/LICENSE.txt for license information +/// See https:///swift.org/CONTRIBUTORS.txt for the list of Swift project authors +/// +///===----------------------------------------------------------------------===/// +/// +/// Routines related to the global concurrent execution service. +/// +/// The execution side of Swift's concurrency model centers around +/// scheduling work onto various execution services ("executors"). +/// Executors vary in several different dimensions: +/// +/// First, executors may be exclusive or concurrent. An exclusive +/// executor can only execute one job at once; a concurrent executor +/// can execute many. Exclusive executors are usually used to achieve +/// some higher-level requirement, like exclusive access to some +/// resource or memory. Concurrent executors are usually used to +/// manage a pool of threads and prevent the number of allocated +/// threads from growing without limit. +/// +/// Second, executors may own dedicated threads, or they may schedule +/// work onto some some underlying executor. Dedicated threads can +/// improve the responsiveness of a subsystem *locally*, but they impose +/// substantial costs which can drive down performance *globally* +/// if not used carefully. When an executor relies on running work +/// on its own dedicated threads, jobs that need to run briefly on +/// that executor may need to suspend and restart. Dedicating threads +/// to an executor is a decision that should be made carefully +/// and holistically. +/// +/// If most executors should not have dedicated threads, they must +/// be backed by some underlying executor, typically a concurrent +/// executor. The purpose of most concurrent executors is to +/// manage threads and prevent excessive growth in the number +/// of threads. Having multiple independent concurrent executors +/// with their own dedicated threads would undermine that. +/// Therefore, it is sensible to have a single, global executor +/// that will ultimately schedule most of the work in the system. +/// With that as a baseline, special needs can be recognized and +/// carved out from the global executor with its cooperation. +/// +/// This file defines Swift's interface to that global executor. +/// +/// The default implementation is backed by libdispatch, but there +/// may be good reasons to provide alternatives (e.g. when building +/// a single-threaded runtime). +/// +///===----------------------------------------------------------------------===/// + +#include "CompatibilityOverride.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/EnvironmentVariables.h" +#include "TaskPrivate.h" +#include "Error.h" + +using namespace swift; + +SWIFT_CC(swift) +void (*swift::swift_task_enqueueGlobal_hook)( + Job *job, swift_task_enqueueGlobal_original original) = nullptr; + +SWIFT_CC(swift) +void (*swift::swift_task_enqueueGlobalWithDelay_hook)( + JobDelay delay, Job *job, + swift_task_enqueueGlobalWithDelay_original original) = nullptr; + +SWIFT_CC(swift) +void (*swift::swift_task_enqueueMainExecutor_hook)( + Job *job, swift_task_enqueueMainExecutor_original original) = nullptr; + +#if SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR +#include "CooperativeGlobalExecutor.inc" +#elif SWIFT_CONCURRENCY_ENABLE_DISPATCH +#include "DispatchGlobalExecutor.inc" +#else +#include "NonDispatchGlobalExecutor.inc" +#endif + +void swift::swift_task_enqueueGlobal(Job *job) { + _swift_tsan_release(job); + + if (swift_task_enqueueGlobal_hook) + swift_task_enqueueGlobal_hook(job, swift_task_enqueueGlobalImpl); + else + swift_task_enqueueGlobalImpl(job); +} + +void swift::swift_task_enqueueGlobalWithDelay(JobDelay delay, Job *job) { + if (swift_task_enqueueGlobalWithDelay_hook) + swift_task_enqueueGlobalWithDelay_hook( + delay, job, swift_task_enqueueGlobalWithDelayImpl); + else + swift_task_enqueueGlobalWithDelayImpl(delay, job); +} + +void swift::swift_task_enqueueMainExecutor(Job *job) { + if (swift_task_enqueueMainExecutor_hook) + swift_task_enqueueMainExecutor_hook(job, + swift_task_enqueueMainExecutorImpl); + else + swift_task_enqueueMainExecutorImpl(job); +} + +ExecutorRef swift::swift_task_getMainExecutor() { +#if !SWIFT_CONCURRENCY_ENABLE_DISPATCH + // FIXME: this isn't right for the non-cooperative environment + return ExecutorRef::generic(); +#else + return ExecutorRef::forOrdinary( + reinterpret_cast(&_dispatch_main_q), + _swift_task_getDispatchQueueSerialExecutorWitnessTable()); +#endif +} + +bool ExecutorRef::isMainExecutor() const { +#if !SWIFT_CONCURRENCY_ENABLE_DISPATCH + // FIXME: this isn't right for the non-cooperative environment + return isGeneric(); +#else + return Identity == reinterpret_cast(&_dispatch_main_q); +#endif +} + +#define OVERRIDE_GLOBAL_EXECUTOR COMPATIBILITY_OVERRIDE +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH diff --git a/stdlib/public/BackDeployConcurrency/MainActor.swift b/stdlib/public/BackDeployConcurrency/MainActor.swift new file mode 100644 index 0000000000000..215b6f08629b5 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/MainActor.swift @@ -0,0 +1,68 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift + +/// A singleton actor whose executor is equivalent to the main +/// dispatch queue. +@available(SwiftStdlib 5.1, *) +@globalActor public final actor MainActor: GlobalActor { + public static let shared = MainActor() + + @inlinable + public nonisolated var unownedExecutor: UnownedSerialExecutor { + #if compiler(>=5.5) && $BuiltinBuildMainExecutor + return UnownedSerialExecutor(Builtin.buildMainActorExecutorRef()) + #else + fatalError("Swift compiler is incompatible with this SDK version") + #endif + } + + @inlinable + public static var sharedUnownedExecutor: UnownedSerialExecutor { + #if compiler(>=5.5) && $BuiltinBuildMainExecutor + return UnownedSerialExecutor(Builtin.buildMainActorExecutorRef()) + #else + fatalError("Swift compiler is incompatible with this SDK version") + #endif + } + + @inlinable + public nonisolated func enqueue(_ job: UnownedJob) { + _enqueueOnMain(job) + } +} + +@available(SwiftStdlib 5.1, *) +extension MainActor { + /// Execute the given body closure on the main actor. + /// + /// Historical ABI entry point, superceded by the Sendable version that is + /// also inlined to back-deploy a semantic fix where this operation would + /// not hop back at the end. + @usableFromInline + static func run( + resultType: T.Type = T.self, + body: @MainActor @Sendable () throws -> T + ) async rethrows -> T { + return try await body() + } + + /// Execute the given body closure on the main actor. + @_alwaysEmitIntoClient + public static func run( + resultType: T.Type = T.self, + body: @MainActor @Sendable () throws -> T + ) async rethrows -> T { + return try await body() + } +} diff --git a/stdlib/public/BackDeployConcurrency/Metadata.cpp b/stdlib/public/BackDeployConcurrency/Metadata.cpp index 409dfb783b1ae..ee7f670dabc69 100644 --- a/stdlib/public/BackDeployConcurrency/Metadata.cpp +++ b/stdlib/public/BackDeployConcurrency/Metadata.cpp @@ -16,8 +16,10 @@ //===----------------------------------------------------------------------===// #include +#include "ConcurrencyRuntime.h" #include "swift/Runtime/Concurrent.h" #include "swift/Runtime/Metadata.h" +#include "llvm/Support/AllocatorBase.h" using namespace swift; diff --git a/stdlib/public/BackDeployConcurrency/Mutex.cpp b/stdlib/public/BackDeployConcurrency/Mutex.cpp new file mode 100644 index 0000000000000..f42d21ee0a385 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Mutex.cpp @@ -0,0 +1,78 @@ +//===--- Mutex.cpp - Mutex support code -----------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "Error.h" + +#define SWIFT_FATAL_ERROR swift_Concurrency_fatalError + +// Include the runtime's mutex support code. +// FIXME: figure out some reasonable way to share this stuff + +#include "ConditionVariable.h" +#include "../runtime/MutexPThread.cpp" +#include "../runtime/MutexWin32.cpp" +#ifdef SWIFT_STDLIB_SINGLE_THREADED_RUNTIME + #include "swift/Runtime/MutexSingleThreaded.h" +#endif + +using namespace swift; + +void ConditionPlatformHelper::init(pthread_cond_t &condition) { + reportError(pthread_cond_init(&condition, nullptr)); +} + +void ConditionPlatformHelper::destroy(pthread_cond_t &condition) { + reportError(pthread_cond_destroy(&condition)); +} + +void ConditionPlatformHelper::notifyOne(pthread_cond_t &condition) { + reportError(pthread_cond_signal(&condition)); +} + +void ConditionPlatformHelper::notifyAll(pthread_cond_t &condition) { + reportError(pthread_cond_broadcast(&condition)); +} + +void ConditionPlatformHelper::wait(pthread_cond_t &condition, + pthread_mutex_t &mutex) { + reportError(pthread_cond_wait(&condition, &mutex)); +} + +ConditionVariable::Mutex::Mutex(bool checked) { + pthread_mutexattr_t attr; + int kind = (checked ? PTHREAD_MUTEX_ERRORCHECK : PTHREAD_MUTEX_NORMAL); + reportError(pthread_mutexattr_init(&attr)); + reportError(pthread_mutexattr_settype(&attr, kind)); + reportError(pthread_mutex_init(&Handle, &attr)); + reportError(pthread_mutexattr_destroy(&attr)); +} + +ConditionVariable::Mutex::~Mutex() { + reportError(pthread_mutex_destroy(&Handle)); +} + +void ConditionVariable::Mutex::lock() { + reportError(pthread_mutex_lock(&Handle)); +} + +void ConditionVariable::Mutex::unlock() { + reportError(pthread_mutex_unlock(&Handle)); +} + +bool ConditionVariable::Mutex::try_lock() { + returnTrueOrReportError(pthread_mutex_trylock(&Handle), + /* returnFalseOnEBUSY = */ true); +} + +void ConditionVariable::Mutex::wait(ConditionVariable &condition) { + reportError(pthread_cond_wait(&condition.Handle, &Handle)); +} diff --git a/stdlib/public/BackDeployConcurrency/NonDispatchGlobalExecutor.inc b/stdlib/public/BackDeployConcurrency/NonDispatchGlobalExecutor.inc new file mode 100644 index 0000000000000..32061e786898f --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/NonDispatchGlobalExecutor.inc @@ -0,0 +1,51 @@ +///===--- NonDispatchGlobalExecutor.inc ---------------------*- C++ -*--===/// +/// +/// This source file is part of the Swift.org open source project +/// +/// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +/// Licensed under Apache License v2.0 with Runtime Library Exception +/// +/// See https:///swift.org/LICENSE.txt for license information +/// See https:///swift.org/CONTRIBUTORS.txt for the list of Swift project authors +/// +///===------------------------------------------------------------------===/// +/// +/// The implementation of the global executor when not using Dispatch but +/// also not using the cooperative global executor. The general assumption +/// is that clients will be installing the appropriate hooks when all of +/// the functions here are called. +/// +/// This file is included into GlobalExecutor.cpp only when both +/// Dispatch integration and the cooperative global executor are disabled. +/// It is expected to define the following functions: +/// swift_task_enqueueGlobalImpl +/// swift_task_enqueueGlobalWithDelayImpl +/// swift_task_enqueueMainExecutorImpl +/// +///===------------------------------------------------------------------===/// + +SWIFT_CC(swift) +static void swift_task_enqueueGlobalImpl(Job *job) { + assert(job && "no job provided"); + + swift_reportError(0, "operation unsupported without libdispatch: " + "swift_task_enqueueGlobal"); +} + +SWIFT_CC(swift) +static void swift_task_enqueueGlobalWithDelayImpl(JobDelay delay, + Job *job) { + assert(job && "no job provided"); + + swift_reportError(0, "operation unsupported without libdispatch: " + "swift_task_enqueueGlobalWithDelay"); +} + +/// Enqueues a task on the main executor. +SWIFT_CC(swift) +static void swift_task_enqueueMainExecutorImpl(Job *job) { + assert(job && "no job provided"); + + swift_reportError(0, "operation unsupported without libdispatch: " + "swift_task_enqueueMainExecutor"); +} diff --git a/stdlib/public/BackDeployConcurrency/PartialAsyncTask.swift b/stdlib/public/BackDeployConcurrency/PartialAsyncTask.swift new file mode 100644 index 0000000000000..4c2618a104570 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/PartialAsyncTask.swift @@ -0,0 +1,294 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_job_run") +@usableFromInline +internal func _swiftJobRun(_ job: UnownedJob, + _ executor: UnownedSerialExecutor) -> () + +/// A unit of scheduleable work. +/// +/// Unless you're implementing a scheduler, +/// you don't generally interact with jobs directly. +@available(SwiftStdlib 5.1, *) +@frozen +public struct UnownedJob: Sendable { + private var context: Builtin.Job + + @_alwaysEmitIntoClient + @inlinable + public func _runSynchronously(on executor: UnownedSerialExecutor) { + _swiftJobRun(self, executor) + } +} + +/// A mechanism to interface +/// between synchronous and asynchronous code, +/// without correctness checking. +/// +/// A *continuation* is an opaque representation of program state. +/// To create a continuation in asynchronous code, +/// call the `withUnsafeContinuation(_:)` or +/// `withUnsafeThrowingContinuation(_:)` function. +/// To resume the asynchronous task, +/// call the `resume(returning:)`, +/// `resume(throwing:)`, +/// `resume(with:)`, +/// or `resume()` method. +/// +/// - Important: You must call a resume method exactly once +/// on every execution path throughout the program. +/// Resuming from a continuation more than once is undefined behavior. +/// Never resuming leaves the task in a suspended state indefinitely, +/// and leaks any associated resources. +/// +/// `CheckedContinuation` performs runtime checks +/// for missing or multiple resume operations. +/// `UnsafeContinuation` avoids enforcing these invariants at runtime +/// because it aims to be a low-overhead mechanism +/// for interfacing Swift tasks with +/// event loops, delegate methods, callbacks, +/// and other non-`async` scheduling mechanisms. +/// However, during development, the ability to verify that the +/// invariants are being upheld in testing is important. +/// Because both types have the same interface, +/// you can replace one with the other in most circumstances, +/// without making other changes. +@available(SwiftStdlib 5.1, *) +@frozen +public struct UnsafeContinuation { + @usableFromInline internal var context: Builtin.RawUnsafeContinuation + + @_alwaysEmitIntoClient + internal init(_ context: Builtin.RawUnsafeContinuation) { + self.context = context + } + + /// Resume the task that's awaiting the continuation + /// by returning the given value. + /// + /// - Parameter value: The value to return from the continuation. + /// + /// A continuation must be resumed exactly once. + /// If the continuation has already resumed, + /// then calling this method results in undefined behavior. + /// + /// After calling this method, + /// control immediately returns to the caller. + /// The task continues executing + /// when its executor schedules it. + @_alwaysEmitIntoClient + public func resume(returning value: __owned T) where E == Never { + #if compiler(>=5.5) && $BuiltinContinuation + Builtin.resumeNonThrowingContinuationReturning(context, value) + #else + fatalError("Swift compiler is incompatible with this SDK version") + #endif + } + + /// Resume the task that's awaiting the continuation + /// by returning the given value. + /// + /// - Parameter value: The value to return from the continuation. + /// + /// A continuation must be resumed exactly once. + /// If the continuation has already resumed, + /// then calling this method results in undefined behavior. + /// + /// After calling this method, + /// control immediately returns to the caller. + /// The task continues executing + /// when its executor schedules it. + @_alwaysEmitIntoClient + public func resume(returning value: __owned T) { + #if compiler(>=5.5) && $BuiltinContinuation + Builtin.resumeThrowingContinuationReturning(context, value) + #else + fatalError("Swift compiler is incompatible with this SDK version") + #endif + } + + /// Resume the task that's awaiting the continuation + /// by throwing the given error. + /// + /// - Parameter error: The error to throw from the continuation. + /// + /// A continuation must be resumed exactly once. + /// If the continuation has already resumed, + /// then calling this method results in undefined behavior. + /// + /// After calling this method, + /// control immediately returns to the caller. + /// The task continues executing + /// when its executor schedules it. + @_alwaysEmitIntoClient + public func resume(throwing error: __owned E) { +#if compiler(>=5.5) && $BuiltinContinuation + Builtin.resumeThrowingContinuationThrowing(context, error) +#else + fatalError("Swift compiler is incompatible with this SDK version") +#endif + } +} + +@available(SwiftStdlib 5.1, *) +extension UnsafeContinuation: Sendable where T: Sendable { } + +@available(SwiftStdlib 5.1, *) +extension UnsafeContinuation { + /// Resume the task that's awaiting the continuation + /// by returning or throwing the given result value. + /// + /// - Parameter result: The result. + /// If it contains a `.success` value, + /// the continuation returns that value; + /// otherwise, it throws the `.error` value. + /// + /// A continuation must be resumed exactly once. + /// If the continuation has already resumed, + /// then calling this method results in undefined behavior. + /// + /// After calling this method, + /// control immediately returns to the caller. + /// The task continues executing + /// when its executor schedules it. + @_alwaysEmitIntoClient + public func resume(with result: Result) where E == Error { + switch result { + case .success(let val): + self.resume(returning: val) + case .failure(let err): + self.resume(throwing: err) + } + } + + /// Resume the task that's awaiting the continuation + /// by returning or throwing the given result value. + /// + /// - Parameter result: The result. + /// If it contains a `.success` value, + /// the continuation returns that value; + /// otherwise, it throws the `.error` value. + /// + /// A continuation must be resumed exactly once. + /// If the continuation has already resumed, + /// then calling this method results in undefined behavior. + /// + /// After calling this method, + /// control immediately returns to the caller. + /// The task continues executing + /// when its executor schedules it. + @_alwaysEmitIntoClient + public func resume(with result: Result) { + switch result { + case .success(let val): + self.resume(returning: val) + case .failure(let err): + self.resume(throwing: err) + } + } + + /// Resume the task that's awaiting the continuation by returning. + /// + /// A continuation must be resumed exactly once. + /// If the continuation has already resumed, + /// then calling this method results in undefined behavior. + /// + /// After calling this method, + /// control immediately returns to the caller. + /// The task continues executing + /// when its executor schedules it. + @_alwaysEmitIntoClient + public func resume() where T == Void { + self.resume(returning: ()) + } +} + +#if _runtime(_ObjC) + +// Intrinsics used by SILGen to resume or fail continuations. +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +internal func _resumeUnsafeContinuation( + _ continuation: UnsafeContinuation, + _ value: __owned T +) { + continuation.resume(returning: value) +} + +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +internal func _resumeUnsafeThrowingContinuation( + _ continuation: UnsafeContinuation, + _ value: __owned T +) { + continuation.resume(returning: value) +} + +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +internal func _resumeUnsafeThrowingContinuationWithError( + _ continuation: UnsafeContinuation, + _ error: __owned Error +) { + continuation.resume(throwing: error) +} + +#endif + +/// Suspends the current task, +/// then calls the given closure with an unsafe continuation for the current task. +/// +/// - Parameter fn: A closure that takes an `UnsafeContinuation` parameter. +/// You must resume the continuation exactly once. +/// +/// - Returns: The value passed to the continuation by the closure. +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +public func withUnsafeContinuation( + _ fn: (UnsafeContinuation) -> Void +) async -> T { + return await Builtin.withUnsafeContinuation { + fn(UnsafeContinuation($0)) + } +} + +/// Suspends the current task, +/// then calls the given closure with an unsafe throwing continuation for the current task. +/// +/// - Parameter fn: A closure that takes an `UnsafeContinuation` parameter. +/// You must resume the continuation exactly once. +/// +/// - Returns: The value passed to the continuation by the closure. +/// +/// If `resume(throwing:)` is called on the continuation, +/// this function throws that error. +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +public func withUnsafeThrowingContinuation( + _ fn: (UnsafeContinuation) -> Void +) async throws -> T { + return try await Builtin.withUnsafeThrowingContinuation { + fn(UnsafeContinuation($0)) + } +} + +/// A hack to mark an SDK that supports swift_continuation_await. +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +public func _abiEnableAwaitContinuation() { + fatalError("never use this function") +} diff --git a/stdlib/public/BackDeployConcurrency/SourceCompatibilityShims.swift b/stdlib/public/BackDeployConcurrency/SourceCompatibilityShims.swift new file mode 100644 index 0000000000000..15b71fee3f173 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/SourceCompatibilityShims.swift @@ -0,0 +1,303 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// This file provides source compatibility shims to help migrate code +// using earlier versions of the concurrency library to the latest syntax. +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + @available(*, deprecated, message: "Task.Priority has been removed; use TaskPriority") + public typealias Priority = TaskPriority + + @available(*, deprecated, message: "Task.Handle has been removed; use Task") + public typealias Handle = _Concurrency.Task + + @available(*, deprecated, message: "Task.CancellationError has been removed; use CancellationError") + @_alwaysEmitIntoClient + public static func CancellationError() -> _Concurrency.CancellationError { + return _Concurrency.CancellationError() + } + + @available(*, deprecated, renamed: "yield()") + @_alwaysEmitIntoClient + public static func suspend() async { + await yield() + } +} + +@available(SwiftStdlib 5.1, *) +extension TaskPriority { + @available(*, deprecated, message: "unspecified priority will be removed; use nil") + @_alwaysEmitIntoClient + public static var unspecified: TaskPriority { + .init(rawValue: 0x00) + } + + @available(*, deprecated, message: "userInteractive priority will be removed") + @_alwaysEmitIntoClient + public static var userInteractive: TaskPriority { + .init(rawValue: 0x21) + } +} + +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +public func withTaskCancellationHandler( + handler: @Sendable () -> Void, + operation: () async throws -> T +) async rethrows -> T { + try await withTaskCancellationHandler(operation: operation, onCancel: handler) +} + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + @available(*, deprecated, message: "`Task.withCancellationHandler` has been replaced by `withTaskCancellationHandler` and will be removed shortly.") + @_alwaysEmitIntoClient + public static func withCancellationHandler( + handler: @Sendable () -> Void, + operation: () async throws -> T + ) async rethrows -> T { + try await withTaskCancellationHandler(handler: handler, operation: operation) + } +} + +@available(SwiftStdlib 5.1, *) +extension Task where Failure == Error { + @discardableResult + @_alwaysEmitIntoClient + @available(*, deprecated, message: "`Task.runDetached` was replaced by `Task.detached` and will be removed shortly.") + public static func runDetached( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> Success + ) -> Task { + detached(priority: priority, operation: operation) + } +} + +@discardableResult +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, message: "`detach` was replaced by `Task.detached` and will be removed shortly.") +@_alwaysEmitIntoClient +public func detach( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> T +) -> Task { + Task.detached(priority: priority, operation: operation) +} + +@discardableResult +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, message: "`detach` was replaced by `Task.detached` and will be removed shortly.") +@_alwaysEmitIntoClient +public func detach( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> T +) -> Task { + Task.detached(priority: priority, operation: operation) +} + +@discardableResult +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, message: "`asyncDetached` was replaced by `Task.detached` and will be removed shortly.") +@_alwaysEmitIntoClient +public func asyncDetached( + priority: TaskPriority? = nil, + @_implicitSelfCapture operation: __owned @Sendable @escaping () async -> T +) -> Task { + return Task.detached(priority: priority, operation: operation) +} + +@discardableResult +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, message: "`asyncDetached` was replaced by `Task.detached` and will be removed shortly.") +@_alwaysEmitIntoClient +public func asyncDetached( + priority: TaskPriority? = nil, + @_implicitSelfCapture operation: __owned @Sendable @escaping () async throws -> T +) -> Task { + return Task.detached(priority: priority, operation: operation) +} + +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, message: "`async` was replaced by `Task.init` and will be removed shortly.") +@discardableResult +@_alwaysEmitIntoClient +public func async( + priority: TaskPriority? = nil, + @_inheritActorContext @_implicitSelfCapture operation: __owned @Sendable @escaping () async -> T +) -> Task { + .init(priority: priority, operation: operation) +} + +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, message: "`async` was replaced by `Task.init` and will be removed shortly.") +@discardableResult +@_alwaysEmitIntoClient +public func async( + priority: TaskPriority? = nil, + @_inheritActorContext @_implicitSelfCapture operation: __owned @Sendable @escaping () async throws -> T +) -> Task { + .init(priority: priority, operation: operation) +} + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + @available(*, deprecated, message: "`Task.Group` was replaced by `ThrowingTaskGroup` and `TaskGroup` and will be removed shortly.") + public typealias Group = ThrowingTaskGroup + + @available(*, deprecated, message: "`Task.withGroup` was replaced by `withThrowingTaskGroup` and `withTaskGroup` and will be removed shortly.") + @_alwaysEmitIntoClient + public static func withGroup( + resultType: TaskResult.Type, + returning returnType: BodyResult.Type = BodyResult.self, + body: (inout Task.Group) async throws -> BodyResult + ) async rethrows -> BodyResult { + try await withThrowingTaskGroup(of: resultType) { group in + try await body(&group) + } + } +} + +@available(SwiftStdlib 5.1, *) +extension Task { + @available(*, deprecated, message: "get() has been replaced by .value") + @_alwaysEmitIntoClient + public func get() async throws -> Success { + return try await value + } + + @available(*, deprecated, message: "getResult() has been replaced by .result") + @_alwaysEmitIntoClient + public func getResult() async -> Result { + return await result + } +} + +@available(SwiftStdlib 5.1, *) +extension Task where Failure == Never { + @available(*, deprecated, message: "get() has been replaced by .value") + @_alwaysEmitIntoClient + public func get() async -> Success { + return await value + } +} + +@available(SwiftStdlib 5.1, *) +extension TaskGroup { + @available(*, deprecated, renamed: "addTask(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func add( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> ChildTaskResult + ) async -> Bool { + return self.addTaskUnlessCancelled(priority: priority) { + await operation() + } + } + + @available(*, deprecated, renamed: "addTask(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func spawn( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> ChildTaskResult + ) { + addTask(priority: priority, operation: operation) + } + + @available(*, deprecated, renamed: "addTaskUnlessCancelled(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func spawnUnlessCancelled( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> ChildTaskResult + ) -> Bool { + addTaskUnlessCancelled(priority: priority, operation: operation) + } + + @available(*, deprecated, renamed: "addTask(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func async( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> ChildTaskResult + ) { + addTask(priority: priority, operation: operation) + } + + @available(*, deprecated, renamed: "addTaskUnlessCancelled(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func asyncUnlessCancelled( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> ChildTaskResult + ) -> Bool { + addTaskUnlessCancelled(priority: priority, operation: operation) + } +} + +@available(SwiftStdlib 5.1, *) +extension ThrowingTaskGroup { + @available(*, deprecated, renamed: "addTask(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func add( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> ChildTaskResult + ) async -> Bool { + return self.addTaskUnlessCancelled(priority: priority) { + try await operation() + } + } + + @available(*, deprecated, renamed: "addTask(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func spawn( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> ChildTaskResult + ) { + addTask(priority: priority, operation: operation) + } + + @available(*, deprecated, renamed: "addTaskUnlessCancelled(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func spawnUnlessCancelled( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> ChildTaskResult + ) -> Bool { + addTaskUnlessCancelled(priority: priority, operation: operation) + } + + @available(*, deprecated, renamed: "addTask(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func async( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> ChildTaskResult + ) { + addTask(priority: priority, operation: operation) + } + + @available(*, deprecated, renamed: "addTaskUnlessCancelled(priority:operation:)") + @_alwaysEmitIntoClient + public mutating func asyncUnlessCancelled( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> ChildTaskResult + ) -> Bool { + addTaskUnlessCancelled(priority: priority, operation: operation) + } +} + +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, message: "please use UnsafeContinuation<..., Error>") +public typealias UnsafeThrowingContinuation = UnsafeContinuation + +@available(SwiftStdlib 5.1, *) +@available(*, deprecated, renamed: "UnownedJob") +public typealias PartialAsyncTask = UnownedJob diff --git a/stdlib/public/BackDeployConcurrency/Task.cpp b/stdlib/public/BackDeployConcurrency/Task.cpp new file mode 100644 index 0000000000000..86f0f5e5958d4 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Task.cpp @@ -0,0 +1,1146 @@ +//===--- Task.cpp - Task object and management ----------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Object management routines for asynchronous task objects. +// +//===----------------------------------------------------------------------===// + +#include "CompatibilityOverride.h" +#include "ConcurrencyRuntime.h" +#include "swift/ABI/Metadata.h" +#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/HeapObject.h" +#include "Task.h" +#include "TaskGroupPrivate.h" +#include "TaskLocal.h" +#include "TaskOptions.h" +#include "TaskPrivate.h" +#include "Debug.h" +#include "Error.h" + +#if SWIFT_CONCURRENCY_ENABLE_DISPATCH +#include +#endif + +#if !defined(_WIN32) +#include +#endif + +#if defined(SWIFT_CONCURRENCY_BACK_DEPLOYMENT) +#include +#include +#if TARGET_OS_WATCH +// Bitcode compilation for the watch device precludes defining the following asm +// symbols, so we don't use them... but simulators are okay. +#if TARGET_OS_SIMULATOR +asm("\n .globl _swift_async_extendedFramePointerFlags" \ + "\n _swift_async_extendedFramePointerFlags = 0x0"); +#endif +#else +asm("\n .globl _swift_async_extendedFramePointerFlags" \ + "\n _swift_async_extendedFramePointerFlags = 0x0"); +#endif +#else +#ifdef __APPLE__ +#if __POINTER_WIDTH__ == 64 +asm("\n .globl _swift_async_extendedFramePointerFlags" \ + "\n _swift_async_extendedFramePointerFlags = 0x1000000000000000"); +#elif __ARM64_ARCH_8_32__ +asm("\n .globl _swift_async_extendedFramePointerFlags" \ + "\n _swift_async_extendedFramePointerFlags = 0x10000000"); +#else +asm("\n .globl _swift_async_extendedFramePointerFlags" \ + "\n _swift_async_extendedFramePointerFlags = 0x0"); +#endif +#endif // __APPLE__ +#endif // !defined(SWIFT_CONCURRENCY_BACK_DEPLOYMENT) + +using namespace swift; +using FutureFragment = AsyncTask::FutureFragment; +using TaskGroup = swift::TaskGroup; + +Metadata swift::TaskAllocatorSlabMetadata; +const void *const swift::_swift_concurrency_debug_asyncTaskSlabMetadata = + &TaskAllocatorSlabMetadata; + +void FutureFragment::destroy() { + auto queueHead = waitQueue.load(std::memory_order_acquire); + switch (queueHead.getStatus()) { + case Status::Executing: + assert(false && "destroying a task that never completed"); + + case Status::Success: + resultType->vw_destroy(getStoragePtr()); + break; + + case Status::Error: + swift_errorRelease(getError()); + break; + } +} + +FutureFragment::Status AsyncTask::waitFuture(AsyncTask *waitingTask, + AsyncContext *waitingTaskContext, + TaskContinuationFunction *resumeFn, + AsyncContext *callerContext, + OpaqueValue *result) { + using Status = FutureFragment::Status; + using WaitQueueItem = FutureFragment::WaitQueueItem; + + assert(isFuture()); + auto fragment = futureFragment(); + + auto queueHead = fragment->waitQueue.load(std::memory_order_acquire); + bool contextIntialized = false; + while (true) { + switch (queueHead.getStatus()) { + case Status::Error: + case Status::Success: + SWIFT_TASK_DEBUG_LOG("task %p waiting on task %p, completed immediately", + waitingTask, this); + _swift_tsan_acquire(static_cast(this)); + if (contextIntialized) waitingTask->flagAsRunning(); + // The task is done; we don't need to wait. + return queueHead.getStatus(); + + case Status::Executing: + SWIFT_TASK_DEBUG_LOG("task %p waiting on task %p, going to sleep", + waitingTask, this); + _swift_tsan_release(static_cast(waitingTask)); + // Task is not complete. We'll need to add ourselves to the queue. + break; + } + + if (!contextIntialized) { + contextIntialized = true; + auto context = + reinterpret_cast(waitingTaskContext); + context->errorResult = nullptr; + context->successResultPointer = result; + context->ResumeParent = resumeFn; + context->Parent = callerContext; + waitingTask->flagAsSuspended(); + } + + // Put the waiting task at the beginning of the wait queue. + waitingTask->getNextWaitingTask() = queueHead.getTask(); + auto newQueueHead = WaitQueueItem::get(Status::Executing, waitingTask); + if (fragment->waitQueue.compare_exchange_weak( + queueHead, newQueueHead, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_acquire)) { + // Escalate the priority of this task based on the priority + // of the waiting task. + swift_task_escalate(this, waitingTask->Flags.getPriority()); + _swift_task_clearCurrent(); + return FutureFragment::Status::Executing; + } + } +} + +void NullaryContinuationJob::process(Job *_job) { + auto *job = cast(_job); + + auto *task = job->Task; + auto *continuation = job->Continuation; + + _swift_task_dealloc_specific(task, job); + + auto *context = cast(continuation->ResumeContext); + + context->setErrorResult(nullptr); + swift_continuation_resume(continuation); +} + +void AsyncTask::completeFuture(AsyncContext *context) { + using Status = FutureFragment::Status; + using WaitQueueItem = FutureFragment::WaitQueueItem; + SWIFT_TASK_DEBUG_LOG("complete future = %p", this); + assert(isFuture()); + auto fragment = futureFragment(); + + // If an error was thrown, save it in the future fragment. + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(context) - sizeof(FutureAsyncContextPrefix)); + bool hadErrorResult = false; + auto errorObject = asyncContextPrefix->errorResult; + fragment->getError() = errorObject; + if (errorObject) { + hadErrorResult = true; + } + + _swift_tsan_release(static_cast(this)); + + // Update the status to signal completion. + auto newQueueHead = WaitQueueItem::get( + hadErrorResult ? Status::Error : Status::Success, + nullptr + ); + auto queueHead = fragment->waitQueue.exchange( + newQueueHead, std::memory_order_acquire); + assert(queueHead.getStatus() == Status::Executing); + + // If this is task group child, notify the parent group about the completion. + if (hasGroupChildFragment()) { + // then we must offer into the parent group that we completed, + // so it may `next()` poll completed child tasks in completion order. + auto group = groupChildFragment()->getGroup(); + group->offer(this, context); + } + + // Schedule every waiting task on the executor. + auto waitingTask = queueHead.getTask(); + + if (!waitingTask) + SWIFT_TASK_DEBUG_LOG("task %p had no waiting tasks", this); + + while (waitingTask) { + // Find the next waiting task before we invalidate it by resuming + // the task. + auto nextWaitingTask = waitingTask->getNextWaitingTask(); + + SWIFT_TASK_DEBUG_LOG("waking task %p from future of task %p", waitingTask, + this); + + // Fill in the return context. + auto waitingContext = + static_cast(waitingTask->ResumeContext); + if (hadErrorResult) { + waitingContext->fillWithError(fragment); + } else { + waitingContext->fillWithSuccess(fragment); + } + + _swift_tsan_acquire(static_cast(waitingTask)); + + // Enqueue the waiter on the global executor. + // TODO: allow waiters to fill in a suggested executor + swift_task_enqueueGlobal(waitingTask); + + // Move to the next task. + waitingTask = nextWaitingTask; + } +} + +SWIFT_CC(swift) +static void destroyJob(SWIFT_CONTEXT HeapObject *obj) { + assert(false && "A non-task job should never be destroyed as heap metadata."); +} + +AsyncTask::~AsyncTask() { + flagAsCompleted(); + + // For a future, destroy the result. + if (isFuture()) { + futureFragment()->destroy(); + } + + Private.destroy(); +} + +void AsyncTask::setTaskId() { + static std::atomic NextId(1); + + // We want the 32-bit Job::Id to be non-zero, so loop if we happen upon zero. + uint64_t Fetched; + do { + Fetched = NextId.fetch_add(1, std::memory_order_relaxed); + Id = Fetched & 0xffffffff; + } while (Id == 0); + + _private().Id = (Fetched >> 32) & 0xffffffff; +} + +SWIFT_CC(swift) +static void destroyTask(SWIFT_CONTEXT HeapObject *obj) { + auto task = static_cast(obj); + task->~AsyncTask(); + + // The task execution itself should always hold a reference to it, so + // if we get here, we know the task has finished running, which means + // swift_task_complete should have been run, which will have torn down + // the task-local allocator. There's actually nothing else to clean up + // here. + + SWIFT_TASK_DEBUG_LOG("destroy task %p", task); + free(task); +} + +static ExecutorRef executorForEnqueuedJob(Job *job) { +#if !SWIFT_CONCURRENCY_ENABLE_DISPATCH + return ExecutorRef::generic(); +#else + void *jobQueue = job->SchedulerPrivate[Job::DispatchQueueIndex]; + if (jobQueue == DISPATCH_QUEUE_GLOBAL_EXECUTOR) + return ExecutorRef::generic(); + else + return ExecutorRef::forOrdinary(reinterpret_cast(jobQueue), + _swift_task_getDispatchQueueSerialExecutorWitnessTable()); +#endif +} + +static void jobInvoke(void *obj, void *unused, uint32_t flags) { + (void)unused; + Job *job = reinterpret_cast(obj); + + swift_job_run(job, executorForEnqueuedJob(job)); +} + +// Magic constant to identify Swift Job vtables to Dispatch. +static const unsigned long dispatchSwiftObjectType = 1; + +FullMetadata swift::jobHeapMetadata = { + { + { + &destroyJob + }, + { + /*value witness table*/ nullptr + } + }, + { + MetadataKind::Job, + dispatchSwiftObjectType, + jobInvoke + } +}; + +/// Heap metadata for an asynchronous task. +static FullMetadata taskHeapMetadata = { + { + { + &destroyTask + }, + { + /*value witness table*/ nullptr + } + }, + { + MetadataKind::Task, + dispatchSwiftObjectType, + jobInvoke + } +}; + +const void *const swift::_swift_concurrency_debug_jobMetadata = + static_cast(&jobHeapMetadata); +const void *const swift::_swift_concurrency_debug_asyncTaskMetadata = + static_cast(&taskHeapMetadata); + +static void completeTaskImpl(AsyncTask *task, + AsyncContext *context, + SwiftError *error) { + assert(task && "completing task, but there is no active task registered"); + + // Store the error result. + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(context) - sizeof(AsyncContextPrefix)); + asyncContextPrefix->errorResult = error; + + task->Private.complete(task); + + SWIFT_TASK_DEBUG_LOG("task %p completed", task); + + // Complete the future. + // Warning: This deallocates the task in case it's an async let task. + // The task must not be accessed afterwards. + if (task->isFuture()) { + task->completeFuture(context); + } + + // TODO: set something in the status? + // if (task->hasChildFragment()) { + // TODO: notify the parent somehow? + // TODO: remove this task from the child-task chain? + // } +} + +/// The function that we put in the context of a simple task +/// to handle the final return. +SWIFT_CC(swiftasync) +static void completeTask(SWIFT_ASYNC_CONTEXT AsyncContext *context, + SWIFT_CONTEXT SwiftError *error) { + // Set that there's no longer a running task in the current thread. + auto task = _swift_task_clearCurrent(); + assert(task && "completing task, but there is no active task registered"); + + completeTaskImpl(task, context, error); +} + +/// The function that we put in the context of a simple task +/// to handle the final return. +SWIFT_CC(swiftasync) +static void completeTaskAndRelease(SWIFT_ASYNC_CONTEXT AsyncContext *context, + SWIFT_CONTEXT SwiftError *error) { + // Set that there's no longer a running task in the current thread. + auto task = _swift_task_clearCurrent(); + assert(task && "completing task, but there is no active task registered"); + + completeTaskImpl(task, context, error); + + // Release the task, balancing the retain that a running task has on itself. + // If it was a group child task, it will remain until the group returns it. + swift_release(task); +} + +/// The function that we put in the context of a simple task +/// to handle the final return from a closure. +SWIFT_CC(swiftasync) +static void completeTaskWithClosure(SWIFT_ASYNC_CONTEXT AsyncContext *context, + SWIFT_CONTEXT SwiftError *error) { + // Release the closure context. + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(context) - sizeof(AsyncContextPrefix)); + + swift_release((HeapObject *)asyncContextPrefix->closureContext); + + // Clean up the rest of the task. + return completeTaskAndRelease(context, error); +} + +SWIFT_CC(swiftasync) +static void non_future_adapter(SWIFT_ASYNC_CONTEXT AsyncContext *_context) { + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(_context) - sizeof(AsyncContextPrefix)); + return asyncContextPrefix->asyncEntryPoint( + _context, asyncContextPrefix->closureContext); +} + +SWIFT_CC(swiftasync) +static void future_adapter(SWIFT_ASYNC_CONTEXT AsyncContext *_context) { + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(_context) - sizeof(FutureAsyncContextPrefix)); + return asyncContextPrefix->asyncEntryPoint( + asyncContextPrefix->indirectResult, _context, + asyncContextPrefix->closureContext); +} + +SWIFT_CC(swiftasync) +static void task_wait_throwing_resume_adapter(SWIFT_ASYNC_CONTEXT AsyncContext *_context) { + + auto context = static_cast(_context); + auto resumeWithError = + reinterpret_cast(context->ResumeParent); + return resumeWithError(context->Parent, context->errorResult); +} + +SWIFT_CC(swiftasync) +static void +task_future_wait_resume_adapter(SWIFT_ASYNC_CONTEXT AsyncContext *_context) { + return _context->ResumeParent(_context->Parent); +} + +/// Implementation of task creation. +SWIFT_CC(swift) +static AsyncTaskAndContext swift_task_create_commonImpl( + size_t rawTaskCreateFlags, + TaskOptionRecord *options, + const Metadata *futureResultType, + FutureAsyncSignature::FunctionType *function, void *closureContext, + size_t initialContextSize) { + TaskCreateFlags taskCreateFlags(rawTaskCreateFlags); + + // Propagate task-creation flags to job flags as appropriate. + JobFlags jobFlags(JobKind::Task, taskCreateFlags.getRequestedPriority()); + jobFlags.task_setIsChildTask(taskCreateFlags.isChildTask()); + if (futureResultType) { + jobFlags.task_setIsFuture(true); + assert(initialContextSize >= sizeof(FutureAsyncContext)); + } + + // Collect the options we know about. + ExecutorRef executor = ExecutorRef::generic(); + TaskGroup *group = nullptr; + AsyncLet *asyncLet = nullptr; + bool hasAsyncLetResultBuffer = false; + for (auto option = options; option; option = option->getParent()) { + switch (option->getKind()) { + case TaskOptionRecordKind::Executor: + executor = cast(option)->getExecutor(); + break; + + case TaskOptionRecordKind::TaskGroup: + group = cast(option)->getGroup(); + assert(group && "Missing group"); + jobFlags.task_setIsGroupChildTask(true); + break; + + case TaskOptionRecordKind::AsyncLet: + asyncLet = cast(option)->getAsyncLet(); + assert(asyncLet && "Missing async let storage"); + jobFlags.task_setIsAsyncLetTask(true); + jobFlags.task_setIsChildTask(true); + break; + + case TaskOptionRecordKind::AsyncLetWithBuffer: + auto *aletRecord = cast(option); + asyncLet = aletRecord->getAsyncLet(); + // TODO: Actually digest the result buffer into the async let task + // context, so that we can emplace the eventual result there instead + // of in a FutureFragment. + hasAsyncLetResultBuffer = true; + assert(asyncLet && "Missing async let storage"); + + jobFlags.task_setIsAsyncLetTask(true); + jobFlags.task_setIsChildTask(true); + break; + } + } + + // Add to the task group, if requested. + if (taskCreateFlags.addPendingGroupTaskUnconditionally()) { + assert(group && "Missing group"); + swift_taskGroup_addPending(group, /*unconditionally=*/true); + } + + AsyncTask *parent = nullptr; + if (jobFlags.task_isChildTask()) { + parent = swift_task_getCurrent(); + assert(parent != nullptr && "creating a child task with no active task"); + } + + // Inherit the priority of the currently-executing task if unspecified and + // we want to inherit. + if (jobFlags.getPriority() == JobPriority::Unspecified && + (jobFlags.task_isChildTask() || taskCreateFlags.inheritContext())) { + AsyncTask *currentTask = parent; + if (!currentTask) + currentTask = swift_task_getCurrent(); + + if (currentTask) + jobFlags.setPriority(currentTask->getPriority()); + else if (taskCreateFlags.inheritContext()) + jobFlags.setPriority(swift_task_getCurrentThreadPriority()); + } + + // Adjust user-interactive priorities down to user-initiated. + if (jobFlags.getPriority() == JobPriority::UserInteractive) + jobFlags.setPriority(JobPriority::UserInitiated); + + // If there is still no job priority, use the default priority. + if (jobFlags.getPriority() == JobPriority::Unspecified) + jobFlags.setPriority(JobPriority::Default); + + // Figure out the size of the header. + size_t headerSize = sizeof(AsyncTask); + if (parent) { + headerSize += sizeof(AsyncTask::ChildFragment); + } + if (group) { + headerSize += sizeof(AsyncTask::GroupChildFragment); + } + if (futureResultType) { + headerSize += FutureFragment::fragmentSize(headerSize, futureResultType); + // Add the future async context prefix. + headerSize += sizeof(FutureAsyncContextPrefix); + } else { + // Add the async context prefix. + headerSize += sizeof(AsyncContextPrefix); + } + + headerSize = llvm::alignTo(headerSize, llvm::Align(alignof(AsyncContext))); + + // Allocate the initial context together with the job. + // This means that we never get rid of this allocation. + size_t amountToAllocate = headerSize + initialContextSize; + + assert(amountToAllocate % MaximumAlignment == 0); + + unsigned initialSlabSize = 512; + + void *allocation = nullptr; + if (asyncLet) { + assert(parent); + + // If there isn't enough room in the fixed async let allocation to + // set up the initial context, then we'll have to allocate more space + // from the parent. + if (asyncLet->getSizeOfPreallocatedSpace() < amountToAllocate) { + hasAsyncLetResultBuffer = false; + } + + // DEPRECATED. This is separated from the above condition because we + // also have to handle an older async let ABI that did not provide + // space for the initial slab in the compiler-generated preallocation. + if (!hasAsyncLetResultBuffer) { + allocation = _swift_task_alloc_specific(parent, + amountToAllocate + initialSlabSize); + } else { + allocation = asyncLet->getPreallocatedSpace(); + assert(asyncLet->getSizeOfPreallocatedSpace() >= amountToAllocate + && "async let does not preallocate enough space for child task"); + initialSlabSize = asyncLet->getSizeOfPreallocatedSpace() + - amountToAllocate; + } + } else { + allocation = malloc(amountToAllocate); + } + SWIFT_TASK_DEBUG_LOG("allocate task %p, parent = %p, slab %u", allocation, + parent, initialSlabSize); + + AsyncContext *initialContext = + reinterpret_cast( + reinterpret_cast(allocation) + headerSize); + + // We can't just use `function` because it uses the new async function entry + // ABI -- passing parameters, closure context, indirect result addresses + // directly -- but AsyncTask->ResumeTask expects the signature to be + // `void (*, *, swiftasync *)`. + // Instead we use an adapter. This adaptor should use the storage prefixed to + // the async context to get at the parameters. + // See e.g. FutureAsyncContextPrefix. + + if (!futureResultType) { + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(allocation) + headerSize - + sizeof(AsyncContextPrefix)); + asyncContextPrefix->asyncEntryPoint = + reinterpret_cast(function); + asyncContextPrefix->closureContext = closureContext; + function = non_future_adapter; + assert(sizeof(AsyncContextPrefix) == 3 * sizeof(void *)); + } else { + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(allocation) + headerSize - + sizeof(FutureAsyncContextPrefix)); + asyncContextPrefix->asyncEntryPoint = + reinterpret_cast(function); + function = future_adapter; + asyncContextPrefix->closureContext = closureContext; + assert(sizeof(FutureAsyncContextPrefix) == 4 * sizeof(void *)); + } + + // Initialize the task so that resuming it will run the given + // function on the initial context. + AsyncTask *task = nullptr; + bool captureCurrentVoucher = taskCreateFlags.copyTaskLocals() || jobFlags.task_isChildTask(); + if (asyncLet) { + // Initialize the refcount bits to "immortal", so that + // ARC operations don't have any effect on the task. + task = new(allocation) AsyncTask(&taskHeapMetadata, + InlineRefCounts::Immortal, jobFlags, + function, initialContext, + captureCurrentVoucher); + } else { + task = new(allocation) AsyncTask(&taskHeapMetadata, jobFlags, + function, initialContext, + captureCurrentVoucher); + } + + // Initialize the child fragment if applicable. + if (parent) { + auto childFragment = task->childFragment(); + new (childFragment) AsyncTask::ChildFragment(parent); + } + + // Initialize the group child fragment if applicable. + if (group) { + auto groupChildFragment = task->groupChildFragment(); + new (groupChildFragment) AsyncTask::GroupChildFragment(group); + } + + // Initialize the future fragment if applicable. + if (futureResultType) { + assert(task->isFuture()); + auto futureFragment = task->futureFragment(); + new (futureFragment) FutureFragment(futureResultType); + + // Set up the context for the future so there is no error, and a successful + // result will be written into the future fragment's storage. + auto futureAsyncContextPrefix = + reinterpret_cast( + reinterpret_cast(allocation) + headerSize - + sizeof(FutureAsyncContextPrefix)); + futureAsyncContextPrefix->indirectResult = futureFragment->getStoragePtr(); + } + + SWIFT_TASK_DEBUG_LOG("creating task %p with parent %p", task, parent); + + // Initialize the task-local allocator. + initialContext->ResumeParent = reinterpret_cast( + asyncLet ? &completeTask + : closureContext ? &completeTaskWithClosure + : &completeTaskAndRelease); + if (asyncLet && initialSlabSize > 0) { + assert(parent); + void *initialSlab = (char*)allocation + amountToAllocate; + task->Private.initializeWithSlab(task, initialSlab, initialSlabSize); + } else { + task->Private.initialize(task); + } + + // Perform additional linking between parent and child task. + if (parent) { + // If the parent was already cancelled, we carry this flag forward to the child. + // + // In a task group we would not have allowed the `add` to create a child anymore, + // however better safe than sorry and `async let` are not expressed as task groups, + // so they may have been spawned in any case still. + if (swift_task_isCancelled(parent) || + (group && group->isCancelled())) + swift_task_cancel(task); + + // Initialize task locals with a link to the parent task. + task->_private().Local.initializeLinkParent(task, parent); + } + + // Configure the initial context. + // + // FIXME: if we store a null pointer here using the standard ABI for + // signed null pointers, then we'll have to authenticate context pointers + // as if they might be null, even though the only time they ever might + // be is the final hop. Store a signed null instead. + initialContext->Parent = nullptr; + initialContext->Flags = AsyncContextKind::Ordinary; + + // Attach to the group, if needed. + if (group) { + swift_taskGroup_attachChild(group, task); + } + + // If we're supposed to copy task locals, do so now. + if (taskCreateFlags.copyTaskLocals()) { + swift_task_localsCopyTo(task); + } + + // Push the async let task status record. + if (asyncLet) { + asyncLet_addImpl(task, asyncLet, !hasAsyncLetResultBuffer); + } + + // If we're supposed to enqueue the task, do so now. + if (taskCreateFlags.enqueueJob()) { + swift_retain(task); + swift_task_enqueue(task, executor); + } + + return {task, initialContext}; +} + +/// Extract the entry point address and initial context size from an async closure value. +template +SWIFT_ALWAYS_INLINE // so this doesn't hang out as a ptrauth gadget +std::pair +getAsyncClosureEntryPointAndContextSize(void *function, + HeapObject *functionContext) { + auto fnPtr = + reinterpret_cast *>(function); +#if SWIFT_PTRAUTH + fnPtr = (const AsyncFunctionPointer *)ptrauth_auth_data( + (void *)fnPtr, ptrauth_key_process_independent_data, AuthDiscriminator); +#endif + return {reinterpret_cast( + fnPtr->Function.get()), + fnPtr->ExpectedContextSize}; +} + +SWIFT_CC(swift) +AsyncTaskAndContext swift::swift_task_create( + size_t taskCreateFlags, + TaskOptionRecord *options, + const Metadata *futureResultType, + void *closureEntry, HeapObject *closureContext) { + FutureAsyncSignature::FunctionType *taskEntry; + size_t initialContextSize; + std::tie(taskEntry, initialContextSize) + = getAsyncClosureEntryPointAndContextSize< + FutureAsyncSignature, + SpecialPointerAuthDiscriminators::AsyncFutureFunction + >(closureEntry, closureContext); + + return swift_task_create_common( + taskCreateFlags, options, futureResultType, taskEntry, closureContext, + initialContextSize); +} + +#ifdef __ARM_ARCH_7K__ +__attribute__((noinline)) +SWIFT_CC(swiftasync) static void workaround_function_swift_task_future_waitImpl( + OpaqueValue *result, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncTask *task, TaskContinuationFunction resumeFunction, + AsyncContext *callContext) { + // Make sure we don't eliminate calls to this function. + asm volatile("" // Do nothing. + : // Output list, empty. + : "r"(result), "r"(callerContext), "r"(task) // Input list. + : // Clobber list, empty. + ); + return; +} +#endif + +SWIFT_CC(swiftasync) +static void swift_task_future_waitImpl( + OpaqueValue *result, + SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncTask *task, + TaskContinuationFunction *resumeFn, + AsyncContext *callContext) { + // Suspend the waiting task. + auto waitingTask = swift_task_getCurrent(); + waitingTask->ResumeTask = task_future_wait_resume_adapter; + waitingTask->ResumeContext = callContext; + + // Wait on the future. + assert(task->isFuture()); + + switch (task->waitFuture(waitingTask, callContext, resumeFn, callerContext, + result)) { + case FutureFragment::Status::Executing: + // The waiting task has been queued on the future. +#ifdef __ARM_ARCH_7K__ + return workaround_function_swift_task_future_waitImpl( + result, callerContext, task, resumeFn, callContext); +#else + return; +#endif + + case FutureFragment::Status::Success: { + // Run the task with a successful result. + auto future = task->futureFragment(); + future->getResultType()->vw_initializeWithCopy(result, + future->getStoragePtr()); + return resumeFn(callerContext); + } + + case FutureFragment::Status::Error: + swift_Concurrency_fatalError(0, "future reported an error, but wait cannot throw"); + } +} + +#ifdef __ARM_ARCH_7K__ +__attribute__((noinline)) +SWIFT_CC(swiftasync) static void workaround_function_swift_task_future_wait_throwingImpl( + OpaqueValue *result, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncTask *task, ThrowingTaskFutureWaitContinuationFunction resumeFunction, + AsyncContext *callContext) { + // Make sure we don't eliminate calls to this function. + asm volatile("" // Do nothing. + : // Output list, empty. + : "r"(result), "r"(callerContext), "r"(task) // Input list. + : // Clobber list, empty. + ); + return; +} +#endif + +SWIFT_CC(swiftasync) +void swift_task_future_wait_throwingImpl( + OpaqueValue *result, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + AsyncTask *task, + ThrowingTaskFutureWaitContinuationFunction *resumeFunction, + AsyncContext *callContext) { + auto waitingTask = swift_task_getCurrent(); + // Suspend the waiting task. + waitingTask->ResumeTask = task_wait_throwing_resume_adapter; + waitingTask->ResumeContext = callContext; + + auto resumeFn = reinterpret_cast(resumeFunction); + + // Wait on the future. + assert(task->isFuture()); + + switch (task->waitFuture(waitingTask, callContext, resumeFn, callerContext, + result)) { + case FutureFragment::Status::Executing: + // The waiting task has been queued on the future. +#ifdef __ARM_ARCH_7K__ + return workaround_function_swift_task_future_wait_throwingImpl( + result, callerContext, task, resumeFunction, callContext); +#else + return; +#endif + + case FutureFragment::Status::Success: { + auto future = task->futureFragment(); + future->getResultType()->vw_initializeWithCopy(result, + future->getStoragePtr()); + return resumeFunction(callerContext, nullptr /*error*/); + } + + case FutureFragment::Status::Error: { + // Run the task with an error result. + auto future = task->futureFragment(); + auto error = future->getError(); + swift_errorRetain(error); + return resumeFunction(callerContext, error); + } + } +} + +size_t swift::swift_task_getJobFlags(AsyncTask *task) { + return task->Flags.getOpaqueValue(); +} + +SWIFT_CC(swift) +static AsyncTask *swift_task_suspendImpl() { + auto task = _swift_task_clearCurrent(); + task->flagAsSuspended(); + return task; +} + +SWIFT_CC(swift) +static AsyncTask *swift_continuation_initImpl(ContinuationAsyncContext *context, + AsyncContinuationFlags flags) { + context->Flags = AsyncContextKind::Continuation; + if (flags.canThrow()) context->Flags.setCanThrow(true); + if (flags.isExecutorSwitchForced()) + context->Flags.continuation_setIsExecutorSwitchForced(true); + context->ErrorResult = nullptr; + + // Set the current executor as the target executor unless there's + // an executor override. + if (!flags.hasExecutorOverride()) + context->ResumeToExecutor = ExecutorRef::generic(); + + // We can initialize this with a relaxed store because resumption + // must happen-after this call. + context->AwaitSynchronization.store(flags.isPreawaited() + ? ContinuationStatus::Awaited + : ContinuationStatus::Pending, + std::memory_order_relaxed); + + AsyncTask *task; + + // A preawait immediately suspends the task. + if (flags.isPreawaited()) { + task = _swift_task_clearCurrent(); + assert(task && "initializing a continuation with no current task"); + task->flagAsSuspended(); + } else { + task = swift_task_getCurrent(); + assert(task && "initializing a continuation with no current task"); + } + + task->ResumeContext = context; + task->ResumeTask = context->ResumeParent; + + return task; +} + +SWIFT_CC(swiftasync) +static void swift_continuation_awaitImpl(ContinuationAsyncContext *context) { +#ifndef NDEBUG + auto task = swift_task_getCurrent(); + assert(task && "awaiting continuation without a task"); + assert(task->ResumeContext == context); + assert(task->ResumeTask == context->ResumeParent); +#endif + + auto &sync = context->AwaitSynchronization; + + auto oldStatus = sync.load(std::memory_order_acquire); + assert((oldStatus == ContinuationStatus::Pending || + oldStatus == ContinuationStatus::Resumed) && + "awaiting a corrupt or already-awaited continuation"); + + // If the status is already Resumed, we can resume immediately. + // Comparing against Pending may be very slightly more compact. + if (oldStatus != ContinuationStatus::Pending) { + if (context->isExecutorSwitchForced()) + return swift_task_switch(context, context->ResumeParent, + context->ResumeToExecutor); + return context->ResumeParent(context); + } + + // Load the current task (we alreaady did this in assertions builds). +#ifdef NDEBUG + auto task = swift_task_getCurrent(); +#endif + + // Flag the task as suspended. + task->flagAsSuspended(); + + // Try to transition to Awaited. + bool success = + sync.compare_exchange_strong(oldStatus, ContinuationStatus::Awaited, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_acquire); + + // If that succeeded, we have nothing to do. + if (success) { + _swift_task_clearCurrent(); + return; + } + + // If it failed, it should be because someone concurrently resumed + // (note that the compare-exchange above is strong). + assert(oldStatus == ContinuationStatus::Resumed && + "continuation was concurrently corrupted or awaited"); + + // Restore the running state of the task and resume it. + task->flagAsRunning(); + if (context->isExecutorSwitchForced()) + return swift_task_switch(context, context->ResumeParent, + context->ResumeToExecutor); + return context->ResumeParent(context); +} + +static void resumeTaskAfterContinuation(AsyncTask *task, + ContinuationAsyncContext *context) { + auto &sync = context->AwaitSynchronization; + auto status = sync.load(std::memory_order_acquire); + assert(status != ContinuationStatus::Resumed && + "continuation was already resumed"); + + // Make sure TSan knows that the resume call happens-before the task + // restarting. + _swift_tsan_release(static_cast(task)); + + // The status should be either Pending or Awaited. If it's Awaited, + // which is probably the most likely option, then we should immediately + // enqueue; we don't need to update the state because there shouldn't + // be a racing attempt to resume the continuation. If it's Pending, + // we need to set it to Resumed; if that fails (with a strong cmpxchg), + // it should be because the original thread concurrently set it to + // Awaited, and so we need to enqueue. + if (status == ContinuationStatus::Pending && + sync.compare_exchange_strong(status, ContinuationStatus::Resumed, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_relaxed)) { + return; + } + assert(status == ContinuationStatus::Awaited && + "detected concurrent attempt to resume continuation"); + + // TODO: maybe in some mode we should set the status to Resumed here + // to make a stronger best-effort attempt to catch racing attempts to + // resume the continuation? + + swift_task_enqueue(task, context->ResumeToExecutor); +} + +SWIFT_CC(swift) +static void swift_continuation_resumeImpl(AsyncTask *task) { + auto context = cast(task->ResumeContext); + resumeTaskAfterContinuation(task, context); +} + +SWIFT_CC(swift) +static void swift_continuation_throwingResumeImpl(AsyncTask *task) { + auto context = cast(task->ResumeContext); + resumeTaskAfterContinuation(task, context); +} + + +SWIFT_CC(swift) +static void swift_continuation_throwingResumeWithErrorImpl(AsyncTask *task, + /* +1 */ SwiftError *error) { + auto context = cast(task->ResumeContext); + context->ErrorResult = error; + resumeTaskAfterContinuation(task, context); +} + +bool swift::swift_task_isCancelled(AsyncTask *task) { + return task->isCancelled(); +} + +SWIFT_CC(swift) +static CancellationNotificationStatusRecord* +swift_task_addCancellationHandlerImpl( + CancellationNotificationStatusRecord::FunctionType handler, + void *context) { + void *allocation = + swift_task_alloc(sizeof(CancellationNotificationStatusRecord)); + auto unsigned_handler = swift_auth_code(handler, 3848); + auto *record = new (allocation) + CancellationNotificationStatusRecord(unsigned_handler, context); + + if (swift_task_addStatusRecord(record)) + return record; + + // else, the task was already cancelled, so while the record was added, + // we must run it immediately here since no other task will trigger it. + record->run(); + return record; +} + +SWIFT_CC(swift) +static void swift_task_removeCancellationHandlerImpl( + CancellationNotificationStatusRecord *record) { + swift_task_removeStatusRecord(record); + swift_task_dealloc(record); +} + +SWIFT_CC(swift) +static NullaryContinuationJob* +swift_task_createNullaryContinuationJobImpl( + size_t priority, + AsyncTask *continuation) { + void *allocation = + swift_task_alloc(sizeof(NullaryContinuationJob)); + auto *job = + new (allocation) NullaryContinuationJob( + swift_task_getCurrent(), static_cast(priority), + continuation); + + return job; +} + +SWIFT_CC(swift) +void swift::swift_continuation_logFailedCheck(const char *message) { + swift_reportError(0, message); +} + +SWIFT_RUNTIME_ATTRIBUTE_NORETURN +SWIFT_CC(swift) +static void swift_task_asyncMainDrainQueueImpl() { +#if SWIFT_CONCURRENCY_COOPERATIVE_GLOBAL_EXECUTOR + bool Finished = false; + swift_task_donateThreadToGlobalExecutorUntil([](void *context) { + return *reinterpret_cast(context); + }, &Finished); +#elif !SWIFT_CONCURRENCY_ENABLE_DISPATCH + // FIXME: consider implementing a concurrent global main queue for + // these environments? + swift_reportError(0, "operation unsupported without libdispatch: " + "swift_task_asyncMainDrainQueue"); +#else +#if defined(_WIN32) + static void(FAR *pfndispatch_main)(void) = NULL; + + if (pfndispatch_main) + return pfndispatch_main(); + + HMODULE hModule = LoadLibraryW(L"dispatch.dll"); + if (hModule == NULL) + swift_reportError(0, "unable to load dispatch.dll"); + + pfndispatch_main = + reinterpret_cast(GetProcAddress(hModule, + "dispatch_main")); + if (pfndispatch_main == NULL) + swift_reportError(0, "unable to locate dispatch_main in dispatch.dll"); + + pfndispatch_main(); + exit(0); +#else + // CFRunLoop is not available on non-Darwin targets. Foundation has an + // implementation, but CoreFoundation is not meant to be exposed. We can only + // assume the existence of `CFRunLoopRun` on Darwin platforms, where the + // system provides an implementation of CoreFoundation. +#if defined(__APPLE__) + auto runLoop = + reinterpret_cast(dlsym(RTLD_DEFAULT, "CFRunLoopRun")); + if (runLoop) { + runLoop(); + exit(0); + } +#endif + + dispatch_main(); +#endif +#endif +} + +#define OVERRIDE_TASK COMPATIBILITY_OVERRIDE +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH diff --git a/stdlib/public/BackDeployConcurrency/Task.h b/stdlib/public/BackDeployConcurrency/Task.h new file mode 100644 index 0000000000000..63b3149a12e6e --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Task.h @@ -0,0 +1,798 @@ +//===--- Task.h - ABI structures for asynchronous tasks ---------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing tasks. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_TASK_BACKDEPLOYED_H +#define SWIFT_ABI_TASK_BACKDEPLOYED_H + +#include "swift/ABI/HeapObject.h" +#include "swift/ABI/Metadata.h" +#include "swift/ABI/MetadataValues.h" +#include "swift/Runtime/Config.h" +#include "swift/Runtime/VoucherShims.h" +#include "swift/Basic/STLExtras.h" +#include "Executor.h" +#include "TaskLocal.h" +#include "bitset" +#include "queue" // TODO: remove and replace with our own mpsc + +namespace swift { +class AsyncTask; +class AsyncContext; +class Job; +struct OpaqueValue; +struct SwiftError; +class TaskStatusRecord; +class TaskOptionRecord; +class TaskGroup; + +extern FullMetadata jobHeapMetadata; + +/// A schedulable job. +class alignas(2 * alignof(void*)) Job : + // For async-let tasks, the refcount bits are initialized as "immortal" + // because such a task is allocated with the parent's stack allocator. + public HeapObject { +public: + // Indices into SchedulerPrivate, for use by the runtime. + enum { + /// The next waiting task link, an AsyncTask that is waiting on a future. + NextWaitingTaskIndex = 0, + + // The Dispatch object header is one pointer and two ints, which is + // equivalent to three pointers on 32-bit and two pointers 64-bit. Set the + // indexes accordingly so that DispatchLinkageIndex points to where Dispatch + // expects. + DispatchHasLongObjectHeader = sizeof(void *) == sizeof(int), + + /// An opaque field used by Dispatch when enqueueing Jobs directly. + DispatchLinkageIndex = DispatchHasLongObjectHeader ? 1 : 0, + + /// The dispatch queue being used when enqueueing a Job directly with + /// Dispatch. + DispatchQueueIndex = DispatchHasLongObjectHeader ? 0 : 1, + }; + + // Reserved for the use of the scheduler. + void *SchedulerPrivate[2]; + + JobFlags Flags; + + // Derived classes can use this to store a Job Id. + uint32_t Id = 0; + + /// The voucher associated with the job. Note: this is currently unused on + /// non-Darwin platforms, with stub implementations of the functions for + /// consistency. + voucher_t Voucher = nullptr; + + /// Reserved for future use. + void *Reserved = nullptr; + + // We use this union to avoid having to do a second indirect branch + // when resuming an asynchronous task, which we expect will be the + // common case. + union { + // A function to run a job that isn't an AsyncTask. + JobInvokeFunction * __ptrauth_swift_job_invoke_function RunJob; + + // A function to resume an AsyncTask. + TaskContinuationFunction * __ptrauth_swift_task_resume_function ResumeTask; + }; + + Job(JobFlags flags, JobInvokeFunction *invoke, + const HeapMetadata *metadata = &jobHeapMetadata) + : HeapObject(metadata), Flags(flags), RunJob(invoke) { + Voucher = voucher_copy(); + assert(!isAsyncTask() && "wrong constructor for a task"); + } + + Job(JobFlags flags, TaskContinuationFunction *invoke, + const HeapMetadata *metadata = &jobHeapMetadata, + bool captureCurrentVoucher = true) + : HeapObject(metadata), Flags(flags), ResumeTask(invoke) { + if (captureCurrentVoucher) + Voucher = voucher_copy(); + assert(isAsyncTask() && "wrong constructor for a non-task job"); + } + + /// Create a job with "immortal" reference counts. + /// Used for async let tasks. + Job(JobFlags flags, TaskContinuationFunction *invoke, + const HeapMetadata *metadata, InlineRefCounts::Immortal_t immortal, + bool captureCurrentVoucher = true) + : HeapObject(metadata, immortal), Flags(flags), ResumeTask(invoke) { + if (captureCurrentVoucher) + Voucher = voucher_copy(); + assert(isAsyncTask() && "wrong constructor for a non-task job"); + } + + ~Job() { swift_voucher_release(Voucher); } + + bool isAsyncTask() const { + return Flags.isAsyncTask(); + } + + JobPriority getPriority() const { + return Flags.getPriority(); + } + + /// Given that we've fully established the job context in the current + /// thread, actually start running this job. To establish the context + /// correctly, call swift_job_run or runJobInExecutorContext. + SWIFT_CC(swiftasync) + void runInFullyEstablishedContext(); + + /// Given that we've fully established the job context in the + /// current thread, and that the job is a simple (non-task) job, + /// actually start running this job. + SWIFT_CC(swiftasync) + void runSimpleInFullyEstablishedContext() { + return RunJob(this); // 'return' forces tail call + } +}; + +// The compiler will eventually assume these. +#if SWIFT_POINTER_IS_8_BYTES +static_assert(sizeof(Job) == 8 * sizeof(void*), + "Job size is wrong"); +#else +static_assert(sizeof(Job) == 10 * sizeof(void*), + "Job size is wrong"); +#endif +static_assert(alignof(Job) == 2 * alignof(void*), + "Job alignment is wrong"); + +class NullaryContinuationJob : public Job { + +private: + AsyncTask* Task; + AsyncTask* Continuation; + +public: + NullaryContinuationJob(AsyncTask *task, JobPriority priority, AsyncTask *continuation) + : Job({JobKind::NullaryContinuation, priority}, &process), + Task(task), Continuation(continuation) {} + + SWIFT_CC(swiftasync) + static void process(Job *job); + + static bool classof(const Job *job) { + return job->Flags.getKind() == JobKind::NullaryContinuation; + } +}; + +/// An asynchronous task. Tasks are the analogue of threads for +/// asynchronous functions: that is, they are a persistent identity +/// for the overall async computation. +/// +/// ### Fragments +/// An AsyncTask may have the following fragments: +/// +/// +--------------------------+ +/// | childFragment? | +/// | groupChildFragment? | +/// | futureFragment? |* +/// +--------------------------+ +/// +/// * The future fragment is dynamic in size, based on the future result type +/// it can hold, and thus must be the *last* fragment. +class AsyncTask : public Job { +public: + // On 32-bit targets, there is a word of tail padding remaining + // in Job, and ResumeContext will fit into that, at offset 28. + // Private then has offset 32. + // On 64-bit targets, there is no tail padding in Job, and so + // ResumeContext has offset 48. There is therefore another word + // of reserved storage prior to Private (which needs to have + // double-word alignment), which has offset 64. + // We therefore converge and end up with 16 words of storage on + // all platforms. + + /// The context for resuming the job. When a task is scheduled + /// as a job, the next continuation should be installed as the + /// ResumeTask pointer in the job header, with this serving as + /// the context pointer. + /// + /// We can't protect the data in the context from being overwritten + /// by attackers, but we can at least sign the context pointer to + /// prevent it from being corrupted in flight. + AsyncContext * __ptrauth_swift_task_resume_context ResumeContext; + +#if SWIFT_POINTER_IS_8_BYTES + void *Reserved64; +#endif + + struct PrivateStorage; + + /// Private storage for the use of the runtime. + struct alignas(2 * alignof(void*)) OpaquePrivateStorage { + void *Storage[14]; + + /// Initialize this storage during the creation of a task. + void initialize(AsyncTask *task); + void initializeWithSlab(AsyncTask *task, + void *slab, size_t slabCapacity); + + /// React to the completion of the enclosing task's execution. + void complete(AsyncTask *task); + + /// React to the final destruction of the enclosing task. + void destroy(); + + PrivateStorage &get(); + const PrivateStorage &get() const; + }; + PrivateStorage &_private(); + const PrivateStorage &_private() const; + + OpaquePrivateStorage Private; + + /// Create a task. + /// This does not initialize Private; callers must call + /// Private.initialize separately. + AsyncTask(const HeapMetadata *metadata, JobFlags flags, + TaskContinuationFunction *run, + AsyncContext *initialContext, + bool captureCurrentVoucher) + : Job(flags, run, metadata, captureCurrentVoucher), + ResumeContext(initialContext) { + assert(flags.isAsyncTask()); + setTaskId(); + } + + /// Create a task with "immortal" reference counts. + /// Used for async let tasks. + /// This does not initialize Private; callers must call + /// Private.initialize separately. + AsyncTask(const HeapMetadata *metadata, InlineRefCounts::Immortal_t immortal, + JobFlags flags, + TaskContinuationFunction *run, + AsyncContext *initialContext, + bool captureCurrentVoucher) + : Job(flags, run, metadata, immortal, captureCurrentVoucher), + ResumeContext(initialContext) { + assert(flags.isAsyncTask()); + setTaskId(); + } + + ~AsyncTask(); + + /// Set the task's ID field to the next task ID. + void setTaskId(); + + /// Given that we've already fully established the job context + /// in the current thread, start running this task. To establish + /// the job context correctly, call swift_job_run or + /// runInExecutorContext. + SWIFT_CC(swiftasync) + void runInFullyEstablishedContext() { + return ResumeTask(ResumeContext); // 'return' forces tail call + } + + /// Flag that this task is now running. This can update + /// the priority stored in the job flags if the priority has been + /// escalated. + /// + /// Generally this should be done immediately after updating + /// ActiveTask. + void flagAsRunning(); + void flagAsRunning_slow(); + + /// Flag that this task is now suspended. This can update the + /// priority stored in the job flags if the priority hsa been + /// escalated. Generally this should be done immediately after + /// clearing ActiveTask and immediately before enqueuing the task + /// somewhere. TODO: record where the task is enqueued if + /// possible. + void flagAsSuspended(); + void flagAsSuspended_slow(); + + /// Flag that this task is now completed. This normally does not do anything + /// but can be used to locally insert logging. + void flagAsCompleted(); + + /// Check whether this task has been cancelled. + /// Checking this is, of course, inherently race-prone on its own. + bool isCancelled() const; + + // ==== Task Local Values ---------------------------------------------------- + + void localValuePush(const HeapObject *key, + /* +1 */ OpaqueValue *value, + const Metadata *valueType); + + OpaqueValue *localValueGet(const HeapObject *key); + + /// Returns true if storage has still more bindings. + bool localValuePop(); + + // ==== Child Fragment ------------------------------------------------------- + + /// A fragment of an async task structure that happens to be a child task. + class ChildFragment { + /// The parent task of this task. + AsyncTask *Parent; + + // TODO: Document more how this is used from the `TaskGroupTaskStatusRecord` + + /// The next task in the singly-linked list of child tasks. + /// The list must start in a `ChildTaskStatusRecord` registered + /// with the parent task. + /// + /// Note that the parent task may have multiple such records. + /// + /// WARNING: Access can only be performed by the `Parent` of this task. + AsyncTask *NextChild = nullptr; + + public: + ChildFragment(AsyncTask *parent) : Parent(parent) {} + + AsyncTask *getParent() const { + return Parent; + } + + AsyncTask *getNextChild() const { + return NextChild; + } + + /// Set the `NextChild` to to the passed task. + /// + /// WARNING: This must ONLY be invoked from the parent of both + /// (this and the passed-in) tasks for thread-safety reasons. + void setNextChild(AsyncTask *task) { + NextChild = task; + } + }; + + bool hasChildFragment() const { + return Flags.task_isChildTask(); + } + + ChildFragment *childFragment() { + assert(hasChildFragment()); + + auto offset = reinterpret_cast(this); + offset += sizeof(AsyncTask); + + return reinterpret_cast(offset); + } + + // ==== TaskGroup Child ------------------------------------------------------ + + /// A child task created by `group.add` is called a "task group child." + /// Upon completion, in addition to the usual future notifying all its waiters, + /// it must also `group->offer` itself to the group. + /// + /// This signalling is necessary to correctly implement the group's `next()`. + class GroupChildFragment { + private: + TaskGroup* Group; + + friend class AsyncTask; + friend class TaskGroup; + + public: + explicit GroupChildFragment(TaskGroup *group) + : Group(group) {} + + /// Return the group this task should offer into when it completes. + TaskGroup* getGroup() { + return Group; + } + }; + + // Checks if task is a child of a TaskGroup task. + // + // A child task that is a group child knows that it's parent is a group + // and therefore may `groupOffer` to it upon completion. + bool hasGroupChildFragment() const { return Flags.task_isGroupChildTask(); } + + GroupChildFragment *groupChildFragment() { + assert(hasGroupChildFragment()); + + auto offset = reinterpret_cast(this); + offset += sizeof(AsyncTask); + if (hasChildFragment()) + offset += sizeof(ChildFragment); + + return reinterpret_cast(offset); + } + + // ==== Future --------------------------------------------------------------- + + class FutureFragment { + public: + /// Describes the status of the future. + /// + /// Futures always begin in the "Executing" state, and will always + /// make a single state change to either Success or Error. + enum class Status : uintptr_t { + /// The future is executing or ready to execute. The storage + /// is not accessible. + Executing = 0, + + /// The future has completed with result (of type \c resultType). + Success, + + /// The future has completed by throwing an error (an \c Error + /// existential). + Error, + }; + + /// An item within the wait queue, which includes the status and the + /// head of the list of tasks. + struct WaitQueueItem { + /// Mask used for the low status bits in a wait queue item. + static const uintptr_t statusMask = 0x03; + + uintptr_t storage; + + Status getStatus() const { + return static_cast(storage & statusMask); + } + + AsyncTask *getTask() const { + return reinterpret_cast(storage & ~statusMask); + } + + static WaitQueueItem get(Status status, AsyncTask *task) { + return WaitQueueItem{ + reinterpret_cast(task) | static_cast(status)}; + } + }; + + private: + /// Queue containing all of the tasks that are waiting in `get()`. + /// + /// The low bits contain the status, the rest of the pointer is the + /// AsyncTask. + std::atomic waitQueue; + + /// The type of the result that will be produced by the future. + const Metadata *resultType; + + SwiftError *error = nullptr; + + // Trailing storage for the result itself. The storage will be + // uninitialized, contain an instance of \c resultType. + + friend class AsyncTask; + + public: + explicit FutureFragment(const Metadata *resultType) + : waitQueue(WaitQueueItem::get(Status::Executing, nullptr)), + resultType(resultType) { } + + /// Destroy the storage associated with the future. + void destroy(); + + const Metadata *getResultType() const { + return resultType; + } + + /// Retrieve a pointer to the storage of the result. + OpaqueValue *getStoragePtr() { + // The result storage starts at the first aligned offset following + // the fragment header. This offset will agree with the abstract + // calculation for `resultOffset` in the fragmentSize function below + // because the entire task is aligned to at least the target + // alignment (because it's aligned to MaxAlignment), which means + // `this` must have the same value modulo that alignment as + // `fragmentOffset` has in that function. + char *fragmentAddr = reinterpret_cast(this); + uintptr_t alignment = resultType->vw_alignment(); + char *resultAddr = fragmentAddr + sizeof(FutureFragment); + uintptr_t unalignedResultAddrInt = + reinterpret_cast(resultAddr); + uintptr_t alignedResultAddrInt = + (unalignedResultAddrInt + alignment - 1) & ~(alignment - 1); + // We could just cast alignedResultAddrInt back to a pointer, but + // doing pointer arithmetic is more strictly conformant and less + // likely to annoy the optimizer. + resultAddr += (alignedResultAddrInt - unalignedResultAddrInt); + return reinterpret_cast(resultAddr); + } + + /// Retrieve the error. + SwiftError *&getError() { return error; } + + /// Determine the size of the future fragment given the result type + /// of the future. + static size_t fragmentSize(size_t fragmentOffset, + const Metadata *resultType) { + assert((fragmentOffset & (alignof(FutureFragment) - 1)) == 0); + size_t alignment = resultType->vw_alignment(); + size_t resultOffset = fragmentOffset + sizeof(FutureFragment); + resultOffset = (resultOffset + alignment - 1) & ~(alignment - 1); + size_t endOffset = resultOffset + resultType->vw_size(); + return (endOffset - fragmentOffset); + } + }; + + bool isFuture() const { return Flags.task_isFuture(); } + + FutureFragment *futureFragment() { + assert(isFuture()); + auto offset = reinterpret_cast(this); + offset += sizeof(AsyncTask); + if (hasChildFragment()) + offset += sizeof(ChildFragment); + if (hasGroupChildFragment()) + offset += sizeof(GroupChildFragment); + + return reinterpret_cast(offset); + } + + /// Wait for this future to complete. + /// + /// \returns the status of the future. If this result is + /// \c Executing, then \c waitingTask has been added to the + /// wait queue and will be scheduled when the future completes. Otherwise, + /// the future has completed and can be queried. + /// The waiting task's async context will be intialized with the parameters if + /// the current's task state is executing. + FutureFragment::Status waitFuture(AsyncTask *waitingTask, + AsyncContext *waitingTaskContext, + TaskContinuationFunction *resumeFn, + AsyncContext *callerContext, + OpaqueValue *result); + + /// Complete this future. + /// + /// Upon completion, any waiting tasks will be scheduled on the given + /// executor. + void completeFuture(AsyncContext *context); + + // ==== ---------------------------------------------------------------------- + + static bool classof(const Job *job) { + return job->isAsyncTask(); + } + +private: + /// Access the next waiting task, which establishes a singly linked list of + /// tasks that are waiting on a future. + AsyncTask *&getNextWaitingTask() { + return reinterpret_cast( + SchedulerPrivate[NextWaitingTaskIndex]); + } +}; + +// The compiler will eventually assume these. +static_assert(sizeof(AsyncTask) == NumWords_AsyncTask * sizeof(void*), + "AsyncTask size is wrong"); +static_assert(alignof(AsyncTask) == 2 * alignof(void*), + "AsyncTask alignment is wrong"); +// Libc hardcodes this offset to extract the TaskID +static_assert(offsetof(AsyncTask, Id) == 4 * sizeof(void *) + 4, + "AsyncTask::Id offset is wrong"); + +SWIFT_CC(swiftasync) +inline void Job::runInFullyEstablishedContext() { + if (auto task = dyn_cast(this)) + return task->runInFullyEstablishedContext(); // 'return' forces tail call + else + return runSimpleInFullyEstablishedContext(); // 'return' forces tail call +} + +/// Kinds of async context. +enum class AsyncContextKind { + /// An ordinary asynchronous function. + Ordinary = 0, + + /// A context which can yield to its caller. + Yielding = 1, + + /// A continuation context. + Continuation = 2, + + // Other kinds are reserved for interesting special + // intermediate contexts. + + // Kinds >= 192 are private to the implementation. + First_Reserved = 192 +}; + +/// Flags for async contexts. +class AsyncContextFlags : public FlagSet { +public: + enum { + Kind = 0, + Kind_width = 8, + + CanThrow = 8, + + // Kind-specific flags should grow down from 31. + + Continuation_IsExecutorSwitchForced = 31, + }; + + explicit AsyncContextFlags(uint32_t bits) : FlagSet(bits) {} + constexpr AsyncContextFlags() {} + AsyncContextFlags(AsyncContextKind kind) { + setKind(kind); + } + + /// The kind of context this represents. + FLAGSET_DEFINE_FIELD_ACCESSORS(Kind, Kind_width, AsyncContextKind, + getKind, setKind) + + /// Whether this context is permitted to throw. + FLAGSET_DEFINE_FLAG_ACCESSORS(CanThrow, canThrow, setCanThrow) + + /// See AsyncContinuationFlags::isExecutorSwitchForced. + FLAGSET_DEFINE_FLAG_ACCESSORS(Continuation_IsExecutorSwitchForced, + continuation_isExecutorSwitchForced, + continuation_setIsExecutorSwitchForced) +}; + +// ==== ------------------------------------------------------------------------ + +/// An asynchronous context within a task. Generally contexts are +/// allocated using the task-local stack alloc/dealloc operations, but +/// there's no guarantee of that, and the ABI is designed to permit +/// contexts to be allocated within their caller's frame. +class alignas(MaximumAlignment) AsyncContext { +public: + /// The parent context. + AsyncContext * __ptrauth_swift_async_context_parent Parent; + + /// The function to call to resume running in the parent context. + /// Generally this means a semantic return, but for some temporary + /// translation contexts it might mean initiating a call. + /// + /// Eventually, the actual type here will depend on the types + /// which need to be passed to the parent. For now, arguments + /// are always written into the context, and so the type is + /// always the same. + TaskContinuationFunction * __ptrauth_swift_async_context_resume + ResumeParent; + + /// Flags describing this context. + /// + /// Note that this field is only 32 bits; any alignment padding + /// following this on 64-bit platforms can be freely used by the + /// function. If the function is a yielding function, that padding + /// is of course interrupted by the YieldToParent field. + AsyncContextFlags Flags; + + AsyncContext(AsyncContextFlags flags, + TaskContinuationFunction *resumeParent, + AsyncContext *parent) + : Parent(parent), ResumeParent(resumeParent), + Flags(flags) {} + + AsyncContext(const AsyncContext &) = delete; + AsyncContext &operator=(const AsyncContext &) = delete; + + /// Perform a return from this context. + /// + /// Generally this should be tail-called. + SWIFT_CC(swiftasync) + void resumeParent() { + // TODO: destroy context before returning? + // FIXME: force tail call + return ResumeParent(Parent); + } +}; + +/// An async context that supports yielding. +class YieldingAsyncContext : public AsyncContext { +public: + /// The function to call to temporarily resume running in the + /// parent context. Generally this means a semantic yield. + TaskContinuationFunction * __ptrauth_swift_async_context_yield + YieldToParent; + + YieldingAsyncContext(AsyncContextFlags flags, + TaskContinuationFunction *resumeParent, + TaskContinuationFunction *yieldToParent, + AsyncContext *parent) + : AsyncContext(flags, resumeParent, parent), + YieldToParent(yieldToParent) {} + + static bool classof(const AsyncContext *context) { + return context->Flags.getKind() == AsyncContextKind::Yielding; + } +}; + +/// An async context that can be resumed as a continuation. +class ContinuationAsyncContext : public AsyncContext { +public: + /// An atomic object used to ensure that a continuation is not + /// scheduled immediately during a resume if it hasn't yet been + /// awaited by the function which set it up. + std::atomic AwaitSynchronization; + + /// The error result value of the continuation. + /// This should be null-initialized when setting up the continuation. + /// Throwing resumers must overwrite this with a non-null value. + SwiftError *ErrorResult; + + /// A pointer to the normal result value of the continuation. + /// Normal resumers must initialize this before resuming. + OpaqueValue *NormalResult; + + /// The executor that should be resumed to. + ExecutorRef ResumeToExecutor; + + void setErrorResult(SwiftError *error) { + ErrorResult = error; + } + + bool isExecutorSwitchForced() const { + return Flags.continuation_isExecutorSwitchForced(); + } + + static bool classof(const AsyncContext *context) { + return context->Flags.getKind() == AsyncContextKind::Continuation; + } +}; + +/// An asynchronous context within a task that describes a general "Future". +/// task. +/// +/// This type matches the ABI of a function ` () async throws -> T`, which +/// is the type used by `detach` and `Task.group.add` to create +/// futures. +class FutureAsyncContext : public AsyncContext { +public: + using AsyncContext::AsyncContext; +}; + +/// This matches the ABI of a closure `() async throws -> ()` +using AsyncVoidClosureEntryPoint = + SWIFT_CC(swiftasync) + void (SWIFT_ASYNC_CONTEXT AsyncContext *, SWIFT_CONTEXT void *); + +/// This matches the ABI of a closure `() async throws -> T` +using AsyncGenericClosureEntryPoint = + SWIFT_CC(swiftasync) + void(OpaqueValue *, + SWIFT_ASYNC_CONTEXT AsyncContext *, SWIFT_CONTEXT void *); + +/// This matches the ABI of the resume function of a closure +/// `() async throws -> ()`. +using AsyncVoidClosureResumeEntryPoint = + SWIFT_CC(swiftasync) + void(SWIFT_ASYNC_CONTEXT AsyncContext *, SWIFT_CONTEXT SwiftError *); + +class AsyncContextPrefix { +public: + // Async closure entry point adhering to compiler calling conv (e.g directly + // passing the closure context instead of via the async context) + AsyncVoidClosureEntryPoint *__ptrauth_swift_task_resume_function + asyncEntryPoint; + void *closureContext; + SwiftError *errorResult; +}; + +/// Storage that is allocated before the AsyncContext to be used by an adapter +/// of Swift's async convention and the ResumeTask interface. +class FutureAsyncContextPrefix { +public: + OpaqueValue *indirectResult; + // Async closure entry point adhering to compiler calling conv (e.g directly + // passing the closure context instead of via the async context) + AsyncGenericClosureEntryPoint *__ptrauth_swift_task_resume_function + asyncEntryPoint; + void *closureContext; + SwiftError *errorResult; +}; + +} // end namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/Task.swift b/stdlib/public/BackDeployConcurrency/Task.swift new file mode 100644 index 0000000000000..fe421c65159f9 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/Task.swift @@ -0,0 +1,896 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +// ==== Task ------------------------------------------------------------------- + +/// A unit of asynchronous work. +/// +/// When you create an instance of `Task`, +/// you provide a closure that contains the work for that task to perform. +/// Tasks can start running immediately after creation; +/// you don't explicitly start or schedule them. +/// After creating a task, you use the instance to interact with it --- +/// for example, to wait for it to complete or to cancel it. +/// It's not a programming error to discard a reference to a task +/// without waiting for that task to finish or canceling it. +/// A task runs regardless of whether you keep a reference to it. +/// However, if you discard the reference to a task, +/// you give up the ability +/// to wait for that task's result or cancel the task. +/// +/// To support operations on the current task, +/// which can be either a detached task or child task, +/// `Task` also exposes class methods like `yield()`. +/// Because these methods are asynchronous, +/// they're always invoked as part of an existing task. +/// +/// Only code that's running as part of the task can interact with that task. +/// To interact with the current task, +/// you call one of the static methods on `Task`. +/// +/// A task's execution can be seen as a series of periods where the task ran. +/// Each such period ends at a suspension point or the +/// completion of the task. +/// These periods of execution are represented by instances of `PartialAsyncTask`. +/// Unless you're implementing a custom executor, +/// you don't directly interact with partial tasks. +/// +/// For information about the language-level concurrency model that `Task` is part of, +/// see [Concurrency][concurrency] in [The Swift Programming Language][tspl]. +/// +/// [concurrency]: https://docs.swift.org/swift-book/LanguageGuide/Concurrency.html +/// [tspl]: https://docs.swift.org/swift-book/ +/// +/// Task Cancellation +/// ================= +/// +/// Tasks include a shared mechanism for indicating cancellation, +/// but not a shared implementation for how to handle cancellation. +/// Depending on the work you're doing in the task, +/// the correct way to stop that work varies. +/// Likewise, +/// it's the responsibility of the code running as part of the task +/// to check for cancellation whenever stopping is appropriate. +/// In a long-task that includes multiple pieces, +/// you might need to check for cancellation at several points, +/// and handle cancellation differently at each point. +/// If you only need to throw an error to stop the work, +/// call the `Task.checkCancellation()` function to check for cancellation. +/// Other responses to cancellation include +/// returning the work completed so far, returning an empty result, or returning `nil`. +/// +/// Cancellation is a purely Boolean state; +/// there's no way to include additional information +/// like the reason for cancellation. +/// This reflects the fact that a task can be canceled for many reasons, +/// and additional reasons can accrue during the cancellation process. +@available(SwiftStdlib 5.1, *) +@frozen +public struct Task: Sendable { + @usableFromInline + internal let _task: Builtin.NativeObject + + @_alwaysEmitIntoClient + internal init(_ task: Builtin.NativeObject) { + self._task = task + } +} + +@available(SwiftStdlib 5.1, *) +extension Task { + /// The result from a throwing task, after it completes. + /// + /// If the task hasn't completed, + /// accessing this property waits for it to complete + /// and its priority increases to that of the current task. + /// Note that this might not be as effective as + /// creating the task with the correct priority, + /// depending on the executor's scheduling details. + /// + /// If the task throws an error, this property propagates that error. + /// Tasks that respond to cancellation by throwing `CancellationError` + /// have that error propagated here upon cancellation. + /// + /// - Returns: The task's result. + public var value: Success { + get async throws { + return try await _taskFutureGetThrowing(_task) + } + } + + /// The result or error from a throwing task, after it completes. + /// + /// If the task hasn't completed, + /// accessing this property waits for it to complete + /// and its priority increases to that of the current task. + /// Note that this might not be as effective as + /// creating the task with the correct priority, + /// depending on the executor's scheduling details. + /// + /// - Returns: If the task succeeded, + /// `.success` with the task's result as the associated value; + /// otherwise, `.failure` with the error as the associated value. + public var result: Result { + get async { + do { + return .success(try await value) + } catch { + return .failure(error as! Failure) // as!-safe, guaranteed to be Failure + } + } + } + + /// Indicates that the task should stop running. + /// + /// Task cancellation is cooperative: + /// a task that supports cancellation + /// checks whether it has been canceled at various points during its work. + /// + /// Calling this method on a task that doesn't support cancellation + /// has no effect. + /// Likewise, if the task has already run + /// past the last point where it would stop early, + /// calling this method has no effect. + /// + /// - SeeAlso: `Task.checkCancellation()` + public func cancel() { + Builtin.cancelAsyncTask(_task) + } +} + +@available(SwiftStdlib 5.1, *) +extension Task where Failure == Never { + /// The result from a nonthrowing task, after it completes. + /// + /// If the task hasn't completed yet, + /// accessing this property waits for it to complete + /// and its priority increases to that of the current task. + /// Note that this might not be as effective as + /// creating the task with the correct priority, + /// depending on the executor's scheduling details. + /// + /// Tasks that never throw an error can still check for cancellation, + /// but they need to use an approach like returning `nil` + /// instead of throwing an error. + public var value: Success { + get async { + return await _taskFutureGet(_task) + } + } +} + +@available(SwiftStdlib 5.1, *) +extension Task: Hashable { + public func hash(into hasher: inout Hasher) { + UnsafeRawPointer(Builtin.bridgeToRawPointer(_task)).hash(into: &hasher) + } +} + +@available(SwiftStdlib 5.1, *) +extension Task: Equatable { + public static func ==(lhs: Self, rhs: Self) -> Bool { + UnsafeRawPointer(Builtin.bridgeToRawPointer(lhs._task)) == + UnsafeRawPointer(Builtin.bridgeToRawPointer(rhs._task)) + } +} + +// ==== Task Priority ---------------------------------------------------------- + +/// The priority of a task. +/// +/// The executor determines how priority information affects the way tasks are scheduled. +/// The behavior varies depending on the executor currently being used. +/// Typically, executors attempt to run tasks with a higher priority +/// before tasks with a lower priority. +/// However, the semantics of how priority is treated are left up to each +/// platform and `Executor` implementation. +/// +/// Child tasks automatically inherit their parent task's priority. +/// Detached tasks created by `detach(priority:operation:)` don't inherit task priority +/// because they aren't attached to the current task. +/// +/// In some situations the priority of a task is elevated --- +/// that is, the task is treated as it if had a higher priority, +/// without actually changing the priority of the task: +/// +/// - If a task runs on behalf of an actor, +/// and a new higher-priority task is enqueued to the actor, +/// then the actor's current task is temporarily elevated +/// to the priority of the enqueued task. +/// This priority elevation allows the new task +/// to be processed at the priority it was enqueued with. +/// - If a a higher-priority task calls the `get()` method, +/// then the priority of this task increases until the task completes. +/// +/// In both cases, priority elevation helps you prevent a low-priority task +/// from blocking the execution of a high priority task, +/// which is also known as *priority inversion*. +@available(SwiftStdlib 5.1, *) +public struct TaskPriority: RawRepresentable, Sendable { + public typealias RawValue = UInt8 + public var rawValue: UInt8 + + public init(rawValue: UInt8) { + self.rawValue = rawValue + } + + public static let high: TaskPriority = .init(rawValue: 0x19) + + @_alwaysEmitIntoClient + public static var medium: TaskPriority { + .init(rawValue: 0x15) + } + + public static let low: TaskPriority = .init(rawValue: 0x11) + + public static let userInitiated: TaskPriority = high + public static let utility: TaskPriority = low + public static let background: TaskPriority = .init(rawValue: 0x09) + + @available(*, deprecated, renamed: "medium") + public static let `default`: TaskPriority = .init(rawValue: 0x15) +} + +@available(SwiftStdlib 5.1, *) +extension TaskPriority: Equatable { + public static func == (lhs: TaskPriority, rhs: TaskPriority) -> Bool { + lhs.rawValue == rhs.rawValue + } + + public static func != (lhs: TaskPriority, rhs: TaskPriority) -> Bool { + lhs.rawValue != rhs.rawValue + } +} + +@available(SwiftStdlib 5.1, *) +extension TaskPriority: Comparable { + public static func < (lhs: TaskPriority, rhs: TaskPriority) -> Bool { + lhs.rawValue < rhs.rawValue + } + + public static func <= (lhs: TaskPriority, rhs: TaskPriority) -> Bool { + lhs.rawValue <= rhs.rawValue + } + + public static func > (lhs: TaskPriority, rhs: TaskPriority) -> Bool { + lhs.rawValue > rhs.rawValue + } + + public static func >= (lhs: TaskPriority, rhs: TaskPriority) -> Bool { + lhs.rawValue >= rhs.rawValue + } +} + +@available(SwiftStdlib 5.1, *) +extension TaskPriority: Codable { } + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + + /// The current task's priority. + /// + /// If you access this property outside of any task, + /// this queries the system to determine the + /// priority at which the current function is running. + /// If the system can't provide a priority, + /// this property's value is `Priority.default`. + public static var currentPriority: TaskPriority { + withUnsafeCurrentTask { task in + // If we are running on behalf of a task, use that task's priority. + if let task = task { + return task.priority + } + + // Otherwise, query the system. + return TaskPriority(rawValue: UInt8(_getCurrentThreadPriority())) + } + } +} + +@available(SwiftStdlib 5.1, *) +extension TaskPriority { + /// Downgrade user-interactive to user-initiated. + var _downgradeUserInteractive: TaskPriority { + return self + } +} + +// ==== Job Flags -------------------------------------------------------------- + +/// Flags for schedulable jobs. +/// +/// This is a port of the C++ FlagSet. +@available(SwiftStdlib 5.1, *) +struct JobFlags { + /// Kinds of schedulable jobs. + enum Kind: Int32 { + case task = 0 + } + + /// The actual bit representation of these flags. + var bits: Int32 = 0 + + /// The kind of job described by these flags. + var kind: Kind { + get { + Kind(rawValue: bits & 0xFF)! + } + + set { + bits = (bits & ~0xFF) | newValue.rawValue + } + } + + /// Whether this is an asynchronous task. + var isAsyncTask: Bool { kind == .task } + + /// The priority given to the job. + var priority: TaskPriority? { + get { + let value = (Int(bits) & 0xFF00) >> 8 + + if value == 0 { + return nil + } + + return TaskPriority(rawValue: UInt8(value)) + } + + set { + bits = (bits & ~0xFF00) | Int32((Int(newValue?.rawValue ?? 0) << 8)) + } + } + + /// Whether this is a child task. + var isChildTask: Bool { + get { + (bits & (1 << 24)) != 0 + } + + set { + if newValue { + bits = bits | 1 << 24 + } else { + bits = (bits & ~(1 << 24)) + } + } + } + + /// Whether this is a future. + var isFuture: Bool { + get { + (bits & (1 << 25)) != 0 + } + + set { + if newValue { + bits = bits | 1 << 25 + } else { + bits = (bits & ~(1 << 25)) + } + } + } + + /// Whether this is a group child. + var isGroupChildTask: Bool { + get { + (bits & (1 << 26)) != 0 + } + + set { + if newValue { + bits = bits | 1 << 26 + } else { + bits = (bits & ~(1 << 26)) + } + } + } + + /// Whether this is a task created by the 'async' operation, which + /// conceptually continues the work of the synchronous code that invokes + /// it. + var isContinuingAsyncTask: Bool { + get { + (bits & (1 << 27)) != 0 + } + + set { + if newValue { + bits = bits | 1 << 27 + } else { + bits = (bits & ~(1 << 27)) + } + } + } +} + +// ==== Task Creation Flags -------------------------------------------------- + +/// Form task creation flags for use with the createAsyncTask builtins. +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +func taskCreateFlags( + priority: TaskPriority?, isChildTask: Bool, copyTaskLocals: Bool, + inheritContext: Bool, enqueueJob: Bool, + addPendingGroupTaskUnconditionally: Bool +) -> Int { + var bits = 0 + bits |= (bits & ~0xFF) | Int(priority?.rawValue ?? 0) + if isChildTask { + bits |= 1 << 8 + } + if copyTaskLocals { + bits |= 1 << 10 + } + if inheritContext { + bits |= 1 << 11 + } + if enqueueJob { + bits |= 1 << 12 + } + if addPendingGroupTaskUnconditionally { + bits |= 1 << 13 + } + return bits +} + +// ==== Task Creation ---------------------------------------------------------- +@available(SwiftStdlib 5.1, *) +extension Task where Failure == Never { + /// Runs the given nonthrowing operation asynchronously + /// as part of a new top-level task on behalf of the current actor. + /// + /// Use this function when creating asynchronous work + /// that operates on behalf of the synchronous function that calls it. + /// Like `Task.detached(priority:operation:)`, + /// this function creates a separate, top-level task. + /// Unlike `Task.detached(priority:operation:)`, + /// the task created by `Task.init(priority:operation:)` + /// inherits the priority and actor context of the caller, + /// so the operation is treated more like an asynchronous extension + /// to the synchronous operation. + /// + /// You need to keep a reference to the task + /// if you want to cancel it by calling the `Task.cancel()` method. + /// Discarding your reference to a detached task + /// doesn't implicitly cancel that task, + /// it only makes it impossible for you to explicitly cancel the task. + /// + /// - Parameters: + /// - priority: The priority of the task. + /// Pass `nil` to use the priority from `Task.currentPriority`. + /// - operation: The operation to perform. + @discardableResult + @_alwaysEmitIntoClient + public init( + priority: TaskPriority? = nil, + @_inheritActorContext @_implicitSelfCapture operation: __owned @Sendable @escaping () async -> Success + ) { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + // Set up the job flags for a new task. + let flags = taskCreateFlags( + priority: priority, isChildTask: false, copyTaskLocals: true, + inheritContext: true, enqueueJob: true, + addPendingGroupTaskUnconditionally: false) + + // Create the asynchronous task. + let (task, _) = Builtin.createAsyncTask(flags, operation) + + self._task = task +#else + fatalError("Unsupported Swift compiler") +#endif + } +} + +@available(SwiftStdlib 5.1, *) +extension Task where Failure == Error { + /// Runs the given throwing operation asynchronously + /// as part of a new top-level task on behalf of the current actor. + /// + /// Use this function when creating asynchronous work + /// that operates on behalf of the synchronous function that calls it. + /// Like `Task.detached(priority:operation:)`, + /// this function creates a separate, top-level task. + /// Unlike `detach(priority:operation:)`, + /// the task created by `Task.init(priority:operation:)` + /// inherits the priority and actor context of the caller, + /// so the operation is treated more like an asynchronous extension + /// to the synchronous operation. + /// + /// You need to keep a reference to the task + /// if you want to cancel it by calling the `Task.cancel()` method. + /// Discarding your reference to a detached task + /// doesn't implicitly cancel that task, + /// it only makes it impossible for you to explicitly cancel the task. + /// + /// - Parameters: + /// - priority: The priority of the task. + /// Pass `nil` to use the priority from `Task.currentPriority`. + /// - operation: The operation to perform. + @discardableResult + @_alwaysEmitIntoClient + public init( + priority: TaskPriority? = nil, + @_inheritActorContext @_implicitSelfCapture operation: __owned @Sendable @escaping () async throws -> Success + ) { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + // Set up the task flags for a new task. + let flags = taskCreateFlags( + priority: priority, isChildTask: false, copyTaskLocals: true, + inheritContext: true, enqueueJob: true, + addPendingGroupTaskUnconditionally: false + ) + + // Create the asynchronous task future. + let (task, _) = Builtin.createAsyncTask(flags, operation) + + self._task = task +#else + fatalError("Unsupported Swift compiler") +#endif + } +} + +// ==== Detached Tasks --------------------------------------------------------- +@available(SwiftStdlib 5.1, *) +extension Task where Failure == Never { + /// Runs the given nonthrowing operation asynchronously + /// as part of a new top-level task. + /// + /// Don't use a detached task if it's possible + /// to model the operation using structured concurrency features like child tasks. + /// Child tasks inherit the parent task's priority and task-local storage, + /// and canceling a parent task automatically cancels all of its child tasks. + /// You need to handle these considerations manually with a detached task. + /// + /// You need to keep a reference to the detached task + /// if you want to cancel it by calling the `Task.cancel()` method. + /// Discarding your reference to a detached task + /// doesn't implicitly cancel that task, + /// it only makes it impossible for you to explicitly cancel the task. + /// + /// - Parameters: + /// - priority: The priority of the task. + /// - operation: The operation to perform. + /// + /// - Returns: A reference to the task. + @discardableResult + @_alwaysEmitIntoClient + public static func detached( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> Success + ) -> Task { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + // Set up the job flags for a new task. + let flags = taskCreateFlags( + priority: priority, isChildTask: false, copyTaskLocals: false, + inheritContext: false, enqueueJob: true, + addPendingGroupTaskUnconditionally: false) + + // Create the asynchronous task future. + let (task, _) = Builtin.createAsyncTask(flags, operation) + + return Task(task) +#else + fatalError("Unsupported Swift compiler") +#endif + } +} + +@available(SwiftStdlib 5.1, *) +extension Task where Failure == Error { + /// Runs the given throwing operation asynchronously + /// as part of a new top-level task. + /// + /// If the operation throws an error, this method propagates that error. + /// + /// Don't use a detached task if it's possible + /// to model the operation using structured concurrency features like child tasks. + /// Child tasks inherit the parent task's priority and task-local storage, + /// and canceling a parent task automatically cancels all of its child tasks. + /// You need to handle these considerations manually with a detached task. + /// + /// You need to keep a reference to the detached task + /// if you want to cancel it by calling the `Task.cancel()` method. + /// Discarding your reference to a detached task + /// doesn't implicitly cancel that task, + /// it only makes it impossible for you to explicitly cancel the task. + /// + /// - Parameters: + /// - priority: The priority of the task. + /// - operation: The operation to perform. + /// + /// - Returns: A reference to the task. + @discardableResult + @_alwaysEmitIntoClient + public static func detached( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> Success + ) -> Task { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + // Set up the job flags for a new task. + let flags = taskCreateFlags( + priority: priority, isChildTask: false, copyTaskLocals: false, + inheritContext: false, enqueueJob: true, + addPendingGroupTaskUnconditionally: false + ) + + // Create the asynchronous task future. + let (task, _) = Builtin.createAsyncTask(flags, operation) + + return Task(task) +#else + fatalError("Unsupported Swift compiler") +#endif + } +} + +// ==== Voluntary Suspension ----------------------------------------------------- + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + + /// Suspends the current task and allows other tasks to execute. + /// + /// A task can voluntarily suspend itself + /// in the middle of a long-running operation + /// that doesn't contain any suspension points, + /// to let other tasks run for a while + /// before execution returns to this task. + /// + /// If this task is the highest-priority task in the system, + /// the executor immediately resumes execution of the same task. + /// As such, + /// this method isn't necessarily a way to avoid resource starvation. + public static func yield() async { + return await Builtin.withUnsafeContinuation { (continuation: Builtin.RawUnsafeContinuation) -> Void in + let job = _taskCreateNullaryContinuationJob( + priority: Int(Task.currentPriority.rawValue), + continuation: continuation) + _enqueueJobGlobal(job) + } + } +} + +// ==== UnsafeCurrentTask ------------------------------------------------------ + +/// Calls a closure with an unsafe reference to the current task. +/// +/// If you call this function from the body of an asynchronous function, +/// the unsafe task handle passed to the closure is always non-`nil` +/// because an asynchronous function always runs in the context of a task. +/// However, if you call this function from the body of a synchronous function, +/// and that function isn't executing in the context of any task, +/// the unsafe task handle is `nil`. +/// +/// Don't store an unsafe task reference +/// for use outside this method's closure. +/// Storing an unsafe reference doesn't affect the task's actual life cycle, +/// and the behavior of accessing an unsafe task reference +/// outside of the `withUnsafeCurrentTask(body:)` method's closure isn't defined. +/// There's no safe way to retrieve a reference to the current task +/// and save it for long-term use. +/// To query the current task without saving a reference to it, +/// use properties like `currentPriority`. +/// If you need to store a reference to a task, +/// create an unstructured task using `Task.detached(priority:operation:)` instead. +/// +/// - Parameters: +/// - body: A closure that takes an `UnsafeCurrentTask` parameter. +/// If `body` has a return value, +/// that value is also used as the return value +/// for the `withUnsafeCurrentTask(body:)` function. +/// +/// - Returns: The return value, if any, of the `body` closure. +@available(SwiftStdlib 5.1, *) +public func withUnsafeCurrentTask(body: (UnsafeCurrentTask?) throws -> T) rethrows -> T { + guard let _task = _getCurrentAsyncTask() else { + return try body(nil) + } + + // FIXME: This retain seems pretty wrong, however if we don't we WILL crash + // with "destroying a task that never completed" in the task's destroy. + // How do we solve this properly? + Builtin.retain(_task) + + return try body(UnsafeCurrentTask(_task)) +} + +/// An unsafe reference to the current task. +/// +/// To get an instance of `UnsafeCurrentTask` for the current task, +/// call the `withUnsafeCurrentTask(body:)` method. +/// Don't store an unsafe task reference +/// for use outside that method's closure. +/// Storing an unsafe reference doesn't affect the task's actual life cycle, +/// and the behavior of accessing an unsafe task reference +/// outside of the `withUnsafeCurrentTask(body:)` method's closure isn't defined. +/// +/// Only APIs on `UnsafeCurrentTask` that are also part of `Task` +/// are safe to invoke from a task other than +/// the task that this `UnsafeCurrentTask` instance refers to. +/// Calling other APIs from another task is undefined behavior, +/// breaks invariants in other parts of the program running on this task, +/// and may lead to crashes or data loss. +/// +/// For information about the language-level concurrency model that `UnsafeCurrentTask` is part of, +/// see [Concurrency][concurrency] in [The Swift Programming Language][tspl]. +/// +/// [concurrency]: https://docs.swift.org/swift-book/LanguageGuide/Concurrency.html +/// [tspl]: https://docs.swift.org/swift-book/ +@available(SwiftStdlib 5.1, *) +public struct UnsafeCurrentTask { + internal let _task: Builtin.NativeObject + + // May only be created by the standard library. + internal init(_ task: Builtin.NativeObject) { + self._task = task + } + + /// A Boolean value that indicates whether the current task was canceled. + /// + /// After the value of this property becomes `true`, it remains `true` indefinitely. + /// There is no way to uncancel a task. + /// + /// - SeeAlso: `checkCancellation()` + public var isCancelled: Bool { + _taskIsCancelled(_task) + } + + /// The current task's priority. + /// + /// - SeeAlso: `TaskPriority` + /// - SeeAlso: `Task.currentPriority` + public var priority: TaskPriority { + getJobFlags(_task).priority ?? TaskPriority( + rawValue: UInt8(_getCurrentThreadPriority())) + } + + /// Cancel the current task. + public func cancel() { + _taskCancel(_task) + } +} + +@available(SwiftStdlib 5.1, *) +@available(*, unavailable) +extension UnsafeCurrentTask: Sendable { } + +@available(SwiftStdlib 5.1, *) +extension UnsafeCurrentTask: Hashable { + public func hash(into hasher: inout Hasher) { + UnsafeRawPointer(Builtin.bridgeToRawPointer(_task)).hash(into: &hasher) + } +} + +@available(SwiftStdlib 5.1, *) +extension UnsafeCurrentTask: Equatable { + public static func ==(lhs: Self, rhs: Self) -> Bool { + UnsafeRawPointer(Builtin.bridgeToRawPointer(lhs._task)) == + UnsafeRawPointer(Builtin.bridgeToRawPointer(rhs._task)) + } +} + +// ==== Internal --------------------------------------------------------------- +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_getCurrent") +func _getCurrentAsyncTask() -> Builtin.NativeObject? + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_getJobFlags") +func getJobFlags(_ task: Builtin.NativeObject) -> JobFlags + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_enqueueGlobal") +@usableFromInline +func _enqueueJobGlobal(_ task: Builtin.Job) + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_enqueueGlobalWithDelay") +@usableFromInline +func _enqueueJobGlobalWithDelay(_ delay: UInt64, _ task: Builtin.Job) + +@available(SwiftStdlib 5.1, *) +@usableFromInline +@_silgen_name("swift_task_asyncMainDrainQueue") +internal func _asyncMainDrainQueue() -> Never + +@available(SwiftStdlib 5.1, *) +@usableFromInline +@_silgen_name("swift_task_getMainExecutor") +internal func _getMainExecutor() -> Builtin.Executor + +@available(SwiftStdlib 5.1, *) +public func _runAsyncMain(_ asyncFun: @escaping () async throws -> ()) { + Task.detached { + do { +#if !os(Windows) +#if compiler(>=5.5) && $BuiltinHopToActor + Builtin.hopToActor(MainActor.shared) +#else + fatalError("Swift compiler is incompatible with this SDK version") +#endif +#endif + try await asyncFun() + exit(0) + } catch { + _errorInMain(error) + } + } + _asyncMainDrainQueue() +} + +// FIXME: both of these ought to take their arguments _owned so that +// we can do a move out of the future in the common case where it's +// unreferenced +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_future_wait") +public func _taskFutureGet(_ task: Builtin.NativeObject) async -> T + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_future_wait_throwing") +public func _taskFutureGetThrowing(_ task: Builtin.NativeObject) async throws -> T + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_cancel") +func _taskCancel(_ task: Builtin.NativeObject) + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_isCancelled") +@usableFromInline +func _taskIsCancelled(_ task: Builtin.NativeObject) -> Bool + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_createNullaryContinuationJob") +func _taskCreateNullaryContinuationJob(priority: Int, continuation: Builtin.RawUnsafeContinuation) -> Builtin.Job + +@available(SwiftStdlib 5.1, *) +@usableFromInline +@_silgen_name("swift_task_isCurrentExecutor") +func _taskIsCurrentExecutor(_ executor: Builtin.Executor) -> Bool + +@available(SwiftStdlib 5.1, *) +@usableFromInline +@_silgen_name("swift_task_reportUnexpectedExecutor") +func _reportUnexpectedExecutor(_ _filenameStart: Builtin.RawPointer, + _ _filenameLength: Builtin.Word, + _ _filenameIsASCII: Builtin.Int1, + _ _line: Builtin.Word, + _ _executor: Builtin.Executor) + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_getCurrentThreadPriority") +func _getCurrentThreadPriority() -> Int + +#if _runtime(_ObjC) + +/// Intrinsic used by SILGen to launch a task for bridging a Swift async method +/// which was called through its ObjC-exported completion-handler-based API. +@available(SwiftStdlib 5.1, *) +@_alwaysEmitIntoClient +@usableFromInline +internal func _runTaskForBridgedAsyncMethod(@_inheritActorContext _ body: __owned @Sendable @escaping () async -> Void) { +#if compiler(>=5.6) + Task(operation: body) +#else + Task { + await body() + return 0 + } +#endif +} + +#endif diff --git a/stdlib/public/BackDeployConcurrency/TaskAlloc.cpp b/stdlib/public/BackDeployConcurrency/TaskAlloc.cpp new file mode 100644 index 0000000000000..610fc5b11775b --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskAlloc.cpp @@ -0,0 +1,64 @@ +//===--- TaskAlloc.cpp - Task-local stack allocator -----------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// A task-local allocator that obeys a stack discipline. +// +// Because allocation is task-local, and there's at most one thread +// running a task at once, no synchronization is required. +// +//===----------------------------------------------------------------------===// + +#include "ConcurrencyRuntime.h" +#include "Task.h" +#include "TaskPrivate.h" + +#include + +using namespace swift; + +namespace { + +struct GlobalAllocator { + TaskAllocator allocator; + void *spaceForFirstSlab[64]; + + GlobalAllocator() : allocator(spaceForFirstSlab, sizeof(spaceForFirstSlab)) {} +}; + +} // end anonymous namespace + +static TaskAllocator &allocator(AsyncTask *task) { + if (task) + return task->Private.get().Allocator; + + // FIXME: this fall-back shouldn't be necessary, but it's useful + // for now, since the current execution tests aren't setting up a task + // properly. + static GlobalAllocator global; + return global.allocator; +} + +void *swift::swift_task_alloc(size_t size) { + return allocator(swift_task_getCurrent()).alloc(size); +} + +void *swift::_swift_task_alloc_specific(AsyncTask *task, size_t size) { + return allocator(task).alloc(size); +} + +void swift::swift_task_dealloc(void *ptr) { + allocator(swift_task_getCurrent()).dealloc(ptr); +} + +void swift::_swift_task_dealloc_specific(AsyncTask *task, void *ptr) { + allocator(task).dealloc(ptr); +} diff --git a/stdlib/public/BackDeployConcurrency/TaskCancellation.swift b/stdlib/public/BackDeployConcurrency/TaskCancellation.swift new file mode 100644 index 0000000000000..538b85adee8b6 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskCancellation.swift @@ -0,0 +1,103 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +// ==== Task Cancellation ------------------------------------------------------ + +/// Execute an operation with a cancellation handler that's immediately +/// invoked if the current task is canceled. +/// +/// This differs from the operation cooperatively checking for cancellation +/// and reacting to it in that the cancellation handler is _always_ and +/// _immediately_ invoked when the task is canceled. For example, even if the +/// operation is running code that never checks for cancellation, a cancellation +/// handler still runs and provides a chance to run some cleanup code. +/// +/// Doesn't check for cancellation, and always executes the passed `operation`. +/// +/// This function returns immediately and never suspends. +@available(SwiftStdlib 5.1, *) +public func withTaskCancellationHandler( + operation: () async throws -> T, + onCancel handler: @Sendable () -> Void +) async rethrows -> T { + // unconditionally add the cancellation record to the task. + // if the task was already cancelled, it will be executed right away. + let record = _taskAddCancellationHandler(handler: handler) + defer { _taskRemoveCancellationHandler(record: record) } + + return try await operation() +} + +@available(SwiftStdlib 5.1, *) +extension Task { + /// A Boolean value that indicates whether the task should stop executing. + /// + /// After the value of this property becomes `true`, it remains `true` indefinitely. + /// There is no way to uncancel a task. + /// + /// - SeeAlso: `checkCancellation()` + @_transparent public var isCancelled: Bool { + _taskIsCancelled(_task) + } +} + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + /// A Boolean value that indicates whether the task should stop executing. + /// + /// After the value of this property becomes `true`, it remains `true` indefinitely. + /// There is no way to uncancel a task. + /// + /// - SeeAlso: `checkCancellation()` + public static var isCancelled: Bool { + withUnsafeCurrentTask { task in + task?.isCancelled ?? false + } + } +} + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + /// Throws an error if the task was canceled. + /// + /// The error is always an instance of `CancellationError`. + /// + /// - SeeAlso: `isCancelled()` + public static func checkCancellation() throws { + if Task.isCancelled { + throw _Concurrency.CancellationError() + } + } +} + +/// An error that indicates a task was canceled. +/// +/// This error is also thrown automatically by `Task.checkCancellation()`, +/// if the current task has been canceled. +@available(SwiftStdlib 5.1, *) +public struct CancellationError: Error { + // no extra information, cancellation is intended to be light-weight + public init() {} +} + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_addCancellationHandler") +func _taskAddCancellationHandler(handler: () -> Void) -> UnsafeRawPointer /*CancellationNotificationStatusRecord*/ + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_removeCancellationHandler") +func _taskRemoveCancellationHandler( + record: UnsafeRawPointer /*CancellationNotificationStatusRecord*/ +) diff --git a/stdlib/public/BackDeployConcurrency/TaskGroup.cpp b/stdlib/public/BackDeployConcurrency/TaskGroup.cpp new file mode 100644 index 0000000000000..a0bfd8768f401 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskGroup.cpp @@ -0,0 +1,886 @@ +//===--- TaskGroup.cpp - Task Groups --------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Object management for child tasks that are children of a task group. +// +//===----------------------------------------------------------------------===// + +#include "CompatibilityOverride.h" + +#include "swift/ABI/Metadata.h" +#include "swift/ABI/HeapObject.h" +#include "Task.h" +#include "TaskPrivate.h" +#include "TaskGroup.h" +#include "TaskGroupPrivate.h" +#include "swift/Basic/RelativePointer.h" +#include "swift/Basic/STLExtras.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/Config.h" +#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/HeapObject.h" +#include "Debug.h" +#include "bitset" +#include "string" +#include "queue" // TODO: remove and replace with usage of our mpsc queue +#include +#include +#if SWIFT_CONCURRENCY_ENABLE_DISPATCH +#include +#endif + +#if !defined(_WIN32) +#include +#endif + +using namespace swift; + +/******************************************************************************/ +/*************************** TASK GROUP ***************************************/ +/******************************************************************************/ + +using FutureFragment = AsyncTask::FutureFragment; + +namespace { +class TaskStatusRecord; + +class TaskGroupImpl: public TaskGroupTaskStatusRecord { +public: + /// Describes the status of the group. + enum class ReadyStatus : uintptr_t { + /// The task group is empty, no tasks are pending. + /// Return immediately, there is no point in suspending. + /// + /// The storage is not accessible. + Empty = 0b00, + + // not used: 0b01; same value as the PollStatus MustWait, + // which does not make sense for the ReadyStatus + + /// The future has completed with result (of type \c resultType). + Success = 0b10, + + /// The future has completed by throwing an error (an \c Error + /// existential). + Error = 0b11, + }; + + enum class PollStatus : uintptr_t { + /// The group is known to be empty and we can immediately return nil. + Empty = 0b00, + + /// The task has been enqueued to the groups wait queue. + MustWait = 0b01, + + /// The task has completed with result (of type \c resultType). + Success = 0b10, + + /// The task has completed by throwing an error (an \c Error existential). + Error = 0b11, + }; + + /// The result of waiting on the TaskGroupImpl. + struct PollResult { + PollStatus status; // TODO: pack it into storage pointer or not worth it? + + /// Storage for the result of the future. + /// + /// When the future completed normally, this is a pointer to the storage + /// of the result value, which lives inside the future task itself. + /// + /// When the future completed by throwing an error, this is the error + /// object itself. + OpaqueValue *storage; + + const Metadata *successType; + + /// The completed task, if necessary to keep alive until consumed by next(). + /// + /// # Important: swift_release + /// If if a task is returned here, the task MUST be swift_released + /// once we are done with it, to balance out the retain made before + /// when the task was enqueued into the ready queue to keep it alive + /// until a next() call eventually picks it up. + AsyncTask *retainedTask; + + bool isStorageAccessible() { + return status == PollStatus::Success || + status == PollStatus::Error || + status == PollStatus::Empty; + } + + static PollResult get(AsyncTask *asyncTask, bool hadErrorResult) { + auto fragment = asyncTask->futureFragment(); + return PollResult{ + /*status*/ hadErrorResult ? + PollStatus::Error : + PollStatus::Success, + /*storage*/ hadErrorResult ? + reinterpret_cast(fragment->getError()) : + fragment->getStoragePtr(), + /*successType*/fragment->getResultType(), + /*task*/ asyncTask + }; + } + }; + + /// An item within the message queue of a group. + struct ReadyQueueItem { + /// Mask used for the low status bits in a message queue item. + static const uintptr_t statusMask = 0x03; + + uintptr_t storage; + + ReadyStatus getStatus() const { + return static_cast(storage & statusMask); + } + + AsyncTask *getTask() const { + return reinterpret_cast(storage & ~statusMask); + } + + static ReadyQueueItem get(ReadyStatus status, AsyncTask *task) { + assert(task == nullptr || task->isFuture()); + return ReadyQueueItem{ + reinterpret_cast(task) | static_cast(status)}; + } + }; + + /// An item within the pending queue. + struct PendingQueueItem { + uintptr_t storage; + + AsyncTask *getTask() const { + return reinterpret_cast(storage); + } + + static ReadyQueueItem get(AsyncTask *task) { + assert(task == nullptr || task->isFuture()); + return ReadyQueueItem{reinterpret_cast(task)}; + } + }; + + struct GroupStatus { + static const uint64_t cancelled = 0b1000000000000000000000000000000000000000000000000000000000000000; + static const uint64_t waiting = 0b0100000000000000000000000000000000000000000000000000000000000000; + + // 31 bits for ready tasks counter + static const uint64_t maskReady = 0b0011111111111111111111111111111110000000000000000000000000000000; + static const uint64_t oneReadyTask = 0b0000000000000000000000000000000010000000000000000000000000000000; + + // 31 bits for pending tasks counter + static const uint64_t maskPending = 0b0000000000000000000000000000000001111111111111111111111111111111; + static const uint64_t onePendingTask = 0b0000000000000000000000000000000000000000000000000000000000000001; + + uint64_t status; + + bool isCancelled() { + return (status & cancelled) > 0; + } + + bool hasWaitingTask() { + return (status & waiting) > 0; + } + + unsigned int readyTasks() { + return (status & maskReady) >> 31; + } + + unsigned int pendingTasks() { + return (status & maskPending); + } + + bool isEmpty() { + return pendingTasks() == 0; + } + + /// Status value decrementing the Ready, Pending and Waiting counters by one. + GroupStatus completingPendingReadyWaiting() { + assert(pendingTasks() && + "can only complete waiting task when pending tasks available"); + assert(readyTasks() && + "can only complete waiting task when ready tasks available"); + assert(hasWaitingTask() && + "can only complete waiting task when waiting task available"); + return GroupStatus{status - waiting - oneReadyTask - onePendingTask}; + } + + GroupStatus completingPendingReady() { + assert(pendingTasks() && + "can only complete waiting task when pending tasks available"); + assert(readyTasks() && + "can only complete waiting task when ready tasks available"); + return GroupStatus{status - oneReadyTask - onePendingTask}; + } + + /// Pretty prints the status, as follows: + /// GroupStatus{ P:{pending tasks} W:{waiting tasks} {binary repr} } + std::string to_string() { + std::string str; + str.append("GroupStatus{ "); + str.append("C:"); // cancelled + str.append(isCancelled() ? "y " : "n "); + str.append("W:"); // has waiting task + str.append(hasWaitingTask() ? "y " : "n "); + str.append("R:"); // ready + str.append(std::to_string(readyTasks())); + str.append(" P:"); // pending + str.append(std::to_string(pendingTasks())); + str.append(" " + std::bitset<64>(status).to_string()); + str.append(" }"); + return str; + } + + /// Initially there are no waiting and no pending tasks. + static const GroupStatus initial() { + return GroupStatus{0}; + }; + }; + + template + class NaiveQueue { + std::queue queue; + + public: + NaiveQueue() = default; + + NaiveQueue(const NaiveQueue &) = delete; + + NaiveQueue &operator=(const NaiveQueue &) = delete; + + NaiveQueue(NaiveQueue &&other) { + queue = std::move(other.queue); + } + + virtual ~NaiveQueue() {} + + bool dequeue(T &output) { + if (queue.empty()) { + return false; + } + output = queue.front(); + queue.pop(); + return true; + } + + void enqueue(const T item) { + queue.push(item); + } + }; + +private: + + // TODO: move to lockless via the status atomic (make readyQueue an mpsc_queue_t) + mutable std::mutex mutex; + + /// Used for queue management, counting number of waiting and ready tasks + std::atomic status; + + /// Queue containing completed tasks offered into this group. + /// + /// The low bits contain the status, the rest of the pointer is the + /// AsyncTask. + NaiveQueue readyQueue; + + /// Single waiting `AsyncTask` currently waiting on `group.next()`, + /// or `nullptr` if no task is currently waiting. + std::atomic waitQueue; + + const Metadata *successType; + + friend class ::swift::AsyncTask; + +public: + explicit TaskGroupImpl(const Metadata *T) + : TaskGroupTaskStatusRecord(), + status(GroupStatus::initial().status), + readyQueue(), + waitQueue(nullptr), successType(T) {} + + TaskGroupTaskStatusRecord *getTaskRecord() { + return reinterpret_cast(this); + } + + /// Destroy the storage associated with the group. + void destroy(); + + bool isEmpty() { + auto oldStatus = GroupStatus{status.load(std::memory_order_relaxed)}; + return oldStatus.pendingTasks() == 0; + } + + bool isCancelled() { + auto oldStatus = GroupStatus{status.load(std::memory_order_relaxed)}; + return oldStatus.isCancelled(); + } + + /// Cancel the task group and all tasks within it. + /// + /// Returns `true` if this is the first time cancelling the group, false otherwise. + bool cancelAll(); + + GroupStatus statusCancel() { + auto old = status.fetch_or(GroupStatus::cancelled, + std::memory_order_relaxed); + return GroupStatus{old}; + } + + /// Returns *assumed* new status, including the just performed +1. + GroupStatus statusMarkWaitingAssumeAcquire() { + auto old = status.fetch_or(GroupStatus::waiting, std::memory_order_acquire); + return GroupStatus{old | GroupStatus::waiting}; + } + + GroupStatus statusRemoveWaiting() { + auto old = status.fetch_and(~GroupStatus::waiting, + std::memory_order_release); + return GroupStatus{old}; + } + + /// Returns *assumed* new status, including the just performed +1. + GroupStatus statusAddReadyAssumeAcquire() { + auto old = status.fetch_add(GroupStatus::oneReadyTask, + std::memory_order_acquire); + auto s = GroupStatus{old + GroupStatus::oneReadyTask}; + assert(s.readyTasks() <= s.pendingTasks()); + return s; + } + + /// Add a single pending task to the status counter. + /// This is used to implement next() properly, as we need to know if there + /// are pending tasks worth suspending/waiting for or not. + /// + /// Note that the group does *not* store child tasks at all, as they are + /// stored in the `TaskGroupTaskStatusRecord` inside the current task, that + /// is currently executing the group. Here we only need the counts of + /// pending/ready tasks. + /// + /// If the `unconditionally` parameter is `true` the operation always successfully + /// adds a pending task, even if the group is cancelled. If the unconditionally + /// flag is `false`, the added pending count will be *reverted* before returning. + /// This is because we will NOT add a task to a cancelled group, unless doing + /// so unconditionally. + /// + /// Returns *assumed* new status, including the just performed +1. + GroupStatus statusAddPendingTaskRelaxed(bool unconditionally) { + auto old = status.fetch_add(GroupStatus::onePendingTask, + std::memory_order_relaxed); + auto s = GroupStatus{old + GroupStatus::onePendingTask}; + + if (!unconditionally && s.isCancelled()) { + // revert that add, it was meaningless + auto o = status.fetch_sub(GroupStatus::onePendingTask, + std::memory_order_relaxed); + s = GroupStatus{o - GroupStatus::onePendingTask}; + } + + return s; + } + + GroupStatus statusLoadRelaxed() { + return GroupStatus{status.load(std::memory_order_relaxed)}; + } + + /// Compare-and-set old status to a status derived from the old one, + /// by simultaneously decrementing one Pending and one Waiting tasks. + /// + /// This is used to atomically perform a waiting task completion. + bool statusCompletePendingReadyWaiting(GroupStatus &old) { + return status.compare_exchange_strong( + old.status, old.completingPendingReadyWaiting().status, + /*success*/ std::memory_order_relaxed, + /*failure*/ std::memory_order_relaxed); + } + + bool statusCompletePendingReady(GroupStatus &old) { + return status.compare_exchange_strong( + old.status, old.completingPendingReady().status, + /*success*/ std::memory_order_relaxed, + /*failure*/ std::memory_order_relaxed); + } + + + /// Offer result of a task into this task group. + /// + /// If possible, and an existing task is already waiting on next(), this will + /// schedule it immediately. If not, the result is enqueued and will be picked + /// up whenever a task calls next() the next time. + void offer(AsyncTask *completed, AsyncContext *context); + + /// Attempt to dequeue ready tasks and complete the waitingTask. + /// + /// If unable to complete the waiting task immediately (with an readily + /// available completed task), either returns an `PollStatus::Empty` + /// result if it is known that no pending tasks in the group, + /// or a `PollStatus::MustWait` result if there are tasks in flight + /// and the waitingTask eventually be woken up by a completion. + PollResult poll(AsyncTask *waitingTask); +}; + +} // end anonymous namespace + +/******************************************************************************/ +/************************ TASK GROUP IMPLEMENTATION ***************************/ +/******************************************************************************/ + +using ReadyQueueItem = TaskGroupImpl::ReadyQueueItem; +using ReadyStatus = TaskGroupImpl::ReadyStatus; +using PollResult = TaskGroupImpl::PollResult; +using PollStatus = TaskGroupImpl::PollStatus; + +static_assert(sizeof(TaskGroupImpl) <= sizeof(TaskGroup) && + alignof(TaskGroupImpl) <= alignof(TaskGroup), + "TaskGroupImpl doesn't fit in TaskGroup"); + +static TaskGroupImpl *asImpl(TaskGroup *group) { + return reinterpret_cast(group); +} + +static TaskGroup *asAbstract(TaskGroupImpl *group) { + return reinterpret_cast(group); +} + +TaskGroupTaskStatusRecord * TaskGroup::getTaskRecord() { + return asImpl(this)->getTaskRecord(); +} + +// ============================================================================= +// ==== initialize ------------------------------------------------------------- + +// Initializes into the preallocated _group an actual TaskGroupImpl. +SWIFT_CC(swift) +static void swift_taskGroup_initializeImpl(TaskGroup *group, const Metadata *T) { + SWIFT_TASK_DEBUG_LOG("creating task group = %p", group); + + TaskGroupImpl *impl = new (group) TaskGroupImpl(T); + auto record = impl->getTaskRecord(); + assert(impl == record && "the group IS the task record"); + + // ok, now that the group actually is initialized: attach it to the task + bool notCancelled = swift_task_addStatusRecord(record); + + // If the task has already been cancelled, reflect that immediately in + // the group status. + if (!notCancelled) impl->statusCancel(); +} + +// ============================================================================= +// ==== add / attachChild ------------------------------------------------------ + +void TaskGroup::addChildTask(AsyncTask *child) { + SWIFT_TASK_DEBUG_LOG("attach child task = %p to group = %p", child, group); + + // The counterpart of this (detachChild) is performed by the group itself, + // when it offers the completed (child) task's value to a waiting task - + // during the implementation of `await group.next()`. + auto groupRecord = asImpl(this)->getTaskRecord(); + groupRecord->attachChild(child); +} + +// ============================================================================= +// ==== destroy ---------------------------------------------------------------- +SWIFT_CC(swift) +static void swift_taskGroup_destroyImpl(TaskGroup *group) { + asImpl(group)->destroy(); +} + +void TaskGroupImpl::destroy() { + SWIFT_TASK_DEBUG_LOG("destroying task group = %p", this); + + // First, remove the group from the task and deallocate the record + swift_task_removeStatusRecord(getTaskRecord()); + + // No need to drain our queue here, as by the time we call destroy, + // all tasks inside the group must have been awaited on already. + // This is done in Swift's withTaskGroup function explicitly. + + // destroy the group's storage + this->~TaskGroupImpl(); +} + +// ============================================================================= +// ==== offer ------------------------------------------------------------------ + +void TaskGroup::offer(AsyncTask *completedTask, AsyncContext *context) { + asImpl(this)->offer(completedTask, context); +} + +bool TaskGroup::isCancelled() { + return asImpl(this)->isCancelled(); +} + +static void fillGroupNextResult(TaskFutureWaitAsyncContext *context, + PollResult result) { + /// Fill in the result value + switch (result.status) { + case PollStatus::MustWait: + assert(false && "filling a waiting status?"); + return; + + case PollStatus::Error: { + context->fillWithError(reinterpret_cast(result.storage)); + return; + } + + case PollStatus::Success: { + // Initialize the result as an Optional. + const Metadata *successType = result.successType; + OpaqueValue *destPtr = context->successResultPointer; + // TODO: figure out a way to try to optimistically take the + // value out of the finished task's future, if there are no + // remaining references to it. + successType->vw_initializeWithCopy(destPtr, result.storage); + successType->vw_storeEnumTagSinglePayload(destPtr, 0, 1); + return; + } + + case PollStatus::Empty: { + // Initialize the result as a nil Optional. + const Metadata *successType = result.successType; + OpaqueValue *destPtr = context->successResultPointer; + successType->vw_storeEnumTagSinglePayload(destPtr, 1, 1); + return; + } + } +} + +void TaskGroupImpl::offer(AsyncTask *completedTask, AsyncContext *context) { + assert(completedTask); + assert(completedTask->isFuture()); + assert(completedTask->hasChildFragment()); + assert(completedTask->hasGroupChildFragment()); + assert(completedTask->groupChildFragment()->getGroup() == asAbstract(this)); + SWIFT_TASK_DEBUG_LOG("offer task %p to group %p", completedTask, this); + + mutex.lock(); // TODO: remove fragment lock, and use status for synchronization + + // Immediately increment ready count and acquire the status + // Examples: + // W:n R:0 P:3 -> W:n R:1 P:3 // no waiter, 2 more pending tasks + // W:n R:0 P:1 -> W:n R:1 P:1 // no waiter, no more pending tasks + // W:n R:0 P:1 -> W:y R:1 P:1 // complete immediately + // W:n R:0 P:1 -> W:y R:1 P:3 // complete immediately, 2 more pending tasks + auto assumed = statusAddReadyAssumeAcquire(); + + auto asyncContextPrefix = reinterpret_cast( + reinterpret_cast(context) - sizeof(FutureAsyncContextPrefix)); + bool hadErrorResult = false; + auto errorObject = asyncContextPrefix->errorResult; + if (errorObject) { + // instead, we need to enqueue this result: + hadErrorResult = true; + } + + // ==== a) has waiting task, so let us complete it right away + if (assumed.hasWaitingTask()) { + auto waitingTask = waitQueue.load(std::memory_order_acquire); + SWIFT_TASK_DEBUG_LOG("group has waiting task = %p, complete with = %p", + waitingTask, completedTask); + while (true) { + // ==== a) run waiting task directly ------------------------------------- + assert(assumed.hasWaitingTask()); + assert(assumed.pendingTasks() && "offered to group with no pending tasks!"); + // We are the "first" completed task to arrive, + // and since there is a task waiting we immediately claim and complete it. + if (waitQueue.compare_exchange_strong( + waitingTask, nullptr, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_acquire) && + statusCompletePendingReadyWaiting(assumed)) { + // Run the task. + auto result = PollResult::get(completedTask, hadErrorResult); + + mutex.unlock(); // TODO: remove fragment lock, and use status for synchronization + + auto waitingContext = + static_cast( + waitingTask->ResumeContext); + + fillGroupNextResult(waitingContext, result); + detachChild(result.retainedTask); + + _swift_tsan_acquire(static_cast(waitingTask)); + + // TODO: allow the caller to suggest an executor + swift_task_enqueueGlobal(waitingTask); + return; + } // else, try again + } + + llvm_unreachable("should have enqueued and returned."); + } + + // ==== b) enqueue completion ------------------------------------------------ + // + // else, no-one was waiting (yet), so we have to instead enqueue to the message + // queue when a task polls during next() it will notice that we have a value + // ready for it, and will process it immediately without suspending. + assert(!waitQueue.load(std::memory_order_relaxed)); + SWIFT_TASK_DEBUG_LOG("group has no waiting tasks, RETAIN and store ready task = %p", + completedTask); + // Retain the task while it is in the queue; + // it must remain alive until the task group is alive. + swift_retain(completedTask); + + auto readyItem = ReadyQueueItem::get( + hadErrorResult ? ReadyStatus::Error : ReadyStatus::Success, + completedTask + ); + + assert(completedTask == readyItem.getTask()); + assert(readyItem.getTask()->isFuture()); + readyQueue.enqueue(readyItem); + mutex.unlock(); // TODO: remove fragment lock, and use status for synchronization + return; +} + +SWIFT_CC(swiftasync) +static void +task_group_wait_resume_adapter(SWIFT_ASYNC_CONTEXT AsyncContext *_context) { + + auto context = static_cast(_context); + auto resumeWithError = + reinterpret_cast(context->ResumeParent); + return resumeWithError(context->Parent, context->errorResult); +} + +#ifdef __ARM_ARCH_7K__ +__attribute__((noinline)) +SWIFT_CC(swiftasync) static void workaround_function_swift_taskGroup_wait_next_throwingImpl( + OpaqueValue *result, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + TaskGroup *_group, + ThrowingTaskFutureWaitContinuationFunction resumeFunction, + AsyncContext *callContext) { + // Make sure we don't eliminate calls to this function. + asm volatile("" // Do nothing. + : // Output list, empty. + : "r"(result), "r"(callerContext), "r"(_group) // Input list. + : // Clobber list, empty. + ); + return; +} +#endif + +// ============================================================================= +// ==== group.next() implementation (wait_next and groupPoll) ------------------ + +SWIFT_CC(swiftasync) +static void swift_taskGroup_wait_next_throwingImpl( + OpaqueValue *resultPointer, SWIFT_ASYNC_CONTEXT AsyncContext *callerContext, + TaskGroup *_group, + ThrowingTaskFutureWaitContinuationFunction *resumeFunction, + AsyncContext *rawContext) { + auto waitingTask = swift_task_getCurrent(); + waitingTask->ResumeTask = task_group_wait_resume_adapter; + waitingTask->ResumeContext = rawContext; + + auto context = static_cast(rawContext); + context->ResumeParent = + reinterpret_cast(resumeFunction); + context->Parent = callerContext; + context->errorResult = nullptr; + context->successResultPointer = resultPointer; + + auto group = asImpl(_group); + assert(group && "swift_taskGroup_wait_next_throwing was passed context without group!"); + + PollResult polled = group->poll(waitingTask); + switch (polled.status) { + case PollStatus::MustWait: + SWIFT_TASK_DEBUG_LOG("poll group = %p, no ready tasks, waiting task = %p", + group, waitingTask); + // The waiting task has been queued on the channel, + // there were pending tasks so it will be woken up eventually. +#ifdef __ARM_ARCH_7K__ + return workaround_function_swift_taskGroup_wait_next_throwingImpl( + resultPointer, callerContext, _group, resumeFunction, rawContext); +#else + return; +#endif + + case PollStatus::Empty: + case PollStatus::Error: + case PollStatus::Success: + SWIFT_TASK_DEBUG_LOG("poll group = %p, task = %p, ready task available = %p", + group, waitingTask, polled.retainedTask); + fillGroupNextResult(context, polled); + if (auto completedTask = polled.retainedTask) { + // it would be null for PollStatus::Empty, then we don't need to release + group->detachChild(polled.retainedTask); + swift_release(polled.retainedTask); + } + + return waitingTask->runInFullyEstablishedContext(); + } +} + +PollResult TaskGroupImpl::poll(AsyncTask *waitingTask) { + mutex.lock(); // TODO: remove group lock, and use status for synchronization + SWIFT_TASK_DEBUG_LOG("poll group = %p", this); + auto assumed = statusMarkWaitingAssumeAcquire(); + + PollResult result; + result.storage = nullptr; + result.successType = nullptr; + result.retainedTask = nullptr; + + // ==== 1) bail out early if no tasks are pending ---------------------------- + if (assumed.isEmpty()) { + SWIFT_TASK_DEBUG_LOG("poll group = %p, group is empty, no pending tasks", this); + // No tasks in flight, we know no tasks were submitted before this poll + // was issued, and if we parked here we'd potentially never be woken up. + // Bail out and return `nil` from `group.next()`. + statusRemoveWaiting(); + result.status = PollStatus::Empty; + result.successType = this->successType; + mutex.unlock(); // TODO: remove group lock, and use status for synchronization + return result; + } + + // Have we suspended the task? + bool hasSuspended = false; + + auto waitHead = waitQueue.load(std::memory_order_acquire); + + // ==== 2) Ready task was polled, return with it immediately ----------------- + if (assumed.readyTasks()) { + SWIFT_TASK_DEBUG_LOG("poll group = %p, group has ready tasks = %d", + this, assumed.readyTasks()); + + auto assumedStatus = assumed.status; + auto newStatus = TaskGroupImpl::GroupStatus{assumedStatus}; + if (status.compare_exchange_strong( + assumedStatus, newStatus.completingPendingReadyWaiting().status, + /*success*/ std::memory_order_relaxed, + /*failure*/ std::memory_order_acquire)) { + + // Success! We are allowed to poll. + ReadyQueueItem item; + bool taskDequeued = readyQueue.dequeue(item); + assert(taskDequeued); (void) taskDequeued; + + // We're going back to running the task, so if we suspended before, + // we need to flag it as running again. + if (hasSuspended) { + waitingTask->flagAsRunning(); + } + + assert(item.getTask()->isFuture()); + auto futureFragment = item.getTask()->futureFragment(); + + // Store the task in the result, so after we're done processing it may + // be swift_release'd; we kept it alive while it was in the readyQueue by + // an additional retain issued as we enqueued it there. + result.retainedTask = item.getTask(); + switch (item.getStatus()) { + case ReadyStatus::Success: + // Immediately return the polled value + result.status = PollStatus::Success; + result.storage = futureFragment->getStoragePtr(); + result.successType = futureFragment->getResultType(); + assert(result.retainedTask && "polled a task, it must be not null"); + _swift_tsan_acquire(static_cast(result.retainedTask)); + mutex.unlock(); // TODO: remove fragment lock, and use status for synchronization + return result; + + case ReadyStatus::Error: + // Immediately return the polled value + result.status = PollStatus::Error; + result.storage = + reinterpret_cast(futureFragment->getError()); + result.successType = nullptr; + assert(result.retainedTask && "polled a task, it must be not null"); + _swift_tsan_acquire(static_cast(result.retainedTask)); + mutex.unlock(); // TODO: remove fragment lock, and use status for synchronization + return result; + + case ReadyStatus::Empty: + result.status = PollStatus::Empty; + result.storage = nullptr; + result.retainedTask = nullptr; + result.successType = this->successType; + mutex.unlock(); // TODO: remove fragment lock, and use status for synchronization + return result; + } + assert(false && "must return result when status compare-and-swap was successful"); + } // else, we failed status-cas (some other waiter claimed a ready pending task, try again) + } + + // ==== 3) Add to wait queue ------------------------------------------------- + assert(assumed.readyTasks() == 0); + _swift_tsan_release(static_cast(waitingTask)); + while (true) { + if (!hasSuspended) { + hasSuspended = true; + waitingTask->flagAsSuspended(); + } + // Put the waiting task at the beginning of the wait queue. + if (waitQueue.compare_exchange_strong( + waitHead, waitingTask, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_acquire)) { + mutex.unlock(); // TODO: remove fragment lock, and use status for synchronization + // no ready tasks, so we must wait. + result.status = PollStatus::MustWait; + _swift_task_clearCurrent(); + return result; + } // else, try again + } +} + +// ============================================================================= +// ==== isEmpty ---------------------------------------------------------------- +SWIFT_CC(swift) +static bool swift_taskGroup_isEmptyImpl(TaskGroup *group) { + return asImpl(group)->isEmpty(); +} + +// ============================================================================= +// ==== isCancelled ------------------------------------------------------------ +SWIFT_CC(swift) +static bool swift_taskGroup_isCancelledImpl(TaskGroup *group) { + return asImpl(group)->isCancelled(); +} + +// ============================================================================= +// ==== cancelAll -------------------------------------------------------------- +SWIFT_CC(swift) +static void swift_taskGroup_cancelAllImpl(TaskGroup *group) { + asImpl(group)->cancelAll(); +} + +bool TaskGroupImpl::cancelAll() { + SWIFT_TASK_DEBUG_LOG("cancel all tasks in group = %p", this); + + // store the cancelled bit + auto old = statusCancel(); + if (old.isCancelled()) { + // already was cancelled previously, nothing to do? + return false; + } + + // FIXME: must also remove the records!!!! + // cancel all existing tasks within the group + swift_task_cancel_group_child_tasks(asAbstract(this)); + return true; +} + +// ============================================================================= +// ==== addPending ------------------------------------------------------------- +SWIFT_CC(swift) +static bool swift_taskGroup_addPendingImpl(TaskGroup *group, bool unconditionally) { + auto assumedStatus = asImpl(group)->statusAddPendingTaskRelaxed(unconditionally); + return !assumedStatus.isCancelled(); +} + +#define OVERRIDE_TASK_GROUP COMPATIBILITY_OVERRIDE +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH diff --git a/stdlib/public/BackDeployConcurrency/TaskGroup.h b/stdlib/public/BackDeployConcurrency/TaskGroup.h new file mode 100644 index 0000000000000..847545969e86c --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskGroup.h @@ -0,0 +1,57 @@ +//===--- TaskGroup.h - ABI structures for task groups -00--------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing task groups. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_TASK_GROUP_BACKDEPLOYED_H +#define SWIFT_ABI_TASK_GROUP_BACKDEPLOYED_H + +#include "swift/ABI/HeapObject.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/Config.h" +#include "swift/Basic/RelativePointer.h" +#include "swift/Basic/STLExtras.h" +#include "Task.h" +#include "TaskStatus.h" + +namespace swift { + +/// The task group is responsible for maintaining dynamically created child tasks. +class alignas(Alignment_TaskGroup) TaskGroup { +public: + // These constructors do not initialize the group instance, and the + // destructor does not destroy the group instance; you must call + // swift_taskGroup_{initialize,destroy} yourself. + constexpr TaskGroup() + : PrivateData{} {} + + void *PrivateData[NumWords_TaskGroup]; + + /// Upon a future task's completion, offer it to the task group it belongs to. + void offer(AsyncTask *completed, AsyncContext *context); + + /// Checks the cancellation status of the group. + bool isCancelled(); + + // Add a child task to the group. Always called with the status record lock of + // the parent task held + void addChildTask(AsyncTask *task); + + // Provide accessor for task group's status record + TaskGroupTaskStatusRecord *getTaskRecord(); +}; + +} // end namespace swift + +#endif // SWIFT_ABI_TASK_GROUP_H diff --git a/stdlib/public/BackDeployConcurrency/TaskGroup.swift b/stdlib/public/BackDeployConcurrency/TaskGroup.swift new file mode 100644 index 0000000000000..a10a20571fe52 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskGroup.swift @@ -0,0 +1,919 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +// ==== TaskGroup -------------------------------------------------------------- + +/// Starts a new scope that can contain a dynamic number of child tasks. +/// +/// A group waits for all of its child tasks +/// to complete or be canceled before it returns. +/// After this function returns, the task group is always empty. +/// +/// To collect the results of the group's child tasks, +/// you can use a `for`-`await`-`in` loop: +/// +/// var sum = 0 +/// for await result in group { +/// sum += result +/// } +/// +/// If you need more control or only a few results, +/// you can call `next()` directly: +/// +/// guard let first = await group.next() else { +/// group.cancelAll() +/// return 0 +/// } +/// let second = await group.next() ?? 0 +/// group.cancelAll() +/// return first + second +/// +/// Task Group Cancellation +/// ======================= +/// +/// You can cancel a task group and all of its child tasks +/// by calling the `cancellAll()` method on the task group, +/// or by canceling the task in which the group is running. +/// +/// If you call `async(priority:operation:)` to create a new task in a canceled group, +/// that task is immediately canceled after creation. +/// Alternatively, you can call `asyncUnlessCancelled(priority:operation:)`, +/// which doesn't create the task if the group has already been canceled +/// Choosing between these two functions +/// lets you control how to react to cancellation within a group: +/// some child tasks need to run regardless of cancellation, +/// but other tasks are better not even being created +/// when you know they can't produce useful results. +/// +/// Because the tasks you add to a group with this method are nonthrowing, +/// those tasks can't respond to cancellation by throwing `CancellationError`. +/// The tasks must handle cancellation in some other way, +/// such as returning the work completed so far, returning an empty result, or returning `nil`. +/// For tasks that need to handle cancellation by throwing an error, +/// use the `withThrowingTaskGroup(of:returning:body:)` method instead. +@available(SwiftStdlib 5.1, *) +@_silgen_name("$ss13withTaskGroup2of9returning4bodyq_xm_q_mq_ScGyxGzYaXEtYar0_lF") +@inlinable +public func withTaskGroup( + of childTaskResultType: ChildTaskResult.Type, + returning returnType: GroupResult.Type = GroupResult.self, + body: (inout TaskGroup) async -> GroupResult +) async -> GroupResult { + #if compiler(>=5.5) && $BuiltinTaskGroupWithArgument + + let _group = Builtin.createTaskGroup(ChildTaskResult.self) + var group = TaskGroup(group: _group) + + // Run the withTaskGroup body. + let result = await body(&group) + + await group.awaitAllRemainingTasks() + + Builtin.destroyTaskGroup(_group) + return result + + #else + fatalError("Swift compiler is incompatible with this SDK version") + #endif +} + +/// Starts a new scope that can contain a dynamic number of throwing child tasks. +/// +/// A group waits for all of its child tasks +/// to complete, throw an error, or be canceled before it returns. +/// After this function returns, the task group is always empty. +/// +/// To collect the results of the group's child tasks, +/// you can use a `for`-`await`-`in` loop: +/// +/// var sum = 0 +/// for await result in group { +/// sum += result +/// } +/// +/// If you need more control or only a few results, +/// you can call `next()` directly: +/// +/// guard let first = await group.next() else { +/// group.cancelAll() +/// return 0 +/// } +/// let second = await group.next() ?? 0 +/// group.cancelAll() +/// return first + second +/// +/// Task Group Cancellation +/// ======================= +/// +/// You can cancel a task group and all of its child tasks +/// by calling the `cancellAll()` method on the task group, +/// or by canceling the task in which the group is running. +/// +/// If you call `async(priority:operation:)` to create a new task in a canceled group, +/// that task is immediately canceled after creation. +/// Alternatively, you can call `asyncUnlessCancelled(priority:operation:)`, +/// which doesn't create the task if the group has already been canceled +/// Choosing between these two functions +/// lets you control how to react to cancellation within a group: +/// some child tasks need to run regardless of cancellation, +/// but other tasks are better not even being created +/// when you know they can't produce useful results. +/// +/// Throwing an error in one of the tasks of a task group +/// doesn't immediately cancel the other tasks in that group. +/// However, +/// if you call `next()` in the task group and propagate its error, +/// all other tasks are canceled. +/// For example, in the code below, +/// nothing is canceled and the group doesn't throw an error: +/// +/// withThrowingTaskGroup { group in +/// group.addTask { throw SomeError() } +/// } +/// +/// In contrast, this example throws `SomeError` +/// and cancels all of the tasks in the group: +/// +/// withThrowingTaskGroup { group in +/// group.addTask { throw SomeError() } +/// try group.next() +/// } +/// +/// An individual task throws its error +/// in the corresponding call to `Group.next()`, +/// which gives you a chance to handle the individual error +/// or to let the group rethrow the error. +@available(SwiftStdlib 5.1, *) +@_silgen_name("$ss21withThrowingTaskGroup2of9returning4bodyq_xm_q_mq_Scgyxs5Error_pGzYaKXEtYaKr0_lF") +@inlinable +public func withThrowingTaskGroup( + of childTaskResultType: ChildTaskResult.Type, + returning returnType: GroupResult.Type = GroupResult.self, + body: (inout ThrowingTaskGroup) async throws -> GroupResult +) async rethrows -> GroupResult { + #if compiler(>=5.5) && $BuiltinTaskGroupWithArgument + + let _group = Builtin.createTaskGroup(ChildTaskResult.self) + var group = ThrowingTaskGroup(group: _group) + + do { + // Run the withTaskGroup body. + let result = try await body(&group) + + await group.awaitAllRemainingTasks() + Builtin.destroyTaskGroup(_group) + + return result + } catch { + group.cancelAll() + + await group.awaitAllRemainingTasks() + Builtin.destroyTaskGroup(_group) + + throw error + } + + #else + fatalError("Swift compiler is incompatible with this SDK version") + #endif +} + +/// A group that contains dynamically created child tasks. +/// +/// To create a task group, +/// call the `withTaskGroup(of:returning:body:)` method. +/// +/// Don't use a task group from outside the task where you created it. +/// In most cases, +/// the Swift type system prevents a task group from escaping like that +/// because adding a child task to a task group is a mutating operation, +/// and mutation operations can't be performed +/// from a concurrent execution context like a child task. +/// +/// For information about the language-level concurrency model that `TaskGroup` is part of, +/// see [Concurrency][concurrency] in [The Swift Programming Language][tspl]. +/// +/// [concurrency]: https://docs.swift.org/swift-book/LanguageGuide/Concurrency.html +/// [tspl]: https://docs.swift.org/swift-book/ +/// +@available(SwiftStdlib 5.1, *) +@frozen +public struct TaskGroup { + + /// Group task into which child tasks offer their results, + /// and the `next()` function polls those results from. + @usableFromInline + internal let _group: Builtin.RawPointer + + // No public initializers + @inlinable + init(group: Builtin.RawPointer) { + self._group = group + } + + /// Adds a child task to the group. + /// + /// - Parameters: + /// - overridingPriority: The priority of the operation task. + /// Omit this parameter or pass `.unspecified` + /// to set the child task's priority to the priority of the group. + /// - operation: The operation to execute as part of the task group. + @_alwaysEmitIntoClient + public mutating func addTask( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> ChildTaskResult + ) { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + let flags = taskCreateFlags( + priority: priority, isChildTask: true, copyTaskLocals: false, + inheritContext: false, enqueueJob: true, + addPendingGroupTaskUnconditionally: true + ) + + // Create the task in this group. + _ = Builtin.createAsyncTaskInGroup(flags, _group, operation) +#else + fatalError("Unsupported Swift compiler") +#endif + } + + /// Adds a child task to the group, unless the group has been canceled. + /// + /// - Parameters: + /// - overridingPriority: The priority of the operation task. + /// Omit this parameter or pass `.unspecified` + /// to set the child task's priority to the priority of the group. + /// - operation: The operation to execute as part of the task group. + /// - Returns: `true` if the child task was added to the group; + /// otherwise `false`. + @_alwaysEmitIntoClient + public mutating func addTaskUnlessCancelled( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async -> ChildTaskResult + ) -> Bool { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + let canAdd = _taskGroupAddPendingTask(group: _group, unconditionally: false) + + guard canAdd else { + // the group is cancelled and is not accepting any new work + return false + } + + let flags = taskCreateFlags( + priority: priority, isChildTask: true, copyTaskLocals: false, + inheritContext: false, enqueueJob: true, + addPendingGroupTaskUnconditionally: false + ) + + // Create the task in this group. + _ = Builtin.createAsyncTaskInGroup(flags, _group, operation) + + return true +#else + fatalError("Unsupported Swift compiler") +#endif + } + + /// Wait for the next child task to complete, + /// and return the value it returned. + /// + /// The values returned by successive calls to this method + /// appear in the order that the tasks *completed*, + /// not in the order that those tasks were added to the task group. + /// For example: + /// + /// group.addTask { 1 } + /// group.addTask { 2 } + /// + /// print(await group.next()) + /// // Prints either "2" or "1". + /// + /// If there aren't any pending tasks in the task group, + /// this method returns `nil`, + /// which lets you write the following + /// to wait for a single task to complete: + /// + /// if let first = try await group.next() { + /// return first + /// } + /// + /// It also lets you write code like the following + /// to wait for all the child tasks to complete, + /// collecting the values they returned: + /// + /// while let first = try await group.next() { + /// collected += value + /// } + /// return collected + /// + /// Awaiting on an empty group + /// immediate returns `nil` without suspending. + /// + /// You can also use a `for`-`await`-`in` loop to collect results of a task group: + /// + /// for await try value in group { + /// collected += value + /// } + /// + /// Don't call this method from outside the task + /// where you created this task group. + /// In most cases, the Swift type system prevents this mistake. + /// For example, because the `add(priority:operation:)` method is mutating, + /// that method can't be called from a concurrent execution context like a child task. + /// + /// - Returns: The value returned by the next child task that completes. + public mutating func next() async -> ChildTaskResult? { + // try!-safe because this function only exists for Failure == Never, + // and as such, it is impossible to spawn a throwing child task. + return try! await _taskGroupWaitNext(group: _group) + } + + /// Await all of the remaining tasks on this group. + @usableFromInline + internal mutating func awaitAllRemainingTasks() async { + while let _ = await next() {} + } + + /// Wait for all of the group's remaining tasks to complete. + @_alwaysEmitIntoClient + public mutating func waitForAll() async { + await awaitAllRemainingTasks() + } + + /// A Boolean value that indicates whether the group has any remaining tasks. + /// + /// At the start of the body of a `withTaskGroup(of:returning:body:)` call, + /// the task group is always empty. + /// It`s guaranteed to be empty when returning from that body + /// because a task group waits for all child tasks to complete before returning. + /// + /// - Returns: `true` if the group has no pending tasks; otherwise `false`. + public var isEmpty: Bool { + _taskGroupIsEmpty(_group) + } + + /// Cancel all of the remaining tasks in the group. + /// + /// After cancellation, + /// any new results from the tasks in this group + /// are silently discarded. + /// + /// If you add a task to a group after canceling the group, + /// that task is canceled immediately after being added to the group. + /// + /// This method can only be called by the parent task that created the task + /// group. + /// + /// - SeeAlso: `Task.isCancelled` + /// - SeeAlso: `TaskGroup.isCancelled` + public func cancelAll() { + _taskGroupCancelAll(group: _group) + } + + /// A Boolean value that indicates whether the group was canceled. + /// + /// To cancel a group, call the `TaskGroup.cancelAll()` method. + /// + /// If the task that's currently running this group is canceled, + /// the group is also implicitly canceled, + /// which is also reflected in this property's value. + public var isCancelled: Bool { + return _taskGroupIsCancelled(group: _group) + } +} + +@available(SwiftStdlib 5.1, *) +@available(*, unavailable) +extension TaskGroup: Sendable { } + +// Implementation note: +// We are unable to just™ abstract over Failure == Error / Never because of the +// complicated relationship between `group.spawn` which dictates if `group.next` +// AND the AsyncSequence conformances would be throwing or not. +// +// We would be able to abstract over TaskGroup<..., Failure> equal to Never +// or Error, and specifically only add the `spawn` and `next` functions for +// those two cases. However, we are not able to conform to AsyncSequence "twice" +// depending on if the Failure is Error or Never, as we'll hit: +// conflicting conformance of 'TaskGroup' to protocol +// 'AsyncSequence'; there cannot be more than one conformance, even with +// different conditional bounds +// So, sadly we're forced to duplicate the entire implementation of TaskGroup +// to TaskGroup and ThrowingTaskGroup. +// +// The throwing task group is parameterized with failure only because of future +// proofing, in case we'd ever have typed errors, however unlikely this may be. +// Today the throwing task group failure is simply automatically bound to `Error`. + +/// A group that contains throwing, dynamically created child tasks. +/// +/// To create a throwing task group, +/// call the `withThrowingTaskGroup(of:returning:body:)` method. +/// +/// Don't use a task group from outside the task where you created it. +/// In most cases, +/// the Swift type system prevents a task group from escaping like that +/// because adding a child task to a task group is a mutating operation, +/// and mutation operations can't be performed +/// from concurrent execution contexts like a child task. +/// +/// For information about the language-level concurrency model that `ThrowingTaskGroup` is part of, +/// see [Concurrency][concurrency] in [The Swift Programming Language][tspl]. +/// +/// [concurrency]: https://docs.swift.org/swift-book/LanguageGuide/Concurrency.html +/// [tspl]: https://docs.swift.org/swift-book/ +/// +@available(SwiftStdlib 5.1, *) +@frozen +public struct ThrowingTaskGroup { + + /// Group task into which child tasks offer their results, + /// and the `next()` function polls those results from. + @usableFromInline + internal let _group: Builtin.RawPointer + + // No public initializers + @inlinable + init(group: Builtin.RawPointer) { + self._group = group + } + + /// Await all the remaining tasks on this group. + @usableFromInline + internal mutating func awaitAllRemainingTasks() async { + while true { + do { + guard let _ = try await next() else { + return + } + } catch {} + } + } + + @usableFromInline + internal mutating func _waitForAll() async throws { + while let _ = try await next() { } + } + + /// Wait for all of the group's remaining tasks to complete. + @_alwaysEmitIntoClient + public mutating func waitForAll() async throws { + while let _ = try await next() { } + } + + /// Adds a child task to the group. + /// + /// This method doesn't throw an error, even if the child task does. + /// Instead, the corresponding call to `ThrowingTaskGroup.next()` rethrows that error. + /// + /// - overridingPriority: The priority of the operation task. + /// Omit this parameter or pass `.unspecified` + /// to set the child task's priority to the priority of the group. + /// - operation: The operation to execute as part of the task group. + @_alwaysEmitIntoClient + public mutating func addTask( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> ChildTaskResult + ) { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + let flags = taskCreateFlags( + priority: priority, isChildTask: true, copyTaskLocals: false, + inheritContext: false, enqueueJob: true, + addPendingGroupTaskUnconditionally: true + ) + + // Create the task in this group. + _ = Builtin.createAsyncTaskInGroup(flags, _group, operation) +#else + fatalError("Unsupported Swift compiler") +#endif + } + + /// Adds a child task to the group, unless the group has been canceled. + /// + /// This method doesn't throw an error, even if the child task does. + /// Instead, the corresponding call to `ThrowingTaskGroup.next()` rethrows that error. + /// + /// - Parameters: + /// - overridingPriority: The priority of the operation task. + /// Omit this parameter or pass `.unspecified` + /// to set the child task's priority to the priority of the group. + /// - operation: The operation to execute as part of the task group. + /// - Returns: `true` if the child task was added to the group; + /// otherwise `false`. + @_alwaysEmitIntoClient + public mutating func addTaskUnlessCancelled( + priority: TaskPriority? = nil, + operation: __owned @Sendable @escaping () async throws -> ChildTaskResult + ) -> Bool { +#if compiler(>=5.5) && $BuiltinCreateAsyncTaskInGroup + let canAdd = _taskGroupAddPendingTask(group: _group, unconditionally: false) + + guard canAdd else { + // the group is cancelled and is not accepting any new work + return false + } + + let flags = taskCreateFlags( + priority: priority, isChildTask: true, copyTaskLocals: false, + inheritContext: false, enqueueJob: true, + addPendingGroupTaskUnconditionally: false + ) + + // Create the task in this group. + _ = Builtin.createAsyncTaskInGroup(flags, _group, operation) + + return true +#else + fatalError("Unsupported Swift compiler") +#endif + } + + /// Wait for the next child task to complete, + /// and return the value it returned or rethrow the error it threw. + /// + /// The values returned by successive calls to this method + /// appear in the order that the tasks *completed*, + /// not in the order that those tasks were added to the task group. + /// For example: + /// + /// group.addTask { 1 } + /// group.addTask { 2 } + /// + /// print(await group.next()) + /// // Prints either "2" or "1". + /// + /// If there aren't any pending tasks in the task group, + /// this method returns `nil`, + /// which lets you write the following + /// to wait for a single task to complete: + /// + /// if let first = try await group.next() { + /// return first + /// } + /// + /// It also lets you write code like the following + /// to wait for all the child tasks to complete, + /// collecting the values they returned: + /// + /// while let first = try await group.next() { + /// collected += value + /// } + /// return collected + /// + /// Awaiting on an empty group + /// immediate returns `nil` without suspending. + /// + /// You can also use a `for`-`await`-`in` loop to collect results of a task group: + /// + /// for await try value in group { + /// collected += value + /// } + /// + /// If the next child task throws an error + /// and you propagate that error from this method + /// out of the body of a call to the + /// `ThrowingTaskGroup.withThrowingTaskGroup(of:returning:body:)` method, + /// then all remaining child tasks in that group are implicitly canceled. + /// + /// Don't call this method from outside the task + /// where this task group was created. + /// In most cases, the Swift type system prevents this mistake; + /// for example, because the `add(priority:operation:)` method is mutating, + /// that method can't be called from a concurrent execution context like a child task. + /// + /// - Returns: The value returned by the next child task that completes. + /// + /// - Throws: The error thrown by the next child task that completes. + /// + /// - SeeAlso: `nextResult()` + public mutating func next() async throws -> ChildTaskResult? { + return try await _taskGroupWaitNext(group: _group) + } + + @_silgen_name("$sScg10nextResults0B0Oyxq_GSgyYaKF") + @usableFromInline + mutating func nextResultForABI() async throws -> Result? { + do { + guard let success: ChildTaskResult = try await _taskGroupWaitNext(group: _group) else { + return nil + } + + return .success(success) + } catch { + return .failure(error as! Failure) // as!-safe, because we are only allowed to throw Failure (Error) + } + } + + /// Wait for the next child task to complete, + /// and return a result containing either + /// the value that the child task returned or the error that it threw. + /// + /// The values returned by successive calls to this method + /// appear in the order that the tasks *completed*, + /// not in the order that those tasks were added to the task group. + /// For example: + /// + /// group.addTask { 1 } + /// group.addTask { 2 } + /// + /// guard let result = await group.nextResult() else { + /// return // No task to wait on, which won't happen in this example. + /// } + /// + /// switch result { + /// case .success(let value): print(value) + /// case .failure(let error): print("Failure: \(error)") + /// } + /// // Prints either "2" or "1". + /// + /// If the next child task throws an error + /// and you propagate that error from this method + /// out of the body of a call to the + /// `ThrowingTaskGroup.withThrowingTaskGroup(of:returning:body:)` method, + /// then all remaining child tasks in that group are implicitly canceled. + /// + /// - Returns: A `Result.success` value + /// containing the value that the child task returned, + /// or a `Result.failure` value + /// containing the error that the child task threw. + /// + /// - SeeAlso: `next()` + @_alwaysEmitIntoClient + public mutating func nextResult() async -> Result? { + return try! await nextResultForABI() + } + + /// A Boolean value that indicates whether the group has any remaining tasks. + /// + /// At the start of the body of a `withThrowingTaskGroup(of:returning:body:)` call, + /// the task group is always empty. + /// It's guaranteed to be empty when returning from that body + /// because a task group waits for all child tasks to complete before returning. + /// + /// - Returns: `true` if the group has no pending tasks; otherwise `false`. + public var isEmpty: Bool { + _taskGroupIsEmpty(_group) + } + + /// Cancel all of the remaining tasks in the group. + /// + /// After cancellation, + /// any new results or errors from the tasks in this group + /// are silently discarded. + /// + /// If you add a task to a group after canceling the group, + /// that task is canceled immediately after being added to the group. + /// + /// There are no restrictions on where you can call this method. + /// Code inside a child task or even another task can cancel a group. + /// + /// - SeeAlso: `Task.isCancelled` + /// - SeeAlso: `ThrowingTaskGroup.isCancelled` + public func cancelAll() { + _taskGroupCancelAll(group: _group) + } + + /// A Boolean value that indicates whether the group was canceled. + /// + /// To cancel a group, call the `ThrowingTaskGroup.cancelAll()` method. + /// + /// If the task that's currently running this group is canceled, + /// the group is also implicitly canceled, + /// which is also reflected in this property's value. + public var isCancelled: Bool { + return _taskGroupIsCancelled(group: _group) + } +} + +@available(SwiftStdlib 5.1, *) +@available(*, unavailable) +extension ThrowingTaskGroup: Sendable { } + +/// ==== TaskGroup: AsyncSequence ---------------------------------------------- + +@available(SwiftStdlib 5.1, *) +extension TaskGroup: AsyncSequence { + public typealias AsyncIterator = Iterator + public typealias Element = ChildTaskResult + + public func makeAsyncIterator() -> Iterator { + return Iterator(group: self) + } + + /// A type that provides an iteration interface + /// over the results of tasks added to the group. + /// + /// The elements returned by this iterator + /// appear in the order that the tasks *completed*, + /// not in the order that those tasks were added to the task group. + /// + /// This iterator terminates after all tasks have completed. + /// After iterating over the results of each task, + /// it's valid to make a new iterator for the task group, + /// which you can use to iterate over the results of new tasks you add to the group. + /// For example: + /// + /// group.addTask { 1 } + /// for await r in group { print(r) } + /// + /// // Add a new child task and iterate again. + /// group.addTask { 2 } + /// for await r in group { print(r) } + /// + /// - SeeAlso: `TaskGroup.next()` + @available(SwiftStdlib 5.1, *) + public struct Iterator: AsyncIteratorProtocol { + public typealias Element = ChildTaskResult + + @usableFromInline + var group: TaskGroup + + @usableFromInline + var finished: Bool = false + + // no public constructors + init(group: TaskGroup) { + self.group = group + } + + /// Advances to and returns the result of the next child task. + /// + /// The elements returned from this method + /// appear in the order that the tasks *completed*, + /// not in the order that those tasks were added to the task group. + /// After this method returns `nil`, + /// this iterator is guaranteed to never produce more values. + /// + /// For more information about the iteration order and semantics, + /// see `TaskGroup.next()`. + /// + /// - Returns: The value returned by the next child task that completes, + /// or `nil` if there are no remaining child tasks, + public mutating func next() async -> Element? { + guard !finished else { return nil } + guard let element = await group.next() else { + finished = true + return nil + } + return element + } + + public mutating func cancel() { + finished = true + group.cancelAll() + } + } +} + +@available(SwiftStdlib 5.1, *) +extension ThrowingTaskGroup: AsyncSequence { + public typealias AsyncIterator = Iterator + public typealias Element = ChildTaskResult + + public func makeAsyncIterator() -> Iterator { + return Iterator(group: self) + } + + /// A type that provides an iteration interface + /// over the results of tasks added to the group. + /// + /// The elements returned by this iterator + /// appear in the order that the tasks *completed*, + /// not in the order that those tasks were added to the task group. + /// + /// This iterator terminates after all tasks have completed successfully, + /// or after any task completes by throwing an error. + /// If a task completes by throwing an error, + /// it doesn't return any further task results. + /// After iterating over the results of each task, + /// it's valid to make a new iterator for the task group, + /// which you can use to iterate over the results of new tasks you add to the group. + /// You can also make a new iterator to resume iteration + /// after a child task thows an error. + /// For example: + /// + /// group.addTask { 1 } + /// group.addTask { throw SomeError } + /// group.addTask { 2 } + /// + /// do { + /// // Assuming the child tasks complete in order, this prints "1" + /// // and then throws an error. + /// for try await r in group { print(r) } + /// } catch { + /// // Resolve the error. + /// } + /// + /// // Assuming the child tasks complete in order, this prints "2". + /// for try await r in group { print(r) } + /// + /// - SeeAlso: `ThrowingTaskGroup.next()` + @available(SwiftStdlib 5.1, *) + public struct Iterator: AsyncIteratorProtocol { + public typealias Element = ChildTaskResult + + @usableFromInline + var group: ThrowingTaskGroup + + @usableFromInline + var finished: Bool = false + + // no public constructors + init(group: ThrowingTaskGroup) { + self.group = group + } + + /// Advances to and returns the result of the next child task. + /// + /// The elements returned from this method + /// appear in the order that the tasks *completed*, + /// not in the order that those tasks were added to the task group. + /// After this method returns `nil`, + /// this iterator is guaranteed to never produce more values. + /// + /// For more information about the iteration order and semantics, + /// see `ThrowingTaskGroup.next()` + /// + /// - Throws: The error thrown by the next child task that completes. + /// + /// - Returns: The value returned by the next child task that completes, + /// or `nil` if there are no remaining child tasks, + public mutating func next() async throws -> Element? { + guard !finished else { return nil } + do { + guard let element = try await group.next() else { + finished = true + return nil + } + return element + } catch { + finished = true + throw error + } + } + + public mutating func cancel() { + finished = true + group.cancelAll() + } + } +} + +/// ==== ----------------------------------------------------------------------- + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_taskGroup_destroy") +func _taskGroupDestroy(group: __owned Builtin.RawPointer) + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_taskGroup_addPending") +@usableFromInline +func _taskGroupAddPendingTask( + group: Builtin.RawPointer, + unconditionally: Bool +) -> Bool + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_taskGroup_cancelAll") +func _taskGroupCancelAll(group: Builtin.RawPointer) + +/// Checks ONLY if the group was specifically canceled. +/// The task itself being canceled must be checked separately. +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_taskGroup_isCancelled") +func _taskGroupIsCancelled(group: Builtin.RawPointer) -> Bool + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_taskGroup_wait_next_throwing") +func _taskGroupWaitNext(group: Builtin.RawPointer) async throws -> T? + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_hasTaskGroupStatusRecord") +func _taskHasTaskGroupStatusRecord() -> Bool + +@available(SwiftStdlib 5.1, *) +enum PollStatus: Int { + case empty = 0 + case waiting = 1 + case success = 2 + case error = 3 +} + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_taskGroup_isEmpty") +func _taskGroupIsEmpty( + _ group: Builtin.RawPointer +) -> Bool diff --git a/stdlib/public/BackDeployConcurrency/TaskGroupPrivate.h b/stdlib/public/BackDeployConcurrency/TaskGroupPrivate.h new file mode 100644 index 0000000000000..195eaa63fd395 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskGroupPrivate.h @@ -0,0 +1,27 @@ +//===--- TaskGroupPrivate.h - TaskGroup internal interface -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Internal functions for the concurrency library. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_CONCURRENCY_TASKGROUPPRIVATE_H +#define SWIFT_CONCURRENCY_TASKGROUPPRIVATE_H + +#include "Task.h" +#include "TaskGroup.h" + +namespace swift { + +} // end namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/TaskLocal.cpp b/stdlib/public/BackDeployConcurrency/TaskLocal.cpp new file mode 100644 index 0000000000000..31ba53a669931 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskLocal.cpp @@ -0,0 +1,417 @@ +//===--- TaskLocal.cpp - Task Local Values --------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "CompatibilityOverride.h" +#include "swift/Runtime/Atomic.h" +#include "swift/Runtime/Casting.h" +#include "swift/Runtime/Once.h" +#include "swift/Runtime/Mutex.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/ThreadLocal.h" +#include "swift/Runtime/ThreadLocalStorage.h" +#include "swift/ABI/Metadata.h" +#include "llvm/ADT/PointerIntPair.h" +#include "Actor.h" +#include "TaskLocal.h" +#include "TaskLocal.h" +#include "TaskPrivate.h" +#include + +#if defined(__APPLE__) +#include +#elif defined(__ANDROID__) +#include +#endif + +#if HAVE_PTHREAD_H +#include +#endif + +#if defined(_WIN32) +#include +#include +#include +#endif + +using namespace swift; + +// ============================================================================= + +/// An extremely silly class which exists to make pointer +/// default-initialization constexpr. +template struct Pointer { + T *Value; + constexpr Pointer() : Value(nullptr) {} + constexpr Pointer(T *value) : Value(value) {} + operator T *() const { return Value; } + T *operator->() const { return Value; } +}; + +/// THIS IS RUNTIME INTERNAL AND NOT ABI. +class FallbackTaskLocalStorage { + static SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( + Pointer, Value, + SWIFT_CONCURRENCY_FALLBACK_TASK_LOCAL_STORAGE_KEY); + +public: + static void set(TaskLocal::Storage *task) { Value.set(task); } + static TaskLocal::Storage *get() { return Value.get(); } +}; + +/// Define the thread-locals. +SWIFT_RUNTIME_DECLARE_THREAD_LOCAL( + Pointer, FallbackTaskLocalStorage::Value, + SWIFT_CONCURRENCY_FALLBACK_TASK_LOCAL_STORAGE_KEY); + +// ==== ABI -------------------------------------------------------------------- + +SWIFT_CC(swift) +static void swift_task_localValuePushImpl(const HeapObject *key, + /* +1 */ OpaqueValue *value, + const Metadata *valueType) { + if (AsyncTask *task = swift_task_getCurrent()) { + task->localValuePush(key, value, valueType); + return; + } + + // no AsyncTask available so we must check the fallback + TaskLocal::Storage *Local = nullptr; + if (auto storage = FallbackTaskLocalStorage::get()) { + Local = storage; + } else { + void *allocation = malloc(sizeof(TaskLocal::Storage)); + auto *freshStorage = new(allocation) TaskLocal::Storage(); + + FallbackTaskLocalStorage::set(freshStorage); + Local = freshStorage; + } + + Local->pushValue(/*task=*/nullptr, key, value, valueType); +} + +SWIFT_CC(swift) +static OpaqueValue* swift_task_localValueGetImpl(const HeapObject *key) { + if (AsyncTask *task = swift_task_getCurrent()) { + // we're in the context of a task and can use the task's storage + return task->localValueGet(key); + } + + // no AsyncTask available so we must check the fallback + if (auto Local = FallbackTaskLocalStorage::get()) { + return Local->getValue(/*task*/nullptr, key); + } + + // no value found in task-local or fallback thread-local storage. + return nullptr; +} + +SWIFT_CC(swift) +static void swift_task_localValuePopImpl() { + if (AsyncTask *task = swift_task_getCurrent()) { + task->localValuePop(); + return; + } + + if (TaskLocal::Storage *Local = FallbackTaskLocalStorage::get()) { + bool hasRemainingBindings = Local->popValue(nullptr); + if (!hasRemainingBindings) { + // We clean up eagerly, it may be that this non-swift-concurrency thread + // never again will use task-locals, and as such we better remove the storage. + FallbackTaskLocalStorage::set(nullptr); + free(Local); + } + return; + } + + assert(false && "Attempted to pop value but no task or thread-local storage available!"); +} + +SWIFT_CC(swift) +static void swift_task_localsCopyToImpl(AsyncTask *task) { + TaskLocal::Storage *Local = nullptr; + + if (AsyncTask *task = swift_task_getCurrent()) { + Local = &task->_private().Local; + } else if (auto *storage = FallbackTaskLocalStorage::get()) { + Local = storage; + } else { + // bail out, there are no values to copy + return; + } + + Local->copyTo(task); +} + +// ============================================================================= +// ==== Initialization --------------------------------------------------------- + +void TaskLocal::Storage::initializeLinkParent(AsyncTask* task, + AsyncTask* parent) { + assert(!head && "initial task local storage was already initialized"); + assert(parent && "parent must be provided to link to it"); + head = TaskLocal::Item::createParentLink(task, parent); +} + +TaskLocal::Item* +TaskLocal::Item::createParentLink(AsyncTask *task, AsyncTask *parent) { + size_t amountToAllocate = Item::itemSize(/*valueType*/nullptr); + void *allocation = _swift_task_alloc_specific(task, amountToAllocate); + Item *item = new(allocation) Item(); + + auto parentHead = parent->_private().Local.head; + if (parentHead) { + if (parentHead->isEmpty()) { + switch (parentHead->getNextLinkType()) { + case NextLinkType::IsParent: + // it has no values, and just points to its parent, + // therefore skip also skip pointing to that parent and point + // to whichever parent it was pointing to as well, it may be its + // immediate parent, or some super-parent. + item->next = reinterpret_cast(parentHead->getNext()) | + static_cast(NextLinkType::IsParent); + break; + case NextLinkType::IsNext: + if (parentHead->getNext()) { + assert(false && "empty taskValue head in parent task, yet parent's 'head' is `IsNext`, " + "this should not happen, as it implies the parent must have stored some value."); + } else { + // is terminal pointer + item->next = reinterpret_cast(parentHead->getNext()); + } + break; + } + } else { + item->next = reinterpret_cast(parentHead) | + static_cast(NextLinkType::IsParent); + } + } else { + item->next = reinterpret_cast(parentHead); + } + + return item; +} + +TaskLocal::Item* +TaskLocal::Item::createLink(AsyncTask *task, + const HeapObject *key, + const Metadata *valueType) { + size_t amountToAllocate = Item::itemSize(valueType); + void *allocation = task ? _swift_task_alloc_specific(task, amountToAllocate) + : malloc(amountToAllocate); + Item *item = new (allocation) Item(key, valueType); + + auto next = task ? task->_private().Local.head + : FallbackTaskLocalStorage::get()->head; + item->next = reinterpret_cast(next) | + static_cast(NextLinkType::IsNext); + + return item; +} + + +void TaskLocal::Item::copyTo(AsyncTask *target) { + assert(target && "TaskLocal item attempt to copy to null target task!"); + + // 'parent' pointers are signified by null valueType. + // We must not copy parent pointers, but rather perform a deep copy of all values, + // as such, we skip parent pointers here entirely. + if (isParentPointer()) + return; + + auto item = Item::createLink(target, key, valueType); + valueType->vw_initializeWithCopy(item->getStoragePtr(), getStoragePtr()); + + /// A `copyTo` may ONLY be invoked BEFORE the task is actually scheduled, + /// so right now we can safely copy the value into the task without additional + /// synchronization. + target->_private().Local.head = item; +} + +// ============================================================================= +// ==== checks ----------------------------------------------------------------- + +SWIFT_CC(swift) +static void swift_task_reportIllegalTaskLocalBindingWithinWithTaskGroupImpl( + const unsigned char *file, uintptr_t fileLength, + bool fileIsASCII, uintptr_t line) { + + char *message; + swift_asprintf( + &message, + "error: task-local: detected illegal task-local value binding at %.*s:%d.\n" + "Task-local values must only be set in a structured-context, such as: " + "around any (synchronous or asynchronous function invocation), " + "around an 'async let' declaration, or around a 'with(Throwing)TaskGroup(...){ ... }' " + "invocation. Notably, binding a task-local value is illegal *within the body* " + "of a withTaskGroup invocation.\n" + "\n" + "The following example is illegal:\n\n" + " await withTaskGroup(...) { group in \n" + " await .withValue(1234) {\n" + " group.spawn { ... }\n" + " }\n" + " }\n" + "\n" + "And should be replaced by, either: setting the value for the entire group:\n" + "\n" + " // bind task-local for all tasks spawned within the group\n" + " await .withValue(1234) {\n" + " await withTaskGroup(...) { group in\n" + " group.spawn { ... }\n" + " }\n" + " }\n" + "\n" + "or, inside the specific task-group child task:\n" + "\n" + " // bind-task-local for only specific child-task\n" + " await withTaskGroup(...) { group in\n" + " group.spawn {\n" + " await .withValue(1234) {\n" + " ... \n" + " }\n" + " }\n" + "\n" + " group.spawn { ... }\n" + " }\n", + (int)fileLength, file, + (int)line); + + if (_swift_shouldReportFatalErrorsToDebugger()) { + RuntimeErrorDetails details = { + .version = RuntimeErrorDetails::currentVersion, + .errorType = "task-local-violation", + .currentStackDescription = "Task-local bound in illegal context", + .framesToSkip = 1, + }; + _swift_reportToDebugger(RuntimeErrorFlagFatal, message, &details); + } + +#if defined(_WIN32) + #define STDERR_FILENO 2 + _write(STDERR_FILENO, message, strlen(message)); +#else + write(STDERR_FILENO, message, strlen(message)); +#endif +#if defined(__APPLE__) + asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", message); +#elif defined(__ANDROID__) + __android_log_print(ANDROID_LOG_FATAL, "SwiftRuntime", "%s", message); +#endif + + free(message); + abort(); +} + +// ============================================================================= +// ==== destroy ---------------------------------------------------------------- + +void TaskLocal::Item::destroy(AsyncTask *task) { + // otherwise it was task-local allocated, so we can safely destroy it right away + if (valueType) { + valueType->vw_destroy(getStoragePtr()); + } + + // if task is available, we must have used the task allocator to allocate this item, + // so we must deallocate it using the same. Otherwise, we must have used malloc. + if (task) _swift_task_dealloc_specific(task, this); + else free(this); +} + +void TaskLocal::Storage::destroy(AsyncTask *task) { + auto item = head; + head = nullptr; + TaskLocal::Item *next; + while (item) { + auto linkType = item->getNextLinkType(); + switch (linkType) { + case TaskLocal::NextLinkType::IsNext: + next = item->getNext(); + item->destroy(task); + item = next; + break; + + case TaskLocal::NextLinkType::IsParent: + // we're done here; as we must not proceed into the parent owned values. + // we do have to destroy the item pointing at the parent/edge itself though. + item->destroy(task); + return; + } + } +} + +// ============================================================================= +// ==== Task Local Storage: operations ----------------------------------------- + +void TaskLocal::Storage::pushValue(AsyncTask *task, + const HeapObject *key, + /* +1 */ OpaqueValue *value, + const Metadata *valueType) { + assert(value && "Task local value must not be nil"); + + auto item = Item::createLink(task, key, valueType); + valueType->vw_initializeWithTake(item->getStoragePtr(), value); + head = item; +} + +bool TaskLocal::Storage::popValue(AsyncTask *task) { + assert(head && "attempted to pop value off empty task-local stack"); + auto old = head; + head = head->getNext(); + old->destroy(task); + + /// if pointing at not-null next item, there are remaining bindings. + return head != nullptr; +} + +OpaqueValue* TaskLocal::Storage::getValue(AsyncTask *task, + const HeapObject *key) { + assert(key && "TaskLocal key must not be null."); + + auto item = head; + while (item) { + if (item->key == key) { + return item->getStoragePtr(); + } + + item = item->getNext(); + } + + return nullptr; +} + + +void TaskLocal::Storage::copyTo(AsyncTask *target) { + assert(target && "task must not be null when copying values into it"); + assert(!(target->_private().Local.head) && + "Task must not have any task-local values bound before copying into it"); + + // Set of keys for which we already have copied to the new task. + // We only ever need to copy the *first* encounter of any given key, + // because it is the most "specific"/"recent" binding and any other binding + // of a key does not matter for the target task as it will never be able to + // observe it. + std::set copied = {}; + + auto item = head; + while (item) { + // we only have to copy an item if it is the most recent binding of a key. + // i.e. if we've already seen an item for this key, we can skip it. + if (copied.emplace(item->key).second) { + item->copyTo(target); + } + + item = item->getNext(); + } +} + +#define OVERRIDE_TASK_LOCAL COMPATIBILITY_OVERRIDE +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH diff --git a/stdlib/public/BackDeployConcurrency/TaskLocal.h b/stdlib/public/BackDeployConcurrency/TaskLocal.h new file mode 100644 index 0000000000000..cbe17932d6236 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskLocal.h @@ -0,0 +1,224 @@ +//===--- TaskLocal.h - ABI of task local values -----------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing task locals. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_TASKLOCAL_H +#define SWIFT_ABI_TASKLOCAL_H + +#include "swift/ABI/HeapObject.h" +#include "swift/ABI/Metadata.h" +#include "swift/ABI/MetadataValues.h" + +namespace swift { +class AsyncTask; +struct OpaqueValue; +struct SwiftError; +class TaskStatusRecord; +class TaskGroup; + +// ==== Task Locals Values --------------------------------------------------- + +class TaskLocal { +public: + /// Type of the pointed at `next` task local item. + enum class NextLinkType : uintptr_t { + /// The storage pointer points at the next TaskLocal::Item in this task. + IsNext = 0b00, + /// The storage pointer points at a item stored by another AsyncTask. + /// + /// Note that this may not necessarily be the same as the task's parent + /// task -- we may point to a super-parent if we know / that the parent + /// does not "contribute" any task local values. This is to speed up + /// lookups by skipping empty parent tasks during get(), and explained + /// in depth in `createParentLink`. + IsParent = 0b01, + }; + + class Item { + private: + /// Mask used for the low status bits in a task local chain item. + static const uintptr_t statusMask = 0x03; + + /// Pointer to one of the following: + /// - next task local item as OpaqueValue* if it is task-local allocated + /// - next task local item as HeapObject* if it is heap allocated "heavy" + /// - the parent task's TaskLocal::Storage + /// + /// Low bits encode `NextLinkType`, based on which the type of the pointer + /// is determined. + uintptr_t next; + + public: + /// The type of the key with which this value is associated. + const HeapObject *key; + /// The type of the value stored by this item. + const Metadata *valueType; + + // Trailing storage for the value itself. The storage will be + // uninitialized or contain an instance of \c valueType. + + /// Returns true if this item is a 'parent pointer'. + /// + /// A parent pointer is special kind of `Item` is created when pointing at + /// the parent storage, forming a chain of task local items spanning multiple + /// tasks. + bool isParentPointer() const { + return !valueType; + } + + protected: + explicit Item() + : next(0), + key(nullptr), + valueType(nullptr) {} + + explicit Item(const HeapObject *key, const Metadata *valueType) + : next(0), + key(key), + valueType(valueType) {} + + public: + /// Item which does not by itself store any value, but only points + /// to the nearest task-local-value containing parent's first task item. + /// + /// This item type is used to link to the appropriate parent task's item, + /// when the current task itself does not have any task local values itself. + /// + /// When a task actually has its own task locals, it should rather point + /// to the parent's *first* task-local item in its *last* item, extending + /// the Item linked list into the appropriate parent. + static Item *createParentLink(AsyncTask *task, AsyncTask *parent); + + static Item *createLink(AsyncTask *task, + const HeapObject *key, + const Metadata *valueType); + + void destroy(AsyncTask *task); + + Item *getNext() { + return reinterpret_cast(next & ~statusMask); + } + + NextLinkType getNextLinkType() const { + return static_cast(next & statusMask); + } + + /// Item does not contain any actual value, and is only used to point at + /// a specific parent item. + bool isEmpty() const { + return !valueType; + } + + /// Retrieve a pointer to the storage of the value. + OpaqueValue *getStoragePtr() { + return reinterpret_cast( + reinterpret_cast(this) + storageOffset(valueType)); + } + + void copyTo(AsyncTask *task); + + /// Compute the offset of the storage from the base of the item. + static size_t storageOffset(const Metadata *valueType) { + size_t offset = sizeof(Item); + + if (valueType) { + size_t alignment = valueType->vw_alignment(); + return (offset + alignment - 1) & ~(alignment - 1); + } else { + return offset; + } + } + + /// Determine the size of the item given a particular value type. + static size_t itemSize(const Metadata *valueType) { + size_t offset = storageOffset(valueType); + if (valueType) { + offset += valueType->vw_size(); + } + return offset; + } + }; + + class Storage { + friend class TaskLocal::Item; + private: + /// A stack (single-linked list) of task local values. + /// + /// Once task local values within this task are traversed, the list continues + /// to the "next parent that contributes task local values," or if no such + /// parent exists it terminates with null. + /// + /// If the TaskLocalValuesFragment was allocated, it is expected that this + /// value should be NOT null; it either has own values, or at least one + /// parent that has values. If this task does not have any values, the head + /// pointer MAY immediately point at this task's parent task which has values. + /// + /// ### Concurrency + /// Access to the head is only performed from the task itself, when it + /// creates child tasks, the child during creation will inspect its parent's + /// task local value stack head, and point to it. This is done on the calling + /// task, and thus needs not to be synchronized. Subsequent traversal is + /// performed by child tasks concurrently, however they use their own + /// pointers/stack and can never mutate the parent's stack. + /// + /// The stack is only pushed/popped by the owning task, at the beginning and + /// end a `body` block of `withLocal(_:boundTo:body:)` respectively. + /// + /// Correctness of the stack strongly relies on the guarantee that tasks + /// never outline a scope in which they are created. Thanks to this, if + /// tasks are created inside the `body` of `withLocal(_:,boundTo:body:)` + /// all tasks created inside the `withLocal` body must complete before it + /// returns, as such, any child tasks potentially accessing the value stack + /// are guaranteed to be completed by the time we pop values off the stack + /// (after the body has completed). + TaskLocal::Item *head = nullptr; + + public: + + void initializeLinkParent(AsyncTask *task, AsyncTask *parent); + + void pushValue(AsyncTask *task, + const HeapObject *key, + /* +1 */ OpaqueValue *value, const Metadata *valueType); + + OpaqueValue* getValue(AsyncTask *task, const HeapObject *key); + + /// Returns `true` of more bindings remain in this storage, + /// and `false` if the just popped value was the last one and the storage + /// can be safely disposed of. + bool popValue(AsyncTask *task); + + /// Copy all task-local bindings to the target task. + /// + /// The new bindings allocate their own items and can out-live the current task. + /// + /// ### Optimizations + /// Only the most recent binding of a value is copied over, i.e. given + /// a key bound to `A` and then `B`, only the `B` binding will be copied. + /// This is safe and correct because the new task would never have a chance + /// to observe the `A` value, because it semantically will never observe a + /// "pop" of the `B` value - it was spawned from a scope where only B was observable. + void copyTo(AsyncTask *target); + + /// Destroy and deallocate all items stored by this specific task. + /// + /// Items owned by a parent task are left untouched, since we do not own them. + void destroy(AsyncTask *task); + }; +}; + +} // end namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/TaskLocal.swift b/stdlib/public/BackDeployConcurrency/TaskLocal.swift new file mode 100644 index 0000000000000..d5eeb4d89f647 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskLocal.swift @@ -0,0 +1,257 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020-2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +/// Property wrapper that defines a task-local value key. +/// +/// A task-local value is a value that can be bound and read in the context of a +/// `Task`. It is implicitly carried with the task, and is accessible by any +/// child tasks the task creates (such as TaskGroup or `async let` created tasks). +/// +/// ### Task-local declarations +/// +/// Task locals must be declared as static properties (or global properties, +/// once property wrappers support these), like this: +/// +/// enum TracingExample { +/// @TaskLocal +/// static let traceID: TraceID? +/// } +/// +/// ### Default values +/// Task local values of optional types default to `nil`. It is possible to define +/// not-optional task-local values, and an explicit default value must then be +/// defined instead. +/// +/// The default value is returned whenever the task-local is read +/// from a context which either: has no task available to read the value from +/// (e.g. a synchronous function, called without any asynchronous function in its call stack), +/// +/// +/// ### Reading task-local values +/// Reading task local values is simple and looks the same as-if reading a normal +/// static property: +/// +/// guard let traceID = TracingExample.traceID else { +/// print("no trace id") +/// return +/// } +/// print(traceID) +/// +/// It is possible to perform task-local value reads from either asynchronous +/// or synchronous functions. Within asynchronous functions, as a "current" task +/// is always guaranteed to exist, this will perform the lookup in the task local context. +/// +/// A lookup made from the context of a synchronous function, that is not called +/// from an asynchronous function (!), will immediately return the task-local's +/// default value. +/// +/// ### Binding task-local values +/// Task local values cannot be `set` directly and must instead be bound using +/// the scoped `$traceID.withValue() { ... }` operation. The value is only bound +/// for the duration of that scope, and is available to any child tasks which +/// are created within that scope. +/// +/// Detached tasks do not inherit task-local values, however tasks created using +/// the `Task { ... }` initializer do inherit task-locals by copying them to the +/// new asynchronous task, even though it is an un-structured task. +/// +/// ### Examples +/// +/// @TaskLocal +/// static var traceID: TraceID? +/// +/// print("traceID: \(traceID)") // traceID: nil +/// +/// $traceID.withValue(1234) { // bind the value +/// print("traceID: \(traceID)") // traceID: 1234 +/// call() // traceID: 1234 +/// +/// Task { // unstructured tasks do inherit task locals by copying +/// call() // traceID: 1234 +/// } +/// +/// Task.detached { // detached tasks do not inherit task-local values +/// call() // traceID: nil +/// } +/// } +/// +/// func call() { +/// print("traceID: \(traceID)") // 1234 +/// } +/// +/// This type must be a `class` so it has a stable identity, that is used as key +/// value for lookups in the task local storage. +@propertyWrapper +@available(SwiftStdlib 5.1, *) +public final class TaskLocal: Sendable, CustomStringConvertible { + let defaultValue: Value + + public init(wrappedValue defaultValue: Value) { + self.defaultValue = defaultValue + } + + var key: Builtin.RawPointer { + unsafeBitCast(self, to: Builtin.RawPointer.self) + } + + /// Gets the value currently bound to this task-local from the current task. + /// + /// If no current task is available in the context where this call is made, + /// or if the task-local has no value bound, this will return the `defaultValue` + /// of the task local. + public func get() -> Value { + guard let rawValue = _taskLocalValueGet(key: key) else { + return self.defaultValue + } + + // Take the value; The type should be correct by construction + let storagePtr = + rawValue.bindMemory(to: Value.self, capacity: 1) + return UnsafeMutablePointer(mutating: storagePtr).pointee + } + + /// Binds the task-local to the specific value for the duration of the asynchronous operation. + /// + /// The value is available throughout the execution of the operation closure, + /// including any `get` operations performed by child-tasks created during the + /// execution of the operation closure. + /// + /// If the same task-local is bound multiple times, be it in the same task, or + /// in specific child tasks, the more specific (i.e. "deeper") binding is + /// returned when the value is read. + /// + /// If the value is a reference type, it will be retained for the duration of + /// the operation closure. + @discardableResult + public func withValue(_ valueDuringOperation: Value, operation: () async throws -> R, + file: String = #file, line: UInt = #line) async rethrows -> R { + // check if we're not trying to bind a value from an illegal context; this may crash + _checkIllegalTaskLocalBindingWithinWithTaskGroup(file: file, line: line) + + _taskLocalValuePush(key: key, value: valueDuringOperation) + defer { _taskLocalValuePop() } + + return try await operation() + } + + /// Binds the task-local to the specific value for the duration of the + /// synchronous operation. + /// + /// The value is available throughout the execution of the operation closure, + /// including any `get` operations performed by child-tasks created during the + /// execution of the operation closure. + /// + /// If the same task-local is bound multiple times, be it in the same task, or + /// in specific child tasks, the "more specific" binding is returned when the + /// value is read. + /// + /// If the value is a reference type, it will be retained for the duration of + /// the operation closure. + @discardableResult + public func withValue(_ valueDuringOperation: Value, operation: () throws -> R, + file: String = #file, line: UInt = #line) rethrows -> R { + // check if we're not trying to bind a value from an illegal context; this may crash + _checkIllegalTaskLocalBindingWithinWithTaskGroup(file: file, line: line) + + _taskLocalValuePush(key: key, value: valueDuringOperation) + defer { _taskLocalValuePop() } + + return try operation() + } + + public var projectedValue: TaskLocal { + get { + self + } + + @available(*, unavailable, message: "use '$myTaskLocal.withValue(_:do:)' instead") + set { + fatalError("Illegal attempt to set a \(Self.self) value, use `withValue(...) { ... }` instead.") + } + } + + // This subscript is used to enforce that the property wrapper may only be used + // on static (or rather, "without enclosing instance") properties. + // This is done by marking the `_enclosingInstance` as `Never` which informs + // the type-checker that this property-wrapper never wants to have an enclosing + // instance (it is impossible to declare a property wrapper inside the `Never` + // type). + @available(*, unavailable, message: "property wrappers cannot be instance members") + public static subscript( + _enclosingInstance object: Never, + wrapped wrappedKeyPath: ReferenceWritableKeyPath, + storage storageKeyPath: ReferenceWritableKeyPath> + ) -> Value { + get { + fatalError("Will never be executed, since enclosing instance is Never") + } + } + + public var wrappedValue: Value { + self.get() + } + + public var description: String { + "\(Self.self)(defaultValue: \(self.defaultValue))" + } + +} + +// ==== ------------------------------------------------------------------------ + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_localValuePush") +func _taskLocalValuePush( + key: Builtin.RawPointer/*: Key*/, + value: __owned Value +) // where Key: TaskLocal + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_localValuePop") +func _taskLocalValuePop() + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_localValueGet") +func _taskLocalValueGet( + key: Builtin.RawPointer/*Key*/ +) -> UnsafeMutableRawPointer? // where Key: TaskLocal + +@available(SwiftStdlib 5.1, *) +@_silgen_name("swift_task_localsCopyTo") +func _taskLocalsCopy( + to target: Builtin.NativeObject +) + +// ==== Checks ----------------------------------------------------------------- + +@available(SwiftStdlib 5.1, *) +@usableFromInline +func _checkIllegalTaskLocalBindingWithinWithTaskGroup(file: String, line: UInt) { + if _taskHasTaskGroupStatusRecord() { + file.withCString { _fileStart in + _reportIllegalTaskLocalBindingWithinWithTaskGroup( + _fileStart, file.count, true, line) + } + } +} + +@available(SwiftStdlib 5.1, *) +@usableFromInline +@_silgen_name("swift_task_reportIllegalTaskLocalBindingWithinWithTaskGroup") +func _reportIllegalTaskLocalBindingWithinWithTaskGroup( + _ _filenameStart: UnsafePointer, + _ _filenameLength: Int, + _ _filenameIsASCII: Bool, + _ _line: UInt) diff --git a/stdlib/public/BackDeployConcurrency/TaskOptions.h b/stdlib/public/BackDeployConcurrency/TaskOptions.h new file mode 100644 index 0000000000000..125dd424c56d3 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskOptions.h @@ -0,0 +1,146 @@ +//===--- TaskOptions.h - ABI structures for task options --------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing task options. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_TASK_OPTIONS_H +#define SWIFT_ABI_TASK_OPTIONS_H + +#include "swift/ABI/HeapObject.h" +#include "swift/ABI/Metadata.h" +#include "swift/ABI/MetadataValues.h" +#include "swift/Runtime/Config.h" +#include "swift/Basic/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "TaskLocal.h" +#include "Executor.h" + +namespace swift { + +// ==== ------------------------------------------------------------------------ +// ==== Task Options, for creating and waiting on tasks + +/// The abstract base class for all options that may be used +/// to configure a newly spawned task. +class TaskOptionRecord { +public: + const TaskOptionRecordFlags Flags; + TaskOptionRecord *Parent; + + TaskOptionRecord(TaskOptionRecordKind kind, + TaskOptionRecord *parent = nullptr) + : Flags(kind), Parent(parent) { } + + TaskOptionRecord(const TaskOptionRecord &) = delete; + TaskOptionRecord &operator=(const TaskOptionRecord &) = delete; + + TaskOptionRecordKind getKind() const { + return Flags.getKind(); + } + + TaskOptionRecord *getParent() const { + return Parent; + } +}; + +/******************************************************************************/ +/****************************** TASK OPTIONS **********************************/ +/******************************************************************************/ + +class TaskGroupTaskOptionRecord : public TaskOptionRecord { + TaskGroup * const Group; + + public: + TaskGroupTaskOptionRecord(TaskGroup *group) + : TaskOptionRecord(TaskOptionRecordKind::TaskGroup), + Group(group) {} + + TaskGroup *getGroup() const { + return Group; + } + + static bool classof(const TaskOptionRecord *record) { + return record->getKind() == TaskOptionRecordKind::TaskGroup; + } +}; + + +/// Task option to specify on what executor the task should be executed. +/// +/// Not passing this option implies that that a "best guess" or good default +/// executor should be used instead, most often this may mean the global +/// concurrent executor, or the enclosing actor's executor. +class ExecutorTaskOptionRecord : public TaskOptionRecord { + const ExecutorRef Executor; + +public: + ExecutorTaskOptionRecord(ExecutorRef executor) + : TaskOptionRecord(TaskOptionRecordKind::Executor), + Executor(executor) {} + + ExecutorRef getExecutor() const { + return Executor; + } + + static bool classof(const TaskOptionRecord *record) { + return record->getKind() == TaskOptionRecordKind::Executor; + } +}; + +/// DEPRECATED. AsyncLetWithBufferTaskOptionRecord is used instead. +/// Task option to specify that the created task is for an 'async let'. +class AsyncLetTaskOptionRecord : public TaskOptionRecord { + AsyncLet *asyncLet; + +public: + AsyncLetTaskOptionRecord(AsyncLet *asyncLet) + : TaskOptionRecord(TaskOptionRecordKind::AsyncLet), + asyncLet(asyncLet) {} + + AsyncLet *getAsyncLet() const { + return asyncLet; + } + + static bool classof(const TaskOptionRecord *record) { + return record->getKind() == TaskOptionRecordKind::AsyncLet; + } +}; + +class AsyncLetWithBufferTaskOptionRecord : public TaskOptionRecord { + AsyncLet *asyncLet; + void *resultBuffer; + +public: + AsyncLetWithBufferTaskOptionRecord(AsyncLet *asyncLet, + void *resultBuffer) + : TaskOptionRecord(TaskOptionRecordKind::AsyncLetWithBuffer), + asyncLet(asyncLet), + resultBuffer(resultBuffer) {} + + AsyncLet *getAsyncLet() const { + return asyncLet; + } + + void *getResultBuffer() const { + return resultBuffer; + } + + static bool classof(const TaskOptionRecord *record) { + return record->getKind() == TaskOptionRecordKind::AsyncLetWithBuffer; + } +}; + +} // end namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/TaskPrivate.h b/stdlib/public/BackDeployConcurrency/TaskPrivate.h new file mode 100644 index 0000000000000..71a773e40a287 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskPrivate.h @@ -0,0 +1,443 @@ +//===--- TaskPrivate.h - Concurrency library internal interface -*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Internal functions for the concurrency library. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_CONCURRENCY_TASKPRIVATE_H +#define SWIFT_CONCURRENCY_TASKPRIVATE_H + +#include "Error.h" +#include "swift/ABI/Metadata.h" +#include "swift/Runtime/Atomic.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/Error.h" +#include "swift/Runtime/Exclusivity.h" +#include "swift/Runtime/HeapObject.h" +#include "Task.h" + +#define SWIFT_FATAL_ERROR swift_Concurrency_fatalError +#include "../runtime/StackAllocator.h" + +#if HAVE_PTHREAD_H +#include +#endif +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#define VC_EXTRA_LEAN +#define NOMINMAX +#include +#endif + +namespace swift { + +// Set to 1 to enable helpful debug spew to stderr +// If this is enabled, tests with `swift_task_debug_log` requirement can run. +#if 0 +#define SWIFT_TASK_DEBUG_LOG(fmt, ...) \ + fprintf(stderr, "[%lu] [%s:%d](%s) " fmt "\n", \ + (unsigned long)_swift_get_thread_id(), \ + __FILE__, __LINE__, __FUNCTION__, \ + __VA_ARGS__) +#else +#define SWIFT_TASK_DEBUG_LOG(fmt, ...) (void)0 +#endif + +#if defined(_WIN32) +using ThreadID = decltype(GetCurrentThreadId()); +#else +using ThreadID = decltype(pthread_self()); +#endif + +inline ThreadID _swift_get_thread_id() { +#if defined(_WIN32) + return GetCurrentThreadId(); +#else + return pthread_self(); +#endif +} + +class AsyncTask; +class TaskGroup; + +/// Allocate task-local memory on behalf of a specific task, +/// not necessarily the current one. Generally this should only be +/// done on behalf of a child task. +void *_swift_task_alloc_specific(AsyncTask *task, size_t size); + +/// dellocate task-local memory on behalf of a specific task, +/// not necessarily the current one. Generally this should only be +/// done on behalf of a child task. +void _swift_task_dealloc_specific(AsyncTask *task, void *ptr); + +/// Given that we've already set the right executor as the active +/// executor, run the given job. This does additional bookkeeping +/// related to the active task. +void runJobInEstablishedExecutorContext(Job *job); + +/// Adopt the voucher stored in `task`. This removes the voucher from the task +/// and adopts it on the current thread. +void adoptTaskVoucher(AsyncTask *task); + +/// Restore the voucher for `task`. This un-adopts the current thread's voucher +/// and stores it back into the task again. +void restoreTaskVoucher(AsyncTask *task); + +/// Initialize the async let storage for the given async-let child task. +void asyncLet_addImpl(AsyncTask *task, AsyncLet *asyncLet, + bool didAllocateInParentTask); + +/// Clear the active task reference for the current thread. +AsyncTask *_swift_task_clearCurrent(); + +/// release() establishes a happens-before relation with a preceding acquire() +/// on the same address. +void _swift_tsan_acquire(void *addr); +void _swift_tsan_release(void *addr); + +/// Special values used with DispatchQueueIndex to indicate the global and main +/// executors. +#define DISPATCH_QUEUE_GLOBAL_EXECUTOR (void *)1 + +#if !defined(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME) +inline SerialExecutorWitnessTable * +_swift_task_getDispatchQueueSerialExecutorWitnessTable() { + extern SerialExecutorWitnessTable wtable + SWIFT_ASM_LABEL_WITH_PREFIX("$ss17DispatchQueueShimCScfsWP"); + return &wtable; +} +#endif + +// ==== ------------------------------------------------------------------------ + +namespace { + +/// The layout of a context to call one of the following functions: +/// +/// @_silgen_name("swift_task_future_wait") +/// func _taskFutureGet(_ task: Builtin.NativeObject) async -> T +/// +/// @_silgen_name("swift_task_future_wait_throwing") +/// func _taskFutureGetThrowing(_ task: Builtin.NativeObject) async throws -> T +/// +/// @_silgen_name("swift_asyncLet_wait") +/// func _asyncLetGet(_ task: Builtin.RawPointer) async -> T +/// +/// @_silgen_name("swift_asyncLet_waitThrowing") +/// func _asyncLetGetThrowing(_ task: Builtin.RawPointer) async throws -> T +/// +/// @_silgen_name("swift_taskGroup_wait_next_throwing") +/// func _taskGroupWaitNext(group: Builtin.RawPointer) async throws -> T? +/// +class TaskFutureWaitAsyncContext : public AsyncContext { +public: + SwiftError *errorResult; + + OpaqueValue *successResultPointer; + + void fillWithSuccess(AsyncTask::FutureFragment *future) { + fillWithSuccess(future->getStoragePtr(), future->getResultType(), + successResultPointer); + } + void fillWithSuccess(OpaqueValue *src, const Metadata *successType, + OpaqueValue *result) { + successType->vw_initializeWithCopy(result, src); + } + + void fillWithError(AsyncTask::FutureFragment *future) { + fillWithError(future->getError()); + } + void fillWithError(SwiftError *error) { + errorResult = error; + swift_errorRetain(error); + } +}; + +} // end anonymous namespace + +/// The current state of a task's status records. +class alignas(sizeof(void*) * 2) ActiveTaskStatus { + enum : uintptr_t { + /// The current running priority of the task. + PriorityMask = 0xFF, + + /// Has the task been cancelled? + IsCancelled = 0x100, + + /// Whether the task status is "locked", meaning that further + /// accesses need to wait on the task status record lock + IsLocked = 0x200, + + /// Whether the running priority has been escalated above the + /// priority recorded in the Job header. + IsEscalated = 0x400, + + /// Whether the task is actively running. + /// We don't really need to be tracking this in the runtime right + /// now, but we will need to eventually track enough information to + /// escalate the thread that's running a task, so doing the stores + /// necessary to maintain this gives us a more realistic baseline + /// for performance. + IsRunning = 0x800, + }; + + TaskStatusRecord *Record; + uintptr_t Flags; + + ActiveTaskStatus(TaskStatusRecord *record, uintptr_t flags) + : Record(record), Flags(flags) {} + +public: +#ifdef __GLIBCXX__ + /// We really don't want to provide this constructor, but in old + /// versions of libstdc++, std::atomic::load incorrectly requires + /// the type to be default-constructible. + ActiveTaskStatus() = default; +#endif + + constexpr ActiveTaskStatus(JobFlags flags) + : Record(nullptr), Flags(uintptr_t(flags.getPriority())) {} + + /// Is the task currently cancelled? + bool isCancelled() const { return Flags & IsCancelled; } + ActiveTaskStatus withCancelled() const { + return ActiveTaskStatus(Record, Flags | IsCancelled); + } + + /// Is the task currently running? + /// Eventually we'll track this with more specificity, like whether + /// it's running on a specific thread, enqueued on a specific actor, + /// etc. + bool isRunning() const { return Flags & IsRunning; } + ActiveTaskStatus withRunning(bool isRunning) const { + return ActiveTaskStatus(Record, isRunning ? (Flags | IsRunning) + : (Flags & ~IsRunning)); + } + + /// Is there an active lock on the cancellation information? + bool isLocked() const { return Flags & IsLocked; } + ActiveTaskStatus withLockingRecord(TaskStatusRecord *lockRecord) const { + assert(!isLocked()); + assert(lockRecord->Parent == Record); + return ActiveTaskStatus(lockRecord, Flags | IsLocked); + } + + JobPriority getStoredPriority() const { + return JobPriority(Flags & PriorityMask); + } + bool isStoredPriorityEscalated() const { + return Flags & IsEscalated; + } + ActiveTaskStatus withEscalatedPriority(JobPriority priority) const { + assert(priority > getStoredPriority()); + return ActiveTaskStatus(Record, + (Flags & ~PriorityMask) + | IsEscalated | uintptr_t(priority)); + } + ActiveTaskStatus withoutStoredPriorityEscalation() const { + assert(isStoredPriorityEscalated()); + return ActiveTaskStatus(Record, Flags & ~IsEscalated); + } + + /// Return the innermost cancellation record. Code running + /// asynchronously with this task should not access this record + /// without having first locked it; see swift_taskCancel. + TaskStatusRecord *getInnermostRecord() const { + return Record; + } + ActiveTaskStatus withInnermostRecord(TaskStatusRecord *newRecord) { + return ActiveTaskStatus(newRecord, Flags); + } + + static TaskStatusRecord *getStatusRecordParent(TaskStatusRecord *ptr); + + using record_iterator = + LinkedListIterator; + llvm::iterator_range records() const { + return record_iterator::rangeBeginning(getInnermostRecord()); + } +}; + +/// The size of an allocator slab. +static constexpr size_t SlabCapacity = 1000; +extern Metadata TaskAllocatorSlabMetadata; + +using TaskAllocator = StackAllocator; + +/// Private storage in an AsyncTask object. +struct AsyncTask::PrivateStorage { + /// The currently-active information about cancellation. + /// Currently two words. + swift::atomic Status; + + /// The allocator for the task stack. + /// Currently 2 words + 8 bytes. + TaskAllocator Allocator; + + /// Storage for task-local values. + /// Currently one word. + TaskLocal::Storage Local; + + /// State inside the AsyncTask whose state is only managed by the exclusivity + /// runtime in stdlibCore. We zero initialize to provide a safe initial value, + /// but actually initialize its bit state to a const global provided by + /// libswiftCore so that libswiftCore can control the layout of our initial + /// state. + uintptr_t ExclusivityAccessSet[2] = {0, 0}; + + /// The top 32 bits of the task ID. The bottom 32 bits are in Job::Id. + uint32_t Id; + + PrivateStorage(JobFlags flags) + : Status(ActiveTaskStatus(flags)), Local(TaskLocal::Storage()) {} + + PrivateStorage(JobFlags flags, void *slab, size_t slabCapacity) + : Status(ActiveTaskStatus(flags)), Allocator(slab, slabCapacity), + Local(TaskLocal::Storage()) {} + + void complete(AsyncTask *task) { + // Destroy and deallocate any remaining task local items. + // We need to do this before we destroy the task local deallocator. + Local.destroy(task); + + this->~PrivateStorage(); + } +}; + +static_assert(sizeof(AsyncTask::PrivateStorage) + <= sizeof(AsyncTask::OpaquePrivateStorage) && + alignof(AsyncTask::PrivateStorage) + <= alignof(AsyncTask::OpaquePrivateStorage), + "Task-private storage doesn't fit in reserved space"); + +inline AsyncTask::PrivateStorage & +AsyncTask::OpaquePrivateStorage::get() { + return reinterpret_cast(*this); +} +inline const AsyncTask::PrivateStorage & +AsyncTask::OpaquePrivateStorage::get() const { + return reinterpret_cast(*this); +} +inline void AsyncTask::OpaquePrivateStorage::initialize(AsyncTask *task) { + new (this) PrivateStorage(task->Flags); +} +inline void +AsyncTask::OpaquePrivateStorage::initializeWithSlab(AsyncTask *task, + void *slab, + size_t slabCapacity) { + new (this) PrivateStorage(task->Flags, slab, slabCapacity); +} +inline void AsyncTask::OpaquePrivateStorage::complete(AsyncTask *task) { + get().complete(task); +} +inline void AsyncTask::OpaquePrivateStorage::destroy() { + // nothing else to do +} + +inline AsyncTask::PrivateStorage &AsyncTask::_private() { + return Private.get(); +} +inline const AsyncTask::PrivateStorage &AsyncTask::_private() const { + return Private.get(); +} + +inline bool AsyncTask::isCancelled() const { + return _private().Status.load(std::memory_order_relaxed) + .isCancelled(); +} + +inline void AsyncTask::flagAsRunning() { + SWIFT_TASK_DEBUG_LOG("%p->flagAsRunning()", this); + auto oldStatus = _private().Status.load(std::memory_order_relaxed); + while (true) { + assert(!oldStatus.isRunning()); + if (oldStatus.isLocked()) { + flagAsRunning_slow(); + adoptTaskVoucher(this); + swift_task_enterThreadLocalContext( + (char *)&_private().ExclusivityAccessSet[0]); + return; + } + + auto newStatus = oldStatus.withRunning(true); + if (newStatus.isStoredPriorityEscalated()) { + newStatus = newStatus.withoutStoredPriorityEscalation(); + Flags.setPriority(oldStatus.getStoredPriority()); + } + + if (_private().Status.compare_exchange_weak(oldStatus, newStatus, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + adoptTaskVoucher(this); + swift_task_enterThreadLocalContext( + (char *)&_private().ExclusivityAccessSet[0]); + return; + } + } +} + +inline void AsyncTask::flagAsSuspended() { + SWIFT_TASK_DEBUG_LOG("%p->flagAsSuspended()", this); + auto oldStatus = _private().Status.load(std::memory_order_relaxed); + while (true) { + assert(oldStatus.isRunning()); + if (oldStatus.isLocked()) { + flagAsSuspended_slow(); + swift_task_exitThreadLocalContext( + (char *)&_private().ExclusivityAccessSet[0]); + restoreTaskVoucher(this); + return; + } + + auto newStatus = oldStatus.withRunning(false); + if (newStatus.isStoredPriorityEscalated()) { + newStatus = newStatus.withoutStoredPriorityEscalation(); + Flags.setPriority(oldStatus.getStoredPriority()); + } + + if (_private().Status.compare_exchange_weak(oldStatus, newStatus, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + swift_task_exitThreadLocalContext( + (char *)&_private().ExclusivityAccessSet[0]); + restoreTaskVoucher(this); + return; + } + } +} + +// READ ME: This is not a dead function! Do not remove it! This is a function +// that can be used when debugging locally to instrument when a task actually is +// dealloced. +inline void AsyncTask::flagAsCompleted() { + SWIFT_TASK_DEBUG_LOG("task completed %p", this); +} + +inline void AsyncTask::localValuePush(const HeapObject *key, + /* +1 */ OpaqueValue *value, + const Metadata *valueType) { + _private().Local.pushValue(this, key, value, valueType); +} + +inline OpaqueValue *AsyncTask::localValueGet(const HeapObject *key) { + return _private().Local.getValue(this, key); +} + +/// Returns true if storage has still more bindings. +inline bool AsyncTask::localValuePop() { + return _private().Local.popValue(this); +} + +} // end namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/TaskSleep.swift b/stdlib/public/BackDeployConcurrency/TaskSleep.swift new file mode 100644 index 0000000000000..4ac6eeb0709cf --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskSleep.swift @@ -0,0 +1,296 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Swift +@_implementationOnly import _SwiftConcurrencyShims + +@available(SwiftStdlib 5.1, *) +extension Task where Success == Never, Failure == Never { + @available(*, deprecated, renamed: "Task.sleep(nanoseconds:)") + /// Suspends the current task for at least the given duration + /// in nanoseconds. + /// + /// This function doesn't block the underlying thread. + public static func sleep(_ duration: UInt64) async { + return await Builtin.withUnsafeContinuation { (continuation: Builtin.RawUnsafeContinuation) -> Void in + let job = _taskCreateNullaryContinuationJob( + priority: Int(Task.currentPriority.rawValue), + continuation: continuation) + _enqueueJobGlobalWithDelay(duration, job) + } + } + + /// The type of continuation used in the implementation of + /// sleep(nanoseconds:). + private typealias SleepContinuation = UnsafeContinuation<(), Error> + + /// Describes the state of a sleep() operation. + private enum SleepState { + /// The sleep continuation has not yet begun. + case notStarted + + // The sleep continuation has been created and is available here. + case activeContinuation(SleepContinuation) + + /// The sleep has finished. + case finished + + /// The sleep was canceled. + case cancelled + + /// The sleep was canceled before it even got started. + case cancelledBeforeStarted + + /// Decode sleep state from the word of storage. + init(word: Builtin.Word) { + switch UInt(word) & 0x03 { + case 0: + let continuationBits = UInt(word) & ~0x03 + if continuationBits == 0 { + self = .notStarted + } else { + let continuation = unsafeBitCast( + continuationBits, to: SleepContinuation.self) + self = .activeContinuation(continuation) + } + + case 1: + self = .finished + + case 2: + self = .cancelled + + case 3: + self = .cancelledBeforeStarted + + default: + fatalError("Bitmask failure") + } + } + + /// Decode sleep state by loading from the given pointer + init(loading wordPtr: UnsafeMutablePointer) { + self.init(word: Builtin.atomicload_seqcst_Word(wordPtr._rawValue)) + } + + /// Encode sleep state into a word of storage. + var word: UInt { + switch self { + case .notStarted: + return 0 + + case .activeContinuation(let continuation): + let continuationBits = unsafeBitCast(continuation, to: UInt.self) + return continuationBits + + case .finished: + return 1 + + case .cancelled: + return 2 + + case .cancelledBeforeStarted: + return 3 + } + } + } + + /// Called when the sleep(nanoseconds:) operation woke up without being + /// canceled. + private static func onSleepWake( + _ wordPtr: UnsafeMutablePointer + ) { + while true { + let state = SleepState(loading: wordPtr) + switch state { + case .notStarted: + fatalError("Cannot wake before we even started") + + case .activeContinuation(let continuation): + // We have an active continuation, so try to transition to the + // "finished" state. + let (_, won) = Builtin.cmpxchg_seqcst_seqcst_Word( + wordPtr._rawValue, + state.word._builtinWordValue, + SleepState.finished.word._builtinWordValue) + if Bool(_builtinBooleanLiteral: won) { + // The sleep finished, so invoke the continuation: we're done. + continuation.resume() + return + } + + // Try again! + continue + + case .finished: + fatalError("Already finished normally, can't do that again") + + case .cancelled: + // The task was cancelled, which means the continuation was + // called by the cancellation handler. We need to deallocate the flag + // word, because it was left over for this task to complete. + wordPtr.deallocate() + return + + case .cancelledBeforeStarted: + // Nothing to do; + return + } + } + } + + /// Called when the sleep(nanoseconds:) operation has been canceled before + /// the sleep completed. + private static func onSleepCancel( + _ wordPtr: UnsafeMutablePointer + ) { + while true { + let state = SleepState(loading: wordPtr) + switch state { + case .notStarted: + // We haven't started yet, so try to transition to the cancelled-before + // started state. + let (_, won) = Builtin.cmpxchg_seqcst_seqcst_Word( + wordPtr._rawValue, + state.word._builtinWordValue, + SleepState.cancelledBeforeStarted.word._builtinWordValue) + if Bool(_builtinBooleanLiteral: won) { + return + } + + // Try again! + continue + + case .activeContinuation(let continuation): + // We have an active continuation, so try to transition to the + // "cancelled" state. + let (_, won) = Builtin.cmpxchg_seqcst_seqcst_Word( + wordPtr._rawValue, + state.word._builtinWordValue, + SleepState.cancelled.word._builtinWordValue) + if Bool(_builtinBooleanLiteral: won) { + // We recorded the task cancellation before the sleep finished, so + // invoke the continuation with the cancellation error. + continuation.resume(throwing: _Concurrency.CancellationError()) + return + } + + // Try again! + continue + + case .finished, .cancelled, .cancelledBeforeStarted: + // The operation already finished, so there is nothing more to do. + return + } + } + } + + /// Suspends the current task for at least the given duration + /// in nanoseconds. + /// + /// If the task is canceled before the time ends, + /// this function throws `CancellationError`. + /// + /// This function doesn't block the underlying thread. + public static func sleep(nanoseconds duration: UInt64) async throws { + // Allocate storage for the storage word. + let wordPtr = UnsafeMutablePointer.allocate(capacity: 1) + + // Initialize the flag word to "not started", which means the continuation + // has neither been created nor completed. + Builtin.atomicstore_seqcst_Word( + wordPtr._rawValue, SleepState.notStarted.word._builtinWordValue) + + do { + // Install a cancellation handler to resume the continuation by + // throwing CancellationError. + try await withTaskCancellationHandler { + let _: () = try await withUnsafeThrowingContinuation { continuation in + while true { + let state = SleepState(loading: wordPtr) + switch state { + case .notStarted: + // The word that describes the active continuation state. + let continuationWord = + SleepState.activeContinuation(continuation).word + + // Try to swap in the continuation word. + let (_, won) = Builtin.cmpxchg_seqcst_seqcst_Word( + wordPtr._rawValue, + state.word._builtinWordValue, + continuationWord._builtinWordValue) + if !Bool(_builtinBooleanLiteral: won) { + // Keep trying! + continue + } + + // Create a task that resumes the continuation normally if it + // finishes first. Enqueue it directly with the delay, so it fires + // when we're done sleeping. + let sleepTaskFlags = taskCreateFlags( + priority: nil, isChildTask: false, copyTaskLocals: false, + inheritContext: false, enqueueJob: false, + addPendingGroupTaskUnconditionally: false) + let (sleepTask, _) = Builtin.createAsyncTask(sleepTaskFlags) { + onSleepWake(wordPtr) + } + _enqueueJobGlobalWithDelay( + duration, Builtin.convertTaskToJob(sleepTask)) + return + + case .activeContinuation, .finished: + fatalError("Impossible to have multiple active continuations") + + case .cancelled: + fatalError("Impossible to have cancelled before we began") + + case .cancelledBeforeStarted: + // Finish the continuation normally. We'll throw later, after + // we clean up. + continuation.resume() + return + } + } + } + } onCancel: { + onSleepCancel(wordPtr) + } + + // Determine whether we got cancelled before we even started. + let cancelledBeforeStarted: Bool + switch SleepState(loading: wordPtr) { + case .notStarted, .activeContinuation, .cancelled: + fatalError("Invalid state for non-cancelled sleep task") + + case .cancelledBeforeStarted: + cancelledBeforeStarted = true + + case .finished: + cancelledBeforeStarted = false + } + + // We got here without being cancelled, so deallocate the storage for + // the flag word and continuation. + wordPtr.deallocate() + + // If we got cancelled before we even started, through the cancellation + // error now. + if cancelledBeforeStarted { + throw _Concurrency.CancellationError() + } + } catch { + // The task was cancelled; propagate the error. The "on wake" task is + // responsible for deallocating the flag word and continuation, if it's + // still running. + throw error + } + } +} diff --git a/stdlib/public/BackDeployConcurrency/TaskStatus.cpp b/stdlib/public/BackDeployConcurrency/TaskStatus.cpp new file mode 100644 index 0000000000000..b85f378913a0b --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskStatus.cpp @@ -0,0 +1,674 @@ +//===--- TaskStatus.cpp - Asynchronous task status tracking ---------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Routines for maintaining and interacting with the current state of a +// task, including tracking child tasks, deadlines, and cancellation. +// +//===----------------------------------------------------------------------===// + +#include "CompatibilityOverride.h" +#include "ConcurrencyRuntime.h" +#include "swift/Runtime/Mutex.h" +#include "swift/Runtime/AtomicWaitQueue.h" +#include "TaskStatus.h" +#include "TaskPrivate.h" +#include + +using namespace swift; + +inline TaskStatusRecord * +ActiveTaskStatus::getStatusRecordParent(TaskStatusRecord *ptr) { + return ptr->getParent(); +} + +/**************************************************************************/ +/************************* RECORD LOCK MANAGEMENT *************************/ +/**************************************************************************/ + +/// A lock used to protect management of task-specific status +/// record locks. +static StaticMutex StatusRecordLockLock; + +namespace { + +/// A lock record which can be used to protect a task's active +/// status records. +/// +/// For the most part, the active task status records of a task are +/// only accessed by the task itself. If that were always true, +/// no synchronization would be required to change them. However, +/// cancellation and escalation can occur asynchronously, and they +/// must be able to inspect the status records without worrying about +/// their concurrent modification or destruction of the records. +/// Therefore, these operations freeze the active status records +/// for their duration. They do this by (1) setting a bit in the +/// task's `Status` field state which says that the records are +/// locked and (2) creating a lock record as the new innermost +/// status record. When the operation is complete, it removes this +/// record and clears the lock bit, then notifies the lock record that +/// the locking operation is complete. +/// +/// When a task wants to change its active status record, but +/// it sees that the locked bit is set in the `Status` field, it +/// must acquire the global status-record lock, find this record +/// (which should be the innermost record), and wait for an unlock. +class StatusRecordLockRecord : + public AtomicWaitQueue, + public TaskStatusRecord { +public: + StatusRecordLockRecord(TaskStatusRecord *parent) + : TaskStatusRecord(TaskStatusRecordKind::Private_RecordLock, parent) { + } + + void updateForNewArguments(TaskStatusRecord *parent) { + Parent = parent; + } + + static bool classof(const TaskStatusRecord *record) { + return record->getKind() == TaskStatusRecordKind::Private_RecordLock; + } +}; + +} + +/// Wait for a task's status record lock to be unlocked. +/// +/// When this function returns, `oldStatus` will have been updated +/// to the last value read and `isLocked()` will be false. +/// Of course, another thread may still be concurrently trying +/// to acquire the record lock. +static void waitForStatusRecordUnlock(AsyncTask *task, + ActiveTaskStatus &oldStatus) { + // Acquire the lock. + StatusRecordLockRecord::Waiter waiter(StatusRecordLockLock); + + while (true) { + assert(oldStatus.isLocked()); + + bool waited = waiter.tryReloadAndWait([&]() -> StatusRecordLockRecord* { + // Check that oldStatus is still correct. + oldStatus = task->_private().Status.load(std::memory_order_acquire); + if (!oldStatus.isLocked()) + return nullptr; + + // The innermost entry should be a record lock record; wait + // for it to be unlocked. + auto record = oldStatus.getInnermostRecord(); + return cast(record); + }); + if (!waited) + return; + + // Reload the status before trying to relock. + oldStatus = task->_private().Status.load(std::memory_order_acquire); + if (!oldStatus.isLocked()) + return; + } +} + +enum class LockContext { + /// The lock is being acquired from within the running task. + OnTask, + + /// The lock is being acquired asynchronously in order to cancel the + /// task. + Cancellation, + + /// The lock is being acquired asynchronously in order to read the + /// status records for some other reason. + OtherAsynchronous +}; + +static std::memory_order getLoadOrdering(LockContext lockContext) { + return lockContext != LockContext::OnTask + ? std::memory_order_acquire + : std::memory_order_relaxed; +} + +/// Call the given function while holding the task status record lock. +/// +/// The value in `status` will be updated with the current status value +/// (ignoring the `TaskStatusLockRecord`) before calling the function, +/// and the value there will be written back into the task status after +/// calling the function. +/// +/// As a special case, if `lockContext` is `Cancellation` and the task +/// is either already cancelled or can be cancelled without acquiring +/// the lock, then cancellation is performed, the lock is not taken, +/// and the function is not called. `withStatusRecordLock` will return +/// false in this case, and `status` will still contain the updated +/// status value, for which `isCancelled()` will be true. +template +static bool withStatusRecordLock(AsyncTask *task, + LockContext lockContext, + ActiveTaskStatus &status, + Fn &&fn) { + StatusRecordLockRecord::Worker worker(StatusRecordLockLock); + + auto loadOrdering = getLoadOrdering(lockContext); + bool forCancellation = lockContext == LockContext::Cancellation; + + // Load the current state. We can use relaxed loads if this isn't + // for cancellation because (1) this operation should be synchronous + // with the task, so the only thing that can modify it asynchronously + // is a cancelling thread, and (2) we'll reload with acquire ordering + // if a cancelling thread forces us to wait for an unlock. + + while (true) { + // Cancellation should be idempotent: if the task has already + // been cancelled (or is being cancelled concurrently), there + // shouldn't be any need to do this work again. + if (status.isCancelled() && forCancellation) + return false; + + // If the old info says we're locked, wait for the lock to clear. + if (status.isLocked()) { + waitForStatusRecordUnlock(task, status); + continue; + } + + // If we're cancelling and the task has no active status records, + // try to just set the cancelled bit and return. + auto oldRecord = status.getInnermostRecord(); + if (!oldRecord && forCancellation) { + ActiveTaskStatus newStatus = status.withCancelled(); + if (task->_private().Status.compare_exchange_weak(status, newStatus, + /*success*/ std::memory_order_relaxed, + /*failure*/ loadOrdering)) { + status = newStatus; + return false; + } + + // If that failed, just restart. + continue; + } + + // Make (or reconfigure) a lock record. + auto recordLockRecord = worker.createQueue(oldRecord); + + // Install the lock record as the top of the queue. + ActiveTaskStatus newStatus = + status.withLockingRecord(recordLockRecord); + if (forCancellation) + newStatus = newStatus.withCancelled(); + if (task->_private().Status.compare_exchange_weak(status, newStatus, + /*success*/ std::memory_order_release, + /*failure*/ loadOrdering)) { + + // Update `status` for the purposes of the callback function. + // Note that we don't include the record lock, but do need to + // set the cancelled bit. + if (forCancellation) + status = status.withCancelled(); + + worker.flagQueueIsPublished(recordLockRecord); + break; + } + } + + assert(worker.isWorkerThread()); + + // Call the function. + std::forward(fn)(); + + // We can just unconditionally store because nobody can be modifying + // the state while we've locked it. + // + // As a general matter, the task won't synchronize with anything we've + // done here through the task status; it may not even realize we ever + // acquired the lock. If we need to change the state in a way that the + // task will see, we need to do so in some other way, probably via + // atomic objects in the task status records. Because of this, we can + // actually unpublish the lock with a relaxed store. + assert(!status.isLocked()); + task->_private().Status.store(status, + /*success*/ std::memory_order_relaxed); + + // Unblock any waiters. + worker.finishAndUnpublishQueue([]{}); + + return true; +} + +/// A convenience version of the above for contexts that haven't already +/// done the load. +template +static bool withStatusRecordLock(AsyncTask *task, + LockContext lockContext, + Fn &&fn) { + ActiveTaskStatus status = + task->_private().Status.load(getLoadOrdering(lockContext)); + return withStatusRecordLock(task, lockContext, status, [&] { + fn(status); + }); +} + +/**************************************************************************/ +/*************************** RECORD MANAGEMENT ****************************/ +/**************************************************************************/ + +SWIFT_CC(swift) +static bool swift_task_addStatusRecordImpl(TaskStatusRecord *newRecord) { + auto task = swift_task_getCurrent(); + + // Load the current state. We can use a relaxed load because we're + // synchronous with the task. + auto oldStatus = task->_private().Status.load(std::memory_order_relaxed); + + while (true) { + // Wait for any active lock to be released. + if (oldStatus.isLocked()) + waitForStatusRecordUnlock(task, oldStatus); + + // Reset the parent of the new record. + newRecord->resetParent(oldStatus.getInnermostRecord()); + + // Set the record as the new innermost record. + // We have to use a release on success to make the initialization of + // the new record visible to the cancelling thread. + ActiveTaskStatus newStatus = oldStatus.withInnermostRecord(newRecord); + if (task->_private().Status.compare_exchange_weak(oldStatus, newStatus, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_relaxed)) + return !oldStatus.isCancelled(); + } +} + +SWIFT_CC(swift) +static bool swift_task_tryAddStatusRecordImpl(TaskStatusRecord *newRecord) { + auto task = swift_task_getCurrent(); + + // Load the current state. We can use a relaxed load because we're + // synchronous with the task. + auto oldStatus = task->_private().Status.load(std::memory_order_relaxed); + + while (true) { + // If the old info is already cancelled, do nothing. + if (oldStatus.isCancelled()) + return false; + + // Wait for any active lock to be released. + if (oldStatus.isLocked()) { + waitForStatusRecordUnlock(task, oldStatus); + + if (oldStatus.isCancelled()) + return false; + } + + // Reset the parent of the new record. + newRecord->resetParent(oldStatus.getInnermostRecord()); + + // Set the record as the new innermost record. + // We have to use a release on success to make the initialization of + // the new record visible to the cancelling thread. + ActiveTaskStatus newStatus = oldStatus.withInnermostRecord(newRecord); + if (task->_private().Status.compare_exchange_weak(oldStatus, newStatus, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_relaxed)) + return true; + } +} + +SWIFT_CC(swift) +static bool swift_task_removeStatusRecordImpl(TaskStatusRecord *record) { + auto task = swift_task_getCurrent(); + SWIFT_TASK_DEBUG_LOG("remove status record = %p, from current task = %p", + record, task); + + // Load the current state. + auto &status = task->_private().Status; + auto oldStatus = status.load(std::memory_order_relaxed); + + while (true) { + // Wait for any active lock to be released. + if (oldStatus.isLocked()) + waitForStatusRecordUnlock(task, oldStatus); + + // If the record is the innermost record, try to just pop it off. + if (oldStatus.getInnermostRecord() == record) { + ActiveTaskStatus newStatus = + oldStatus.withInnermostRecord(record->getParent()); + if (status.compare_exchange_weak(oldStatus, newStatus, + /*success*/ std::memory_order_release, + /*failure*/ std::memory_order_relaxed)) { + return !oldStatus.isCancelled(); + } + + // Otherwise, restart. + continue; + } + + // If the record is not the innermost record, we need to acquire the + // record lock; there's no way to splice the record list safely with + // a thread that's attempting to acquire the lock. + break; + } + + // Acquire the status record lock. + withStatusRecordLock(task, LockContext::OnTask, oldStatus, [&] { + // We can't observe the record to be the innermost record here because + // that would require some other thread to be concurrently structurally + // changing the set of status records, but we're running + // synchronously with the task. + auto cur = oldStatus.getInnermostRecord(); + assert(cur != record); + + // Splice the record out. + while (true) { + auto next = cur->getParent(); + if (next == record) { + cur->spliceParent(record->getParent()); + break; + } + } + }); + + return !oldStatus.isCancelled(); +} + +SWIFT_CC(swift) +static bool swift_task_hasTaskGroupStatusRecordImpl() { + auto task = swift_task_getCurrent(); + + // a group must be in a task, so if we're not in a task... + // then, we certainly are not in a group either! + if (!task) + return false; + + bool foundTaskGroupRecord = false; + withStatusRecordLock(task, LockContext::OnTask, + [&](ActiveTaskStatus &status) { + // Scan for the task group record within all the active records. + for (auto record: status.records()) { + if (record->getKind() == TaskStatusRecordKind::TaskGroup) { + foundTaskGroupRecord = true; + return; + } + } + }); + + return foundTaskGroupRecord; +} + +/**************************************************************************/ +/************************** CHILD TASK MANAGEMENT *************************/ +/**************************************************************************/ + +// ==== Child tasks ------------------------------------------------------------ +SWIFT_CC(swift) +static ChildTaskStatusRecord* +swift_task_attachChildImpl(AsyncTask *child) { + void *allocation = malloc(sizeof(swift::ChildTaskStatusRecord)); + auto record = new (allocation) swift::ChildTaskStatusRecord(child); + SWIFT_TASK_DEBUG_LOG("attach child task = %p, record = %p, to current task = %p", + child, record, swift_task_getCurrent()); + swift_task_addStatusRecord(record); + return record; +} + +SWIFT_CC(swift) +static void +swift_task_detachChildImpl(ChildTaskStatusRecord *record) { + swift_task_removeStatusRecord(record); +} + +SWIFT_CC(swift) +static void swift_taskGroup_attachChildImpl(TaskGroup *group, + AsyncTask *child) { + + // We are always called from the context of the parent + // + // Acquire the status record lock of parent - we want to synchronize with + // concurrent cancellation or escalation as we're adding new tasks to the + // group. + + auto parent = swift_task_getCurrent(); + withStatusRecordLock(parent, LockContext::OnTask, + [&](ActiveTaskStatus &status) { + group->addChildTask(child); + }); +} + +/****************************** CANCELLATION ******************************/ +/**************************************************************************/ + +/// Perform any cancellation actions required by the given record. +static void performCancellationAction(TaskStatusRecord *record) { + switch (record->getKind()) { + // Deadlines don't require any special support. + case TaskStatusRecordKind::Deadline: + return; + + // Child tasks need to be recursively cancelled. + case TaskStatusRecordKind::ChildTask: { + auto childRecord = cast(record); + for (AsyncTask *child: childRecord->children()) + swift_task_cancel(child); + return; + } + + case TaskStatusRecordKind::TaskGroup: { + auto childRecord = cast(record); + for (AsyncTask *child: childRecord->children()) + swift_task_cancel(child); + return; + } + + // Cancellation notifications need to be called. + case TaskStatusRecordKind::CancellationNotification: { + auto notification = + cast(record); + notification->run(); + return; + } + + // Escalation notifications can be ignored. + case TaskStatusRecordKind::EscalationNotification: + return; + + // Record locks shouldn't be found this way, but they don't have + // anything to do anyway. + case TaskStatusRecordKind::Private_RecordLock: + return; + } + + // Other cases can fall through here and be ignored. + // FIXME: allow dynamic extension/correction? +} + +SWIFT_CC(swift) +static void swift_task_cancelImpl(AsyncTask *task) { + SWIFT_TASK_DEBUG_LOG("cancel task = %p", task); + + // withStatusRecordLock has some special behavior for + // LockContext::Cancellation; the function only gets called + // when they don't apply. + withStatusRecordLock(task, LockContext::Cancellation, + [&](ActiveTaskStatus &status) { + assert(status.isCancelled()); + + // Carry out the cancellation operations associated with all + // the active records. + for (auto cur: status.records()) { + performCancellationAction(cur); + } + }); +} + +SWIFT_CC(swift) +static void swift_task_cancel_group_child_tasksImpl(TaskGroup *group) { + // Acquire the status record lock. + // + // Guaranteed to be called from the context of the parent task that created + // the task group once we have #40616 + auto task = swift_task_getCurrent(); + withStatusRecordLock(task, LockContext::OnTask, + [&](ActiveTaskStatus &status) { + // We purposefully DO NOT make this a cancellation by itself. + // We are cancelling the task group, and all tasks it contains. + // We are NOT cancelling the entire parent task though. + performCancellationAction(group->getTaskRecord()); + }); +} + +/**************************************************************************/ +/******************************* ESCALATION *******************************/ +/**************************************************************************/ + +/// Perform any escalation actions required by the given record. +static void performEscalationAction(TaskStatusRecord *record, + JobPriority newPriority) { + switch (record->getKind()) { + // Deadlines don't require any special support. + case TaskStatusRecordKind::Deadline: + return; + + // Child tasks need to be recursively escalated. + case TaskStatusRecordKind::ChildTask: { + auto childRecord = cast(record); + for (AsyncTask *child: childRecord->children()) + swift_task_escalate(child, newPriority); + return; + } + case TaskStatusRecordKind::TaskGroup: { + auto childRecord = cast(record); + for (AsyncTask *child: childRecord->children()) + swift_task_escalate(child, newPriority); + return; + } + + // Cancellation notifications can be ignore. + case TaskStatusRecordKind::CancellationNotification: + return; + + // Escalation notifications need to be called. + case TaskStatusRecordKind::EscalationNotification: { + auto notification = + cast(record); + notification->run(newPriority); + return; + } + + // Record locks shouldn't be found this way, but they don't have + // anything to do anyway. + case TaskStatusRecordKind::Private_RecordLock: + return; + } + + // Other cases can fall through here and be ignored. + // FIXME: allow dynamic extension/correction? +} + +SWIFT_CC(swift) +JobPriority +static swift_task_escalateImpl(AsyncTask *task, JobPriority newPriority) { + // Fast path: check that the stored priority is already at least + // as high as the desired priority. + auto status = task->_private().Status.load(std::memory_order_relaxed); + if (status.getStoredPriority() >= newPriority) + return status.getStoredPriority(); + + withStatusRecordLock(task, LockContext::OtherAsynchronous, status, [&] { + // Now that we have the task's status lock, check again that the + // priority is still too low. + if (status.getStoredPriority() >= newPriority) + return; + status = status.withEscalatedPriority(newPriority); + + // TODO: attempt to escalate the thread running the task, if it's + // currently running. This probably requires the task to be enqueued + // on a standard executor. + + // Perform escalation operations for all the status records. + for (auto cur: status.records()) { + performEscalationAction(cur, newPriority); + } + }); + + return status.getStoredPriority(); +} + +void AsyncTask::flagAsRunning_slow() { + withStatusRecordLock(this, LockContext::OnTask, + [&](ActiveTaskStatus &status) { + assert(!status.isRunning()); + + status = status.withRunning(true); + if (status.isStoredPriorityEscalated()) { + status = status.withoutStoredPriorityEscalation(); + Flags.setPriority(status.getStoredPriority()); + } + }); +} + +void AsyncTask::flagAsSuspended_slow() { + withStatusRecordLock(this, LockContext::OnTask, + [&](ActiveTaskStatus &status) { + assert(status.isRunning()); + + status = status.withRunning(false); + if (status.isStoredPriorityEscalated()) { + status = status.withoutStoredPriorityEscalation(); + Flags.setPriority(status.getStoredPriority()); + } + }); +} + +/**************************************************************************/ +/******************************** DEADLINE ********************************/ +/**************************************************************************/ +SWIFT_CC(swift) +static NearestTaskDeadline swift_task_getNearestDeadlineImpl(AsyncTask *task) { + // We don't have to worry about the deadline records being + // concurrently modified, so we can just walk the record chain, + // ignoring the possibility of a concurrent cancelling task. + + // Load the current state. + auto &status = task->_private().Status; + auto oldStatus = status.load(std::memory_order_relaxed); + + NearestTaskDeadline result; + + // If it's already cancelled, we're done. + if (oldStatus.isCancelled()) { + result.ValueKind = NearestTaskDeadline::AlreadyCancelled; + return result; + } + + // If it's locked, wait for the lock; we can't safely step through + // the RecordLockStatusRecord on a different thread. + if (oldStatus.isLocked()) { + waitForStatusRecordUnlock(task, oldStatus); + assert(!oldStatus.isLocked()); + } + + // Walk all the records looking for deadlines. + result.ValueKind = NearestTaskDeadline::None; + for (const auto *record: oldStatus.records()) { + auto deadlineRecord = dyn_cast(record); + if (!deadlineRecord) continue; + auto recordDeadline = deadlineRecord->getDeadline(); + + // If we already have a deadline, pick the earlier. + if (result.ValueKind == NearestTaskDeadline::Active) { + if (recordDeadline < result.Value) + result.Value = recordDeadline; + } else { + result.Value = recordDeadline; + result.ValueKind = NearestTaskDeadline::Active; + } + } + return result; +} + +#define OVERRIDE_TASK_STATUS COMPATIBILITY_OVERRIDE +#include COMPATIBILITY_OVERRIDE_INCLUDE_PATH diff --git a/stdlib/public/BackDeployConcurrency/TaskStatus.h b/stdlib/public/BackDeployConcurrency/TaskStatus.h new file mode 100644 index 0000000000000..abd0277c95789 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/TaskStatus.h @@ -0,0 +1,312 @@ +//===--- TaskStatusRecord.h - Structures to track task status --*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift ABI describing "status records", the mechanism by which +// tasks track dynamic information about their child tasks, custom +// cancellation hooks, and other information which may need to be exposed +// asynchronously outside of the task. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_ABI_TASKSTATUS_H +#define SWIFT_ABI_TASKSTATUS_H + +#include "swift/ABI/MetadataValues.h" +#include "Task.h" + +namespace swift { + +/// The abstract base class for all status records. +/// +/// TaskStatusRecords are typically allocated on the stack (possibly +/// in the task context), partially initialized, and then atomically +/// added to the task with `swift_task_addTaskStatusRecord`. While +/// registered with the task, a status record should only be +/// modified in ways that respect the possibility of asynchronous +/// access by a cancelling thread. In particular, the chain of +/// status records must not be disturbed. When the task leaves +/// the scope that requires the status record, the record can +/// be unregistered from the task with `swift_task_removeStatusRecord`, +/// at which point the memory can be returned to the system. +class TaskStatusRecord { +public: + TaskStatusRecordFlags Flags; + TaskStatusRecord *Parent; + + TaskStatusRecord(TaskStatusRecordKind kind, + TaskStatusRecord *parent = nullptr) + : Flags(kind) { + resetParent(parent); + } + + TaskStatusRecord(const TaskStatusRecord &) = delete; + TaskStatusRecord &operator=(const TaskStatusRecord &) = delete; + + TaskStatusRecordKind getKind() const { return Flags.getKind(); } + + TaskStatusRecord *getParent() const { return Parent; } + + /// Change the parent of this unregistered status record to the + /// given record. + /// + /// This should be used when the record has been previously initialized + /// without knowing what the true parent is. If we decide to cache + /// important information (e.g. the earliest timeout) in the innermost + /// status record, this is the method that should fill that in + /// from the parent. + void resetParent(TaskStatusRecord *newParent) { + Parent = newParent; + // TODO: cache + } + + /// Splice a record out of the status-record chain. + /// + /// Unlike resetParent, this assumes that it's just removing one or + /// more records from the chain and that there's no need to do any + /// extra cache manipulation. + void spliceParent(TaskStatusRecord *newParent) { Parent = newParent; } +}; + +/// A deadline for the task. If this is reached, the task will be +/// automatically cancelled. The deadline can also be queried and used +/// in other ways. +struct TaskDeadline { + // FIXME: I don't really know what this should look like right now. + // It's probably target-specific. + uint64_t Value; + + bool operator==(const TaskDeadline &other) const { + return Value == other.Value; + } + bool operator<(const TaskDeadline &other) const { + return Value < other.Value; + } +}; + +/// A status record which states that there's an active deadline +/// within the task. +class DeadlineStatusRecord : public TaskStatusRecord { + TaskDeadline Deadline; + +public: + DeadlineStatusRecord(TaskDeadline deadline) + : TaskStatusRecord(TaskStatusRecordKind::Deadline), Deadline(deadline) {} + + TaskDeadline getDeadline() const { return Deadline; } + + static bool classof(const TaskStatusRecord *record) { + return record->getKind() == TaskStatusRecordKind::Deadline; + } +}; + +/// A status record which states that a task has one or +/// more active child tasks. +class ChildTaskStatusRecord : public TaskStatusRecord { + AsyncTask *FirstChild; + +public: + ChildTaskStatusRecord(AsyncTask *child) + : TaskStatusRecord(TaskStatusRecordKind::ChildTask), FirstChild(child) {} + + ChildTaskStatusRecord(AsyncTask *child, TaskStatusRecordKind kind) + : TaskStatusRecord(kind), FirstChild(child) { + assert(kind == TaskStatusRecordKind::ChildTask); + assert(!child->hasGroupChildFragment() && + "Group child tasks must be tracked in their respective " + "TaskGroupTaskStatusRecord, and not as independent " + "ChildTaskStatusRecord " + "records."); + } + + /// Return the first child linked by this record. This may be null; + /// if not, it (and all of its successors) are guaranteed to satisfy + /// `isChildTask()`. + AsyncTask *getFirstChild() const { return FirstChild; } + + static AsyncTask *getNextChildTask(AsyncTask *task) { + return task->childFragment()->getNextChild(); + } + + using child_iterator = LinkedListIterator; + llvm::iterator_range children() const { + return child_iterator::rangeBeginning(getFirstChild()); + } + + static bool classof(const TaskStatusRecord *record) { + return record->getKind() == TaskStatusRecordKind::ChildTask; + } +}; + +/// A status record which states that a task has a task group. +/// +/// A record always is a specific `TaskGroupImpl`. +/// +/// The child tasks are stored as an invasive single-linked list, starting +/// from `FirstChild` and continuing through the `NextChild` pointers of all +/// the linked children. +/// +/// All children of the specific `Group` are stored "by" this record, +/// so that they may be cancelled when this task becomes cancelled. +/// +/// When the group exits, it may simply remove this single record from the task +/// running it. As it has guaranteed that the tasks have already completed. +/// +/// Group child tasks DO NOT have their own `ChildTaskStatusRecord` entries, +/// and are only tracked by their respective `TaskGroupTaskStatusRecord`. +class TaskGroupTaskStatusRecord : public TaskStatusRecord { + AsyncTask *FirstChild; + AsyncTask *LastChild; + +public: + TaskGroupTaskStatusRecord() + : TaskStatusRecord(TaskStatusRecordKind::TaskGroup), + FirstChild(nullptr), + LastChild(nullptr) { + } + + TaskGroupTaskStatusRecord(AsyncTask *child) + : TaskStatusRecord(TaskStatusRecordKind::TaskGroup), + FirstChild(child), + LastChild(child) { + assert(!LastChild || !LastChild->childFragment()->getNextChild()); + } + + TaskGroup *getGroup() { return reinterpret_cast(this); } + + /// Return the first child linked by this record. This may be null; + /// if not, it (and all of its successors) are guaranteed to satisfy + /// `isChildTask()`. + AsyncTask *getFirstChild() const { return FirstChild; } + + /// Attach the passed in `child` task to this group. + void attachChild(AsyncTask *child) { + assert(child->hasGroupChildFragment()); + assert(child->groupChildFragment()->getGroup() == getGroup()); + + auto oldLastChild = LastChild; + LastChild = child; + + if (!FirstChild) { + // This is the first child we ever attach, so store it as FirstChild. + FirstChild = child; + return; + } + + oldLastChild->childFragment()->setNextChild(child); + } + + void detachChild(AsyncTask *child) { + assert(child && "cannot remove a null child from group"); + if (FirstChild == child) { + FirstChild = getNextChildTask(child); + if (FirstChild == nullptr) { + LastChild = nullptr; + } + return; + } + + AsyncTask *prev = FirstChild; + // Remove the child from the linked list, i.e.: + // prev -> afterPrev -> afterChild + // == + // child -> afterChild + // Becomes: + // prev --------------> afterChild + while (prev) { + auto afterPrev = getNextChildTask(prev); + + if (afterPrev == child) { + auto afterChild = getNextChildTask(child); + prev->childFragment()->setNextChild(afterChild); + if (child == LastChild) { + LastChild = prev; + } + return; + } + + prev = afterPrev; + } + } + + static AsyncTask *getNextChildTask(AsyncTask *task) { + return task->childFragment()->getNextChild(); + } + + using child_iterator = LinkedListIterator; + llvm::iterator_range children() const { + return child_iterator::rangeBeginning(getFirstChild()); + } + + static bool classof(const TaskStatusRecord *record) { + return record->getKind() == TaskStatusRecordKind::TaskGroup; + } +}; + +/// A cancellation record which states that a task has an arbitrary +/// function that needs to be called if the task is cancelled. +/// +/// The end of any call to the function will be ordered before the +/// end of a call to unregister this record from the task. That is, +/// code may call `swift_task_removeStatusRecord` and freely +/// assume after it returns that this function will not be +/// subsequently used. +class CancellationNotificationStatusRecord : public TaskStatusRecord { +public: + using FunctionType = SWIFT_CC(swift) void(SWIFT_CONTEXT void *); + +private: + FunctionType *__ptrauth_swift_cancellation_notification_function Function; + void *Argument; + +public: + CancellationNotificationStatusRecord(FunctionType *fn, void *arg) + : TaskStatusRecord(TaskStatusRecordKind::CancellationNotification), + Function(fn), Argument(arg) {} + + void run() { Function(Argument); } + + static bool classof(const TaskStatusRecord *record) { + return record->getKind() == TaskStatusRecordKind::CancellationNotification; + } +}; + +/// A status record which says that a task has an arbitrary +/// function that needs to be called if the task's priority is escalated. +/// +/// The end of any call to the function will be ordered before the +/// end of a call to unregister this record from the task. That is, +/// code may call `swift_task_removeStatusRecord` and freely +/// assume after it returns that this function will not be +/// subsequently used. +class EscalationNotificationStatusRecord : public TaskStatusRecord { +public: + using FunctionType = void(void *, JobPriority); + +private: + FunctionType *__ptrauth_swift_escalation_notification_function Function; + void *Argument; + +public: + EscalationNotificationStatusRecord(FunctionType *fn, void *arg) + : TaskStatusRecord(TaskStatusRecordKind::EscalationNotification), + Function(fn), Argument(arg) {} + + void run(JobPriority newPriority) { Function(Argument, newPriority); } + + static bool classof(const TaskStatusRecord *record) { + return record->getKind() == TaskStatusRecordKind::EscalationNotification; + } +}; + +} // end namespace swift + +#endif diff --git a/stdlib/public/BackDeployConcurrency/ThreadSanitizer.cpp b/stdlib/public/BackDeployConcurrency/ThreadSanitizer.cpp new file mode 100644 index 0000000000000..e8c8701b8247c --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/ThreadSanitizer.cpp @@ -0,0 +1,50 @@ +//===--- ThreadSanitizer.cpp - Thread Sanitizer support -------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Thread Sanitizer support for the Swift Task runtime. +// +//===----------------------------------------------------------------------===// + +#include "TaskPrivate.h" + +// Thread Sanitizer is not supported on Windows. +#if defined(_WIN32) +void swift::_swift_tsan_acquire(void *addr) {} +void swift::_swift_tsan_release(void *addr) {} +#else +#include + +namespace { +using TSanFunc = void(void *); +TSanFunc *tsan_acquire, *tsan_release; +} // anonymous namespace + +void swift::_swift_tsan_acquire(void *addr) { + if (tsan_acquire) { + tsan_acquire(addr); + SWIFT_TASK_DEBUG_LOG("tsan_acquire on %p", addr); + } +} + +void swift::_swift_tsan_release(void *addr) { + if (tsan_release) { + tsan_release(addr); + SWIFT_TASK_DEBUG_LOG("tsan_release on %p", addr); + } +} + +SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(c) +void __tsan_on_initialize() { + tsan_acquire = (TSanFunc *)dlsym(RTLD_DEFAULT, "__tsan_acquire"); + tsan_release = (TSanFunc *)dlsym(RTLD_DEFAULT, "__tsan_release"); +} +#endif diff --git a/stdlib/public/BackDeployConcurrency/VoucherSupport.h b/stdlib/public/BackDeployConcurrency/VoucherSupport.h new file mode 100644 index 0000000000000..49bd201dc4b07 --- /dev/null +++ b/stdlib/public/BackDeployConcurrency/VoucherSupport.h @@ -0,0 +1,124 @@ +//===--- VoucherSupport.h - Support code for OS vouchers -----------*- C++ -*-// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Support code for interfacing with OS voucher calls. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_CONCURRENCY_VOUCHERSUPPORT_H +#define SWIFT_CONCURRENCY_VOUCHERSUPPORT_H + +#include "llvm/ADT/Optional.h" +#include "swift/Runtime/VoucherShims.h" +#include "Task.h" +#include "TaskPrivate.h" + +namespace swift { + +/// A class which manages voucher adoption for Job and Task objects. +class VoucherManager { + /// The original voucher that was set on the thread before Swift started + /// doing async work. This must be restored on the thread after we finish + /// async work. + llvm::Optional OriginalVoucher; + +public: + VoucherManager() { + SWIFT_TASK_DEBUG_LOG("[%p] Constructing VoucherManager", this); + } + + /// Clean up after completing async work, restoring the original voucher on + /// the current thread if necessary. This MUST be called before the + /// VoucherManager object is destroyed. It may also be called in other + /// places to restore the original voucher and reset the VoucherManager. + void leave() { + if (OriginalVoucher) { + SWIFT_TASK_DEBUG_LOG("[%p] Restoring original voucher %p", this, + *OriginalVoucher); + if (swift_voucher_needs_adopt(*OriginalVoucher)) { + auto previous = voucher_adopt(*OriginalVoucher); + swift_voucher_release(previous); + } else { + swift_voucher_release(*OriginalVoucher); + } + OriginalVoucher = llvm::None; + } else + SWIFT_TASK_DEBUG_LOG("[%p] Leaving empty VoucherManager", this); + } + + ~VoucherManager() { assert(!OriginalVoucher); } + + /// Set up for a new Job by adopting its voucher on the current thread. This + /// takes over ownership of the voucher from the Job object. For plain Jobs, + /// this is permanent. For Tasks, the voucher must be restored using + /// restoreVoucher if the task suspends. + void swapToJob(Job *job) { + SWIFT_TASK_DEBUG_LOG("[%p] Swapping jobs to %p", this, job); + assert(job); + assert(job->Voucher != SWIFT_DEAD_VOUCHER); + + voucher_t previous; + if (swift_voucher_needs_adopt(job->Voucher)) { + // If we need to adopt the voucher, do so, and grab the old one. + SWIFT_TASK_DEBUG_LOG("[%p] Swapping jobs to %p, adopting voucher %p", + this, job, job->Voucher); + previous = voucher_adopt(job->Voucher); + } else { + // If we don't need to adopt the voucher, take the voucher out of Job + // directly. + SWIFT_TASK_DEBUG_LOG( + "[%p] Swapping jobs to to %p, voucher %p does not need adoption", + this, job, job->Voucher); + previous = job->Voucher; + } + + // Either way, we've taken ownership of the job's voucher, so mark the job + // as having a dead voucher. + job->Voucher = SWIFT_DEAD_VOUCHER; + if (!OriginalVoucher) { + // If we don't yet have an original voucher, then save the one we grabbed + // above to restore later. + OriginalVoucher = previous; + SWIFT_TASK_DEBUG_LOG("[%p] Saved original voucher %p", this, previous); + } else { + // We already have an original voucher. The one we grabbed above is not + // needed. We own it, so destroy it here. + swift_voucher_release(previous); + } + } + + // Take the current thread's adopted voucher and place it back into the task + // that previously owned it, re-adopting the thread's original voucher. + void restoreVoucher(AsyncTask *task) { + SWIFT_TASK_DEBUG_LOG("[%p] Restoring %svoucher on task %p", this, + OriginalVoucher ? "" : "missing ", task); + assert(OriginalVoucher); + assert(task->Voucher == SWIFT_DEAD_VOUCHER); + + if (swift_voucher_needs_adopt(*OriginalVoucher)) { + // Adopt the execution thread's original voucher. The task's voucher is + // the one currently adopted, and is returned by voucher_adopt. + task->Voucher = voucher_adopt(*OriginalVoucher); + } else { + // No need to adopt the voucher, so we can take the one out of + // OriginalVoucher and return it to the task. + task->Voucher = *OriginalVoucher; + } + + // We've given up ownership of OriginalVoucher, clear it out. + OriginalVoucher = llvm::None; + } +}; + +} // end namespace swift + +#endif