Initial implementation of idle notification. (flutter#4012)

 - Assumes only the Dart VM is interested in idle notification.
 - Gives the VM the time remaining in each frame.
 - Gives the VM 100ms if there is no pending frame.

Issue flutter/flutter#9594

runtime/runtime_controller.cc

@@ -76,6 +76,15 @@ void RuntimeController::BeginFrame(ftl::TimePoint frame_time) {
   GetWindow()->BeginFrame(frame_time);
 }
 
+void RuntimeController::NotifyIdle(int64_t deadline) {
+  UIDartState* dart_state = dart_controller_->dart_state();
+  if (!dart_state) {
+    return;
+  }
+  DartState::Scope scope(dart_state);
+  Dart_NotifyIdle(deadline);
+}
+
 void RuntimeController::DispatchPlatformMessage(
     ftl::RefPtr<PlatformMessage> message) {
   TRACE_EVENT0("flutter", "RuntimeController::DispatchPlatformMessage");

runtime/runtime_controller.h

@@ -38,6 +38,7 @@ class RuntimeController : public WindowClient, public IsolateClient {
   void SetSemanticsEnabled(bool enabled);
 
   void BeginFrame(ftl::TimePoint frame_time);
+  void NotifyIdle(int64_t deadline);
 
   void DispatchPlatformMessage(ftl::RefPtr<PlatformMessage> message);
   void DispatchPointerDataPacket(const PointerDataPacket& packet);

shell/common/animator.cc

@@ -7,6 +7,7 @@
 #include "flutter/common/threads.h"
 #include "flutter/fml/trace_event.h"
 #include "lib/ftl/time/stopwatch.h"
+#include "dart/runtime/include/dart_tools_api.h"
 
 namespace shell {
 
@@ -16,10 +17,13 @@ Animator::Animator(ftl::WeakPtr<Rasterizer> rasterizer,
     : rasterizer_(rasterizer),
       waiter_(waiter),
       engine_(engine),
+      last_begin_frame_time_(),
+      dart_frame_deadline_(0),
       layer_tree_pipeline_(ftl::MakeRefCounted<LayerTreePipeline>(2)),
       pending_frame_semaphore_(1),
       frame_number_(1),
       paused_(false),
+      frame_scheduled_(false),
       weak_factory_(this) {}
 
 Animator::~Animator() = default;
@@ -37,9 +41,17 @@ void Animator::Start() {
   RequestFrame();
 }
 
-void Animator::BeginFrame(ftl::TimePoint frame_time) {
+static int64_t FtlToDartOrEarlier(ftl::TimePoint time) {
+  int64_t dart_now = Dart_TimelineGetMicros();
+  ftl::TimePoint ftl_now = ftl::TimePoint::Now();
+  return (time - ftl_now).ToMicroseconds() + dart_now;
+}
+
+void Animator::BeginFrame(ftl::TimePoint frame_start_time,
+                          ftl::TimePoint frame_target_time) {
   TRACE_EVENT_ASYNC_END0("flutter", "Frame Request Pending", frame_number_++);
 
+  frame_scheduled_ = false;
   pending_frame_semaphore_.Signal();
 
   if (!producer_continuation_) {
@@ -62,8 +74,15 @@ void Animator::BeginFrame(ftl::TimePoint frame_time) {
   // to service potential frame.
   FTL_DCHECK(producer_continuation_);
 
-  last_begin_frame_time_ = frame_time;
+  last_begin_frame_time_ = frame_start_time;
+  dart_frame_deadline_ = FtlToDartOrEarlier(frame_target_time);
   engine_->BeginFrame(last_begin_frame_time_);
+
+  if (!frame_scheduled_) {
+    // We don't have another frame pending, so we're waiting on user input
+    // or I/O. Allow the Dart VM 100 ms.
+    engine_->NotifyIdle(dart_frame_deadline_ + 100000);
+  }
 }
 
 void Animator::Render(std::unique_ptr<flow::LayerTree> layer_tree) {
@@ -111,14 +130,17 @@ void Animator::RequestFrame() {
                                  frame_number);
         self->AwaitVSync();
       });
+  frame_scheduled_ = true;
 }
 
 void Animator::AwaitVSync() {
   waiter_->AsyncWaitForVsync([self = weak_factory_.GetWeakPtr()](
-      ftl::TimePoint frame_time) {
+      ftl::TimePoint frame_start_time, ftl::TimePoint frame_target_time) {
     if (self)
-      self->BeginFrame(frame_time);
+      self->BeginFrame(frame_start_time, frame_target_time);
   });
+
+  engine_->NotifyIdle(dart_frame_deadline_);
 }
 
 }  // namespace shell

shell/common/animator.h

@@ -35,7 +35,8 @@ class Animator {
  private:
   using LayerTreePipeline = flutter::Pipeline<flow::LayerTree>;
 
-  void BeginFrame(ftl::TimePoint frame_time);
+  void BeginFrame(ftl::TimePoint frame_start_time,
+                  ftl::TimePoint frame_target_time);
 
   void AwaitVSync();
 
@@ -44,11 +45,13 @@ class Animator {
   Engine* engine_;
 
   ftl::TimePoint last_begin_frame_time_;
+  int64_t dart_frame_deadline_;
   ftl::RefPtr<LayerTreePipeline> layer_tree_pipeline_;
   flutter::Semaphore pending_frame_semaphore_;
   LayerTreePipeline::ProducerContinuation producer_continuation_;
   int64_t frame_number_;
   bool paused_;
+  bool frame_scheduled_;
 
   ftl::WeakPtrFactory<Animator> weak_factory_;
 

shell/common/engine.cc

@@ -243,6 +243,12 @@ void Engine::BeginFrame(ftl::TimePoint frame_time) {
     runtime_->BeginFrame(frame_time);
 }
 
+void Engine::NotifyIdle(int64_t deadline) {
+  TRACE_EVENT0("flutter", "Engine::NotifyIdle");
+  if (runtime_)
+    runtime_->NotifyIdle(deadline);
+}
+
 void Engine::RunFromSource(const std::string& main,
                            const std::string& packages,
                            const std::string& bundle_path) {

shell/common/engine.h

@@ -50,6 +50,7 @@ class Engine : public blink::RuntimeDelegate {
                           const std::string& packages);
 
   void BeginFrame(ftl::TimePoint frame_time);
+  void NotifyIdle(int64_t deadline);
 
   void RunFromSource(const std::string& main,
                      const std::string& packages,

shell/common/vsync_waiter.h

@@ -13,7 +13,8 @@ namespace shell {
 
 class VsyncWaiter {
  public:
-  using Callback = std::function<void(ftl::TimePoint frame_time)>;
+  using Callback = std::function<void(ftl::TimePoint frame_start_time,
+                                      ftl::TimePoint frame_target_time)>;
 
   virtual void AsyncWaitForVsync(Callback callback) = 0;
 

shell/common/vsync_waiter_fallback.cc

@@ -26,12 +26,12 @@ VsyncWaiterFallback::VsyncWaiterFallback()
 
 VsyncWaiterFallback::~VsyncWaiterFallback() = default;
 
+constexpr ftl::TimeDelta interval = ftl::TimeDelta::FromSecondsF(1.0 / 60.0);
+
 void VsyncWaiterFallback::AsyncWaitForVsync(Callback callback) {
   FTL_DCHECK(!callback_);
   callback_ = std::move(callback);
 
-  constexpr ftl::TimeDelta interval = ftl::TimeDelta::FromSecondsF(1.0 / 60.0);
-
   ftl::TimePoint now = ftl::TimePoint::Now();
   ftl::TimePoint next = SnapToNextTick(now, phase_, interval);
 
@@ -42,7 +42,7 @@ void VsyncWaiterFallback::AsyncWaitForVsync(Callback callback) {
         ftl::TimePoint frame_time = ftl::TimePoint::Now();
         Callback callback = std::move(self->callback_);
         self->callback_ = Callback();
-        callback(frame_time);
+        callback(frame_time, frame_time + interval);
       },
       next - now);
 }

shell/platform/android/io/flutter/view/FlutterView.java

@@ -35,6 +35,7 @@
 import io.flutter.plugin.common.MethodChannel;
 import io.flutter.plugin.editing.TextInputPlugin;
 import io.flutter.plugin.platform.PlatformPlugin;
+import io.flutter.view.VsyncWaiter;
 
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -662,6 +663,9 @@ private void updateViewportMetrics() {
             mMetrics.physicalPaddingRight,
             mMetrics.physicalPaddingBottom,
             mMetrics.physicalPaddingLeft);
+
+        float fps = getDisplay().getRefreshRate();
+        VsyncWaiter.refreshPeriodNanos = (long)(1000000000.0 / fps);
     }
 
     // Called by native to send us a platform message.

shell/platform/android/io/flutter/view/VsyncWaiter.java

@@ -7,14 +7,17 @@
 import android.view.Choreographer;
 
 public class VsyncWaiter {
+    // This estimate will be updated by FlutterView when it is attached to a Display.
+    public static long refreshPeriodNanos = 1000000000 / 60;
+
     public static void asyncWaitForVsync(final long cookie) {
         Choreographer.getInstance().postFrameCallback(new Choreographer.FrameCallback() {
             @Override
             public void doFrame(long frameTimeNanos) {
-                nativeOnVsync(frameTimeNanos, cookie);
+                nativeOnVsync(frameTimeNanos, frameTimeNanos + refreshPeriodNanos, cookie);
             }
         });
     }
 
-    private static native void nativeOnVsync(long frameTimeNanos, long cookie);
+    private static native void nativeOnVsync(long frameTimeNanos, long frameTargetTimeNanos, long cookie);
 }

shell/platform/android/vsync_waiter_android.cc

@@ -37,18 +37,22 @@ void VsyncWaiterAndroid::AsyncWaitForVsync(Callback callback) {
   });
 }
 
-void VsyncWaiterAndroid::OnVsync(int64_t frameTimeNanos) {
+void VsyncWaiterAndroid::OnVsync(int64_t frameTimeNanos,
+                                 int64_t frameTargetTimeNanos) {
   Callback callback = std::move(callback_);
   callback_ = Callback();
-  blink::Threads::UI()->PostTask([callback, frameTimeNanos] {
+  blink::Threads::UI()->PostTask([callback, frameTimeNanos, frameTargetTimeNanos] {
     callback(ftl::TimePoint::FromEpochDelta(
-        ftl::TimeDelta::FromNanoseconds(frameTimeNanos)));
+                 ftl::TimeDelta::FromNanoseconds(frameTimeNanos)),
+             ftl::TimePoint::FromEpochDelta(
+                 ftl::TimeDelta::FromNanoseconds(frameTargetTimeNanos)));
   });
 }
 
 static void OnNativeVsync(JNIEnv* env,
                           jclass jcaller,
                           jlong frameTimeNanos,
+                          jlong frameTargetTimeNanos,
                           jlong cookie) {
   // Note: The tag name must be "VSYNC" (it is special) so that the "Highlight
   // Vsync" checkbox in the timeline can be enabled.
@@ -61,14 +65,15 @@ static void OnNativeVsync(JNIEnv* env,
   VsyncWaiterAndroid* waiter = weak->get();
   delete weak;
   if (waiter) {
-    waiter->OnVsync(static_cast<int64_t>(frameTimeNanos));
+    waiter->OnVsync(static_cast<int64_t>(frameTimeNanos),
+                    static_cast<int64_t>(frameTargetTimeNanos));
   }
 }
 
 bool VsyncWaiterAndroid::Register(JNIEnv* env) {
   static const JNINativeMethod methods[] = {{
       .name = "nativeOnVsync",
-      .signature = "(JJ)V",
+      .signature = "(JJJ)V",
       .fnPtr = reinterpret_cast<void*>(&OnNativeVsync),
   }};
 

shell/platform/android/vsync_waiter_android.h

@@ -22,7 +22,7 @@ class VsyncWaiterAndroid : public VsyncWaiter {
 
   void AsyncWaitForVsync(Callback callback) override;
 
-  void OnVsync(int64_t frameTimeNanos);
+  void OnVsync(int64_t frameTimeNanos, int64_t frameTargetTimeNanos);
 
  private:
   Callback callback_;

shell/platform/darwin/desktop/vsync_waiter_mac.cc

@@ -40,13 +40,21 @@ void VsyncWaiterMac::OnDisplayLink(void* context) {
 }
 
 void VsyncWaiterMac::OnDisplayLink() {
-  ftl::TimePoint frame_time = ftl::TimePoint::Now();
+  ftl::TimePoint frame_start_time = ftl::TimePoint::Now();
+  ftl::TimePoint frame_target_time =
+      frame_start_time +
+      ftl::TimeDelta::FromSecondsF(
+          CVDisplayLinkGetActualOutputVideoRefreshPeriod(link_));
+
   CVDisplayLinkStop(link_);
+
   auto callback = std::move(callback_);
   callback_ = Callback();
 
   blink::Threads::UI()->PostTask(
-      [callback, frame_time] { callback(frame_time); });
+      [callback, frame_start_time, frame_target_time] {
+        callback(frame_start_time, frame_target_time);
+      });
 }
 
 void VsyncWaiterMac::AsyncWaitForVsync(Callback callback) {

shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.mm

@@ -47,6 +47,11 @@ - (void)await:(shell::VsyncWaiter::Callback)callback {
 }
 
 - (void)onDisplayLink:(CADisplayLink*)link {
+  // ftl::TimePoint and CATimeInterval both use mach_absolute_time.
+  ftl::TimePoint frame_start_time = ftl::TimePoint::Now();
+  ftl::TimePoint frame_target_time =
+      ftl::TimePoint::FromEpochDelta(ftl::TimeDelta::FromSecondsF(link.targetTimestamp));
+
   _displayLink.paused = YES;
 
   // Note: Even though we know we are on the UI thread already (since the
@@ -57,9 +62,9 @@ - (void)onDisplayLink:(CADisplayLink*)link {
   //
   // We are not using the PostTask for thread switching, but to make task
   // observers work.
-  blink::Threads::UI()->PostTask([callback = _pendingCallback]() {
-    callback(ftl::TimePoint::Now());
-  });
+  blink::Threads::UI()->PostTask([
+    callback = _pendingCallback, frame_start_time, frame_target_time
+  ]() { callback(frame_start_time, frame_target_time); });
 
   _pendingCallback = nullptr;
 }