Skip to content

Commit

Permalink
[fuchsia][a11y] Set explicit hit regions per compositor layer on GFX. (…
Browse files Browse the repository at this point in the history 
…flutter#33852)
  • Loading branch information
@abrush21
abrush21 authored Jun 16, 2022
1 parent 385861e commit 0720dc1
Show file tree
Hide file tree
Showing 3 changed files with 383 additions and 107 deletions.
115 changes: 87 additions & 28 deletions shell/platform/fuchsia/flutter/gfx_external_view_embedder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

#include <algorithm> // For std::clamp

#include "flutter/fml/logging.h"
#include "flutter/fml/trace_event.h"
#include "third_party/skia/include/core/SkPicture.h"
#include "third_party/skia/include/core/SkSurface.h"
Expand Down Expand Up @@ -169,8 +170,9 @@ void GfxExternalViewEmbedder::PrerollCompositeEmbeddedView(
zx_handle_t handle = static_cast<zx_handle_t>(view_id);
FML_CHECK(frame_layers_.count(handle) == 0);

frame_layers_.emplace(std::make_pair(EmbedderLayerId{handle},
EmbedderLayer(frame_size_, *params)));
frame_layers_.emplace(std::make_pair(
EmbedderLayerId{handle},
EmbedderLayer(frame_size_, *params, flutter::RTreeFactory())));
frame_composition_order_.push_back(handle);
}

Expand Down Expand Up @@ -200,8 +202,9 @@ void GfxExternalViewEmbedder::BeginFrame(
frame_dpr_ = device_pixel_ratio;

// Create the root layer.
frame_layers_.emplace(
std::make_pair(kRootLayerId, EmbedderLayer(frame_size, std::nullopt)));
frame_layers_.emplace(std::make_pair(
kRootLayerId,
EmbedderLayer(frame_size, std::nullopt, flutter::RTreeFactory())));
frame_composition_order_.push_back(kRootLayerId);

// Set up the input interceptor at the top of the scene, if applicable.
Expand Down Expand Up @@ -271,6 +274,19 @@ void GfxExternalViewEmbedder::SubmitFrame(
}
}

// Finish recording SkPictures.
{
TRACE_EVENT0("flutter", "FinishRecordingPictures");

for (const auto& surface_index : frame_surface_indices) {
const auto& layer = frame_layers_.find(surface_index.first);
FML_CHECK(layer != frame_layers_.end());
layer->second.picture =
layer->second.recorder->finishRecordingAsPicture();
FML_CHECK(layer->second.picture != nullptr);
}
}

// Submit layers and platform views to Scenic in composition order.
{
TRACE_EVENT0("flutter", "SubmitLayers");
Expand Down Expand Up @@ -437,10 +453,18 @@ void GfxExternalViewEmbedder::SubmitFrame(
FML_CHECK(scenic_layer_index <= scenic_layers_.size());
if (scenic_layer_index == scenic_layers_.size()) {
ScenicLayer new_layer{
.shape_node = scenic::ShapeNode(session_->get()),
.material = scenic::Material(session_->get()),
.layer_node = scenic::EntityNode(session_->get()),
.image =
ScenicImage{
.shape_node = scenic::ShapeNode(session_->get()),
.material = scenic::Material(session_->get()),
},
// We'll set hit regions later.
.hit_regions = {},
};
new_layer.shape_node.SetMaterial(new_layer.material);
new_layer.layer_node.SetLabel("Flutter::Layer");
new_layer.layer_node.AddChild(new_layer.image.shape_node);
new_layer.image.shape_node.SetMaterial(new_layer.image.material);
scenic_layers_.emplace_back(std::move(new_layer));
}

Expand Down Expand Up @@ -491,25 +515,50 @@ void GfxExternalViewEmbedder::SubmitFrame(
embedded_views_height;
auto& scenic_layer = scenic_layers_[scenic_layer_index];
auto& scenic_rect = found_rects->second[rect_index];
scenic_layer.shape_node.SetLabel("Flutter::Layer");
scenic_layer.shape_node.SetShape(scenic_rect);
scenic_layer.shape_node.SetTranslation(
auto& image = scenic_layer.image;
image.shape_node.SetLabel("Flutter::Layer::Image");
image.shape_node.SetShape(scenic_rect);
image.shape_node.SetTranslation(
layer->second.surface_size.width() * 0.5f,
layer->second.surface_size.height() * 0.5f, -layer_elevation);
scenic_layer.material.SetColor(SK_AlphaOPAQUE, SK_AlphaOPAQUE,
SK_AlphaOPAQUE, layer_opacity);
scenic_layer.material.SetTexture(surface_for_layer->GetImageId());

// Only the first (i.e. the bottom-most) layer should receive input.
// TODO: Workaround for invisible overlays stealing input. Remove when
// the underlying bug is fixed.
const fuchsia::ui::gfx::HitTestBehavior layer_hit_test_behavior =
first_layer ? fuchsia::ui::gfx::HitTestBehavior::kDefault
: fuchsia::ui::gfx::HitTestBehavior::kSuppress;
scenic_layer.shape_node.SetHitTestBehavior(layer_hit_test_behavior);
image.material.SetColor(SK_AlphaOPAQUE, SK_AlphaOPAQUE, SK_AlphaOPAQUE,
layer_opacity);
image.material.SetTexture(surface_for_layer->GetImageId());

// We'll set hit regions expliclty on a separate ShapeNode, so the image
// itself should be unhittable and semantically invisible.
image.shape_node.SetHitTestBehavior(
fuchsia::ui::gfx::HitTestBehavior::kSuppress);
image.shape_node.SetSemanticVisibility(false);

// Attach the ScenicLayer to the main scene graph.
layer_tree_node_.AddChild(scenic_layer.shape_node);
layer_tree_node_.AddChild(scenic_layer.layer_node);

// Compute the set of non-overlapping set of bounding boxes for the
// painted content in this layer.
{
FML_CHECK(layer->second.rtree);
std::list<SkRect> intersection_rects =
layer->second.rtree->searchNonOverlappingDrawnRects(
SkRect::Make(layer->second.surface_size));

// SkRect joined_rect = SkRect::MakeEmpty();
for (const SkRect& rect : intersection_rects) {
auto paint_bounds =
scenic::Rectangle(session_->get(), rect.width(), rect.height());
auto hit_region = scenic::ShapeNode(session_->get());
hit_region.SetLabel("Flutter::Layer::HitRegion");
hit_region.SetShape(paint_bounds);
hit_region.SetTranslation(rect.centerX(), rect.centerY(),
-layer_elevation);
hit_region.SetHitTestBehavior(
fuchsia::ui::gfx::HitTestBehavior::kDefault);
hit_region.SetSemanticVisibility(true);

scenic_layer.layer_node.AddChild(hit_region);
scenic_layer.hit_regions.push_back(std::move(hit_region));
}
}
}

// Reset for the next pass:
Expand All @@ -527,7 +576,11 @@ void GfxExternalViewEmbedder::SubmitFrame(
session_->Present();
}

// Render the recorded SkPictures into the surfaces.
// Flush pending skia operations.
// NOTE: This operation MUST occur AFTER the `Present() ` call above. We
// pipeline the Skia rendering work with scenic IPC, and scenic will delay
// internally until Skia is finished. So, doing this work before calling
// `Present()` would adversely affect performance.
{
TRACE_EVENT0("flutter", "RasterizeSurfaces");

Expand All @@ -548,13 +601,10 @@ void GfxExternalViewEmbedder::SubmitFrame(

const auto& layer = frame_layers_.find(surface_index.first);
FML_CHECK(layer != frame_layers_.end());
sk_sp<SkPicture> picture =
layer->second.recorder->finishRecordingAsPicture();
FML_CHECK(picture != nullptr);

canvas->setMatrix(SkMatrix::I());
canvas->clear(SK_ColorTRANSPARENT);
canvas->drawPicture(picture);
canvas->drawPicture(layer->second.picture);
canvas->flush();
}
}
Expand Down Expand Up @@ -636,7 +686,16 @@ void GfxExternalViewEmbedder::Reset() {

// Clear images on all layers so they aren't cached unnecessarily.
for (auto& layer : scenic_layers_) {
layer.material.SetTexture(0);
layer.image.material.SetTexture(0);

// Detach hit regions; otherwise, they may persist across frames
// incorrectly.
for (auto& hit_region : layer.hit_regions) {
hit_region.Detach();
}

// Remove cached hit regions so that we don't recreate stale ones.
layer.hit_regions.clear();
}
}

Expand Down
47 changes: 41 additions & 6 deletions shell/platform/fuchsia/flutter/gfx_external_view_embedder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include <vector>

#include "flutter/flow/embedded_views.h"
#include "flutter/flow/rtree.h"
#include "flutter/fml/logging.h"
#include "flutter/fml/macros.h"
#include "flutter/shell/common/canvas_spy.h"
Expand Down Expand Up @@ -138,18 +139,29 @@ class GfxExternalViewEmbedder final : public flutter::ExternalViewEmbedder {

struct EmbedderLayer {
EmbedderLayer(const SkISize& frame_size,
std::optional<flutter::EmbeddedViewParams> view_params)
: embedded_view_params(std::move(view_params)),
std::optional<flutter::EmbeddedViewParams> view_params,
flutter::RTreeFactory rtree_factory)
: rtree(rtree_factory.getInstance()),
embedded_view_params(std::move(view_params)),
recorder(std::make_unique<SkPictureRecorder>()),
canvas_spy(std::make_unique<flutter::CanvasSpy>(
recorder->beginRecording(frame_size.width(),
frame_size.height()))),
surface_size(frame_size) {}
recorder->beginRecording(SkRect::Make(frame_size),
&rtree_factory))),
surface_size(frame_size),
picture(nullptr) {}

// Records paint operations applied to this layer's `SkCanvas`.
// These records are used to determine which portions of this layer
// contain content. The embedder propagates this information to scenic, so
// that scenic can accurately decide which portions of this layer may
// interact with input.
sk_sp<flutter::RTree> rtree;

std::optional<flutter::EmbeddedViewParams> embedded_view_params;
std::unique_ptr<SkPictureRecorder> recorder;
std::unique_ptr<flutter::CanvasSpy> canvas_spy;
SkISize surface_size;
sk_sp<SkPicture> picture;
};
using EmbedderLayerId = std::optional<uint32_t>;
constexpr static EmbedderLayerId kRootLayerId = EmbedderLayerId{};
Expand All @@ -172,11 +184,34 @@ class GfxExternalViewEmbedder final : public flutter::ExternalViewEmbedder {
bool pending_focusable = true;
};

struct ScenicLayer {
// GFX resources required to render a composited flutter layer (i.e. an
// SkPicture).
struct ScenicImage {
scenic::ShapeNode shape_node;
scenic::Material material;
};

// All resources required to represent a flutter layer in the GFX scene
// graph. The structure of the subgraph for a particular layer is:
//
// layer_node
// / \
// image node hit regions (zero or more)
//
// NOTE: `hit_regions` must be cleared before submitting each new frame;
// otherwise, we will report stale hittable geometry to scenic.
struct ScenicLayer {
// Root of the subtree containing the scenic resources for this layer.
scenic::EntityNode layer_node;

// Scenic resources used to render this layer's image.
ScenicImage image;

// Scenic resources that specify which parts of this layer are responsive
// to input.
std::vector<scenic::ShapeNode> hit_regions;
};

std::shared_ptr<GfxSessionConnection> session_;
std::shared_ptr<SurfaceProducer> surface_producer_;

Expand Down
Loading

0 comments on commit 0720dc1

Please sign in to comment.