Add optional corruption filter settings to EncodedImage.

This is a prerequisite for enabling implementation-specific filter settings for automatic corruption detection. Bug: webrtc:358039777 Change-Id: I363c592aa35164f690dd4ad1204e90afc0277d8b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/368940 Commit-Queue: Erik Språng <[email protected]> Reviewed-by: Sergey Silkin <[email protected]> Cr-Commit-Position: refs/heads/main@{#43443}
ibaoger · Nov 22, 2024 · e5f6f1f · e5f6f1f
1 parent 24992e9
commit e5f6f1f
Show file tree

Hide file tree

Showing 9 changed files with 122 additions and 37 deletions.
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
@@ -142,13 +142,19 @@ rtc_source_set("resolution") {
   deps = [ "../../rtc_base/system:rtc_export" ]
 }
 
+rtc_source_set("corruption_detection_filter_settings") {
+  visibility = [ "*" ]
+  public = [ "corruption_detection_filter_settings.h" ]
+}
+
 rtc_library("encoded_image") {
   visibility = [ "*" ]
   sources = [
     "encoded_image.cc",
     "encoded_image.h",
   ]
   deps = [
+    ":corruption_detection_filter_settings",
     ":video_codec_constants",
     ":video_frame",
     ":video_frame_type",

diff --git a/api/video/corruption_detection_filter_settings.h b/api/video/corruption_detection_filter_settings.h
@@ -0,0 +1,27 @@
+/*
+ *  Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_
+#define API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_
+
+#include <stdint.h>
+
+// Filter settings for automatic corruption detection. See
+// http://www.webrtc.org/experiments/rtp-hdrext/corruption-detection for more
+// information.
+struct CorruptionDetectionFilterSettings {
+  // Size of the blur kernel used.
+  double std_dev = 0.0;
+  // Allowed error thresholds (maps to `Y err` and `UV err` respectively).
+  int luma_error_threshold = 0;
+  int chroma_error_threshold = 0;
+};
+
+#endif  // API_VIDEO_CORRUPTION_DETECTION_FILTER_SETTINGS_H_
diff --git a/api/video/encoded_image.h b/api/video/encoded_image.h
@@ -23,6 +23,7 @@
 #include "api/scoped_refptr.h"
 #include "api/units/timestamp.h"
 #include "api/video/color_space.h"
+#include "api/video/corruption_detection_filter_settings.h"
 #include "api/video/video_codec_constants.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame_type.h"
@@ -228,6 +229,15 @@ class RTC_EXPORT EncodedImage {
   VideoContentType contentType() const { return content_type_; }
   VideoRotation rotation() const { return rotation_; }
 
+  std::optional<CorruptionDetectionFilterSettings>
+  corruption_detection_filter_settings() const {
+    return corruption_detection_filter_settings_;
+  }
+  void set_corruption_detection_filter_settings(
+      const CorruptionDetectionFilterSettings& settings) {
+    corruption_detection_filter_settings_ = settings;
+  }
+
   uint32_t _encodedWidth = 0;
   uint32_t _encodedHeight = 0;
   // NTP time of the capture time in local timebase in milliseconds.
@@ -283,6 +293,12 @@ class RTC_EXPORT EncodedImage {
   // True if the frame that was encoded is a steady-state refresh frame intended
   // to improve the visual quality.
   bool is_steady_state_refresh_frame_ = false;
+
+  // Filter settings for corruption detection suggested by the encoder
+  // implementation, if any. Otherwise generic per-codec-type settings will be
+  // used.
+  std::optional<CorruptionDetectionFilterSettings>
+      corruption_detection_filter_settings_;
 };
 
 }  // namespace webrtc

diff --git a/video/corruption_detection/BUILD.gn b/video/corruption_detection/BUILD.gn
@@ -47,6 +47,7 @@ rtc_library("frame_instrumentation_generator") {
     ":generic_mapping_functions",
     ":halton_frame_sampler",
     "../../api:scoped_refptr",
+    "../../api/video:corruption_detection_filter_settings",
     "../../api/video:encoded_image",
     "../../api/video:video_frame",
     "../../api/video:video_frame_type",
@@ -84,6 +85,7 @@ rtc_library("generic_mapping_functions") {
     "generic_mapping_functions.h",
   ]
   deps = [
+    "../../api/video:corruption_detection_filter_settings",
     "../../api/video:video_frame",
     "../../api/video_codecs:video_codecs_api",
     "../../rtc_base:checks",

diff --git a/video/corruption_detection/frame_instrumentation_generator.cc b/video/corruption_detection/frame_instrumentation_generator.cc
@@ -19,6 +19,7 @@
 #include "absl/algorithm/container.h"
 #include "absl/types/variant.h"
 #include "api/scoped_refptr.h"
+#include "api/video/corruption_detection_filter_settings.h"
 #include "api/video/encoded_image.h"
 #include "api/video/video_codec_type.h"
 #include "api/video/video_frame.h"
@@ -40,36 +41,33 @@ namespace {
 // can lead to frame buffer pools draining.
 constexpr size_t kMaxPendingFrames = 3;
 
-std::optional<FilterSettings> GetCorruptionFilterSettings(
+std::optional<CorruptionDetectionFilterSettings> GetCorruptionFilterSettings(
     const EncodedImage& encoded_image,
     VideoCodecType video_codec_type,
     int layer_id) {
-  /* TODO: bugs.webrtc.org/358039777 - Uncomment when parameters are available
-     in EncodedImage.
-  if (encoded_image.CorruptionDetectionParameters()) {
-    return FilterSettings{
-        .std_dev = encoded_image.CorruptionDetectionParameters()->std_dev,
-        .luma_error_threshold =
-            encoded_image.CorruptionDetectionParameters()->luma_error_threshold,
-        .chroma_error_threshold = encoded_image.CorruptionDetectionParameters()
-                                      ->chroma_error_threshold};
-  }
-  */
-
-  int qp = encoded_image.qp_;
-  if (qp == -1) {
-    std::optional<uint32_t> parsed_qp = QpParser().Parse(
-        video_codec_type, layer_id, encoded_image.data(), encoded_image.size());
-    if (!parsed_qp.has_value()) {
-      RTC_LOG(LS_VERBOSE) << "Missing QP for "
-                          << CodecTypeToPayloadString(video_codec_type)
-                          << " layer " << layer_id << ".";
-      return std::nullopt;
+  std::optional<CorruptionDetectionFilterSettings> filter_settings =
+      encoded_image.corruption_detection_filter_settings();
+
+  if (!filter_settings.has_value()) {
+    // No implementation specific filter settings available, using a generic
+    // QP-based settings instead.
+    int qp = encoded_image.qp_;
+    if (qp == -1) {
+      std::optional<uint32_t> parsed_qp =
+          QpParser().Parse(video_codec_type, layer_id, encoded_image.data(),
+                           encoded_image.size());
+      if (!parsed_qp.has_value()) {
+        RTC_LOG(LS_VERBOSE)
+            << "Missing QP for " << CodecTypeToPayloadString(video_codec_type)
+            << " layer " << layer_id << ".";
+        return std::nullopt;
+      }
+      qp = *parsed_qp;
     }
-    qp = *parsed_qp;
-  }
 
-  return GetCorruptionFilterSettings(qp, video_codec_type);
+    filter_settings = GetCorruptionFilterSettings(qp, video_codec_type);
+  }
+  return filter_settings;
 }
 
 }  // namespace
@@ -158,7 +156,7 @@ FrameInstrumentationGenerator::OnEncodedImage(
                                         .communicate_upper_bits = true};
   }
 
-  std::optional<FilterSettings> filter_settings =
+  std::optional<CorruptionDetectionFilterSettings> filter_settings =
       GetCorruptionFilterSettings(encoded_image, video_codec_type_, layer_id);
   if (!filter_settings.has_value()) {
     return std::nullopt;

diff --git a/video/corruption_detection/frame_instrumentation_generator_unittest.cc b/video/corruption_detection/frame_instrumentation_generator_unittest.cc
@@ -722,5 +722,37 @@ TEST(FrameInstrumentationGeneratorTest, QueuesAtMostThreeInputFrames) {
   EXPECT_THAT(frames_destroyed, ElementsAre(true, true, true, true));
 }
 
+TEST(FrameInstrumentationGeneratorTest,
+     UsesFilterSettingsFromFrameWhenAvailable) {
+  FrameInstrumentationGenerator generator(VideoCodecType::kVideoCodecVP8);
+  VideoFrame frame = VideoFrame::Builder()
+                         .set_video_frame_buffer(MakeDefaultI420FrameBuffer())
+                         .set_rtp_timestamp(1)
+                         .build();
+  // No QP needed when frame provides filter settings.
+  EncodedImage encoded_image;
+  encoded_image.SetRtpTimestamp(1);
+  encoded_image.SetFrameType(VideoFrameType::kVideoFrameKey);
+  encoded_image._encodedWidth = kDefaultScaledWidth;
+  encoded_image._encodedHeight = kDefaultScaledHeight;
+  encoded_image.set_corruption_detection_filter_settings(
+      CorruptionDetectionFilterSettings{.std_dev = 1.0,
+                                        .luma_error_threshold = 2,
+                                        .chroma_error_threshold = 3});
+
+  generator.OnCapturedFrame(frame);
+  std::optional<
+      absl::variant<FrameInstrumentationSyncData, FrameInstrumentationData>>
+      data = generator.OnEncodedImage(encoded_image);
+
+  ASSERT_TRUE(data.has_value());
+  ASSERT_TRUE(absl::holds_alternative<FrameInstrumentationData>(*data));
+  FrameInstrumentationData frame_instrumentation_data =
+      absl::get<FrameInstrumentationData>(*data);
+  EXPECT_EQ(frame_instrumentation_data.std_dev, 1.0);
+  EXPECT_EQ(frame_instrumentation_data.luma_error_threshold, 2);
+  EXPECT_EQ(frame_instrumentation_data.chroma_error_threshold, 3);
+}
+
 }  // namespace
 }  // namespace webrtc
diff --git a/video/corruption_detection/frame_pair_corruption_score.cc b/video/corruption_detection/frame_pair_corruption_score.cc
@@ -77,7 +77,8 @@ double FramePairCorruptionScorer::CalculateScore(
   scoped_refptr<I420Buffer> test_i420_buffer =
       GetAsI420Buffer(test_buffer.ToI420());
 
-  FilterSettings filter_settings = GetCorruptionFilterSettings(qp, codec_type_);
+  CorruptionDetectionFilterSettings filter_settings =
+      GetCorruptionFilterSettings(qp, codec_type_);
 
   const std::vector<FilteredSample> filtered_reference_sample_values =
       GetSampleValuesForFrame(

diff --git a/video/corruption_detection/generic_mapping_functions.cc b/video/corruption_detection/generic_mapping_functions.cc
@@ -73,10 +73,13 @@ double MapQpToOptimalStdDev(int qp, VideoCodecType codec_type) {
 
 }  // namespace
 
-FilterSettings GetCorruptionFilterSettings(int qp, VideoCodecType codec_type) {
-  return FilterSettings{.std_dev = MapQpToOptimalStdDev(qp, codec_type),
-                        .luma_error_threshold = LumaThreshold(codec_type),
-                        .chroma_error_threshold = ChromaThreshold(codec_type)};
+CorruptionDetectionFilterSettings GetCorruptionFilterSettings(
+    int qp,
+    VideoCodecType codec_type) {
+  return CorruptionDetectionFilterSettings{
+      .std_dev = MapQpToOptimalStdDev(qp, codec_type),
+      .luma_error_threshold = LumaThreshold(codec_type),
+      .chroma_error_threshold = ChromaThreshold(codec_type)};
 }
 
 }  // namespace webrtc
diff --git a/video/corruption_detection/generic_mapping_functions.h b/video/corruption_detection/generic_mapping_functions.h
@@ -11,17 +11,17 @@
 #ifndef VIDEO_CORRUPTION_DETECTION_GENERIC_MAPPING_FUNCTIONS_H_
 #define VIDEO_CORRUPTION_DETECTION_GENERIC_MAPPING_FUNCTIONS_H_
 
+#include "api/video/corruption_detection_filter_settings.h"
 #include "api/video/video_codec_type.h"
 
 namespace webrtc {
 
-struct FilterSettings {
-  double std_dev = 0.0;
-  int luma_error_threshold = 0;
-  int chroma_error_threshold = 0;
-};
+// TODO: bugs.webrtc.org/358039777 - Remove when downstream usage is gone.
+using FilterSettings = CorruptionDetectionFilterSettings;
 
-FilterSettings GetCorruptionFilterSettings(int qp, VideoCodecType codec_type);
+CorruptionDetectionFilterSettings GetCorruptionFilterSettings(
+    int qp,
+    VideoCodecType codec_type);
 
 }  // namespace webrtc