Skip to content

Commit

Permalink
Add initial support for RtpEncodingParameters max_framerate.
Browse files Browse the repository at this point in the history 
Add support to set the framerate to the maximum of |max_framerate|.
Different framerates are currently not supported per stream for video.

Bug: webrtc:9597
Change-Id: Ie326617b66bd97be387f809a7f82b97b8f3ff5fe
Reviewed-on: https://webrtc-review.googlesource.com/92392
Reviewed-by: Sebastian Jansson <[email protected]>
Reviewed-by: Erik Språng <[email protected]>
Reviewed-by: Magnus Jedvert <[email protected]>
Reviewed-by: Steve Anton <[email protected]>
Commit-Queue: Åsa Persson <[email protected]>
Cr-Commit-Position: refs/heads/master@{#24270}
  • Loading branch information
Åsa Persson authored and Commit Bot committed Aug 13, 2018
1 parent 553b6f6 commit ced5cfd
Show file tree
Hide file tree
Showing 14 changed files with 387 additions and 167 deletions.
6 changes: 5 additions & 1 deletion api/rtpparameters.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,10 @@ struct RtpEncodingParameters {
// TODO(asapersson): Not implemented for ORTC API.
absl::optional<int> min_bitrate_bps;

// TODO(deadbeef): Not implemented.
// Specifies the maximum framerate in fps for video.
// TODO(asapersson): Different framerates are not supported per stream.
// If set, the maximum |max_framerate| is currently used.
// Not supported for screencast.
absl::optional<int> max_framerate;

// For video, scale the resolution down by this factor.
Expand Down Expand Up @@ -451,6 +454,7 @@ struct RtpEncodingParameters {
fec == o.fec && rtx == o.rtx && dtx == o.dtx &&
bitrate_priority == o.bitrate_priority && ptime == o.ptime &&
max_bitrate_bps == o.max_bitrate_bps &&
min_bitrate_bps == o.min_bitrate_bps &&
max_framerate == o.max_framerate &&
scale_resolution_down_by == o.scale_resolution_down_by &&
scale_framerate_down_by == o.scale_framerate_down_by &&
Expand Down
23 changes: 20 additions & 3 deletions call/call_perf_tests.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,7 @@ TEST_F(CallPerfTest, ReceivesCpuOveruseAndUnderuse) {
class LoadObserver : public test::SendTest,
public test::FrameGeneratorCapturer::SinkWantsObserver {
public:
LoadObserver() : SendTest(kLongTimeoutMs), test_phase_(TestPhase::kStart) {}
LoadObserver() : SendTest(kLongTimeoutMs), test_phase_(TestPhase::kInit) {}

void OnFrameGeneratorCapturerCreated(
test::FrameGeneratorCapturer* frame_generator_capturer) override {
Expand All @@ -507,9 +507,21 @@ TEST_F(CallPerfTest, ReceivesCpuOveruseAndUnderuse) {
// TODO(sprang): Add integration test for maintain-framerate mode?
void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
const rtc::VideoSinkWants& wants) override {
// First expect CPU overuse. Then expect CPU underuse when the encoder
// At kStart expect CPU overuse. Then expect CPU underuse when the encoder
// delay has been decreased.
switch (test_phase_) {
case TestPhase::kInit:
// Max framerate should be set initially.
if (wants.max_framerate_fps != std::numeric_limits<int>::max() &&
wants.max_pixel_count == std::numeric_limits<int>::max()) {
test_phase_ = TestPhase::kStart;
} else {
ADD_FAILURE() << "Got unexpected adaptation request, max res = "
<< wants.max_pixel_count << ", target res = "
<< wants.target_pixel_count.value_or(-1)
<< ", max fps = " << wants.max_framerate_fps;
}
break;
case TestPhase::kStart:
if (wants.max_pixel_count < std::numeric_limits<int>::max()) {
// On adapting down, VideoStreamEncoder::VideoSourceProxy will set
Expand Down Expand Up @@ -553,7 +565,12 @@ TEST_F(CallPerfTest, ReceivesCpuOveruseAndUnderuse) {
EXPECT_TRUE(Wait()) << "Timed out before receiving an overuse callback.";
}

enum class TestPhase { kStart, kAdaptedDown, kAdaptedUp } test_phase_;
enum class TestPhase {
kInit,
kStart,
kAdaptedDown,
kAdaptedUp
} test_phase_;
} test;

RunBaseTest(&test);
Expand Down
15 changes: 6 additions & 9 deletions media/engine/simulcast.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <algorithm>
#include <string>

#include "media/base/mediaconstants.h"
#include "media/base/streamparams.h"
#include "media/engine/constants.h"
#include "media/engine/simulcast.h"
Expand Down Expand Up @@ -189,17 +190,15 @@ std::vector<webrtc::VideoStream> GetSimulcastConfig(
int /*max_bitrate_bps*/,
double bitrate_priority,
int max_qp,
int max_framerate,
int /*max_framerate*/,
bool is_screenshare,
bool temporal_layers_supported) {
if (is_screenshare) {
return GetScreenshareLayers(max_layers, width, height, bitrate_priority,
max_qp, max_framerate,
temporal_layers_supported);
max_qp, temporal_layers_supported);
} else {
return GetNormalSimulcastLayers(max_layers, width, height, bitrate_priority,
max_qp, max_framerate,
temporal_layers_supported);
max_qp, temporal_layers_supported);
}
}

Expand All @@ -209,7 +208,6 @@ std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
int height,
double bitrate_priority,
int max_qp,
int max_framerate,
bool temporal_layers_supported) {
// TODO(bugs.webrtc.org/8785): Currently if the resolution isn't large enough
// (defined in kSimulcastFormats) we scale down the number of simulcast
Expand Down Expand Up @@ -270,7 +268,7 @@ std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
static_cast<int>(layers[s].target_bitrate_bps * rate_factor);
}
layers[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height);
layers[s].max_framerate = max_framerate;
layers[s].max_framerate = kDefaultVideoMaxFramerate;

width /= 2;
height /= 2;
Expand All @@ -294,7 +292,6 @@ std::vector<webrtc::VideoStream> GetScreenshareLayers(
int height,
double bitrate_priority,
int max_qp,
int max_framerate,
bool temporal_layers_supported) {
size_t num_simulcast_layers =
std::min<int>(max_layers, kMaxScreenshareSimulcastLayers);
Expand Down Expand Up @@ -346,7 +343,7 @@ std::vector<webrtc::VideoStream> GetScreenshareLayers(
layers[1].width = width;
layers[1].height = height;
layers[1].max_qp = max_qp;
layers[1].max_framerate = max_framerate;
layers[1].max_framerate = kDefaultVideoMaxFramerate;
layers[1].num_temporal_layers =
temporal_layers_supported ? DefaultNumberOfTemporalLayers(1) : 0;
layers[1].min_bitrate_bps = layers[0].target_bitrate_bps * 2;
Expand Down
6 changes: 2 additions & 4 deletions media/engine/simulcast.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ void BoostMaxSimulcastLayer(int max_bitrate_bps,
std::vector<webrtc::VideoStream>* layers);

// Gets simulcast settings.
// TODO(asapersson): Remove max_bitrate_bps.
// TODO(asapersson): Remove max_bitrate_bps and max_framerate.
std::vector<webrtc::VideoStream> GetSimulcastConfig(
size_t max_layers,
int width,
int height,
int /*max_bitrate_bps*/,
double bitrate_priority,
int max_qp,
int max_framerate,
int /*max_framerate*/,
bool is_screenshare,
bool temporal_layers_supported = true);

Expand All @@ -45,7 +45,6 @@ std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
int height,
double bitrate_priority,
int max_qp,
int max_framerate,
bool temporal_layers_supported = true);

// Gets simulcast config layers for screenshare settings.
Expand All @@ -55,7 +54,6 @@ std::vector<webrtc::VideoStream> GetScreenshareLayers(
int height,
double bitrate_priority,
int max_qp,
int max_framerate,
bool temporal_layers_supported = true);
} // namespace cricket

Expand Down
3 changes: 2 additions & 1 deletion media/engine/simulcast_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

#include "media/engine/simulcast.h"

#include "media/base/mediaconstants.h"
#include "media/engine/constants.h"
#include "test/field_trial.h"
#include "test/gtest.h"
Expand Down Expand Up @@ -94,7 +95,7 @@ TEST(SimulcastTest, GetConfig) {

for (size_t i = 0; i < streams.size(); ++i) {
EXPECT_EQ(size_t{kDefaultTemporalLayers}, streams[i].num_temporal_layers);
EXPECT_EQ(kMaxFps, streams[i].max_framerate);
EXPECT_EQ(cricket::kDefaultVideoMaxFramerate, streams[i].max_framerate);
EXPECT_EQ(kQpMax, streams[i].max_qp);
EXPECT_EQ(kExpected[i].min_bitrate_bps, streams[i].min_bitrate_bps);
EXPECT_EQ(kExpected[i].target_bitrate_bps, streams[i].target_bitrate_bps);
Expand Down
55 changes: 40 additions & 15 deletions media/engine/webrtcvideoengine.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,18 @@ std::vector<VideoCodec> AssignPayloadTypesAndDefaultCodecs(
: std::vector<VideoCodec>();
}

int GetMaxFramerate(const webrtc::VideoEncoderConfig& encoder_config,
size_t num_layers) {
int max_fps = -1;
for (size_t i = 0; i < num_layers; ++i) {
int fps = (encoder_config.simulcast_layers[i].max_framerate > 0)
? encoder_config.simulcast_layers[i].max_framerate
: kDefaultVideoMaxFramerate;
max_fps = std::max(fps, max_fps);
}
return max_fps;
}

static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) {
std::stringstream out;
out << '{';
Expand Down Expand Up @@ -1802,13 +1814,16 @@ webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters(
return error;
}

bool new_bitrate = false;
bool new_param = false;
for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
if ((new_parameters.encodings[i].min_bitrate_bps !=
rtp_parameters_.encodings[i].min_bitrate_bps) ||
(new_parameters.encodings[i].max_bitrate_bps !=
rtp_parameters_.encodings[i].max_bitrate_bps)) {
new_bitrate = true;
rtp_parameters_.encodings[i].max_bitrate_bps) ||
(new_parameters.encodings[i].max_framerate !=
rtp_parameters_.encodings[i].max_framerate)) {
new_param = true;
break;
}
}

Expand All @@ -1822,8 +1837,8 @@ webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters(
// entire encoder reconfiguration, it just needs to update the bitrate
// allocator.
bool reconfigure_encoder =
new_bitrate || (new_parameters.encodings[0].bitrate_priority !=
rtp_parameters_.encodings[0].bitrate_priority);
new_param || (new_parameters.encodings[0].bitrate_priority !=
rtp_parameters_.encodings[0].bitrate_priority);

// TODO(bugs.webrtc.org/8807): The active field as well should not require
// a full encoder reconfiguration, but it needs to update both the bitrate
Expand Down Expand Up @@ -1985,7 +2000,7 @@ WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig(

// Application-controlled state is held in the encoder_config's
// simulcast_layers. Currently this is used to control which simulcast layers
// are active and for configuring the min/max bitrate.
// are active and for configuring the min/max bitrate and max framerate.
// The encoder_config's simulcast_layers is also used for non-simulcast (when
// there is a single layer).
RTC_DCHECK_GE(rtp_parameters_.encodings.size(),
Expand All @@ -2003,14 +2018,17 @@ WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig(
encoder_config.simulcast_layers[i].max_bitrate_bps =
*rtp_parameters_.encodings[i].max_bitrate_bps;
}
if (rtp_parameters_.encodings[i].max_framerate) {
encoder_config.simulcast_layers[i].max_framerate =
*rtp_parameters_.encodings[i].max_framerate;
}
}

int max_qp = kDefaultQpMax;
codec.GetParam(kCodecParamMaxQuantization, &max_qp);
encoder_config.video_stream_factory =
new rtc::RefCountedObject<EncoderStreamFactory>(
codec.name, max_qp, kDefaultVideoMaxFramerate, is_screencast,
parameters_.conference_mode);
codec.name, max_qp, is_screencast, parameters_.conference_mode);
return encoder_config;
}

Expand Down Expand Up @@ -2702,19 +2720,17 @@ WebRtcVideoChannel::MapCodecs(const std::vector<VideoCodec>& codecs) {
return video_codecs;
}

// TODO(bugs.webrtc.org/8785): Consider removing max_qp and max_framerate
// as members of EncoderStreamFactory and instead set these values individually
// for each stream in the VideoEncoderConfig.simulcast_layers.
// TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of
// EncoderStreamFactory and instead set this value individually for each stream
// in the VideoEncoderConfig.simulcast_layers.
EncoderStreamFactory::EncoderStreamFactory(
std::string codec_name,
int max_qp,
int max_framerate,
bool is_screenshare,
bool screenshare_config_explicitly_enabled)

: codec_name_(codec_name),
max_qp_(max_qp),
max_framerate_(max_framerate),
is_screenshare_(is_screenshare),
screenshare_config_explicitly_enabled_(
screenshare_config_explicitly_enabled) {}
Expand All @@ -2738,12 +2754,18 @@ std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
bool temporal_layers_supported = CodecNamesEq(codec_name_, kVp8CodecName);
layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height,
0 /*not used*/, encoder_config.bitrate_priority,
max_qp_, max_framerate_, is_screenshare_,
max_qp_, 0 /*not_used*/, is_screenshare_,
temporal_layers_supported);
// The maximum |max_framerate| is currently used for video.
int max_framerate = GetMaxFramerate(encoder_config, layers.size());
// Update the active simulcast layers and configured bitrates.
bool is_highest_layer_max_bitrate_configured = false;
for (size_t i = 0; i < layers.size(); ++i) {
layers[i].active = encoder_config.simulcast_layers[i].active;
if (!is_screenshare_) {
// Update simulcast framerates with max configured max framerate.
layers[i].max_framerate = max_framerate;
}
// Update simulcast bitrates with configured min and max bitrate.
if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) {
layers[i].min_bitrate_bps =
Expand Down Expand Up @@ -2800,11 +2822,14 @@ std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
if (encoder_config.max_bitrate_bps <= 0)
max_bitrate_bps = std::max(min_bitrate_bps, max_bitrate_bps);
}
int max_framerate = (encoder_config.simulcast_layers[0].max_framerate > 0)
? encoder_config.simulcast_layers[0].max_framerate
: kDefaultVideoMaxFramerate;

webrtc::VideoStream layer;
layer.width = width;
layer.height = height;
layer.max_framerate = max_framerate_;
layer.max_framerate = max_framerate;

// In the case that the application sets a max bitrate that's lower than the
// min bitrate, we adjust it down (see bugs.webrtc.org/9141).
Expand Down
2 changes: 0 additions & 2 deletions media/engine/webrtcvideoengine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,6 @@ class EncoderStreamFactory
public:
EncoderStreamFactory(std::string codec_name,
int max_qp,
int max_framerate,
bool is_screenshare,
bool screenshare_config_explicitly_enabled);

Expand All @@ -517,7 +516,6 @@ class EncoderStreamFactory

const std::string codec_name_;
const int max_qp_;
const int max_framerate_;
const bool is_screenshare_;
// Allows a screenshare specific configuration, which enables temporal
// layering and allows simulcast.
Expand Down
Loading

0 comments on commit ced5cfd

Please sign in to comment.