Skip to content

Commit

Permalink
Bug 1749044 - Make AppleATDecoder capable of decoding ATDS bytestream…
Browse files Browse the repository at this point in the history 
… directly r=chunmin

Differential Revision: https://phabricator.services.mozilla.com/D192363
  • Loading branch information
@padenot
padenot committed Mar 5, 2024
1 parent 738a845 commit 5d24210
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 31 deletions.
90 changes: 70 additions & 20 deletions dom/media/platforms/apple/AppleATDecoder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
#include "mozilla/SyncRunnable.h"
#include "mozilla/UniquePtr.h"
#include "nsTArray.h"
#include "ADTSDemuxer.h"

#include <array>

#define LOG(...) DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, __VA_ARGS__)
#define LOGEX(_this, ...) \
Expand Down Expand Up @@ -62,6 +65,7 @@ AppleATDecoder::~AppleATDecoder() {

RefPtr<MediaDataDecoder::InitPromise> AppleATDecoder::Init() {
if (!mFormatID) {
LOG("AppleATDecoder::Init failure: unknown format ID");
return InitPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("Non recognised format")),
Expand All @@ -85,6 +89,7 @@ RefPtr<MediaDataDecoder::FlushPromise> AppleATDecoder::Flush() {
}
}
if (mErrored) {
LOG("Flush error");
mParsedFramesForAACMagicCookie = 0;
mMagicCookie.Clear();
ProcessShutdown();
Expand Down Expand Up @@ -188,18 +193,28 @@ RefPtr<MediaDataDecoder::DecodePromise> AppleATDecoder::Decode(

MediaResult rv = NS_OK;
if (!mConverter) {
LOG("Lazily initing the decoder");
rv = SetupDecoder(aSample);
if (rv != NS_OK && rv != NS_ERROR_NOT_INITIALIZED) {
LOG("Decoder not initialized");
return DecodePromise::CreateAndReject(rv, __func__);
}
}

if (mIsADTS) {
bool rv = ADTS::StripHeader(aSample);
if (!rv) {
LOG("Stripping the ADTS header in AppleATDecoder failed");
}
}

mQueuedSamples.AppendElement(aSample);

if (rv == NS_OK) {
for (size_t i = 0; i < mQueuedSamples.Length(); i++) {
rv = DecodeSample(mQueuedSamples[i]);
if (NS_FAILED(rv)) {
LOG("Decoding error");
mErrored = true;
return DecodePromise::CreateAndReject(rv, __func__);
}
Expand Down Expand Up @@ -277,7 +292,7 @@ MediaResult AppleATDecoder::DecodeSample(MediaRawData* aSample) {
}

size_t numFrames = outputData.Length() / channels;
int rate = mOutputFormat.mSampleRate;
int rate = AssertedCast<int>(mOutputFormat.mSampleRate);
media::TimeUnit duration(numFrames, rate);
if (!duration.IsValid()) {
NS_WARNING("Invalid count of accumulated audio samples");
Expand Down Expand Up @@ -340,8 +355,8 @@ MediaResult AppleATDecoder::GetInputAudioDescription(
aDesc.mChannelsPerFrame = mConfig.mChannels;
aDesc.mSampleRate = mConfig.mRate;
UInt32 inputFormatSize = sizeof(aDesc);
OSStatus rv = AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL,
&inputFormatSize, &aDesc);
OSStatus rv = AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0,
nullptr, &inputFormatSize, &aDesc);
if (NS_WARN_IF(rv)) {
return MediaResult(
NS_ERROR_FAILURE,
Expand Down Expand Up @@ -419,7 +434,7 @@ nsresult AppleATDecoder::SetupChannelLayout() {
UInt32 propertySize;
UInt32 size;
OSStatus status = AudioConverterGetPropertyInfo(
mConverter, kAudioConverterOutputChannelLayout, &propertySize, NULL);
mConverter, kAudioConverterOutputChannelLayout, &propertySize, nullptr);
if (status || !propertySize) {
LOG("Couldn't get channel layout property (%s)", FourCC2Str(status));
return NS_ERROR_FAILURE;
Expand Down Expand Up @@ -504,15 +519,36 @@ MediaResult AppleATDecoder::SetupDecoder(MediaRawData* aSample) {
MOZ_ASSERT(mThread->IsOnCurrentThread());
static const uint32_t MAX_FRAMES = 2;

bool isADTS =
ADTS::FrameHeader::MatchesSync(Span{aSample->Data(), aSample->Size()});

if (isADTS) {
ADTS::FrameParser parser;
if(!parser.Parse(0, aSample->Data(), aSample->Data() + aSample->Size()) {
LOG("ADTS frame parsing error");
return NS_ERROR_NOT_INITIALIZED;
}

AudioCodecSpecificBinaryBlob blob;
ADTS::InitAudioSpecificConfig(parser.FirstFrame(), blob.mBinaryBlob);
mConfig.mCodecSpecificConfig = AudioCodecSpecificVariant{std::move(blob)};
mConfig.mProfile = mConfig.mExtendedProfile =
parser.FirstFrame().Header().mObjectType;
mIsADTS = true;
}

if (mFormatID == kAudioFormatMPEG4AAC && mConfig.mExtendedProfile == 2 &&
mParsedFramesForAACMagicCookie < MAX_FRAMES) {
LOG("Attempting to get implicit AAC magic cookie");
// Check for implicit SBR signalling if stream is AAC-LC
// This will provide us with an updated magic cookie for use with
// GetInputAudioDescription.
if (NS_SUCCEEDED(GetImplicitAACMagicCookie(aSample)) &&
!mMagicCookie.Length()) {
!mMagicCookie.Length() && !isADTS) {
// nothing found yet, will try again later
LOG("Getting implicit AAC magic cookie failed");
mParsedFramesForAACMagicCookie++;
LOG("Not initialized -- need magic cookie");
return NS_ERROR_NOT_INITIALIZED;
}
// An error occurred, fallback to using default stream description
Expand All @@ -538,6 +574,7 @@ MediaResult AppleATDecoder::SetupDecoder(MediaRawData* aSample) {

MediaResult rv = GetInputAudioDescription(inputFormat, magicCookie);
if (NS_FAILED(rv)) {
LOG("GetInputAudioDescription failure");
return rv;
}
// Fill in the output format manually.
Expand Down Expand Up @@ -617,36 +654,49 @@ static void _SampleCallback(void* aSBR, UInt32 aNumBytes, UInt32 aNumPackets,
const void* aData,
AudioStreamPacketDescription* aPackets) {}

nsresult AppleATDecoder::GetImplicitAACMagicCookie(
const MediaRawData* aSample) {
nsresult AppleATDecoder::GetImplicitAACMagicCookie(MediaRawData* aSample) {
MOZ_ASSERT(mThread->IsOnCurrentThread());

// Prepend ADTS header to AAC audio.
RefPtr<MediaRawData> adtssample(aSample->Clone());
if (!adtssample) {
return NS_ERROR_OUT_OF_MEMORY;
}
int8_t frequency_index = Adts::GetFrequencyIndex(mConfig.mRate);
bool isADTS =
ADTS::FrameHeader::MatchesSync(Span{aSample->Data(), aSample->Size()});

bool rv = Adts::ConvertSample(mConfig.mChannels, frequency_index,
mConfig.mProfile, adtssample);
if (!rv) {
NS_WARNING("Failed to apply ADTS header");
return NS_ERROR_FAILURE;
RefPtr<MediaRawData> adtssample = aSample;

if (!isADTS) {
// Prepend ADTS header to AAC audio.
adtssample = aSample->Clone();
if (!adtssample) {
return NS_ERROR_OUT_OF_MEMORY;
}
auto frequency_index = ADTS::GetFrequencyIndex(mConfig.mRate);

if (frequency_index.isErr()) {
LOG("%d isn't a valid rate for AAC", mConfig.mRate);
return NS_ERROR_FAILURE;
}

// Arbitrarily pick main profile if not specified
int profile = mConfig.mProfile ? mConfig.mProfile : 1;
bool rv = ADTS::ConvertSample(mConfig.mChannels, frequency_index.unwrap(),
profile, adtssample);
if (!rv) {
LOG("Failed to apply ADTS header");
return NS_ERROR_FAILURE;
}
}
if (!mStream) {
OSStatus rv = AudioFileStreamOpen(this, _MetadataCallback, _SampleCallback,
kAudioFileAAC_ADTSType, &mStream);
if (rv) {
NS_WARNING("Couldn't open AudioFileStream");
LOG("Couldn't open AudioFileStream");
return NS_ERROR_FAILURE;
}
}

OSStatus status = AudioFileStreamParseBytes(
mStream, adtssample->Size(), adtssample->Data(), 0 /* discontinuity */);
if (status) {
NS_WARNING("Couldn't parse sample");
LOG("Couldn't parse sample");
}

if (status || mFileStreamError || mMagicCookie.Length()) {
Expand Down
5 changes: 3 additions & 2 deletions dom/media/platforms/apple/AppleATDecoder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class AppleATDecoder final : public MediaDataDecoder,
nsCString GetCodecName() const override;

// Callbacks also need access to the config.
const AudioInfo mConfig;
AudioInfo mConfig;

// Use to extract magic cookie for HE-AAC detection.
nsTArray<uint8_t> mMagicCookie;
Expand Down Expand Up @@ -67,11 +67,12 @@ class AppleATDecoder final : public MediaDataDecoder,
// Setup AudioConverter once all information required has been gathered.
// Will return NS_ERROR_NOT_INITIALIZED if more data is required.
MediaResult SetupDecoder(MediaRawData* aSample);
nsresult GetImplicitAACMagicCookie(const MediaRawData* aSample);
nsresult GetImplicitAACMagicCookie(MediaRawData* aSample);
nsresult SetupChannelLayout();
uint32_t mParsedFramesForAACMagicCookie;
uint32_t mEncoderDelay = 0;
uint64_t mTotalMediaFrames = 0;
bool mIsADTS = false;
bool mErrored;
};

Expand Down
18 changes: 9 additions & 9 deletions dom/media/webcodecs/AudioDecoder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ UniquePtr<AudioDecoderConfigInternal> AudioDecoderConfigInternal::Create(
GetErrorName(rv.unwrapErr(), error);
LOGE(
"Failed to create AudioDecoderConfigInternal due to invalid "
"description data. Error: %s", error.get());
"description data. Error: %s",
error.get());
return nullptr;
}
description.emplace(rv.unwrap());
Expand All @@ -115,10 +116,10 @@ struct AudioMIMECreateParam {
const nsString mParsedCodec;
};


// Map between WebCodecs pcm types as strings and codec numbers
// All other codecs
nsCString ConvertCodecName(const nsCString& aContainer, const nsCString& aCodec) {
nsCString ConvertCodecName(const nsCString& aContainer,
const nsCString& aCodec) {
if (!aContainer.EqualsLiteral("x-wav")) {
return aCodec;
}
Expand Down Expand Up @@ -216,8 +217,8 @@ static Result<Ok, nsresult> CloneConfiguration(
}

// https://w3c.github.io/webcodecs/#create-a-audiodata
static RefPtr<AudioData> CreateAudioData(
nsIGlobalObject* aGlobalObject, mozilla::AudioData* aData) {
static RefPtr<AudioData> CreateAudioData(nsIGlobalObject* aGlobalObject,
mozilla::AudioData* aData) {
MOZ_ASSERT(aGlobalObject);
MOZ_ASSERT(aData);

Expand Down Expand Up @@ -282,7 +283,8 @@ Result<UniquePtr<TrackInfo>, nsresult> AudioDecoderTraits::CreateTrackInfo(

// https://w3c.github.io/webcodecs/#valid-audiodecoderconfig
/* static */
bool AudioDecoderTraits::Validate(const AudioDecoderConfig& aConfig, nsCString& aErrorMessage) {
bool AudioDecoderTraits::Validate(const AudioDecoderConfig& aConfig,
nsCString& aErrorMessage) {
Maybe<nsString> codec = ParseCodecString(aConfig.mCodec);
if (!codec || codec->IsEmpty()) {
LOGE("Validating AudioDecoderConfig: invalid codec string");
Expand Down Expand Up @@ -395,7 +397,6 @@ already_AddRefed<Promise> AudioDecoder::IsConfigSupported(
return p.forget();
}


// TODO: Move the following works to another thread to unblock the current
// thread, as what spec suggests.

Expand All @@ -405,8 +406,7 @@ already_AddRefed<Promise> AudioDecoder::IsConfigSupported(
nsresult e = r.unwrapErr();
nsCString error;
GetErrorName(e, error);
LOGE("Failed to clone AudioDecoderConfig. Error: %s",
error.get());
LOGE("Failed to clone AudioDecoderConfig. Error: %s", error.get());
p->MaybeRejectWithTypeError("Failed to clone AudioDecoderConfig");
aRv.Throw(e);
return p.forget();
Expand Down

0 comments on commit 5d24210

Please sign in to comment.