Bug 1631263: Implement RTCRtpScriptTransform. r=pehrsons,jib,asuth,em…

…ilio,saschanaz

Differential Revision: https://phabricator.services.mozilla.com/D179735

dom/events/EventNameList.h

@@ -295,6 +295,7 @@ WINDOW_EVENT(languagechange, eLanguageChange,
 // need a different macro to flag things like that (IDL, but not content
 // attributes on body/frameset), or is just using EventNameType_None enough?
 WINDOW_EVENT(message, eMessage, EventNameType_None, eBasicEventClass)
+WINDOW_EVENT(rtctransform, eRTCTransform, EventNameType_None, eBasicEventClass)
 WINDOW_EVENT(messageerror, eMessageError, EventNameType_HTMLBodyOrFramesetOnly,
              eBasicEventClass)
 WINDOW_EVENT(offline, eOffline,

dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp

@@ -0,0 +1,94 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#include "jsapi/RTCEncodedAudioFrame.h"
+
+#include <stdint.h>
+
+#include <memory>
+#include <utility>
+
+#include "api/frame_transformer_interface.h"
+
+#include "jsapi/RTCEncodedFrameBase.h"
+#include "jsapi/RTCRtpScriptTransform.h"
+#include "mozilla/dom/RTCRtpScriptTransformer.h"
+#include "mozilla/dom/RTCEncodedAudioFrameBinding.h"
+#include "nsWrapperCache.h"
+#include "nsISupports.h"
+#include "nsCycleCollectionParticipant.h"
+#include "nsIGlobalObject.h"
+#include "nsContentUtils.h"
+#include "mozilla/HoldDropJSObjects.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/Unused.h"
+#include "mozilla/fallible.h"
+#include "js/RootingAPI.h"
+
+namespace mozilla::dom {
+
+NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase,
+                                   mOwner)
+NS_IMPL_ADDREF_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase)
+NS_IMPL_RELEASE_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase)
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedAudioFrame)
+  NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
+NS_INTERFACE_MAP_END_INHERITING(RTCEncodedFrameBase)
+
+RTCEncodedAudioFrame::RTCEncodedAudioFrame(
+    nsIGlobalObject* aGlobal,
+    std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
+    uint64_t aCounter, RTCRtpScriptTransformer* aOwner)
+    : RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter),
+      mOwner(aOwner) {
+  mMetadata.mSynchronizationSource.Construct(mFrame->GetSsrc());
+  mMetadata.mPayloadType.Construct(mFrame->GetPayloadType());
+  // send frames are derived directly from TransformableFrameInterface, not
+  // TransformableAudioFrameInterface! Right now, send frames have no csrcs
+  // or sequence number
+  // TODO(bug 1835076): Fix this
+  if (mFrame->GetDirection() ==
+      webrtc::TransformableFrameInterface::Direction::kReceiver) {
+    const auto& audioFrame(
+        static_cast<webrtc::TransformableAudioFrameInterface&>(*mFrame));
+    mMetadata.mContributingSources.Construct();
+    for (const auto csrc : audioFrame.GetContributingSources()) {
+      Unused << mMetadata.mContributingSources.Value().AppendElement(csrc,
+                                                                     fallible);
+    }
+    mMetadata.mSequenceNumber.Construct(audioFrame.GetHeader().sequenceNumber);
+  }
+
+  // Base class needs this, but can't do it itself because of an assertion in
+  // the cycle-collector.
+  mozilla::HoldJSObjects(this);
+}
+
+RTCEncodedAudioFrame::~RTCEncodedAudioFrame() {
+  // Base class needs this, but can't do it itself because of an assertion in
+  // the cycle-collector.
+  mozilla::DropJSObjects(this);
+}
+
+JSObject* RTCEncodedAudioFrame::WrapObject(JSContext* aCx,
+                                           JS::Handle<JSObject*> aGivenProto) {
+  return RTCEncodedAudioFrame_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+nsIGlobalObject* RTCEncodedAudioFrame::GetParentObject() const {
+  return mGlobal;
+}
+
+void RTCEncodedAudioFrame::GetMetadata(
+    RTCEncodedAudioFrameMetadata& aMetadata) const {
+  aMetadata = mMetadata;
+}
+
+bool RTCEncodedAudioFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const {
+  return aOwner == mOwner;
+}
+}  // namespace mozilla::dom

dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h

@@ -0,0 +1,52 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_
+#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_
+
+#include "mozilla/RefPtr.h"
+#include "nsIGlobalObject.h"
+#include "jsapi/RTCEncodedFrameBase.h"
+#include "mozilla/dom/RTCEncodedAudioFrameBinding.h"
+
+namespace mozilla::dom {
+
+// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and
+// providing read-only access to various metadata. After the libwebrtc frame is
+// extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is
+// detached, but the metadata remains accessible.
+class RTCEncodedAudioFrame final : public RTCEncodedFrameBase {
+ public:
+  explicit RTCEncodedAudioFrame(
+      nsIGlobalObject* aGlobal,
+      std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
+      uint64_t aCounter, RTCRtpScriptTransformer* aOwner);
+
+  // nsISupports
+  NS_DECL_ISUPPORTS_INHERITED
+  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCEncodedAudioFrame,
+                                           RTCEncodedFrameBase)
+
+  // webidl (timestamp and data accessors live in base class)
+  JSObject* WrapObject(JSContext* aCx,
+                       JS::Handle<JSObject*> aGivenProto) override;
+
+  nsIGlobalObject* GetParentObject() const;
+
+  void GetMetadata(RTCEncodedAudioFrameMetadata& aMetadata) const;
+
+  bool CheckOwner(RTCRtpScriptTransformer* aOwner) const override;
+
+  bool IsVideo() const override { return false; }
+
+ private:
+  virtual ~RTCEncodedAudioFrame();
+  RefPtr<RTCRtpScriptTransformer> mOwner;
+  RTCEncodedAudioFrameMetadata mMetadata;
+};
+
+}  // namespace mozilla::dom
+#endif  // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_

dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp

@@ -0,0 +1,71 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#include "jsapi/RTCEncodedFrameBase.h"
+
+#include "nsIGlobalObject.h"
+#include "mozilla/dom/ScriptSettings.h"
+#include "js/ArrayBuffer.h"
+
+namespace mozilla::dom {
+NS_IMPL_CYCLE_COLLECTION_WITH_JS_MEMBERS(RTCEncodedFrameBase, (mGlobal),
+                                         (mData))
+NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCEncodedFrameBase)
+NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCEncodedFrameBase)
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedFrameBase)
+  NS_INTERFACE_MAP_ENTRY(nsISupports)
+NS_INTERFACE_MAP_END
+
+RTCEncodedFrameBase::RTCEncodedFrameBase(
+    nsIGlobalObject* aGlobal,
+    std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
+    uint64_t aCounter)
+    : mGlobal(aGlobal),
+      mFrame(std::move(aFrame)),
+      mCounter(aCounter),
+      mTimestamp(mFrame->GetTimestamp()) {
+  AutoJSAPI jsapi;
+  if (NS_WARN_IF(!jsapi.Init(mGlobal))) {
+    return;
+  }
+
+  // Avoid a copy
+  mData = JS::NewArrayBufferWithUserOwnedContents(
+      jsapi.cx(), mFrame->GetData().size(), (void*)(mFrame->GetData().data()));
+}
+
+RTCEncodedFrameBase::~RTCEncodedFrameBase() = default;
+
+unsigned long RTCEncodedFrameBase::Timestamp() const { return mTimestamp; }
+
+void RTCEncodedFrameBase::SetData(const ArrayBuffer& aData) {
+  mData.set(aData.Obj());
+  if (mFrame) {
+    aData.ComputeState();
+    mFrame->SetData(rtc::ArrayView<const uint8_t>(
+        static_cast<const uint8_t*>(aData.Data()), aData.Length()));
+  }
+}
+
+void RTCEncodedFrameBase::GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj) {
+  aObj->set(mData);
+}
+
+uint64_t RTCEncodedFrameBase::GetCounter() const { return mCounter; }
+
+std::unique_ptr<webrtc::TransformableFrameInterface>
+RTCEncodedFrameBase::TakeFrame() {
+  AutoJSAPI jsapi;
+  if (!jsapi.Init(mGlobal)) {
+    MOZ_CRASH("Could not init JSAPI!");
+  }
+  JS::Rooted<JSObject*> rootedData(jsapi.cx(), mData);
+  JS::DetachArrayBuffer(jsapi.cx(), rootedData);
+  return std::move(mFrame);
+}
+
+}  // namespace mozilla::dom

dom/media/webrtc/jsapi/RTCEncodedFrameBase.h

@@ -0,0 +1,58 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_
+#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_
+
+#include "js/TypeDecls.h"
+#include "mozilla/dom/TypedArray.h"  // ArrayBuffer
+#include "mozilla/Assertions.h"
+#include "api/frame_transformer_interface.h"
+#include <memory>
+
+class nsIGlobalObject;
+
+namespace mozilla::dom {
+
+class RTCRtpScriptTransformer;
+
+class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
+ public:
+  explicit RTCEncodedFrameBase(
+      nsIGlobalObject* aGlobal,
+      std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
+      uint64_t aCounter);
+
+  // nsISupports
+  NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+  NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(RTCEncodedFrameBase)
+
+  // Common webidl for RTCEncodedVideoFrame/RTCEncodedAudioFrame
+  unsigned long Timestamp() const;
+
+  void SetData(const ArrayBuffer& aData);
+
+  void GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj);
+
+  uint64_t GetCounter() const;
+
+  virtual bool CheckOwner(RTCRtpScriptTransformer* aOwner) const = 0;
+
+  std::unique_ptr<webrtc::TransformableFrameInterface> TakeFrame();
+
+  virtual bool IsVideo() const = 0;
+
+ protected:
+  virtual ~RTCEncodedFrameBase();
+  RefPtr<nsIGlobalObject> mGlobal;
+  std::unique_ptr<webrtc::TransformableFrameInterface> mFrame;
+  const uint64_t mCounter = 0;
+  const unsigned long mTimestamp = 0;
+  JS::Heap<JSObject*> mData;
+};
+
+}  // namespace mozilla::dom
+#endif  // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_