diff --git a/LayoutTests/ChangeLog b/LayoutTests/ChangeLog index f24c3d901745f..46fb8f7f9fd47 100644 --- a/LayoutTests/ChangeLog +++ b/LayoutTests/ChangeLog @@ -1,3 +1,14 @@ +2018-06-07 Thibault Saunier + + [GTK][WPE] Start implementing MediaStream API + https://bugs.webkit.org/show_bug.cgi?id=185787 + + Reviewed by Philippe Normand. + + * platform/gtk/TestExpectations: Reactivate MediaStream tests and enable all tests + related to the mediaDevice.enumerateDevices and MediaStream (not RTCPeerConnection nor + webaudio). + 2018-06-07 Brent Fulgham Remove unused debug mode conditions diff --git a/LayoutTests/platform/gtk/TestExpectations b/LayoutTests/platform/gtk/TestExpectations index a7e67477d0191..59752370dce78 100644 --- a/LayoutTests/platform/gtk/TestExpectations +++ b/LayoutTests/platform/gtk/TestExpectations @@ -572,40 +572,31 @@ webkit.org/b/99036 fast/shadow-dom/pointerlockelement-in-slot.html [ Skip ] webkit.org/b/85211 ietestcenter/css3/flexbox/flexbox-align-stretch-001.htm [ ImageOnlyFailure ] webkit.org/b/85212 ietestcenter/css3/flexbox/flexbox-layout-002.htm [ ImageOnlyFailure ] -# Skipping until backend is implemented -fast/mediastream [ Skip ] webrtc [ Skip ] imported/w3c/web-platform-tests/webrtc/ [ Skip ] http/tests/webrtc [ Skip ] # The MediaStream implementation is also still not completed webkit.org/b/79203 fast/mediastream/mock-media-source-webaudio.html [ Timeout ] webkit.org/b/79203 fast/mediastream/getUserMedia-webaudio.html [ Failure ] -webkit.org/b/79203 fast/mediastream/mock-media-source.html [ Failure ] -webkit.org/b/79203 fast/mediastream/MediaStream-video-element-track-stop.html [ Pass Crash ] +webkit.org/b/79203 fast/mediastream/MediaStream-video-element-track-stop.html [ Timeout ] webkit.org/b/79203 fast/mediastream/RTCPeerConnection-dtmf.html [ Timeout ] webkit.org/b/79203 fast/mediastream/RTCPeerConnection-icecandidate-event.html [ Failure Crash ] +webkit.org/b/79203 fast/mediastream/RTCPeerConnection-media-setup-two-dialogs.html [ Failure Crash ] webkit.org/b/79203 fast/mediastream/RTCRtpSender-replaceTrack.html [ Failure ] # Crash is webkit.org/b/184292 webkit.org/b/79203 fast/mediastream/RTCPeerConnection-addIceCandidate.html [ Failure Pass Crash ] -webkit.org/b/79203 fast/mediastream/RTCPeerConnection-closed-state.html [ Timeout ] -webkit.org/b/79203 fast/mediastream/RTCPeerConnection-ice.html [ Timeout ] -webkit.org/b/79203 fast/mediastream/RTCPeerConnection-iceconnectionstatechange-event.html [ Timeout ] +webkit.org/b/79203 fast/mediastream/RTCPeerConnection-iceconnectionstatechange-event.html [ Failure Timeout ] +webkit.org/b/79203 fast/mediastream/RTCPeerConnection-ice.html [ Failure Timeout ] webkit.org/b/79203 fast/mediastream/RTCPeerConnection-inspect-answer.html [ Failure ] webkit.org/b/79203 fast/mediastream/RTCPeerConnection-inspect-offer-bundlePolicy-bundle-only.html [ Failure ] webkit.org/b/79203 fast/mediastream/RTCPeerConnection-inspect-offer.html [ Failure ] webkit.org/b/79203 fast/mediastream/RTCPeerConnection-legacy-stream-based-api.html [ Failure ] +webkit.org/b/79203 fast/mediastream/RTCPeerConnection-media-setup-callbacks-single-dialog.html [ Crash Failure ] webkit.org/b/79203 webaudio/mediastreamaudiosourcenode.html [ Failure ] -webkit.org/b/79203 fast/mediastream/MediaStream-video-element-displays-buffer.html [ Crash ] -webkit.org/b/79203 fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html [ Crash ] -webkit.org/b/79203 fast/mediastream/MediaStream-video-element.html [ Crash ] -webkit.org/b/79203 fast/mediastream/apply-constraints-advanced.html [ Crash ] +webkit.org/b/79203 fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html [ Timeout Failure ] +webkit.org/b/79203 fast/mediastream/MediaStream-video-element-displays-buffer.html [ Failure ] webkit.org/b/79203 http/tests/media/media-stream/getusermedia-with-canvas.html [ Crash ] -webkit.org/b/79203 fast/mediastream/MediaStream-MediaElement-setObject-null.html [ Timeout ] webkit.org/b/79203 fast/mediastream/RTCPeerConnection-stats.html [ Timeout ] -webkit.org/b/79203 fast/mediastream/captureStream/canvas2d-heavy-drawing.html [ Timeout ] -webkit.org/b/79203 fast/mediastream/get-user-media-on-loadedmetadata.html [ Timeout ] -webkit.org/b/79203 fast/mediastream/local-audio-playing-event.html [ Timeout ] -webkit.org/b/79203 fast/mediastream/media-stream-track-source-failure.html [ Timeout ] webkit.org/b/79203 imported/w3c/web-platform-tests/mediacapture-streams/MediaStream-MediaElement-preload-none.https.html [ Failure ] webkit.org/b/79203 imported/w3c/web-platform-tests/mediacapture-streams/MediaStreamTrack-end-manual.https.html [ Failure ] webkit.org/b/151344 fast/mediastream/MediaStream-add-ended-tracks.html [ Timeout ] @@ -616,6 +607,9 @@ webkit.org/b/173257 fast/mediastream/getUserMedia-grant-persistency3.html [ Pass # Canvas captureStream support is not implemented webkit.org/b/169811 fast/mediacapturefromelement [ Skip ] +webkit.org/b/169811 fast/mediastream/captureStream/ [ Failure ] +webkit.org/b/169811 fast/mediastream/captureStream/canvas2d.html [ Crash Timeout ] +webkit.org/b/169811 fast/mediastream/captureStream/canvas3d.html [ Crash Timeout ] # Requires Resolution Media Query support webkit.org/b/100137 fast/media/mq-resolution.html [ Failure ] @@ -1241,25 +1235,18 @@ webkit.org/b/131546 media/track/track-in-band.html [ Crash Timeout Failure ] webkit.org/b/163782 media/video-played-ranges-1.html [ Crash Pass ] -webkit.org/b/169811 fast/mediastream/captureStream/canvas2d.html [ Crash Timeout ] -webkit.org/b/169811 fast/mediastream/captureStream/canvas3d.html [ Crash Timeout ] - webkit.org/b/172281 accessibility/insert-children-assert.html [ Crash ] webkit.org/b/172955 media/video-preload.html [ Crash Pass ] webkit.org/b/175575 imported/w3c/web-platform-tests/html/semantics/embedded-content/media-elements/ready-states/autoplay-with-slow-text-tracks.html [ Crash Pass ] -webkit.org/b/176801 fast/mediastream/apply-constraints-audio.html [ Crash ] webkit.org/b/176801 fast/mediastream/argument-types.html [ Crash Pass ] webkit.org/b/176803 http/tests/local/blob/send-hybrid-blob-using-open-panel.html [ Crash ] -webkit.org/b/176856 fast/mediastream/apply-constraints-video.html [ Crash ] - # See also webkit.org/b/141699 webkit.org/b/177534 fast/attachment/attachment-respects-css-size.html [ Crash ImageOnlyFailure ] -webkit.org/b/177535 fast/mediastream/media-device-info.html [ Crash Pass ] # See also webkit.org/b/163528 Threaded compositor failures caused by one of r203496, r203497, or r203498 webkit.org/b/177536 media/video-object-fit.html [ Crash ImageOnlyFailure ] diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog index 2bd09496e890f..213ec82388bdd 100644 --- a/Source/WebCore/ChangeLog +++ b/Source/WebCore/ChangeLog @@ -1,3 +1,120 @@ +2018-06-07 Thibault Saunier and Alejandro G. Castro + + [GTK][WPE] Start implementing MediaStream API + https://bugs.webkit.org/show_bug.cgi?id=185787 + + Reviewed by Philippe Normand. + + We are adding all the required classes to make the + MediaStream API work, that means our own RealtimeMediaSourceCenterLibWebRTC + for the platform, the GStreamerCaptureDeviceManager, the audio/video capturers + and their respective audio/video sources as well as a dedicated GStreamer Source + that adds support for using MediaStream stream inside playbin3. + We are using the GstDeviceMonitor to list devices on the devices. + + Enable mediastream tests. + + * platform/GStreamer.cmake: Added the new files to the compilation. + * platform/audio/AudioStreamDescription.h: Added new GStreamer type. + * platform/audio/PlatformAudioData.h: Added new GStreamer type for + the GStreamerAudioData class. + * platform/graphics/gstreamer/GStreamerCommon.cpp: + (WebCore::simpleBusMessageCallback): This function and the next + one help us to connect a monitoring callback to a pipeline for + debugging. + (WebCore::connectSimpleBusMessageCallback): Ditto. + * platform/graphics/gstreamer/GStreamerCommon.h: Ditto + * platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp: + (WebCore::MediaPlayerPrivateGStreamer::load): Make use of the loadFull() method. + (WebCore::MediaPlayerPrivateGStreamer::loadFull): Very similar to load() + but allows specifying what pipeline type to use (null to let the function determine + which one should be used). This is required as we force to always use playbin3 for the + mediastream source as it relies on the GstStream API. + (WebCore::MediaPlayerPrivateGStreamer::playbackPosition const): Style fix. + (WebCore::MediaPlayerPrivateGStreamer::naturalSize const): Added, use MediaStream specific information if available. + (WebCore::MediaPlayerPrivateGStreamer::updateTracks): Some style fixes. + (WebCore::MediaPlayerPrivateGStreamer::handleMessage): Enhance dotfiles names. + (WebCore::MediaPlayerPrivateGStreamer::processTableOfContentsEntry): Minor formatting fix. + (WebCore::MediaPlayerPrivateGStreamer::sourceSetup): Set MediaStream on WebKitMediaStreamSource when setting it up. + (WebCore::MediaPlayerPrivateGStreamer::supportsType): Advertise that we support MediaStream if support is built. + (WebCore::MediaPlayerPrivateGStreamer::createGSTPlayBin): Make sure playbin3 is forced when loading a MediaStream. + * platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h: Add a reference to the MediaStream object and declare loadFull and naturalSize override. + * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp: + (WebCore::registerWebKitGStreamerElements): Register the new MediaStreamSrc element. + * platform/graphics/gstreamer/VideoTrackPrivateGStreamer.cpp: + (WebCore::VideoTrackPrivateGStreamer::VideoTrackPrivateGStreamer): Make sure that MediaStream MAIN tracks are selected by default. We have no way to do it in MediaStreamSrc now as the GstStreamCollection is recreated by parsebin. + * platform/mediastream/RealtimeMediaSource.h: Make CaptureFailed a virtual method as in our mocks we require need to make + * platform/mediastream/RealtimeMediaSourceCenter.cpp: + (WebCore::RealtimeMediaSourceCenter::singleton): Remove the code + used for compilation for the platform when we do not have a + RealtimeMediaSourceCenterLibWebRTC. Now we return the proper class + for the platform. + * platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp: + Added class representing the RealtimeMediaSource for the Audio + with GStreamer. + * platform/mediastream/gstreamer/GStreamerAudioCaptureSource.h: + Ditto. + * platform/mediastream/gstreamer/GStreamerAudioCapturer.cpp: Added + this class that represents the GStreamer pipeline that captures + audio from the system devices, it inherits from GStreamerCapturer. + * platform/mediastream/gstreamer/GStreamerAudioCapturer.h: Dito. + * platform/mediastream/gstreamer/GStreamerAudioData.h: Added this + class implementing PlatformAudioData for the GStreamer platform, + used to pass the samples information. + * platform/mediastream/gstreamer/GStreamerAudioStreamDescription.h: + Added this class implementing AudioStreamDescription to export the + information about the audio stream to libwebrtc. + * platform/mediastream/gstreamer/GStreamerCaptureDevice.h: Added + this base class for the audio and video capturing devices, it + implements general WebKit CaptureDevice class. + * platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.cpp: + Added this class that implements the system monitor to get the + list of available devices in the system. It uses GstDeviceMonitor + to handle the operation. It uses two singleton device managers one + for audio and another one for video, as required by the + RealtimeMediaSourceCenter design. + * platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.h: Ditto. + * platform/mediastream/gstreamer/GStreamerCapturer.cpp: Added this + base class representing how GStreamer captures the media from the + input devices in the system. Two classes inherit from this one to + capture audio and video. It setups the GStreamer pipeline and adds + functions to control it. + * platform/mediastream/gstreamer/GStreamerCapturer.h: Ditto. + * platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp: Added. + Implements a subclass of GstBin as a source element that will contain several + GstAppSrc, basically one per MediaStreamTrackPrivate of the MediaStreamPrivate + passed in parameter. It adds Observers on the MediaStreamTracks and + pushes the data to the sources as required. The element implements the GstURIHandler + interface so it can be used in playbin. The MediaPlayerPrivateGStreamer is responsible + for passing the MediaStreamPrivate object to the source when required. + (WebCore::webkitMediaStreamSrcPadProbeCb): Event probe that fixes stream_start events (setting the ID etc) + and finally add src pads to the pipeline. + * platform/mediastream/gstreamer/GStreamerMediaStreamSource.h: Ditto. + * platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp: + Added this RealtimeMediaSource representing the source of the + video data for the GStreamer platform. It handles the settings and + capabilities of the source and creates the capturer used to + control the operation of the stream. + * platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h: Ditto. + * platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp: Added + this class that inherits from the GStreamerCapturer and controls + the GStreamer pipelines of the video streams of the system. + * platform/mediastream/gstreamer/GStreamerVideoCapturer.h: Ditto. + * platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.cpp: Added. Implementation of a Mock capturer for audio stream. + Subclasses GStreamerAudioCapturer and wraps a MockRealtimeAudioSource so that the behaviour is the same a MockRealtimeAudioSource + but still the GStreamer implementation code paths are tested. + * platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.h: Ditto. + * platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.cpp: Added. Implementation of a Mock capturer for video stream. + Subclasses GStreamerVideoCapturer and wraps a MockRealtimeVideoSource so that the behaviour is the same a MockRealtimeVideoSource + but still the GStreamer implementation code paths are tested. + * platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.h: Ditto. + * platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.cpp: + Added this class that implements the key RealtimeMediaSourceCenter + functions to configure the base class for the platform, using the + other GStreamer classes. + * platform/mock/MockRealtimeAudioSource.cpp: Do not build ::create if GStreamer implementation is built + * platform/mock/MockRealtimeVideoSource.cpp: Do not build ::create if GStreamer implementation is built + 2018-06-07 Brent Fulgham Handle Storage Access API calls in the absence of an attached frame diff --git a/Source/WebCore/PlatformGTK.cmake b/Source/WebCore/PlatformGTK.cmake index 823c77ddb218e..c2d4cae0f6025 100644 --- a/Source/WebCore/PlatformGTK.cmake +++ b/Source/WebCore/PlatformGTK.cmake @@ -24,12 +24,14 @@ list(APPEND WebCore_INCLUDE_DIRECTORIES "${WEBCORE_DIR}/platform/gtk" "${WEBCORE_DIR}/platform/graphics/egl" "${WEBCORE_DIR}/platform/graphics/glx" + "${WEBCORE_DIR}/platform/graphics/gstreamer" "${WEBCORE_DIR}/platform/graphics/gtk" "${WEBCORE_DIR}/platform/graphics/opengl" "${WEBCORE_DIR}/platform/graphics/opentype" "${WEBCORE_DIR}/platform/graphics/wayland" "${WEBCORE_DIR}/platform/graphics/x11" "${WEBCORE_DIR}/platform/mediastream/gtk" + "${WEBCORE_DIR}/platform/mediastream/gstreamer" "${WEBCORE_DIR}/platform/mock/mediasource" "${WEBCORE_DIR}/platform/network/gtk" "${WEBCORE_DIR}/platform/network/soup" diff --git a/Source/WebCore/platform/GStreamer.cmake b/Source/WebCore/platform/GStreamer.cmake index 3d5a6e0025a52..50b2d207c6cea 100644 --- a/Source/WebCore/platform/GStreamer.cmake +++ b/Source/WebCore/platform/GStreamer.cmake @@ -33,6 +33,17 @@ if (ENABLE_VIDEO OR ENABLE_WEB_AUDIO) platform/graphics/gstreamer/mse/PlaybackPipeline.cpp platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.cpp platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp + + platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp + platform/mediastream/gstreamer/GStreamerAudioCapturer.cpp + platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.cpp + platform/mediastream/gstreamer/GStreamerCapturer.cpp + platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp + platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp + platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp + platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.cpp + platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.cpp + platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.cpp ) list(APPEND WebCore_SYSTEM_INCLUDE_DIRECTORIES diff --git a/Source/WebCore/platform/audio/AudioStreamDescription.h b/Source/WebCore/platform/audio/AudioStreamDescription.h index e440dc90a3e84..2ba585c9c0567 100644 --- a/Source/WebCore/platform/audio/AudioStreamDescription.h +++ b/Source/WebCore/platform/audio/AudioStreamDescription.h @@ -35,6 +35,7 @@ struct PlatformDescription { enum { None, CAAudioStreamBasicType, + GStreamerAudioStreamDescription, } type; Variant description; }; diff --git a/Source/WebCore/platform/audio/PlatformAudioData.h b/Source/WebCore/platform/audio/PlatformAudioData.h index e4544f63254a8..cda72e8ab3c4c 100644 --- a/Source/WebCore/platform/audio/PlatformAudioData.h +++ b/Source/WebCore/platform/audio/PlatformAudioData.h @@ -34,6 +34,7 @@ class PlatformAudioData { enum class Kind { None, WebAudioBufferList, + GStreamerAudioData, }; virtual Kind kind() const { return Kind::None; } diff --git a/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp b/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp index e514fe3428de5..d5475b74cddcf 100644 --- a/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #if ENABLE(VIDEO_TRACK) && USE(GSTREAMER_MPEGTS) #define GST_USE_UNSTABLE_API @@ -320,6 +321,52 @@ bool gstRegistryHasElementForMediaType(GList* elementFactories, const char* caps return result; } +static void simpleBusMessageCallback(GstBus*, GstMessage* message, GstBin* pipeline) +{ + switch (GST_MESSAGE_TYPE(message)) { + case GST_MESSAGE_ERROR: + GST_ERROR_OBJECT(pipeline, "Got message: %" GST_PTR_FORMAT, message); + { + WTF::String dotFileName = String::format("%s_error", GST_OBJECT_NAME(pipeline)); + GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(pipeline, GST_DEBUG_GRAPH_SHOW_ALL, dotFileName.utf8().data()); + } + break; + case GST_MESSAGE_STATE_CHANGED: + if (GST_MESSAGE_SRC(message) == GST_OBJECT(pipeline)) { + GstState oldState, newState, pending; + gst_message_parse_state_changed(message, &oldState, &newState, &pending); + + GST_INFO_OBJECT(pipeline, "State changed (old: %s, new: %s, pending: %s)", + gst_element_state_get_name(oldState), + gst_element_state_get_name(newState), + gst_element_state_get_name(pending)); + + WTF::String dotFileName = String::format("%s_%s_%s", + GST_OBJECT_NAME(pipeline), + gst_element_state_get_name(oldState), + gst_element_state_get_name(newState)); + + GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, dotFileName.utf8().data()); + } + break; + default: + break; + } +} + +void disconnectSimpleBusMessageCallback(GstElement *pipeline) +{ + GRefPtr bus = adoptGRef(gst_pipeline_get_bus(GST_PIPELINE(pipeline))); + g_signal_handlers_disconnect_by_func(bus.get(), reinterpret_cast(simpleBusMessageCallback), pipeline); +} + +void connectSimpleBusMessageCallback(GstElement *pipeline) +{ + GRefPtr bus = adoptGRef(gst_pipeline_get_bus(GST_PIPELINE(pipeline))); + gst_bus_add_signal_watch_full(bus.get(), RunLoopSourcePriority::RunLoopDispatcher); + g_signal_connect(bus.get(), "message", G_CALLBACK(simpleBusMessageCallback), pipeline); +} + } #endif // USE(GSTREAMER) diff --git a/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.h b/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.h index 259255e741e41..cbb49323a68d2 100644 --- a/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.h +++ b/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.h @@ -82,6 +82,9 @@ inline GstClockTime toGstClockTime(const MediaTime &mediaTime) } bool gstRegistryHasElementForMediaType(GList* elementFactories, const char* capsString); +void connectSimpleBusMessageCallback(GstElement *pipeline); +void disconnectSimpleBusMessageCallback(GstElement *pipeline); + } #ifndef GST_BUFFER_DTS_OR_PTS diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp index 7d847f18d766e..0ec4b13f89154 100644 --- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp @@ -51,6 +51,10 @@ #include #include +#if ENABLE(MEDIA_STREAM) && GST_CHECK_VERSION(1, 10, 0) +#include "GStreamerMediaStreamSource.h" +#endif + #if ENABLE(VIDEO_TRACK) #include "AudioTrackPrivateGStreamer.h" #include "InbandMetadataTextTrackPrivateGStreamer.h" @@ -237,6 +241,11 @@ void MediaPlayerPrivateGStreamer::setPlaybinURL(const URL& url) } void MediaPlayerPrivateGStreamer::load(const String& urlString) +{ + loadFull(urlString, nullptr); +} + +void MediaPlayerPrivateGStreamer::loadFull(const String& urlString, const gchar *playbinName) { // FIXME: This method is still called even if supportsType() returned // IsNotSupported. This would deserve more investigation but meanwhile make @@ -254,7 +263,7 @@ void MediaPlayerPrivateGStreamer::load(const String& urlString) return; if (!m_pipeline) - createGSTPlayBin(isMediaSource() ? "playbin" : nullptr); + createGSTPlayBin(isMediaSource() ? "playbin" : playbinName); if (m_fillTimer.isActive()) m_fillTimer.stop(); @@ -292,12 +301,19 @@ void MediaPlayerPrivateGStreamer::load(const String&, MediaSourcePrivateClient*) #endif #if ENABLE(MEDIA_STREAM) -void MediaPlayerPrivateGStreamer::load(MediaStreamPrivate&) +void MediaPlayerPrivateGStreamer::load(MediaStreamPrivate& stream) { +#if GST_CHECK_VERSION(1, 10, 0) + m_streamPrivate = &stream; + loadFull(String("mediastream://") + stream.id(), "playbin3"); + ensureGLVideoSinkContext(); + m_player->play(); +#else // Properly fail so the global MediaPlayer tries to fallback to the next MediaPlayerPrivate. m_networkState = MediaPlayer::FormatError; m_player->networkStateChanged(); notImplemented(); +#endif } #endif @@ -330,7 +346,7 @@ MediaTime MediaPlayerPrivateGStreamer::playbackPosition() const // Position is only available if no async state change is going on and the state is either paused or playing. gint64 position = GST_CLOCK_TIME_NONE; - GstQuery* query= gst_query_new_position(GST_FORMAT_TIME); + GstQuery* query = gst_query_new_position(GST_FORMAT_TIME); if (gst_element_query(m_pipeline.get(), query)) gst_query_parse_position(query, 0, &position); gst_query_unref(query); @@ -648,6 +664,25 @@ void MediaPlayerPrivateGStreamer::clearTracks() m_current##Type##StreamId = String(gst_stream_get_stream_id(stream.get())); \ } \ } + +FloatSize MediaPlayerPrivateGStreamer::naturalSize() const +{ +#if ENABLE(MEDIA_STREAM) + if (!m_isLegacyPlaybin && !m_currentVideoStreamId.isEmpty()) { + RefPtr videoTrack = m_videoTracks.get(m_currentVideoStreamId); + + if (videoTrack) { + auto tags = gst_stream_get_tags(videoTrack->stream()); + gint width, height; + + if (gst_tag_list_get_int(tags, WEBKIT_MEDIA_TRACK_TAG_WIDTH, &width) && gst_tag_list_get_int(tags, WEBKIT_MEDIA_TRACK_TAG_HEIGHT, &height)) + return FloatSize(width, height); + } + } +#endif // ENABLE(MEDIA_STREAM) + + return MediaPlayerPrivateGStreamerBase::naturalSize(); +} #else #define CREATE_TRACK(type, _id, tracks, method, stream) m_has##Type## = true; #endif // ENABLE(VIDEO_TRACK) @@ -1165,7 +1200,8 @@ void MediaPlayerPrivateGStreamer::handleMessage(GstMessage* message) // Construct a filename for the graphviz dot file output. GstState newState; gst_message_parse_state_changed(message, ¤tState, &newState, nullptr); - CString dotFileName = String::format("webkit-video.%s_%s", gst_element_state_get_name(currentState), gst_element_state_get_name(newState)).utf8(); + CString dotFileName = String::format("%s.%s_%s", GST_OBJECT_NAME(m_pipeline.get()), + gst_element_state_get_name(currentState), gst_element_state_get_name(newState)).utf8(); GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(m_pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, dotFileName.data()); break; @@ -1431,7 +1467,7 @@ void MediaPlayerPrivateGStreamer::processTableOfContentsEntry(GstTocEntry* entry GstTagList* tags = gst_toc_entry_get_tags(entry); if (tags) { - gchar* title = nullptr; + gchar* title = nullptr; gst_tag_list_get_string(tags, GST_TAG_TITLE, &title); if (title) { cue->setContent(title); @@ -1726,6 +1762,12 @@ void MediaPlayerPrivateGStreamer::sourceSetup(GstElement* sourceElement) if (WEBKIT_IS_WEB_SRC(m_source.get())) { webKitWebSrcSetMediaPlayer(WEBKIT_WEB_SRC(m_source.get()), m_player); g_signal_connect(GST_ELEMENT_PARENT(m_source.get()), "element-added", G_CALLBACK(uriDecodeBinElementAddedCallback), this); +#if ENABLE(MEDIA_STREAM) && GST_CHECK_VERSION(1, 10, 0) + } else if (WEBKIT_IS_MEDIA_STREAM_SRC(sourceElement)) { + auto stream = m_streamPrivate.get(); + ASSERT(stream); + webkitMediaStreamSrcSetStream(WEBKIT_MEDIA_STREAM_SRC(sourceElement), stream); +#endif } } @@ -2269,8 +2311,10 @@ MediaPlayer::SupportsType MediaPlayerPrivateGStreamer::supportsType(const MediaE return result; #endif +#if !ENABLE(MEDIA_STREAM) || !GST_CHECK_VERSION(1, 10, 0) if (parameters.isMediaStream) return result; +#endif if (parameters.type.isEmpty()) return result; @@ -2433,17 +2477,20 @@ void MediaPlayerPrivateGStreamer::createGSTPlayBin(const gchar* playbinName) ASSERT(!m_pipeline); #if GST_CHECK_VERSION(1, 10, 0) - m_isLegacyPlaybin = !g_getenv("USE_PLAYBIN3"); - if (!m_isLegacyPlaybin) + if (g_getenv("USE_PLAYBIN3")) playbinName = "playbin3"; +#else + playbinName = "playbin"; #endif if (!playbinName) playbinName = "playbin"; + m_isLegacyPlaybin = !g_strcmp0(playbinName, "playbin"); + // gst_element_factory_make() returns a floating reference so // we should not adopt. - setPipeline(gst_element_factory_make(playbinName, "play")); + setPipeline(gst_element_factory_make(playbinName, String::format("play_%p", this).utf8().data())); setStreamVolumeElement(GST_STREAM_VOLUME(m_pipeline.get())); GST_INFO("Using legacy playbin element: %s", boolForPrinting(m_isLegacyPlaybin)); diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h index 1ae4a54bae0ad..b2ee5e695bdc4 100644 --- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h +++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h @@ -179,6 +179,7 @@ class MediaPlayerPrivateGStreamer : public MediaPlayerPrivateGStreamerBase { static void downloadBufferFileCreatedCallback(MediaPlayerPrivateGStreamer*); void setPlaybinURL(const URL& urlString); + void loadFull(const String& url, const gchar *playbinName); #if GST_CHECK_VERSION(1, 10, 0) void updateTracks(); @@ -262,7 +263,11 @@ class MediaPlayerPrivateGStreamer : public MediaPlayerPrivateGStreamerBase { bool m_isLegacyPlaybin; #if GST_CHECK_VERSION(1, 10, 0) GRefPtr m_streamCollection; -#endif + FloatSize naturalSize() const final; +#if ENABLE(MEDIA_STREAM) + RefPtr m_streamPrivate; +#endif // ENABLE(MEDIA_STREAM) +#endif // GST_CHECK_VERSION(1, 10, 0) String m_currentAudioStreamId; String m_currentVideoStreamId; String m_currentTextStreamId; diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp index 2eb441d2753ae..627c4b2a27b1d 100644 --- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp @@ -81,6 +81,10 @@ #endif // !GST_CHECK_VERSION(1, 14, 0) #endif // USE(LIBEPOXY) +#if ENABLE(MEDIA_STREAM) && GST_CHECK_VERSION(1, 10, 0) +#include "GStreamerMediaStreamSource.h" +#endif + #define GST_USE_UNSTABLE_API #include #undef GST_USE_UNSTABLE_API @@ -143,6 +147,9 @@ void registerWebKitGStreamerElements() if (!clearKeyDecryptorFactory) gst_element_register(nullptr, "webkitclearkey", GST_RANK_PRIMARY + 100, WEBKIT_TYPE_MEDIA_CK_DECRYPT); #endif +#if ENABLE(MEDIA_STREAM) && GST_CHECK_VERSION(1,10,0) + gst_element_register(nullptr, "mediastreamsrc", GST_RANK_PRIMARY, WEBKIT_TYPE_MEDIA_STREAM_SRC); +#endif } bool MediaPlayerPrivateGStreamerBase::initializeGStreamerAndRegisterWebKitElements() diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoTrackPrivateGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/VideoTrackPrivateGStreamer.cpp index c54200eae817d..4d99786e01200 100644 --- a/Source/WebCore/platform/graphics/gstreamer/VideoTrackPrivateGStreamer.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/VideoTrackPrivateGStreamer.cpp @@ -48,6 +48,14 @@ VideoTrackPrivateGStreamer::VideoTrackPrivateGStreamer(WeakPtr(VideoTrackPrivate::Kind::Main)) { + GstStreamFlags streamFlags = gst_stream_get_stream_flags(stream.get()); + gst_stream_set_stream_flags(stream.get(), static_cast(streamFlags | GST_STREAM_FLAG_SELECT)); + } + m_id = gst_stream_get_stream_id(stream.get()); setActive(gst_stream_get_stream_flags(stream.get()) & GST_STREAM_FLAG_SELECT); notifyTrackOfActiveChanged(); diff --git a/Source/WebCore/platform/mediastream/RealtimeMediaSource.h b/Source/WebCore/platform/mediastream/RealtimeMediaSource.h index 9b451205e5b93..5411085c4a326 100644 --- a/Source/WebCore/platform/mediastream/RealtimeMediaSource.h +++ b/Source/WebCore/platform/mediastream/RealtimeMediaSource.h @@ -216,7 +216,7 @@ class WEBCORE_EXPORT RealtimeMediaSource : public ThreadSafeRefCounted center; - return center; -#else return RealtimeMediaSourceCenter::platformCenter(); -#endif } void RealtimeMediaSourceCenter::setSharedStreamCenterOverride(RealtimeMediaSourceCenter* center) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp new file mode 100644 index 0000000000000..9c5aeafffcf12 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerAudioCaptureSource.h" + +#include "GStreamerAudioData.h" +#include "GStreamerAudioStreamDescription.h" +#include "GStreamerCaptureDeviceManager.h" + +#include +#include +#include + +namespace WebCore { + +const static CapabilityValueOrRange defaultVolumeCapability = CapabilityValueOrRange(0.0, 1.0); +const static RealtimeMediaSourceCapabilities::EchoCancellation defaultEchoCancellationCapability = RealtimeMediaSourceCapabilities::EchoCancellation::ReadWrite; + +GST_DEBUG_CATEGORY(webkit_audio_capture_source_debug); +#define GST_CAT_DEFAULT webkit_audio_capture_source_debug + +static void initializeGStreamerDebug() +{ + static std::once_flag debugRegisteredFlag; + std::call_once(debugRegisteredFlag, [] { + GST_DEBUG_CATEGORY_INIT(webkit_audio_capture_source_debug, "webkitaudiocapturesource", 0, "WebKit Audio Capture Source."); + }); +} + +class GStreamerAudioCaptureSourceFactory : public RealtimeMediaSource::AudioCaptureFactory { +public: + CaptureSourceOrError createAudioCaptureSource(const CaptureDevice& device, const MediaConstraints* constraints) final + { + return GStreamerAudioCaptureSource::create(device.persistentId(), constraints); + } +}; + +static GStreamerAudioCaptureSourceFactory& libWebRTCAudioCaptureSourceFactory() +{ + static NeverDestroyed factory; + return factory.get(); +} + +CaptureSourceOrError GStreamerAudioCaptureSource::create(const String& deviceID, const MediaConstraints* constraints) +{ + auto device = GStreamerAudioCaptureDeviceManager::singleton().gstreamerDeviceWithUID(deviceID); + if (!device) { + auto errorMessage = String::format("GStreamerAudioCaptureSource::create(): GStreamer did not find the device: %s.", deviceID.utf8().data()); + return CaptureSourceOrError(WTFMove(errorMessage)); + } + + auto source = adoptRef(*new GStreamerAudioCaptureSource(device.value())); + + if (constraints) { + auto result = source->applyConstraints(*constraints); + if (result) + return WTFMove(result.value().first); + } + return CaptureSourceOrError(WTFMove(source)); +} + +RealtimeMediaSource::AudioCaptureFactory& GStreamerAudioCaptureSource::factory() +{ + return libWebRTCAudioCaptureSourceFactory(); +} + +GStreamerAudioCaptureSource::GStreamerAudioCaptureSource(GStreamerCaptureDevice device) + : RealtimeMediaSource(device.persistentId(), RealtimeMediaSource::Type::Audio, device.label()) + , m_capturer(std::make_unique(device)) +{ + initializeGStreamerDebug(); +} + +GStreamerAudioCaptureSource::GStreamerAudioCaptureSource(const String& deviceID, const String& name) + : RealtimeMediaSource(deviceID, RealtimeMediaSource::Type::Audio, name) + , m_capturer(std::make_unique()) +{ + initializeGStreamerDebug(); +} + +GStreamerAudioCaptureSource::~GStreamerAudioCaptureSource() +{ +} + +void GStreamerAudioCaptureSource::startProducingData() +{ + m_capturer->setupPipeline(); + m_capturer->setSampleRate(sampleRate()); + g_signal_connect(m_capturer->sink(), "new-sample", G_CALLBACK(newSampleCallback), this); + m_capturer->play(); +} + +GstFlowReturn GStreamerAudioCaptureSource::newSampleCallback(GstElement* sink, GStreamerAudioCaptureSource* source) +{ + auto sample = adoptGRef(gst_app_sink_pull_sample(GST_APP_SINK(sink))); + + // FIXME - figure out a way to avoid copying (on write) the data. + GstBuffer* buf = gst_sample_get_buffer(sample.get()); + auto frames(std::unique_ptr(new GStreamerAudioData(WTFMove(sample)))); + auto streamDesc(std::unique_ptr(new GStreamerAudioStreamDescription(frames->getAudioInfo()))); + + source->audioSamplesAvailable( + MediaTime(GST_TIME_AS_USECONDS(GST_BUFFER_PTS(buf)), G_USEC_PER_SEC), + *frames, *streamDesc, gst_buffer_get_size(buf) / frames->getAudioInfo().bpf); + + return GST_FLOW_OK; +} + +void GStreamerAudioCaptureSource::stopProducingData() +{ + g_signal_handlers_disconnect_by_func(m_capturer->sink(), reinterpret_cast(newSampleCallback), this); + m_capturer->stop(); +} + +const RealtimeMediaSourceCapabilities& GStreamerAudioCaptureSource::capabilities() const +{ + if (m_capabilities) + return m_capabilities.value(); + + uint i; + GRefPtr caps = m_capturer->caps(); + int minSampleRate = 0, maxSampleRate = 0; + for (i = 0; i < gst_caps_get_size(caps.get()); i++) { + int capabilityMinSampleRate = 0, capabilityMaxSampleRate = 0; + GstStructure* str = gst_caps_get_structure(caps.get(), i); + + // Only accept raw audio for now. + if (!gst_structure_has_name(str, "audio/x-raw")) + continue; + + gst_structure_get(str, "rate", GST_TYPE_INT_RANGE, &capabilityMinSampleRate, &capabilityMaxSampleRate, nullptr); + if (i > 0) { + minSampleRate = std::min(minSampleRate, capabilityMinSampleRate); + maxSampleRate = std::max(maxSampleRate, capabilityMaxSampleRate); + } else { + minSampleRate = capabilityMinSampleRate; + maxSampleRate = capabilityMaxSampleRate; + } + } + + RealtimeMediaSourceCapabilities capabilities(settings().supportedConstraints()); + capabilities.setDeviceId(id()); + capabilities.setEchoCancellation(defaultEchoCancellationCapability); + capabilities.setVolume(defaultVolumeCapability); + capabilities.setSampleRate(CapabilityValueOrRange(minSampleRate, maxSampleRate)); + m_capabilities = WTFMove(capabilities); + + return m_capabilities.value(); +} + +bool GStreamerAudioCaptureSource::applySampleRate(int sampleRate) +{ + return m_capturer->setSampleRate(sampleRate); +} + +const RealtimeMediaSourceSettings& GStreamerAudioCaptureSource::settings() const +{ + if (!m_currentSettings) { + RealtimeMediaSourceSettings settings; + settings.setDeviceId(id()); + + RealtimeMediaSourceSupportedConstraints supportedConstraints; + supportedConstraints.setSupportsDeviceId(true); + supportedConstraints.setSupportsEchoCancellation(true); + supportedConstraints.setSupportsVolume(true); + supportedConstraints.setSupportsSampleRate(true); + settings.setSupportedConstraints(supportedConstraints); + + m_currentSettings = WTFMove(settings); + } + + m_currentSettings->setVolume(volume()); + m_currentSettings->setSampleRate(sampleRate()); + m_currentSettings->setEchoCancellation(echoCancellation()); + + return m_currentSettings.value(); +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCaptureSource.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCaptureSource.h new file mode 100644 index 0000000000000..3fa4a9c2e3225 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCaptureSource.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerAudioCapturer.h" +#include "GStreamerCaptureDevice.h" +#include "RealtimeMediaSource.h" + +namespace WebCore { + +class GStreamerAudioCaptureSource : public RealtimeMediaSource { +public: + static CaptureSourceOrError create(const String& deviceID, const MediaConstraints*); + WEBCORE_EXPORT static AudioCaptureFactory& factory(); + + const RealtimeMediaSourceCapabilities& capabilities() const override; + const RealtimeMediaSourceSettings& settings() const override; + + GstElement* pipeline() { return m_capturer->pipeline(); } + GStreamerCapturer* capturer() { return m_capturer.get(); } + +protected: + GStreamerAudioCaptureSource(GStreamerCaptureDevice); + GStreamerAudioCaptureSource(const String& deviceID, const String& name); + virtual ~GStreamerAudioCaptureSource(); + void startProducingData() override; + void stopProducingData() override; + + mutable std::optional m_capabilities; + mutable std::optional m_currentSettings; + +private: + bool applySampleRate(int) final; + bool isCaptureSource() const final { return true; } + bool applyVolume(double) final { return true; } + + std::unique_ptr m_capturer; + + static GstFlowReturn newSampleCallback(GstElement*, GStreamerAudioCaptureSource*); + void triggerSampleAvailable(GstSample*); +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCapturer.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCapturer.cpp new file mode 100644 index 0000000000000..64157ef70a6a0 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCapturer.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerAudioCapturer.h" + +#include "LibWebRTCAudioFormat.h" + +#include + +namespace WebCore { + +GStreamerAudioCapturer::GStreamerAudioCapturer(GStreamerCaptureDevice device) + : GStreamerCapturer(device, adoptGRef(gst_caps_new_simple("audio/x-raw", "rate", G_TYPE_INT, LibWebRTCAudioFormat::sampleRate, nullptr))) +{ +} + +GStreamerAudioCapturer::GStreamerAudioCapturer() + : GStreamerCapturer("audiotestsrc", adoptGRef(gst_caps_new_simple("audio/x-raw", "rate", G_TYPE_INT, LibWebRTCAudioFormat::sampleRate, nullptr))) +{ +} + +GstElement* GStreamerAudioCapturer::createConverter() +{ + auto converter = gst_parse_bin_from_description("audioconvert ! audioresample", TRUE, nullptr); + + ASSERT(converter); + + return converter; +} + +bool GStreamerAudioCapturer::setSampleRate(int sampleRate) +{ + + if (sampleRate > 0) { + GST_INFO_OBJECT(m_pipeline.get(), "Setting SampleRate %d", sampleRate); + m_caps = adoptGRef(gst_caps_new_simple("audio/x-raw", "rate", G_TYPE_INT, sampleRate, nullptr)); + } else { + GST_INFO_OBJECT(m_pipeline.get(), "Not forcing sample rate"); + m_caps = adoptGRef(gst_caps_new_empty_simple("audio/x-raw")); + } + + if (!m_capsfilter.get()) + return false; + + g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr); + + return true; +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCapturer.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCapturer.h new file mode 100644 index 0000000000000..9aaf761d6c4b4 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCapturer.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) + +#include "GStreamerCapturer.h" + +namespace WebCore { + +class GStreamerAudioCapturer : public GStreamerCapturer { +public: + GStreamerAudioCapturer(GStreamerCaptureDevice); + GStreamerAudioCapturer(); + + GstElement* createConverter() final; + const char* name() final { return "Audio"; } + + bool setSampleRate(int); +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioData.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioData.h new file mode 100644 index 0000000000000..953f3a8a0e4f6 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioData.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) + +#include "GRefPtrGStreamer.h" +#include "PlatformAudioData.h" + +#include + +namespace WebCore { +class GStreamerAudioData : public PlatformAudioData { +public: + GStreamerAudioData(GRefPtr&& sample, GstAudioInfo info) + : m_sample(WTFMove(sample)) + , m_audioInfo(info) + { + } + + GStreamerAudioData(GRefPtr&& sample) + : m_sample(WTFMove(sample)) + { + gst_audio_info_from_caps(&m_audioInfo, gst_sample_get_caps(m_sample.get())); + } + + GstSample* getSample() { return m_sample.get(); } + GstAudioInfo getAudioInfo() {return m_audioInfo; } + +private: + Kind kind() const { return Kind::GStreamerAudioData; } + GRefPtr m_sample; + GRefPtr m_caps; + + GstAudioInfo m_audioInfo; +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioStreamDescription.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioStreamDescription.h new file mode 100644 index 0000000000000..1ff59b2a1748b --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioStreamDescription.h @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) + +#include "AudioStreamDescription.h" +#include + +namespace WebCore { + +class GStreamerAudioStreamDescription final: public AudioStreamDescription { +public: + GStreamerAudioStreamDescription(GstAudioInfo info) + : m_info(info) + , m_caps(adoptGRef(gst_audio_info_to_caps(&m_info))) + { + } + + GStreamerAudioStreamDescription(GstAudioInfo *info) + : m_info(*info) + , m_caps(adoptGRef(gst_audio_info_to_caps(&m_info))) + { + } + + GStreamerAudioStreamDescription() + { + gst_audio_info_init(&m_info); + } + + WEBCORE_EXPORT ~GStreamerAudioStreamDescription() { }; + + const PlatformDescription& platformDescription() const + { + m_platformDescription = { PlatformDescription::GStreamerAudioStreamDescription, reinterpret_cast(&m_info) }; + + return m_platformDescription; + } + + WEBCORE_EXPORT PCMFormat format() const final { + switch (GST_AUDIO_INFO_FORMAT(&m_info)) { + case GST_AUDIO_FORMAT_S16LE: + case GST_AUDIO_FORMAT_S16BE: + return Int16; + case GST_AUDIO_FORMAT_S32LE: + case GST_AUDIO_FORMAT_S32BE: + return Int32; + case GST_AUDIO_FORMAT_F32LE: + case GST_AUDIO_FORMAT_F32BE: + return Float32; + case GST_AUDIO_FORMAT_F64LE: + case GST_AUDIO_FORMAT_F64BE: + return Float64; + default: + break; + } + return None; + } + + double sampleRate() const final { return GST_AUDIO_INFO_RATE(&m_info); } + bool isPCM() const final { return format() != None; } + bool isInterleaved() const final { return GST_AUDIO_INFO_LAYOUT(&m_info) == GST_AUDIO_LAYOUT_INTERLEAVED; } + bool isSignedInteger() const final { return GST_AUDIO_INFO_IS_INTEGER(&m_info); } + bool isNativeEndian() const final { return GST_AUDIO_INFO_ENDIANNESS(&m_info) == G_BYTE_ORDER; } + bool isFloat() const final { return GST_AUDIO_INFO_IS_FLOAT(&m_info); } + + uint32_t numberOfInterleavedChannels() const final { return isInterleaved() ? GST_AUDIO_INFO_CHANNELS(&m_info) : TRUE; } + uint32_t numberOfChannelStreams() const final { return GST_AUDIO_INFO_CHANNELS(&m_info); } + uint32_t numberOfChannels() const final { return GST_AUDIO_INFO_CHANNELS(&m_info); } + uint32_t sampleWordSize() const final { return GST_AUDIO_INFO_BPS(&m_info); } + + bool operator==(const GStreamerAudioStreamDescription& other) { return gst_audio_info_is_equal(&m_info, &other.m_info); } + bool operator!=(const GStreamerAudioStreamDescription& other) { return !operator == (other); } + + GstCaps* caps() { return m_caps.get(); } + GstAudioInfo* getInfo() { return &m_info; } + +private: + GstAudioInfo m_info; + GRefPtr m_caps; + mutable PlatformDescription m_platformDescription; +}; + +} // WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDevice.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDevice.h new file mode 100644 index 0000000000000..6422ddd83a3fb --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDevice.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(GSTREAMER) + +#include "CaptureDevice.h" +#include "GStreamerCommon.h" + +#include + +namespace WebCore { + +class GStreamerCaptureDevice : public CaptureDevice { +public: + GStreamerCaptureDevice(GRefPtr&& device, const String& persistentId, DeviceType type, const String& label, const String& groupId = emptyString()) + : CaptureDevice(persistentId, type, label, groupId) + , m_device(device) + { + } + + GstCaps* caps() const { return gst_device_get_caps(m_device.get()); } + GstDevice* device() { return m_device.get(); } + +private: + GRefPtr m_device; +}; + +} + +#endif // ENABLE(MEDIA_STREAM) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.cpp new file mode 100644 index 0000000000000..3b78287e7aaf5 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(GSTREAMER) +#include "GStreamerCaptureDeviceManager.h" + +#include "GStreamerCommon.h" + +namespace WebCore { + +GStreamerAudioCaptureDeviceManager& GStreamerAudioCaptureDeviceManager::singleton() +{ + static NeverDestroyed manager; + return manager; +} + +GStreamerVideoCaptureDeviceManager& GStreamerVideoCaptureDeviceManager::singleton() +{ + static NeverDestroyed manager; + return manager; +} + +GStreamerDisplayCaptureDeviceManager& GStreamerDisplayCaptureDeviceManager::singleton() +{ + static NeverDestroyed manager; + return manager; +} + +std::optional GStreamerCaptureDeviceManager::gstreamerDeviceWithUID(const String& deviceID) +{ + captureDevices(); + + for (auto& device : m_gstreamerDevices) { + if (device.persistentId() == deviceID) + return device; + } + return std::nullopt; +} + +const Vector& GStreamerCaptureDeviceManager::captureDevices() +{ + initializeGStreamer(); + if (m_devices.isEmpty()) + refreshCaptureDevices(); + + return m_devices; +} + +void GStreamerCaptureDeviceManager::deviceAdded(GRefPtr&& device) +{ + GUniquePtr properties(gst_device_get_properties(device.get())); + const char* klass = gst_structure_get_string(properties.get(), "device.class"); + + if (klass && !g_strcmp0(klass, "monitor")) + return; + + CaptureDevice::DeviceType type = deviceType(); + GUniquePtr deviceClassChar(gst_device_get_device_class(device.get())); + String deviceClass(String(deviceClassChar.get())); + if (type == CaptureDevice::DeviceType::Microphone && !deviceClass.startsWith("Audio")) + return; + if (type == CaptureDevice::DeviceType::Camera && !deviceClass.startsWith("Video")) + return; + + // This isn't really a UID but should be good enough (libwebrtc + // itself does that at least for pulseaudio devices). + GUniquePtr deviceName(gst_device_get_display_name(device.get())); + String identifier = String::fromUTF8(deviceName.get()); + + auto gstCaptureDevice = GStreamerCaptureDevice(WTFMove(device), identifier, type, identifier); + gstCaptureDevice.setEnabled(true); + m_gstreamerDevices.append(WTFMove(gstCaptureDevice)); + // FIXME: We need a CaptureDevice copy in other vector just for captureDevices API. + auto captureDevice = CaptureDevice(identifier, type, identifier); + captureDevice.setEnabled(true); + m_devices.append(WTFMove(captureDevice)); +} + +void GStreamerCaptureDeviceManager::refreshCaptureDevices() +{ + if (!m_deviceMonitor) { + m_deviceMonitor = adoptGRef(gst_device_monitor_new()); + + CaptureDevice::DeviceType type = deviceType(); + if (type == CaptureDevice::DeviceType::Camera) { + GRefPtr caps = adoptGRef(gst_caps_new_empty_simple("video/x-raw")); + gst_device_monitor_add_filter(m_deviceMonitor.get(), "Video/Source", caps.get()); + } else if (type == CaptureDevice::DeviceType::Microphone) { + GRefPtr caps = adoptGRef(gst_caps_new_empty_simple("audio/x-raw")); + gst_device_monitor_add_filter(m_deviceMonitor.get(), "Audio/Source", caps.get()); + } + // FIXME: Add monitor for added/removed messages on the bus. + } + + if (!gst_device_monitor_start(m_deviceMonitor.get())) { + GST_WARNING_OBJECT(m_deviceMonitor.get(), "Could not start device monitor"); + m_deviceMonitor = nullptr; + + return; + } + + GList* devices = gst_device_monitor_get_devices(m_deviceMonitor.get()); + while (devices) { + GRefPtr device = adoptGRef(GST_DEVICE_CAST(devices->data)); + deviceAdded(WTFMove(device)); + devices = g_list_delete_link(devices, devices); + } + + gst_device_monitor_stop(m_deviceMonitor.get()); +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.h new file mode 100644 index 0000000000000..86392414d0116 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.h @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(GSTREAMER) + +#include "CaptureDeviceManager.h" +#include "GRefPtrGStreamer.h" +#include "GStreamerCaptureDevice.h" + +namespace WebCore { + +class GStreamerCaptureDeviceManager : public CaptureDeviceManager { +public: + std::optional gstreamerDeviceWithUID(const String&); + + void refreshCaptureDevices() final; + const Vector& captureDevices() final; + virtual CaptureDevice::DeviceType deviceType() = 0; + +private: + void deviceAdded(GRefPtr&&); + + GRefPtr m_deviceMonitor; + Vector m_gstreamerDevices; + Vector m_devices; +}; + +class GStreamerAudioCaptureDeviceManager final : public GStreamerCaptureDeviceManager { + friend class NeverDestroyed; +public: + static GStreamerAudioCaptureDeviceManager& singleton(); + CaptureDevice::DeviceType deviceType() final { return CaptureDevice::DeviceType::Microphone; } +private: + GStreamerAudioCaptureDeviceManager() = default; + ~GStreamerAudioCaptureDeviceManager() = default; +}; + +class GStreamerVideoCaptureDeviceManager final : public GStreamerCaptureDeviceManager { + friend class NeverDestroyed; +public: + static GStreamerVideoCaptureDeviceManager& singleton(); + static RealtimeMediaSource::VideoCaptureFactory& videoFactory(); + CaptureDevice::DeviceType deviceType() final { return CaptureDevice::DeviceType::Camera; } +private: + GStreamerVideoCaptureDeviceManager() = default; + ~GStreamerVideoCaptureDeviceManager() = default; +}; + +class GStreamerDisplayCaptureDeviceManager final : public GStreamerCaptureDeviceManager { + friend class NeverDestroyed; +public: + static GStreamerDisplayCaptureDeviceManager& singleton(); + CaptureDevice::DeviceType deviceType() final { return CaptureDevice::DeviceType::Screen; } +private: + GStreamerDisplayCaptureDeviceManager() = default; + ~GStreamerDisplayCaptureDeviceManager() = default; +}; + +} + +#endif // ENABLE(MEDIA_STREAM) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.cpp new file mode 100644 index 0000000000000..92bfa415a2d62 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.cpp @@ -0,0 +1,197 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerCapturer.h" + +#include +#include +#include +#include + +#if GST_CHECK_VERSION(1, 10, 0) + +GST_DEBUG_CATEGORY(webkit_capturer_debug); +#define GST_CAT_DEFAULT webkit_capturer_debug + +namespace WebCore { + +static void initializeGStreamerAndDebug() +{ + initializeGStreamer(); + + static std::once_flag debugRegisteredFlag; + std::call_once(debugRegisteredFlag, [] { + GST_DEBUG_CATEGORY_INIT(webkit_capturer_debug, "webkitcapturer", 0, "WebKit Capturer"); + }); +} + +GStreamerCapturer::GStreamerCapturer(GStreamerCaptureDevice device, GRefPtr caps) + : m_device(device.device()) + , m_caps(caps) + , m_sourceFactory(nullptr) +{ + initializeGStreamerAndDebug(); +} + +GStreamerCapturer::GStreamerCapturer(const char* sourceFactory, GRefPtr caps) + : m_device(nullptr) + , m_caps(caps) + , m_sourceFactory(sourceFactory) +{ + initializeGStreamerAndDebug(); +} + +GStreamerCapturer::~GStreamerCapturer() +{ + if (m_pipeline) + disconnectSimpleBusMessageCallback(pipeline()); +} + +GstElement* GStreamerCapturer::createSource() +{ + if (m_sourceFactory) { + m_src = makeElement(m_sourceFactory); + if (GST_IS_APP_SRC(m_src.get())) + g_object_set(m_src.get(), "is-live", true, "format", GST_FORMAT_TIME, nullptr); + + ASSERT(m_src); + return m_src.get(); + } + + ASSERT(m_device); + GUniquePtr sourceName(g_strdup_printf("%s_%p", name(), this)); + m_src = gst_device_create_element(m_device.get(), sourceName.get()); + ASSERT(m_src); + + return m_src.get(); +} + +GstCaps* GStreamerCapturer::caps() +{ + if (m_sourceFactory) { + GRefPtr element = makeElement(m_sourceFactory); + auto pad = adoptGRef(gst_element_get_static_pad(element.get(), "src")); + + return gst_pad_query_caps(pad.get(), nullptr); + } + + ASSERT(m_device); + return gst_device_get_caps(m_device.get()); +} + +void GStreamerCapturer::setupPipeline() +{ + if (m_pipeline) + disconnectSimpleBusMessageCallback(pipeline()); + + m_pipeline = makeElement("pipeline"); + + GRefPtr source = createSource(); + GRefPtr converter = createConverter(); + + m_capsfilter = makeElement("capsfilter"); + m_tee = makeElement("tee"); + m_sink = makeElement("appsink"); + + gst_app_sink_set_emit_signals(GST_APP_SINK(m_sink.get()), TRUE); + g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr); + + gst_bin_add_many(GST_BIN(m_pipeline.get()), source.get(), converter.get(), m_capsfilter.get(), m_tee.get(), nullptr); + gst_element_link_many(source.get(), converter.get(), m_capsfilter.get(), m_tee.get(), nullptr); + + addSink(m_sink.get()); + + connectSimpleBusMessageCallback(pipeline()); +} + +GstElement* GStreamerCapturer::makeElement(const char* factoryName) +{ + auto element = gst_element_factory_make(factoryName, nullptr); + ASSERT(element); + GUniquePtr capturerName(g_strdup_printf("%s_capturer_%s_%p", name(), GST_OBJECT_NAME(element), this)); + gst_object_set_name(GST_OBJECT(element), capturerName.get()); + + return element; +} + +void GStreamerCapturer::addSink(GstElement* newSink) +{ + ASSERT(m_pipeline); + ASSERT(m_tee); + + auto queue = makeElement("queue"); + gst_bin_add_many(GST_BIN(pipeline()), queue, newSink, nullptr); + gst_element_sync_state_with_parent(queue); + gst_element_sync_state_with_parent(newSink); + + if (!gst_element_link_pads(m_tee.get(), "src_%u", queue, "sink")) { + ASSERT_NOT_REACHED(); + return; + } + + if (!gst_element_link(queue, newSink)) { + ASSERT_NOT_REACHED(); + return; + } + + if (newSink == m_sink.get()) { + GST_INFO_OBJECT(m_pipeline.get(), "Setting queue as leaky upstream" + " so that the player can set the sink to PAUSED without " + " setting the whole capturer to PAUSED"); + g_object_set(queue, "leaky", 2 /* upstream */, nullptr); + } + + GST_INFO_OBJECT(pipeline(), "Adding sink: %" GST_PTR_FORMAT, newSink); + + GUniquePtr dumpName(g_strdup_printf("%s_sink_%s_added", GST_OBJECT_NAME(pipeline()), GST_OBJECT_NAME(newSink))); + GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(pipeline()), GST_DEBUG_GRAPH_SHOW_ALL, dumpName.get()); +} + +void GStreamerCapturer::play() +{ + ASSERT(m_pipeline); + + GST_INFO_OBJECT(pipeline(), "Going to PLAYING!"); + + gst_element_set_state(pipeline(), GST_STATE_PLAYING); +} + +void GStreamerCapturer::stop() +{ + ASSERT(m_pipeline); + + GST_INFO_OBJECT(pipeline(), "Tearing down!"); + + // Make sure to remove sync handler before tearing down, avoiding + // possible deadlocks. + GRefPtr bus = adoptGRef(gst_pipeline_get_bus(GST_PIPELINE(pipeline()))); + gst_bus_set_sync_handler(bus.get(), nullptr, nullptr, nullptr); + + gst_element_set_state(pipeline(), GST_STATE_NULL); +} + +} // namespace WebCore + +#endif // GST_CHECK_VERSION(1, 10, 0) +#endif // ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.h new file mode 100644 index 0000000000000..7bfcedad801df --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerCapturer.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) + +#include "GStreamerCaptureDevice.h" +#include "GStreamerCommon.h" +#include "LibWebRTCMacros.h" + +#include + +namespace WebCore { + +class GStreamerCapturer { +public: + GStreamerCapturer(GStreamerCaptureDevice, GRefPtr); + GStreamerCapturer(const char* sourceFactory, GRefPtr); + ~GStreamerCapturer(); + + void setupPipeline(); + virtual void play(); + virtual void stop(); + GstCaps* caps(); + void addSink(GstElement *newSink); + GstElement* makeElement(const char* factoryName); + GstElement* createSource(); + GstElement* source() { return m_src.get(); } + virtual const char* name() = 0; + + GstElement* sink() const { return m_sink.get(); } + void setSink(GstElement* sink) { m_sink = adoptGRef(sink); }; + + GstElement* pipeline() const { return m_pipeline.get(); } + virtual GstElement* createConverter() = 0; + +protected: + GRefPtr m_sink; + GRefPtr m_src; + GRefPtr m_tee; + GRefPtr m_capsfilter; + GRefPtr m_device; + GRefPtr m_caps; + GRefPtr m_pipeline; + +private: + const char* m_sourceFactory; + +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp new file mode 100644 index 0000000000000..dba1ae6320f8a --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp @@ -0,0 +1,530 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerMediaStreamSource.h" + +#include "AudioTrackPrivate.h" +#include "GStreamerAudioData.h" +#include "GStreamerCommon.h" +#include "GStreamerVideoCaptureSource.h" +#include "MediaSampleGStreamer.h" +#include "VideoTrackPrivate.h" + +#include +#include + +#if GST_CHECK_VERSION(1, 10, 0) + +namespace WebCore { + +static void webkitMediaStreamSrcPushVideoSample(WebKitMediaStreamSrc* self, GstSample* gstsample); +static void webkitMediaStreamSrcPushAudioSample(WebKitMediaStreamSrc* self, GstSample* gstsample); +static void webkitMediaStreamSrcTrackEnded(WebKitMediaStreamSrc* self, MediaStreamTrackPrivate&); + +static GstStaticPadTemplate videoSrcTemplate = GST_STATIC_PAD_TEMPLATE("video_src", + GST_PAD_SRC, + GST_PAD_SOMETIMES, + GST_STATIC_CAPS("video/x-raw")); + +static GstStaticPadTemplate audioSrcTemplate = GST_STATIC_PAD_TEMPLATE("audio_src", + GST_PAD_SRC, + GST_PAD_SOMETIMES, + GST_STATIC_CAPS("audio/x-raw(ANY);")); + +static GstTagList* mediaStreamTrackPrivateGetTags(MediaStreamTrackPrivate* track) +{ + auto taglist = gst_tag_list_new_empty(); + + if (!track->label().isEmpty()) { + gst_tag_list_add(taglist, GST_TAG_MERGE_APPEND, + GST_TAG_TITLE, track->label().utf8().data(), nullptr); + } + + if (track->type() == RealtimeMediaSource::Type::Audio) { + gst_tag_list_add(taglist, GST_TAG_MERGE_APPEND, WEBKIT_MEDIA_TRACK_TAG_KIND, + static_cast(AudioTrackPrivate::Kind::Main), nullptr); + } else if (track->type() == RealtimeMediaSource::Type::Video) { + gst_tag_list_add(taglist, GST_TAG_MERGE_APPEND, WEBKIT_MEDIA_TRACK_TAG_KIND, + static_cast(VideoTrackPrivate::Kind::Main), nullptr); + + if (track->isCaptureTrack()) { + GStreamerVideoCaptureSource& source = static_cast( + track->source()); + + gst_tag_list_add(taglist, GST_TAG_MERGE_APPEND, + WEBKIT_MEDIA_TRACK_TAG_WIDTH, source.size().width(), + WEBKIT_MEDIA_TRACK_TAG_HEIGHT, source.size().height(), nullptr); + } + } + + return taglist; +} + +GstStream* webkitMediaStreamNew(MediaStreamTrackPrivate* track) +{ + GRefPtr caps; + GstStreamType type; + + if (track->type() == RealtimeMediaSource::Type::Audio) { + caps = adoptGRef(gst_static_pad_template_get_caps(&audioSrcTemplate)); + type = GST_STREAM_TYPE_AUDIO; + } else if (track->type() == RealtimeMediaSource::Type::Video) { + caps = adoptGRef(gst_static_pad_template_get_caps(&videoSrcTemplate)); + type = GST_STREAM_TYPE_VIDEO; + } else { + GST_FIXME("Handle %d type", (gint) track->type()); + + return nullptr; + } + + auto gststream = (GstStream*)gst_stream_new(track->id().utf8().data(), + caps.get(), type, GST_STREAM_FLAG_SELECT); + auto tags = adoptGRef(mediaStreamTrackPrivateGetTags(track)); + gst_stream_set_tags(gststream, tags.get()); + + return gststream; +} + +class WebKitMediaStreamTrackObserver + : public MediaStreamTrackPrivate::Observer { +public: + virtual ~WebKitMediaStreamTrackObserver() { }; + WebKitMediaStreamTrackObserver(WebKitMediaStreamSrc* src) + : m_mediaStreamSrc(src) { } + void trackStarted(MediaStreamTrackPrivate&) final { }; + + void trackEnded(MediaStreamTrackPrivate& track) final + { + webkitMediaStreamSrcTrackEnded(m_mediaStreamSrc, track); + } + + void trackMutedChanged(MediaStreamTrackPrivate&) final { }; + void trackSettingsChanged(MediaStreamTrackPrivate&) final { }; + void trackEnabledChanged(MediaStreamTrackPrivate&) final { }; + void readyStateChanged(MediaStreamTrackPrivate&) final { }; + + void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample& sample) final + { + auto gstsample = static_cast(&sample)->platformSample().sample.gstSample; + + webkitMediaStreamSrcPushVideoSample(m_mediaStreamSrc, gstsample); + } + + void audioSamplesAvailable(MediaStreamTrackPrivate&, const MediaTime&, const PlatformAudioData& audioData, const AudioStreamDescription&, size_t) final + { + auto audiodata = static_cast(audioData); + + webkitMediaStreamSrcPushAudioSample(m_mediaStreamSrc, audiodata.getSample()); + } + +private: + WebKitMediaStreamSrc* m_mediaStreamSrc; +}; + +typedef struct _WebKitMediaStreamSrcClass WebKitMediaStreamSrcClass; +struct _WebKitMediaStreamSrc { + GstBin parent_instance; + + gchar* uri; + + GstElement* audioSrc; + GstElement* videoSrc; + + std::unique_ptr observer; + String videoTrackID; + volatile gint npads; + gulong probeid; + RefPtr stream; + + GstFlowCombiner* flowCombiner; + GRefPtr streamCollection; +}; + +struct _WebKitMediaStreamSrcClass { + GstBinClass parent_class; +}; + +enum { + PROP_0, + PROP_IS_LIVE, + PROP_LAST +}; + +static GstURIType webkit_media_stream_src_uri_get_type(GType) +{ + return GST_URI_SRC; +} + +static const gchar* const* webkit_media_stream_src_uri_get_protocols(GType) +{ + static const gchar* protocols[] = { "mediastream", nullptr }; + + return protocols; +} + +static gchar* webkit_media_stream_src_uri_get_uri(GstURIHandler* handler) +{ + WebKitMediaStreamSrc* self = WEBKIT_MEDIA_STREAM_SRC(handler); + + /* FIXME: make thread-safe */ + return g_strdup(self->uri); +} + +static gboolean webkitMediaStreamSrcUriSetUri(GstURIHandler* handler, const gchar* uri, + GError**) +{ + WebKitMediaStreamSrc* self = WEBKIT_MEDIA_STREAM_SRC(handler); + self->uri = g_strdup(uri); + + return TRUE; +} + +static void webkitMediaStreamSrcUriHandlerInit(gpointer g_iface, gpointer) +{ + GstURIHandlerInterface* iface = (GstURIHandlerInterface*)g_iface; + + iface->get_type = webkit_media_stream_src_uri_get_type; + iface->get_protocols = webkit_media_stream_src_uri_get_protocols; + iface->get_uri = webkit_media_stream_src_uri_get_uri; + iface->set_uri = webkitMediaStreamSrcUriSetUri; +} + +GST_DEBUG_CATEGORY_STATIC(webkitMediaStreamSrcDebug); +#define GST_CAT_DEFAULT webkitMediaStreamSrcDebug + +#define doInit \ + G_IMPLEMENT_INTERFACE(GST_TYPE_URI_HANDLER, webkitMediaStreamSrcUriHandlerInit); \ + GST_DEBUG_CATEGORY_INIT(webkitMediaStreamSrcDebug, "webkitwebmediastreamsrc", 0, "mediastreamsrc element"); \ + gst_tag_register_static(WEBKIT_MEDIA_TRACK_TAG_WIDTH, GST_TAG_FLAG_META, G_TYPE_INT, "Webkit MediaStream width", "Webkit MediaStream width", gst_tag_merge_use_first); \ + gst_tag_register_static(WEBKIT_MEDIA_TRACK_TAG_HEIGHT, GST_TAG_FLAG_META, G_TYPE_INT, "Webkit MediaStream height", "Webkit MediaStream height", gst_tag_merge_use_first); \ + gst_tag_register_static(WEBKIT_MEDIA_TRACK_TAG_KIND, GST_TAG_FLAG_META, G_TYPE_INT, "Webkit MediaStream Kind", "Webkit MediaStream Kind", gst_tag_merge_use_first); + +G_DEFINE_TYPE_WITH_CODE(WebKitMediaStreamSrc, webkit_media_stream_src, GST_TYPE_BIN, doInit); + +static void webkitMediaStreamSrcSetProperty(GObject* object, guint prop_id, + const GValue*, GParamSpec* pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void webkitMediaStreamSrcGetProperty(GObject* object, guint prop_id, GValue* value, + GParamSpec* pspec) +{ + switch (prop_id) { + case PROP_IS_LIVE: + g_value_set_boolean(value, TRUE); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void webkitMediaStreamSrcDispose(GObject* object) +{ + WebKitMediaStreamSrc* self = WEBKIT_MEDIA_STREAM_SRC(object); + + if (self->audioSrc) { + gst_bin_remove(GST_BIN(self), self->audioSrc); + self->audioSrc = nullptr; + } + + if (self->videoSrc) { + gst_bin_remove(GST_BIN(self), self->videoSrc); + self->videoSrc = nullptr; + } +} + +static void webkitMediaStreamSrcFinalize(GObject* object) +{ + WebKitMediaStreamSrc* self = WEBKIT_MEDIA_STREAM_SRC(object); + + GST_OBJECT_LOCK(self); + for (auto& track : self->stream->tracks()) + track->removeObserver(*self->observer.get()); + GST_OBJECT_UNLOCK(self); + + g_clear_pointer(&self->uri, g_free); + gst_flow_combiner_free(self->flowCombiner); +} + +static GstStateChangeReturn webkitMediaStreamSrcChangeState(GstElement* element, GstStateChange transition) +{ + GstStateChangeReturn result; + auto* self = WEBKIT_MEDIA_STREAM_SRC(element); + + if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) { + + GST_OBJECT_LOCK(self); + for (auto& track : self->stream->tracks()) + track->removeObserver(*self->observer.get()); + GST_OBJECT_UNLOCK(self); + } + + result = GST_ELEMENT_CLASS(webkit_media_stream_src_parent_class)->change_state(element, transition); + + if (transition == GST_STATE_CHANGE_READY_TO_PAUSED) + result = GST_STATE_CHANGE_NO_PREROLL; + + return result; +} + +static void webkit_media_stream_src_class_init(WebKitMediaStreamSrcClass* klass) +{ + GObjectClass* gobject_class = G_OBJECT_CLASS(klass); + GstElementClass* gstelement_klass = GST_ELEMENT_CLASS(klass); + + gobject_class->finalize = webkitMediaStreamSrcFinalize; + gobject_class->dispose = webkitMediaStreamSrcDispose; + gobject_class->get_property = webkitMediaStreamSrcGetProperty; + gobject_class->set_property = webkitMediaStreamSrcSetProperty; + + g_object_class_install_property(gobject_class, PROP_IS_LIVE, + g_param_spec_boolean("is-live", "Is Live", + "Let playbin3 know we are a live source.", + TRUE, (GParamFlags)(G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + + gstelement_klass->change_state = webkitMediaStreamSrcChangeState; + gst_element_class_add_pad_template(gstelement_klass, + gst_static_pad_template_get(&videoSrcTemplate)); + gst_element_class_add_pad_template(gstelement_klass, + gst_static_pad_template_get(&audioSrcTemplate)); +} + +static void webkit_media_stream_src_init(WebKitMediaStreamSrc* self) +{ + self->observer = std::make_unique(self); + self->flowCombiner = gst_flow_combiner_new(); +} + +typedef struct { + WebKitMediaStreamSrc* self; + RefPtr track; + GstStaticPadTemplate* pad_template; +} ProbeData; + +static GstFlowReturn webkitMediaStreamSrcChain(GstPad* pad, GstObject* object, GstBuffer* buffer) +{ + GstFlowReturn result; + GRefPtr self = adoptGRef(WEBKIT_MEDIA_STREAM_SRC(gst_object_get_parent(object))); + + result = gst_flow_combiner_update_pad_flow(self.get()->flowCombiner, pad, + gst_proxy_pad_chain_default(pad, GST_OBJECT(self.get()), buffer)); + + return result; +} + +static void webkitMediaStreamSrcAddPad(WebKitMediaStreamSrc* self, GstPad* target, GstStaticPadTemplate* pad_template) +{ + auto padname = String::format("src_%u", g_atomic_int_add(&(self->npads), 1)); + auto ghostpad = gst_ghost_pad_new_from_template(padname.utf8().data(), target, + gst_static_pad_template_get(pad_template)); + + GST_DEBUG_OBJECT(self, "%s Ghosting %" GST_PTR_FORMAT, + gst_object_get_path_string(GST_OBJECT_CAST(self)), + target); + + auto proxypad = adoptGRef(GST_PAD(gst_proxy_pad_get_internal(GST_PROXY_PAD(ghostpad)))); + gst_pad_set_chain_function(proxypad.get(), + static_cast(webkitMediaStreamSrcChain)); + gst_pad_set_active(ghostpad, TRUE); + ASSERT(gst_element_add_pad(GST_ELEMENT(self), GST_PAD(ghostpad))); +} + +static GstPadProbeReturn webkitMediaStreamSrcPadProbeCb(GstPad* pad, GstPadProbeInfo* info, ProbeData* data) +{ + GstEvent* event = GST_PAD_PROBE_INFO_EVENT(info); + WebKitMediaStreamSrc* self = data->self; + + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_STREAM_START: { + const gchar* stream_id; + GRefPtr stream = nullptr; + + gst_event_parse_stream_start(event, &stream_id); + if (!g_strcmp0(stream_id, data->track->id().utf8().data())) { + GST_INFO_OBJECT(pad, "Event has been sticked already"); + return GST_PAD_PROBE_OK; + } + + auto stream_start = gst_event_new_stream_start(data->track->id().utf8().data()); + gst_event_set_group_id(stream_start, 1); + gst_event_unref(event); + + gst_pad_push_event(pad, stream_start); + gst_pad_push_event(pad, gst_event_new_stream_collection(self->streamCollection.get())); + gst_pad_push_event(pad, gst_event_new_tag(mediaStreamTrackPrivateGetTags(data->track.get()))); + + webkitMediaStreamSrcAddPad(self, pad, data->pad_template); + + return GST_PAD_PROBE_HANDLED; + } + default: + break; + } + + return GST_PAD_PROBE_OK; +} + +static gboolean webkitMediaStreamSrcSetupSrc(WebKitMediaStreamSrc* self, + MediaStreamTrackPrivate* track, GstElement* element, + GstStaticPadTemplate* pad_template, gboolean observe_track) +{ + auto pad = adoptGRef(gst_element_get_static_pad(element, "src")); + + gst_bin_add(GST_BIN(self), element); + + + ProbeData* data = new ProbeData; + data->self = WEBKIT_MEDIA_STREAM_SRC(self); + data->pad_template = pad_template; + data->track = track; + + self->probeid = gst_pad_add_probe(pad.get(), (GstPadProbeType)GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, + (GstPadProbeCallback)webkitMediaStreamSrcPadProbeCb, data, + [](gpointer data) { + delete (ProbeData*)data; + }); + + if (observe_track) + track->addObserver(*self->observer.get()); + + gst_element_sync_state_with_parent(element); + return TRUE; +} + +static gboolean webkitMediaStreamSrcSetupAppSrc(WebKitMediaStreamSrc* self, + MediaStreamTrackPrivate* track, GstElement** element, + GstStaticPadTemplate* pad_template) +{ + *element = gst_element_factory_make("appsrc", nullptr); + g_object_set(*element, "is-live", true, "format", GST_FORMAT_TIME, nullptr); + + return webkitMediaStreamSrcSetupSrc(self, track, *element, pad_template, TRUE); +} + +static void webkitMediaStreamSrcPostStreamCollection(WebKitMediaStreamSrc* self, MediaStreamPrivate* stream) +{ + GST_OBJECT_LOCK(self); + self->streamCollection = adoptGRef(gst_stream_collection_new(stream->id().utf8().data())); + for (auto& track : stream->tracks()) { + auto gststream = webkitMediaStreamNew(track.get()); + + gst_stream_collection_add_stream(self->streamCollection.get(), gststream); + } + GST_OBJECT_UNLOCK(self); + + gst_element_post_message(GST_ELEMENT(self), + gst_message_new_stream_collection(GST_OBJECT(self), self->streamCollection.get())); +} + +gboolean webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc* self, MediaStreamPrivate* stream) +{ + g_return_val_if_fail(WEBKIT_IS_MEDIA_STREAM_SRC(self), FALSE); + + if (self->audioSrc) { + gst_element_set_state(self->audioSrc, GST_STATE_NULL); + gst_bin_remove(GST_BIN(self), self->audioSrc); + self->audioSrc = nullptr; + } + + if (self->videoSrc) { + gst_element_set_state(self->videoSrc, GST_STATE_NULL); + gst_bin_remove(GST_BIN(self), self->videoSrc); + self->videoSrc = nullptr; + } + + webkitMediaStreamSrcPostStreamCollection(self, stream); + + self->stream = stream; + for (auto& track : stream->tracks()) { + if (track->type() == RealtimeMediaSource::Type::Audio) { + webkitMediaStreamSrcSetupAppSrc(self, track.get(), &self->audioSrc, + &audioSrcTemplate); + } else if (track->type() == RealtimeMediaSource::Type::Video) { + webkitMediaStreamSrcSetupAppSrc(self, track.get(), &self->videoSrc, + &videoSrcTemplate); + } else { + GST_INFO("Unsuported track type: %d", (gint) track->type()); + continue; + } + } + + return TRUE; +} + +static void webkitMediaStreamSrcPushVideoSample(WebKitMediaStreamSrc* self, GstSample* gstsample) +{ + if (self->videoSrc) + gst_app_src_push_sample(GST_APP_SRC(self->videoSrc), gstsample); +} + +static void webkitMediaStreamSrcPushAudioSample(WebKitMediaStreamSrc* self, GstSample* gstsample) +{ + if (self->audioSrc) + gst_app_src_push_sample(GST_APP_SRC(self->audioSrc), gstsample); +} + +static void webkitMediaStreamSrcTrackEnded(WebKitMediaStreamSrc* self, + MediaStreamTrackPrivate& track) +{ + GRefPtr pad = nullptr; + + GST_OBJECT_LOCK(self); + for (auto tmp = GST_ELEMENT(self)->srcpads; tmp; tmp = tmp->next) { + GstPad* tmppad = GST_PAD(tmp->data); + const gchar* stream_id; + + GstEvent* stream_start = gst_pad_get_sticky_event(tmppad, GST_EVENT_STREAM_START, 0); + if (!stream_start) + continue; + + gst_event_parse_stream_start(stream_start, &stream_id); + if (String(stream_id) == track.id()) { + pad = tmppad; + break; + } + } + GST_OBJECT_UNLOCK(self); + + if (!pad) { + GST_ERROR_OBJECT(self, "No pad found for %s", track.id().utf8().data()); + + return; + } + + // Make sure that the video.videoWidth is reset to 0 + webkitMediaStreamSrcPostStreamCollection(self, self->stream.get()); + auto tags = mediaStreamTrackPrivateGetTags(&track); + gst_pad_push_event(pad.get(), gst_event_new_tag(tags)); + gst_pad_push_event(pad.get(), gst_event_new_eos()); +} + +} // WebCore +#endif // GST_CHECK_VERSION(1, 10, 0) +#endif // ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.h new file mode 100644 index 0000000000000..eb4112e707f57 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) + +#include "MediaStreamPrivate.h" +#include "MediaStreamTrackPrivate.h" + +#include + +#define WEBKIT_MEDIA_TRACK_TAG_WIDTH "webkit-media-stream-width" +#define WEBKIT_MEDIA_TRACK_TAG_HEIGHT "webkit-media-stream-height" +#define WEBKIT_MEDIA_TRACK_TAG_KIND "webkit-media-stream-kind" + +namespace WebCore { + +typedef struct _WebKitMediaStreamSrc WebKitMediaStreamSrc; + +#define WEBKIT_MEDIA_STREAM_SRC(o) (G_TYPE_CHECK_INSTANCE_CAST((o), WEBKIT_TYPE_MEDIA_STREAM_SRC, WebKitMediaStreamSrc)) +#define WEBKIT_IS_MEDIA_STREAM_SRC(o) (G_TYPE_CHECK_INSTANCE_TYPE((o), WEBKIT_TYPE_MEDIA_STREAM_SRC)) +#define WEBKIT_TYPE_MEDIA_STREAM_SRC (webkit_media_stream_src_get_type()) +GType webkit_media_stream_src_get_type(void) G_GNUC_CONST; +gboolean webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc*, MediaStreamPrivate*); +} // WebCore + +#endif // ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp new file mode 100644 index 0000000000000..5a9dfb517fa24 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.cpp @@ -0,0 +1,270 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerVideoCaptureSource.h" + +#include "GStreamerCaptureDeviceManager.h" +#include "MediaSampleGStreamer.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace WebCore { + +const static int defaultWidth = 640; +const static int defaultHeight = 480; + +GST_DEBUG_CATEGORY(webkit_video_capture_source_debug); +#define GST_CAT_DEFAULT webkit_video_capture_source_debug + +static void initializeGStreamerDebug() +{ + static std::once_flag debugRegisteredFlag; + std::call_once(debugRegisteredFlag, [] { + GST_DEBUG_CATEGORY_INIT(webkit_video_capture_source_debug, "webkitvideocapturesource", 0, + "WebKit Video Capture Source."); + }); +} + +class GStreamerVideoCaptureSourceFactory final : public RealtimeMediaSource::VideoCaptureFactory { +public: + CaptureSourceOrError createVideoCaptureSource(const CaptureDevice& device, const MediaConstraints* constraints) final + { + return GStreamerVideoCaptureSource::create(device.persistentId(), constraints); + } +}; + +RealtimeMediaSource::VideoCaptureFactory& libWebRTCVideoCaptureSourceFactory() +{ + static NeverDestroyed factory; + return factory.get(); +} + +CaptureSourceOrError GStreamerVideoCaptureSource::create(const String& deviceID, const MediaConstraints* constraints) +{ + auto device = GStreamerVideoCaptureDeviceManager::singleton().gstreamerDeviceWithUID(deviceID); + if (!device) { + auto errorMessage = String::format("GStreamerVideoCaptureSource::create(): GStreamer did not find the device: %s.", deviceID.utf8().data()); + return CaptureSourceOrError(WTFMove(errorMessage)); + } + + auto source = adoptRef(*new GStreamerVideoCaptureSource(device.value())); + + if (constraints) { + auto result = source->applyConstraints(*constraints); + if (result) + return WTFMove(result.value().first); + } + return CaptureSourceOrError(WTFMove(source)); +} + +RealtimeMediaSource::VideoCaptureFactory& GStreamerVideoCaptureSource::factory() +{ + return libWebRTCVideoCaptureSourceFactory(); +} + +GStreamerVideoCaptureSource::GStreamerVideoCaptureSource(const String& deviceID, const String& name, const gchar *source_factory) + : RealtimeMediaSource(deviceID, RealtimeMediaSource::Type::Video, name) + , m_capturer(std::make_unique(source_factory)) +{ + initializeGStreamerDebug(); +} + +GStreamerVideoCaptureSource::GStreamerVideoCaptureSource(GStreamerCaptureDevice device) + : RealtimeMediaSource(device.persistentId(), RealtimeMediaSource::Type::Video, device.label()) + , m_capturer(std::make_unique(device)) +{ + initializeGStreamerDebug(); +} + +GStreamerVideoCaptureSource::~GStreamerVideoCaptureSource() +{ +} + +bool GStreamerVideoCaptureSource::applySize(const IntSize &size) +{ + m_capturer->setSize(size.width(), size.height()); + + return true; +} + +bool GStreamerVideoCaptureSource::applyFrameRate(double framerate) +{ + m_capturer->setFrameRate(framerate); + + return true; +} + +void GStreamerVideoCaptureSource::startProducingData() +{ + m_capturer->setupPipeline(); + m_capturer->setSize(size().width(), size().height()); + m_capturer->setFrameRate(frameRate()); + g_signal_connect(m_capturer->sink(), "new-sample", G_CALLBACK(newSampleCallback), this); + m_capturer->play(); +} + +GstFlowReturn GStreamerVideoCaptureSource::newSampleCallback(GstElement* sink, GStreamerVideoCaptureSource* source) +{ + auto gstSample = adoptGRef(gst_app_sink_pull_sample(GST_APP_SINK(sink))); + auto mediaSample = MediaSampleGStreamer::create(WTFMove(gstSample), WebCore::FloatSize(), String()); + + // FIXME - Check how presentationSize is supposed to be used here. + callOnMainThread([protectedThis = makeRef(*source), mediaSample = WTFMove(mediaSample)] { + protectedThis->videoSampleAvailable(mediaSample.get()); + }); + + return GST_FLOW_OK; +} + +void GStreamerVideoCaptureSource::stopProducingData() +{ + g_signal_handlers_disconnect_by_func(m_capturer->sink(), reinterpret_cast(newSampleCallback), this); + m_capturer->stop(); + + GST_INFO("Reset height and width after stopping source"); + setHeight(0); + setWidth(0); +} + +const RealtimeMediaSourceCapabilities& GStreamerVideoCaptureSource::capabilities() const +{ + if (m_capabilities) + return m_capabilities.value(); + + RealtimeMediaSourceCapabilities capabilities(settings().supportedConstraints()); + GRefPtr caps = adoptGRef(m_capturer->caps()); + int32_t minWidth = G_MAXINT32, maxWidth = 0, minHeight = G_MAXINT32, maxHeight = 0; + double minFramerate = G_MAXDOUBLE, maxFramerate = G_MINDOUBLE; + + for (guint i = 0; i < gst_caps_get_size(caps.get()); i++) { + GstStructure* str = gst_caps_get_structure(caps.get(), i); + + // Only accept raw video for now. + if (!gst_structure_has_name(str, "video/x-raw")) + continue; + + int32_t tmpMinWidth, tmpMinHeight, tmpMinFPSNumerator, tmpMinFPSDenominator; + int32_t tmpMaxWidth, tmpMaxHeight, tmpMaxFPSNumerator, tmpMaxFPSDenominator; + double tmpMinFramerate = G_MAXDOUBLE, tmpMaxFramerate = G_MINDOUBLE; + + if (!gst_structure_get(str, "width", GST_TYPE_INT_RANGE, &tmpMinWidth, &tmpMaxWidth, "height", GST_TYPE_INT_RANGE, &tmpMinHeight, &tmpMaxHeight, nullptr)) { + if (!gst_structure_get(str, "width", G_TYPE_INT, &tmpMinWidth, "height", G_TYPE_INT, &tmpMinHeight, nullptr)) + continue; + + tmpMaxWidth = tmpMinWidth; + tmpMaxHeight = tmpMinHeight; + } + + if (gst_structure_get(str, "framerate", GST_TYPE_FRACTION_RANGE, &tmpMinFPSNumerator, &tmpMinFPSDenominator, &tmpMaxFPSNumerator, &tmpMaxFPSDenominator, nullptr)) { + gst_util_fraction_to_double(tmpMinFPSNumerator, tmpMinFPSDenominator, &tmpMinFramerate); + gst_util_fraction_to_double(tmpMaxFPSNumerator, tmpMaxFPSDenominator, &tmpMaxFramerate); + } else if (gst_structure_get(str, + "framerate", GST_TYPE_FRACTION, &tmpMinFPSNumerator, &tmpMinFPSDenominator, nullptr)) { + tmpMaxFPSNumerator = tmpMinFPSNumerator; + tmpMaxFPSDenominator = tmpMinFPSDenominator; + gst_util_fraction_to_double(tmpMinFPSNumerator, tmpMinFPSDenominator, &tmpMinFramerate); + gst_util_fraction_to_double(tmpMaxFPSNumerator, tmpMaxFPSDenominator, &tmpMaxFramerate); + } else { + const GValue* frameRates(gst_structure_get_value(str, "framerate")); + tmpMinFramerate = G_MAXDOUBLE; + tmpMaxFramerate = 0.0; + + guint frameRatesLength = static_cast(gst_value_list_get_size(frameRates)) - 1; + + for (guint i = 0; i < frameRatesLength; i++) { + double tmpFrameRate; + const GValue* val = gst_value_list_get_value(frameRates, i); + + ASSERT(G_VALUE_TYPE(val) == GST_TYPE_FRACTION); + gst_util_fraction_to_double(gst_value_get_fraction_numerator(val), + gst_value_get_fraction_denominator(val), &tmpFrameRate); + + tmpMinFramerate = std::min(tmpMinFramerate, tmpFrameRate); + tmpMaxFramerate = std::max(tmpMaxFramerate, tmpFrameRate); + } + + if (i > 0) { + minWidth = std::min(tmpMinWidth, minWidth); + minHeight = std::min(tmpMinHeight, minHeight); + minFramerate = std::min(tmpMinFramerate, minFramerate); + + maxWidth = std::max(tmpMaxWidth, maxWidth); + maxHeight = std::max(tmpMaxHeight, maxHeight); + maxFramerate = std::max(tmpMaxFramerate, maxFramerate); + } else { + minWidth = tmpMinWidth; + minHeight = tmpMinHeight; + minFramerate = tmpMinFramerate; + + maxWidth = tmpMaxWidth; + maxHeight = tmpMaxHeight; + maxFramerate = tmpMaxFramerate; + } + } + + capabilities.setDeviceId(id()); + capabilities.setWidth(CapabilityValueOrRange(minWidth, maxWidth)); + capabilities.setHeight(CapabilityValueOrRange(minHeight, maxHeight)); + capabilities.setFrameRate(CapabilityValueOrRange(minFramerate, maxFramerate)); + m_capabilities = WTFMove(capabilities); + } + + return m_capabilities.value(); +} + +const RealtimeMediaSourceSettings& GStreamerVideoCaptureSource::settings() const +{ + if (!m_currentSettings) { + RealtimeMediaSourceSettings settings; + settings.setDeviceId(id()); + + RealtimeMediaSourceSupportedConstraints supportedConstraints; + supportedConstraints.setSupportsDeviceId(true); + supportedConstraints.setSupportsFacingMode(true); + supportedConstraints.setSupportsWidth(true); + supportedConstraints.setSupportsHeight(true); + supportedConstraints.setSupportsAspectRatio(true); + supportedConstraints.setSupportsFrameRate(true); + settings.setSupportedConstraints(supportedConstraints); + + m_currentSettings = WTFMove(settings); + } + + m_currentSettings->setWidth(size().width()); + m_currentSettings->setHeight(size().height()); + m_currentSettings->setFrameRate(frameRate()); + m_currentSettings->setAspectRatio(aspectRatio()); + m_currentSettings->setFacingMode(facingMode()); + return m_currentSettings.value(); +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h new file mode 100644 index 0000000000000..379ba7e0fd6e7 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCaptureSource.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerVideoCapturer.h" +#include "RealtimeMediaSource.h" + +namespace WebCore { + +class GStreamerVideoCaptureSource : public RealtimeMediaSource { +public: + static CaptureSourceOrError create(const String& deviceID, const MediaConstraints*); + WEBCORE_EXPORT static VideoCaptureFactory& factory(); + + const RealtimeMediaSourceCapabilities& capabilities() const override; + const RealtimeMediaSourceSettings& settings() const override; + GstElement* pipeline() { return m_capturer->pipeline(); } + GStreamerCapturer* capturer() { return m_capturer.get(); } + + +protected: + GStreamerVideoCaptureSource(const String& deviceID, const String& name, const gchar * source_factory); + GStreamerVideoCaptureSource(GStreamerCaptureDevice); + virtual ~GStreamerVideoCaptureSource(); + void startProducingData() override; + void stopProducingData() override; + + mutable std::optional m_capabilities; + mutable std::optional m_currentSettings; + +private: + static GstFlowReturn newSampleCallback(GstElement*, GStreamerVideoCaptureSource*); + + bool isCaptureSource() const final { return true; } + bool applySize(const IntSize&) final; + bool applyFrameRate(double) final; + bool applyAspectRatio(double) final { return true; } + + std::unique_ptr m_capturer; +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp new file mode 100644 index 0000000000000..88b6050216f39 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "GStreamerVideoCapturer.h" + +#include + +namespace WebCore { + +GStreamerVideoCapturer::GStreamerVideoCapturer(GStreamerCaptureDevice device) + : GStreamerCapturer(device, adoptGRef(gst_caps_new_empty_simple("video/x-raw"))) +{ +} + +GStreamerVideoCapturer::GStreamerVideoCapturer(const char* sourceFactory) + : GStreamerCapturer(sourceFactory, adoptGRef(gst_caps_new_empty_simple("video/x-raw"))) +{ +} + +GstElement* GStreamerVideoCapturer::createConverter() +{ + auto converter = gst_parse_bin_from_description("videoscale ! videoconvert ! videorate", TRUE, nullptr); + + ASSERT(converter); + + return converter; +} + +GstVideoInfo GStreamerVideoCapturer::getBestFormat() +{ + GRefPtr caps = adoptGRef(gst_caps_fixate(gst_device_get_caps(m_device.get()))); + GstVideoInfo info; + gst_video_info_from_caps(&info, caps.get()); + + return info; +} + +bool GStreamerVideoCapturer::setSize(int width, int height) +{ + if (!width || !height) + return false; + + m_caps = adoptGRef(gst_caps_copy(m_caps.get())); + gst_caps_set_simple(m_caps.get(), "width", G_TYPE_INT, width, "height", G_TYPE_INT, height, nullptr); + + if (!m_capsfilter) + return false; + + g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr); + + return true; +} + +bool GStreamerVideoCapturer::setFrameRate(double frameRate) +{ + int numerator, denominator; + + gst_util_double_to_fraction(frameRate, &numerator, &denominator); + + if (numerator < -G_MAXINT) { + GST_INFO_OBJECT(m_pipeline.get(), "Framerate %f not allowed", frameRate); + return false; + } + + if (!numerator) { + GST_INFO_OBJECT(m_pipeline.get(), "Do not force variable framerate"); + return false; + } + + m_caps = adoptGRef(gst_caps_copy(m_caps.get())); + gst_caps_set_simple(m_caps.get(), "framerate", GST_TYPE_FRACTION, numerator, denominator, nullptr); + + if (!m_capsfilter) + return false; + + g_object_set(m_capsfilter.get(), "caps", m_caps.get(), nullptr); + + return true; +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h new file mode 100644 index 0000000000000..57b0b4686d34a --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerVideoCapturer.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) + +#include "GStreamerCapturer.h" + +#include + +namespace WebCore { + +class GStreamerVideoCapturer : public GStreamerCapturer { +public: + GStreamerVideoCapturer(GStreamerCaptureDevice); + GStreamerVideoCapturer(const char* source_factory); + + GstElement* createConverter() final; + const char* name() final { return "Video"; } + + bool setSize(int width, int height); + bool setFrameRate(double); + GstVideoInfo getBestFormat(); +}; + +} // namespace WebCore +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.cpp b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.cpp new file mode 100644 index 0000000000000..995f05720eb90 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "MockGStreamerAudioCaptureSource.h" + +#include "MockRealtimeAudioSource.h" + +namespace WebCore { + +class WrappedMockRealtimeAudioSource : public MockRealtimeAudioSource { +public: + WrappedMockRealtimeAudioSource(const String& deviceID, const String& name) + : MockRealtimeAudioSource(deviceID, name) + { + } +}; + +CaptureSourceOrError MockRealtimeAudioSource::create(const String& deviceID, + const String& name, const MediaConstraints* constraints) +{ + auto source = adoptRef(*new MockGStreamerAudioCaptureSource(deviceID, name)); + + if (constraints && source->applyConstraints(*constraints)) + return { }; + + return CaptureSourceOrError(WTFMove(source)); +} + +std::optional> MockGStreamerAudioCaptureSource::applyConstraints(const MediaConstraints& constraints) +{ + m_wrappedSource->applyConstraints(constraints); + return GStreamerAudioCaptureSource::applyConstraints(constraints); +} + +void MockGStreamerAudioCaptureSource::applyConstraints(const MediaConstraints& constraints, SuccessHandler&& successHandler, FailureHandler&& failureHandler) +{ + m_wrappedSource->applyConstraints(constraints, WTFMove(successHandler), WTFMove(failureHandler)); +} + +MockGStreamerAudioCaptureSource::MockGStreamerAudioCaptureSource(const String& deviceID, const String& name) + : GStreamerAudioCaptureSource(deviceID, name) + , m_wrappedSource(std::make_unique(deviceID, name)) +{ + m_wrappedSource->addObserver(*this); +} + +MockGStreamerAudioCaptureSource::~MockGStreamerAudioCaptureSource() +{ + m_wrappedSource->removeObserver(*this); +} + +void MockGStreamerAudioCaptureSource::stopProducingData() +{ + m_wrappedSource->stop(); + + GStreamerAudioCaptureSource::stopProducingData(); +} + +void MockGStreamerAudioCaptureSource::startProducingData() +{ + GStreamerAudioCaptureSource::startProducingData(); + m_wrappedSource->start(); +} + +const RealtimeMediaSourceSettings& MockGStreamerAudioCaptureSource::settings() const +{ + return m_wrappedSource->settings(); +} + +const RealtimeMediaSourceCapabilities& MockGStreamerAudioCaptureSource::capabilities() const +{ + m_capabilities = m_wrappedSource->capabilities(); + m_currentSettings = m_wrappedSource->settings(); + return m_wrappedSource->capabilities(); +} + +void MockGStreamerAudioCaptureSource::captureFailed() +{ + stop(); + RealtimeMediaSource::captureFailed(); +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.h b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.h new file mode 100644 index 0000000000000..bc11f179b986b --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerAudioCaptureSource.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) + +#include "GStreamerAudioCaptureSource.h" + +namespace WebCore { + +class MockGStreamerAudioCaptureSource final : public GStreamerAudioCaptureSource, RealtimeMediaSource::Observer { +public: + MockGStreamerAudioCaptureSource(const String& deviceID, const String& name); + ~MockGStreamerAudioCaptureSource(); + std::optional> applyConstraints(const MediaConstraints&); + void applyConstraints(const MediaConstraints&, SuccessHandler&&, FailureHandler&&) final; + +private: + void stopProducingData() final; + void startProducingData() final; + const RealtimeMediaSourceSettings& settings() const final; + const RealtimeMediaSourceCapabilities& capabilities() const final; + + void captureFailed(); + std::unique_ptr m_wrappedSource; + + void videoSampleAvailable(MediaSample&) override { }; +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.cpp b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.cpp new file mode 100644 index 0000000000000..bdfabedd46f8c --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.cpp @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) +#include "MockGStreamerVideoCaptureSource.h" + +#include "MediaSampleGStreamer.h" +#include "MockRealtimeVideoSource.h" + +#include + +namespace WebCore { + +class WrappedMockRealtimeVideoSource : public MockRealtimeVideoSource { +public: + WrappedMockRealtimeVideoSource(const String& deviceID, const String& name) + : MockRealtimeVideoSource(deviceID, name) + { + } + + void updateSampleBuffer() + { + int fpsNumerator, fpsDenominator; + auto imageBuffer = this->imageBuffer(); + + if (!imageBuffer) + return; + + gst_util_double_to_fraction(frameRate(), &fpsNumerator, &fpsDenominator); + auto data = imageBuffer->toBGRAData(); + auto size = data.size(); + auto image_size = imageBuffer->internalSize(); + auto gstsample = gst_sample_new(gst_buffer_new_wrapped(static_cast(data.releaseBuffer().get()), size), + adoptGRef(gst_caps_new_simple("video/x-raw", + "format", G_TYPE_STRING, "BGRA", + "width", G_TYPE_INT, image_size.width(), + "height", G_TYPE_INT, image_size.height(), + "framerate", GST_TYPE_FRACTION, fpsNumerator, fpsDenominator, + nullptr)).get(), + nullptr, nullptr); + + auto sample = MediaSampleGStreamer::create(WTFMove(gstsample), + WebCore::FloatSize(), String()); + videoSampleAvailable(sample); + } +}; + +CaptureSourceOrError MockRealtimeVideoSource::create(const String& deviceID, + const String& name, const MediaConstraints* constraints) +{ + auto source = adoptRef(*new MockGStreamerVideoCaptureSource(deviceID, name)); + + if (constraints && source->applyConstraints(*constraints)) + return { }; + + return CaptureSourceOrError(WTFMove(source)); +} + +void MockGStreamerVideoCaptureSource::startProducingData() +{ + GStreamerVideoCaptureSource::startProducingData(); + m_wrappedSource->start(); +} + +void MockGStreamerVideoCaptureSource::stopProducingData() +{ + m_wrappedSource->stop(); + + GStreamerVideoCaptureSource::stopProducingData(); +} + +void MockGStreamerVideoCaptureSource::videoSampleAvailable(MediaSample& sample) +{ + auto src = capturer()->source(); + + if (src) { + auto gstsample = static_cast(&sample)->platformSample().sample.gstSample; + gst_app_src_push_sample(GST_APP_SRC(src), gstsample); + } +} + +MockGStreamerVideoCaptureSource::MockGStreamerVideoCaptureSource(const String& deviceID, const String& name) + : GStreamerVideoCaptureSource(deviceID, name, "appsrc") + , m_wrappedSource(std::make_unique(deviceID, name)) +{ + m_wrappedSource->addObserver(*this); +} + +MockGStreamerVideoCaptureSource::~MockGStreamerVideoCaptureSource() +{ + m_wrappedSource->removeObserver(*this); +} + +std::optional> MockGStreamerVideoCaptureSource::applyConstraints(const MediaConstraints& constraints) +{ + m_wrappedSource->applyConstraints(constraints); + return GStreamerVideoCaptureSource::applyConstraints(constraints); +} + +void MockGStreamerVideoCaptureSource::applyConstraints(const MediaConstraints& constraints, SuccessHandler&& successHandler, FailureHandler&& failureHandler) +{ + m_wrappedSource->applyConstraints(constraints, WTFMove(successHandler), WTFMove(failureHandler)); +} + +const RealtimeMediaSourceSettings& MockGStreamerVideoCaptureSource::settings() const +{ + return m_wrappedSource->settings(); +} + +const RealtimeMediaSourceCapabilities& MockGStreamerVideoCaptureSource::capabilities() const +{ + m_capabilities = m_wrappedSource->capabilities(); + m_currentSettings = m_wrappedSource->settings(); + return m_capabilities.value(); +} + +void MockGStreamerVideoCaptureSource::captureFailed() +{ + stop(); + + RealtimeMediaSource::captureFailed(); +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.h b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.h new file mode 100644 index 0000000000000..dc802768d7f71 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/MockGStreamerVideoCaptureSource.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) + +#include "GStreamerVideoCaptureSource.h" +#include "MockRealtimeMediaSource.h" + +namespace WebCore { + +class WrappedMockRealtimeVideoSource; +// We are subclassing GStreamerVideoCaptureSource and not MockRealtimeMediaSource +// because the MediaPlayer expects GStreamerVideoCaptureSource to be able to properly +// build the GStreamer pipeline. Still we make it so that it behaves as closely as possible +// to the MockRealtimeMediaSource class by wrapping our own subclass of it. +class MockGStreamerVideoCaptureSource final : public GStreamerVideoCaptureSource, RealtimeMediaSource::Observer { +public: + MockGStreamerVideoCaptureSource(const String& deviceID, const String& name); + ~MockGStreamerVideoCaptureSource(); + std::optional> applyConstraints(const MediaConstraints&); + void applyConstraints(const MediaConstraints&, SuccessHandler&&, FailureHandler&&) final; + +private: + void stopProducingData() final; + void startProducingData() final; + const RealtimeMediaSourceSettings& settings() const final; + std::unique_ptr m_wrappedSource; + const RealtimeMediaSourceCapabilities& capabilities() const final; + void captureFailed() override; + + void videoSampleAvailable(MediaSample&) override; +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER) diff --git a/Source/WebCore/platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.cpp b/Source/WebCore/platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.cpp new file mode 100644 index 0000000000000..bdce1d6e60f30 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.cpp @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) +#include "RealtimeMediaSourceCenterLibWebRTC.h" + +#include "GStreamerAudioCaptureSource.h" +#include "GStreamerCaptureDevice.h" +#include "GStreamerCaptureDeviceManager.h" +#include "GStreamerVideoCaptureSource.h" +#include + +namespace WebCore { + +RealtimeMediaSource::AudioCaptureFactory& RealtimeMediaSourceCenterLibWebRTC::audioCaptureSourceFactory() +{ + return RealtimeMediaSourceCenterLibWebRTC::singleton().audioFactory(); +} + +RealtimeMediaSourceCenterLibWebRTC& RealtimeMediaSourceCenterLibWebRTC::singleton() +{ + ASSERT(isMainThread()); + static NeverDestroyed center; + return center; +} + +RealtimeMediaSourceCenter& RealtimeMediaSourceCenter::platformCenter() +{ + return RealtimeMediaSourceCenterLibWebRTC::singleton(); +} + +RealtimeMediaSourceCenterLibWebRTC::RealtimeMediaSourceCenterLibWebRTC() + : m_libWebRTCProvider(LibWebRTCProvider::create()) +{ +} + +RealtimeMediaSourceCenterLibWebRTC::~RealtimeMediaSourceCenterLibWebRTC() +{ +} + +RealtimeMediaSource::AudioCaptureFactory& RealtimeMediaSourceCenterLibWebRTC::audioFactory() +{ + if (m_audioFactoryOverride) + return *m_audioFactoryOverride; + + return GStreamerAudioCaptureSource::factory(); +} + +RealtimeMediaSource::VideoCaptureFactory& RealtimeMediaSourceCenterLibWebRTC::videoFactory() +{ + return GStreamerVideoCaptureSource::factory(); +} + +CaptureDeviceManager& RealtimeMediaSourceCenterLibWebRTC::audioCaptureDeviceManager() +{ + return GStreamerAudioCaptureDeviceManager::singleton(); +} + +CaptureDeviceManager& RealtimeMediaSourceCenterLibWebRTC::videoCaptureDeviceManager() +{ + return GStreamerVideoCaptureDeviceManager::singleton(); +} + +CaptureDeviceManager& RealtimeMediaSourceCenterLibWebRTC::displayCaptureDeviceManager() +{ + return GStreamerDisplayCaptureDeviceManager::singleton(); +} + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) diff --git a/Source/WebCore/platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.h b/Source/WebCore/platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.h new file mode 100644 index 0000000000000..921284e1780b1 --- /dev/null +++ b/Source/WebCore/platform/mediastream/gstreamer/RealtimeMediaSourceCenterLibWebRTC.h @@ -0,0 +1,68 @@ + /* + * Copyright (C) 2018 Metrological Group B.V. + * Author: Thibault Saunier + * Author: Alejandro G. Castro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public License + * aint with this library; see the file COPYING.LIB. If not, write to + * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#if ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) + +#include "CaptureDeviceManager.h" +#include "LibWebRTCProvider.h" +#include "RealtimeMediaSource.h" +#include "RealtimeMediaSourceCenter.h" + +#include +#include +#include + +namespace WebCore { + +class RealtimeMediaSourceCenterLibWebRTC final : public RealtimeMediaSourceCenter { +public: + WEBCORE_EXPORT static RealtimeMediaSourceCenterLibWebRTC& singleton(); + + webrtc::PeerConnectionFactoryInterface* factory() { return m_libWebRTCProvider->factory(); } + + static RealtimeMediaSource::VideoCaptureFactory& videoCaptureSourceFactory(); + static RealtimeMediaSource::AudioCaptureFactory& audioCaptureSourceFactory(); + +private: + friend class NeverDestroyed; + RealtimeMediaSourceCenterLibWebRTC(); + ~RealtimeMediaSourceCenterLibWebRTC(); + + void setAudioFactory(RealtimeMediaSource::AudioCaptureFactory& factory) final { m_audioFactoryOverride = &factory; } + void unsetAudioFactory(RealtimeMediaSource::AudioCaptureFactory&) final { m_audioFactoryOverride = nullptr; } + + RealtimeMediaSource::AudioCaptureFactory& audioFactory() final; + RealtimeMediaSource::VideoCaptureFactory& videoFactory() final; + + CaptureDeviceManager& audioCaptureDeviceManager() final; + CaptureDeviceManager& videoCaptureDeviceManager() final; + CaptureDeviceManager& displayCaptureDeviceManager() final; + + RealtimeMediaSource::AudioCaptureFactory* m_audioFactoryOverride { nullptr }; + UniqueRef m_libWebRTCProvider; +}; + +} // namespace WebCore + +#endif // ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) + diff --git a/Source/WebCore/platform/mock/MockRealtimeAudioSource.cpp b/Source/WebCore/platform/mock/MockRealtimeAudioSource.cpp index c50abcb7a3256..9c6024e53a8a0 100644 --- a/Source/WebCore/platform/mock/MockRealtimeAudioSource.cpp +++ b/Source/WebCore/platform/mock/MockRealtimeAudioSource.cpp @@ -55,7 +55,7 @@ class MockRealtimeAudioSourceFactory : public RealtimeMediaSource::AudioCaptureF } }; -#if !PLATFORM(MAC) && !PLATFORM(IOS) +#if !PLATFORM(MAC) && !PLATFORM(IOS) && !(USE(GSTREAMER) && USE(LIBWEBRTC)) CaptureSourceOrError MockRealtimeAudioSource::create(const String& deviceID, const String& name, const MediaConstraints* constraints) { auto source = adoptRef(*new MockRealtimeAudioSource(deviceID, name)); diff --git a/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp b/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp index 9e860d6cf5ab7..ededa83a0e3c5 100644 --- a/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp +++ b/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp @@ -82,7 +82,7 @@ class MockRealtimeVideoSourceFactory : public RealtimeMediaSource::VideoCaptureF #endif }; -#if !PLATFORM(MAC) && !PLATFORM(IOS) +#if !PLATFORM(MAC) && !PLATFORM(IOS) && !(USE(GSTREAMER) && USE(LIBWEBRTC)) CaptureSourceOrError MockRealtimeVideoSource::create(const String& deviceID, const String& name, const MediaConstraints* constraints) { auto source = adoptRef(*new MockRealtimeVideoSource(deviceID, name)); diff --git a/Tools/ChangeLog b/Tools/ChangeLog index d31a51785095d..dc87e828e05de 100644 --- a/Tools/ChangeLog +++ b/Tools/ChangeLog @@ -1,3 +1,15 @@ +2018-06-07 Thibault Saunier + + [GTK][WPE] Start implementing MediaStream API + https://bugs.webkit.org/show_bug.cgi?id=185787 + + Reviewed by Philippe Normand. + + * Scripts/webkitpy/style/checker.py: Apply special formatting rules for new GObject subclasses. + * gstreamer/jhbuild.modules: Added a patch for the gst-plugins-base. + * gstreamer/patches/gst-plugins-base-0001-parsebin-Post-STREAM_COLLECTION-on-EVENT_STREAM_COLL.patch: + Added this fix to gst-plugings-base to fix the decodebin3. Merged as 89d0e9cc92a86aa0227ee87406737b6d31670aea + 2018-06-07 Brendan McLoughlin Update web-platform-tests github location in webkitpy.w3c.test_importer diff --git a/Tools/Scripts/webkitpy/style/checker.py b/Tools/Scripts/webkitpy/style/checker.py index 333dce2656799..a5908e6016a21 100644 --- a/Tools/Scripts/webkitpy/style/checker.py +++ b/Tools/Scripts/webkitpy/style/checker.py @@ -207,6 +207,8 @@ os.path.join('Source', 'WebCore', 'platform', 'graphics', 'gstreamer', 'VideoSinkGStreamer.cpp'), os.path.join('Source', 'WebCore', 'platform', 'graphics', 'gstreamer', 'WebKitWebSourceGStreamer.cpp'), os.path.join('Source', 'WebCore', 'platform', 'audio', 'gstreamer', 'WebKitWebAudioSourceGStreamer.cpp'), + os.path.join('Source', 'WebCore', 'platform', 'mediastream', 'gstreamer', 'GStreamerMediaStreamSource.h'), + os.path.join('Source', 'WebCore', 'platform', 'mediastream', 'gstreamer', 'GStreamerMediaStreamSource.cpp'), os.path.join('Source', 'WebCore', 'platform', 'network', 'soup', 'ProxyResolverSoup.cpp'), os.path.join('Source', 'WebCore', 'platform', 'network', 'soup', 'ProxyResolverSoup.h')], ["-readability/naming", diff --git a/Tools/gstreamer/jhbuild.modules b/Tools/gstreamer/jhbuild.modules index 5aeb756ec0a0c..c90a0cb468c21 100644 --- a/Tools/gstreamer/jhbuild.modules +++ b/Tools/gstreamer/jhbuild.modules @@ -57,7 +57,9 @@ - + + + diff --git a/Tools/gstreamer/patches/gst-plugins-base-0001-parsebin-Post-STREAM_COLLECTION-on-EVENT_STREAM_COLL.patch b/Tools/gstreamer/patches/gst-plugins-base-0001-parsebin-Post-STREAM_COLLECTION-on-EVENT_STREAM_COLL.patch new file mode 100644 index 0000000000000..680bc2db46454 --- /dev/null +++ b/Tools/gstreamer/patches/gst-plugins-base-0001-parsebin-Post-STREAM_COLLECTION-on-EVENT_STREAM_COLL.patch @@ -0,0 +1,33 @@ +From 89d0e9cc92a86aa0227ee87406737b6d31670aea Mon Sep 17 00:00:00 2001 +From: Thibault Saunier +Date: Wed, 18 Apr 2018 16:06:07 -0300 +Subject: [PATCH] parsebin: Post STREAM_COLLECTION on EVENT_STREAM_COLLECTION + +Otherwise decodebin won't get notified about STREAM_COLLECTION comming +from the sources and thus will never get informored about it. Without +being informed about the stream collection decodebin won't be able to +select any streams. It ends up not creating any output for the streams +defined from outside parserbin. + +https://bugzilla.gnome.org/show_bug.cgi?id=795364 +--- + gst/playback/gstparsebin.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/gst/playback/gstparsebin.c b/gst/playback/gstparsebin.c +index 4c5bfc19d..f9662eb49 100644 +--- a/gst/playback/gstparsebin.c ++++ b/gst/playback/gstparsebin.c +@@ -4046,6 +4046,9 @@ gst_parse_pad_event (GstPad * pad, GstPadProbeInfo * info, gpointer user_data) + GstStreamCollection *collection = NULL; + gst_event_parse_stream_collection (event, &collection); + gst_parse_pad_update_stream_collection (parsepad, collection); ++ gst_element_post_message (GST_ELEMENT (parsepad->parsebin), ++ gst_message_new_stream_collection (GST_OBJECT (parsepad->parsebin), ++ collection)); + break; + } + case GST_EVENT_EOS:{ +-- +2.17.0 +