Switch back to H.264 video, audio support in progress, switch InputSe…

…rver to .NET 4.7.1
wchill · Jun 3, 2018 · 1eb6e35 · 1eb6e35
1 parent c6f1ab4
commit 1eb6e35
Show file tree

Hide file tree

Showing 24 changed files with 1,746 additions and 30 deletions.
diff --git a/InputServer/InputServer/ClientQueue.cs b/InputServer/InputServer/ClientQueue.cs
@@ -24,7 +24,7 @@ public void Enqueue(T client)
 
         public void RemoveFromQueue(T client)
         {
-            _dict.Remove(client, out var value);
+            _dict.TryRemove(client, out var value);
         }
 
         public bool Dequeue(out T result)
@@ -35,7 +35,7 @@ public bool Dequeue(out T result)
                 {
                     if (_dict.ContainsKey(result))
                     {
-                        _dict.Remove(result, out var value);
+                        _dict.TryRemove(result, out var value);
                         return true;
                     }
                 }

diff --git a/InputServer/InputServer/InputServer.csproj b/InputServer/InputServer/InputServer.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.0</TargetFramework>
+    <TargetFramework>net471</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/InputServer/InputServer/InputWsClient.cs b/InputServer/InputServer/InputWsClient.cs
@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
-using Fleck;
 using Microsoft.IdentityModel.Tokens;
 using WebSocketSharp;
 using WebSocketSharp.Server;
@@ -44,13 +43,13 @@ protected override void OnMessage(MessageEventArgs e)
                 try
                 {
                     MyUser = new TwitchUser(wsArgs[1], wsArgs[2]);
-                    Console.WriteLine($"{MyUser.UserName} logged in");
+                    //Console.WriteLine($"{MyUser.UserName} logged in");
                     Send("TWITCH_VERIFIED");
                 }
                 catch (SecurityTokenValidationException)
                 {
                     MyUser = null;
-                    Console.WriteLine("Invalid login attempt");
+                    //Console.WriteLine("Invalid login attempt");
                     Send("TWITCH_INVALID");
                 }
             }

diff --git a/InputServer/InputServer/InputWsServer.cs b/InputServer/InputServer/InputWsServer.cs
@@ -78,7 +78,7 @@ private void OnUpdate(string stateStr)
                     }
                     else
                     {
-                        _listenerClients.Remove(client, out var value);
+                        _listenerClients.TryRemove(client, out var value);
                     }
                 }
             });

diff --git a/InputServer/InputServer/SwitchInputSink.cs b/InputServer/InputServer/SwitchInputSink.cs
@@ -31,7 +31,7 @@ public class SwitchInputSink : IInputSink
     {
         private SwitchInputState _state;
         private SerialPort _serialPort;
-        private readonly ConcurrentQueue<InputFrame> _queuedFrames;
+        private ConcurrentQueue<InputFrame> _queuedFrames;
         private OnUpdateCallback _callback;
         private readonly object _lock = new object();
 
@@ -133,7 +133,7 @@ public void Reset()
                 RightX = 128,
                 RightY = 128
             };
-            _queuedFrames.Clear();
+            _queuedFrames = new ConcurrentQueue<InputFrame>();
             Update(newFrame);
         }
 

diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html
@@ -6,6 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <link rel="stylesheet" href="{{ '/assets/css/style.css?v=' | append: site.github.build_revision | relative_url }}">
     <link rel="stylesheet" href="assets/css/general.css">
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css" integrity="sha384-DNOHZ68U8hZfKXOrtjWvjxusGo9WQnrNx2sqG0tfsghAvtVlRW3tvkXWZh58N9jp" crossorigin="anonymous">
 
     {% seo %}
 </head>

diff --git a/docs/_layouts/play.html b/docs/_layouts/play.html
@@ -3,12 +3,12 @@
 ---
 
 <link rel="stylesheet" type="text/css" href="assets/css/gamepad.css">
+<link rel="stylesheet" type="text/css" href="assets/css/volume-slider.css">
 
 <div id="app" v-cloak>
     <twitch-auth></twitch-auth>
     <control-mode-select class="center-text min-padding"></control-mode-select>
-    <!-- <joycon-stream-renderer v-bind:endpoint="streamEndpoint"></joycon-stream-renderer> -->
-    <joycon-stream-renderer-webrtc v-bind:endpoint="webRtcEndpoint"></joycon-stream-renderer-webrtc>
+    <joycon-stream-renderer v-bind:video="videoEndpoint" v-bind:audio="audioEndpoint"></joycon-stream-renderer>
     <control-ws v-bind:endpoint="controlEndpoint"></control-ws>
     <server-status></server-status>
     <!--
@@ -35,12 +35,15 @@
     window.vueNotification = window["vue-notification"].default;
     Vue.use(vueNotification);
 </script>
-<script src="https://embed.twitch.tv/embed/v1.js"></script>
-<script src="assets/js/lib/keymaster.js"></script>
-<script src="assets/js/lib/h264-websocket.min.js"></script>
-<script src="assets/js/stats.js"></script>
-<script src="assets/js/oidc-client.min.js"></script>
-<script src="assets/js/twitch-auth-config.js"></script>
-<script src="assets/js/lib/adapter.js"></script>
-<script src="assets/js/lib/kurento-utils.js"></script>
-<script src="assets/js/bundle.js"></script>
+<script defer src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
+<script defer src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.11.2/jquery-ui.min.js"></script>
+<script defer src="https://embed.twitch.tv/embed/v1.js"></script>
+<script defer src="assets/js/lib/keymaster.js"></script>
+<script defer src="assets/js/lib/h264-websocket.min.js"></script>
+<script defer src="assets/js/stats.js"></script>
+<script defer src="assets/js/oidc-client.min.js"></script>
+<script defer src="assets/js/twitch-auth-config.js"></script>
+<script defer src="assets/js/lib/libopus.js"></script>
+<script defer src="assets/js/lib/opus.js"></script>
+<script defer src="assets/js/lib/xaudio.js"></script>
+<script defer src="assets/js/bundle.js"></script>
diff --git a/docs/_layouts/player.html b/docs/_layouts/player.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+<script src="assets/js/lib/libopus.js"></script>
+<script src="assets/js/lib/opus.js"></script>
+<script src="assets/js/lib/xaudio.js"></script>
+<script src="assets/js/lib/ws-audio-api.js"></script>
+<script>
+    var player = new WSAudioAPI.Player({
+        server: {
+            host: 'wss://api.twitchplays.gg/switch/stream/audio', //websockets server addres. In this example - localhost
+        }
+    });
+</script>
+<button onclick="window.testContext.resume();player.start()">Play stream</button>
+<button onclick="player.stop()">Stop playing</button>
+</body>
+</html>
diff --git a/docs/_layouts/streamer.html b/docs/_layouts/streamer.html
@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+<script src="assets/js/lib/libopus.js"></script>
+<script src="assets/js/lib/opus.js"></script>
+<script src="assets/js/lib/xaudio.js"></script>
+<script src="assets/js/lib/ws-audio-api.js"></script>
+<script>
+    var streamer = new WSAudioAPI.Streamer({
+        server: {
+            host: 'wss://api.twitchplays.gg/switch/stream/audio', //websockets server addres. In this example - localhost
+        }
+    });
+</script>
+
+<button onclick="streamer.start()">Start stream</button>
+<button onclick="streamer.stop()">Stop stream</button>
+</body>
+</html>
diff --git a/docs/assets/css/volume-slider.css b/docs/assets/css/volume-slider.css
diff --git a/docs/assets/css/volume-slider.scss b/docs/assets/css/volume-slider.scss
@@ -0,0 +1,58 @@
+$primary-color: #2ecc71;
+
+.volume-slider {
+  width: 350px;
+  height: 40px;
+  position: relative;
+  margin: 0 auto;
+
+  i {
+    position: absolute;
+    top: 50%;
+    transform: translate(0, -50%);
+    color: #666;
+  }
+
+  i.fa-volume-down {
+    margin-left: -8px;
+  }
+
+  i.fa-volume-up {
+    margin-right: -8px;
+    right: 0;
+  }
+}
+
+.slider {
+  position: absolute;
+  left: 24px;
+  margin: auto;
+  height: 5px;
+  width: 300px;
+  background: #555;
+  border-radius: 15px;
+  top: 50%;
+  transform: translate(0, -50%);
+
+  .ui-slider-range-min {
+    height: 5px;
+    width: 300px;
+    position: absolute;
+    background: $primary-color;
+    border: none;
+    border-radius: 10px;
+    outline: none;
+  }
+
+  .ui-slider-handle {
+    width: 20px;
+    height: 20px;
+    border-radius: 20px;
+    background: #FFF;
+    position: absolute;
+    margin-left: -8px;
+    margin-top: -8px;
+    cursor: pointer;
+    outline: none;
+  }
+}
diff --git a/docs/assets/js/JoyconStreamRenderer.js b/docs/assets/js/JoyconStreamRenderer.js
@@ -1,5 +1,6 @@
 import {PlayerState} from "./Common";
 import {H264Player, PlayerBus, PlayerEvents} from "./H264WebSocketPlayer";
+import {OpusWebSocketPlayer} from "./OpusWebSocketPlayer";
 
 export const JoyconSprites = {
     data: function() {
@@ -39,9 +40,10 @@ export const JoyconSprites = {
 
 export const JoyconStreamRenderer = {
     mixins: [JoyconSprites],
-    props: ['endpoint'],
+    props: ['video', 'audio'],
     components: {
-        'h264-ws-player': H264Player
+        'h264-ws-player': H264Player,
+        'opus-ws-player': OpusWebSocketPlayer
     },
     data: function() {
         return {
@@ -217,5 +219,5 @@ export const JoyconStreamRenderer = {
         this.spriteSheetReady = false;
         this.streamReady = false;
     },
-    template: '<div><h264-ws-player v-bind:endpoint="endpoint" v-bind:canvas="streamCanvas"></h264-ws-player><canvas class="controlCanvas" ref="controlCanvas"></canvas><img ref="spriteSheet" v-bind:src="spriteSheetUrl" style="display:none;" @load="imageLoaded"/></div>'
+    template: '<div><h264-ws-player v-bind:endpoint="video" v-bind:canvas="streamCanvas"></h264-ws-player><canvas class="controlCanvas" ref="controlCanvas"></canvas><opus-ws-player v-bind:endpoint="audio"></opus-ws-player><img ref="spriteSheet" v-bind:src="spriteSheetUrl" style="display:none;" @load="imageLoaded"/></div>'
 };
diff --git a/docs/assets/js/JoyconStreamRendererWebRTC.js b/docs/assets/js/JoyconStreamRendererWebRTC.js
@@ -269,6 +269,22 @@ export const JoyconStreamRendererWebRTC = {
             }
         });
 
+        /*
+        window.setInterval(function() {
+            if (self.webRtcPeer) {
+                self.webRtcPeer.peerConnection.getStats(null, function(d) {
+                    let keys = Object.keys(d);
+                    for(let i = 0; i < keys.length; i++) {
+                        if (d[keys[i]].mediaType === 'video') {
+                            let track = d[keys[i]];
+                            console.log(track.googCurrentDelayMs, track.googJitterBufferMs, track.googTargetDelayMs);
+                        }
+                    }
+                })
+            }
+        }, 1000);
+        */
+
         this.renderImage();
     },
     beforeDestroy: function () {
-Original file line number
+Diff line change
@@ Expand Up / @@ -78,7 +78,7 @@ private void OnUpdate(string stateStr) @@
                         }
                         else
                         {
-                            _listenerClients.Remove(client, out var value);
+                            _listenerClients.TryRemove(client, out var value);
                         }
                     }
                 });
@@ Expand Down @@