Merge pull request video-dev#416 from dailymotion/alt-audio

Alternate audio track support
normenster · Jun 20, 2016 · dc71ea7 · dc71ea7
2 parents ea02c09 + 3d4e442
commit dc71ea7
Show file tree

Hide file tree

Showing 28 changed files with 1,297 additions and 284 deletions.
diff --git a/API.md b/API.md
@@ -642,14 +642,20 @@ By default, hls.js will automatically start loading quality level playlists, and
 However if `config.autoStartLoad` is set to `false`, the following method needs to be called to manually start playlist and fragments loading:
 
 #### `hls.startLoad()`
-
 Start/restart playlist/fragment loading. this is only effective if MANIFEST_PARSED event has been triggered and video element has been attached to hls object.
 
 #### `hls.stopLoad()`
-
 Stop playlist/fragment loading. could be resumed later on by calling `hls.startLoad()`
 
 
+## Audio Tracks Control API
+
+#### ```hls.audioTracks```
+get : array of audio tracks exposed in manifest
+
+#### ```hls.audioTrack```
+get/set : audio track id (returned by)
+
 ## Runtime Events
 
 Hls.js fires a bunch of events, that could be registered as below:
@@ -673,7 +679,7 @@ Full list of Events is available below:
   - `Hls.Events.MANIFEST_LOADING`  - fired to signal that a manifest loading starts
     -  data: { url : manifestURL }
   - `Hls.Events.MANIFEST_LOADED`  - fired after manifest has been loaded
-    -  data: { levels : [ available quality levels ], url : manifestURL, stats : { trequest, tfirst, tload, mtime } }
+    -  data: { levels : [available quality levels] , audioTracks : [ available audio tracks], url : manifestURL, stats : { trequest, tfirst, tload, mtime}}
   - `Hls.Events.MANIFEST_PARSED`  - fired after manifest has been parsed
     -  data: { levels : [ available quality levels ], firstLevel : index of first quality level appearing in Manifest }
   - `Hls.Events.LEVEL_LOADING`  - fired when a level playlist loading starts
@@ -695,19 +701,17 @@ Full list of Events is available below:
   - `Hls.Events.FRAG_LOAD_PROGRESS`  - fired when a fragment load is in progress
     - data: { frag : fragment object with frag.loaded=stats.loaded, stats : { trequest, tfirst, loaded, total } }
   - `Hls.Events.FRAG_LOADED`  - fired when a fragment loading is completed
-    -  data: { frag : fragment object, payload : fragment payload, stats : { trequest, tfirst, tload, length } }
-  - `Hls.Events.FRAG_PARSING_INIT_SEGMENT`  - fired when Init Segment has been extracted from fragment
-    -  data: { moov : moov MP4 box, codecs : codecs found while parsing fragment }
-  - `Hls.Events.FRAG_PARSING_USERDATA`  - fired when parsing sei text is completed
-    -  data: { samples : [ sei samples pes ] }
+    -  data: { frag : fragment object, payload : fragment payload, stats : { trequest, tfirst, tload, length}}
+  - `Hls.Events.FRAG_PARSING_INIT_SEGMENT` - fired when Init Segment has been extracted from fragment
+    -  data: { id: demuxer id, moov : moov MP4 box, codecs : codecs found while parsing fragment}
   - `Hls.Events.FRAG_PARSING_METADATA`  - fired when parsing id3 is completed
-    -  data: { samples : [ id3 pes - pts and dts timestamp are relative, values are in seconds ] }
+      -  data: { id: demuxer id, samples : [ id3 pes - pts and dts timestamp are relative, values are in seconds]}
   - `Hls.Events.FRAG_PARSING_DATA`  - fired when moof/mdat have been extracted from fragment
-    -  data: { moof : moof MP4 box, mdat : mdat MP4 box, startPTS : PTS of first sample, endPTS : PTS of last sample, startDTS : DTS of first sample, endDTS : DTS of last sample, type : stream type (audio or video), nb : number of samples }
+    -  data: { id: demuxer id, moof : moof MP4 box, mdat : mdat MP4 box, startPTS : PTS of first sample, endPTS : PTS of last sample, startDTS : DTS of first sample, endDTS : DTS of last sample, type : stream type (audio or video), nb : number of samples}
   - `Hls.Events.FRAG_PARSED`  - fired when fragment parsing is completed
-    -  data: undefined
+    -  data: { id: demuxer id}
   - `Hls.Events.FRAG_BUFFERED`  - fired when fragment remuxed MP4 boxes have all been appended into SourceBuffer
-    -  data: { frag : fragment object, stats : { trequest, tfirst, tload, tparsed, tbuffered, length } }
+    -  data: { id: demuxer id, frag : fragment object, stats : { trequest, tfirst, tload, tparsed, tbuffered, length} }
   - `Hls.Events.FRAG_CHANGED`  - fired when fragment matching with current video position is changing
     -  data: { frag : fragment object }
   - `Hls.Events.FPS_DROP` - triggered when FPS drop in last monitoring period is higher than given threshold

diff --git a/demo/canvas.js b/demo/canvas.js
@@ -515,7 +515,7 @@
       legend += ' ' + event.id2;
     }
     if(event.id !== undefined) {
-      if(event.type === 'fragment') {
+      if(event.type.indexOf('fragment') !== -1) {
       legend += ' @';
       }
       legend += ' ' + event.id;

diff --git a/demo/index.html b/demo/index.html
@@ -42,7 +42,7 @@
 }
 
 #customButtons input { width: 25%; display : inline-block; text-align: center; font-size: 8pt;}
-#toggleButtons button { width: 24%; display : inline-block; text-align: center; font-size: 8pt; background-color: #A0A0A0 }
+#toggleButtons button { width: 19%; display : inline-block; text-align: center; font-size: 8pt; background-color: #A0A0A0 }
 
 </style>
   <title>hls.js demo</title>
@@ -98,6 +98,7 @@ <h1 class="title"><a href="https://github.com/dailymotion/hls.js">hls.js</a> dem
     <div class="center" id="toggleButtons">
       <button type="button" class="btn btn-sm" onclick="$('#PlaybackControl').toggle();">toggle playback controls</button>
       <button type="button" class="btn btn-sm" onclick="$('#QualityLevelControl').toggle();">toggle Quality Level controls</button>
+      <button type="button" class="btn btn-sm" onclick="$('#AudioTrackControl').toggle();">toggle Audio Track controls</button>
       <button type="button" class="btn btn-sm" onclick="$('#MetricsDisplay').toggle();toggleMetricsDisplay();">toggle Metrics Display</button>
       <button type="button" class="btn btn-sm" onclick="$('#StatsDisplay').toggle();">toggle Stats Display</button>
     </div>
@@ -143,6 +144,17 @@ <h4> Quality Control </h4>
       </table>
     </div>
 
+    <div id='AudioTrackControl'>
+    <h4> Audio Track Control </h4>
+      <table>
+        <tr>
+          <td>current audio track</td>
+          <td width=10px></td>
+          <td> <div id="audioTrackControl" style="display: inline;"></div> </td>
+        </tr>
+      </table>
+    </div>
+
     <div id='MetricsDisplay'>
     <h4> Real Time Metrics Display </h4>
       <div id="metricsButton">
@@ -201,6 +213,7 @@ <h4> Stats Display </h4>
   $('#videoSize').change(function()          {  $('#video').width($('#videoSize').val()); $('#buffered_c').width($('#videoSize').val());  });
    $("#PlaybackControl").hide();
    $("#QualityLevelControl").hide();
+   $("#AudioTrackControl").hide();
    $("#MetricsDisplay").hide();
    $("#StatsDisplay").hide();
    $('#metricsButtonWindow').toggle(windowSliding);
@@ -228,7 +241,7 @@ <h4> Stats Display </h4>
   var video = $('#video')[0];
   video.volume = 0.05;
 
-  loadStream(decodeURIComponent(getURLParam('src','http://www.streambox.fr/playlists/x36xhzz/x36xhzz.m3u8')));
+  loadStream(decodeURIComponent(getURLParam('src','https://hls.ted.com/talks/2024.m3u8?qr&network=tv&preroll=Blank')));
 
   function loadStream(url) {
     hideCanvas();
@@ -269,11 +282,11 @@ <h4> Stats Display </h4>
        });
        hls.on(Hls.Events.FRAG_PARSING_INIT_SEGMENT,function(event,data) {
           showCanvas();
-          var event = {time : performance.now() - events.t0, type : "init segment"};
+          var event = {time : performance.now() - events.t0, type : data.id + " init segment"};
           events.video.push(event);
        });
        hls.on(Hls.Events.FRAG_PARSING_METADATA, function(event, data) {
-         console.log("Id3 samples ", data.samples);
+         //console.log("Id3 samples ", data.samples);
        });
        hls.on(Hls.Events.LEVEL_SWITCH,function(event,data) {
           events.level.push({time : performance.now() - events.t0, id : data.level, bitrate : Math.round(hls.levels[data.level].bitrate/1000)});
@@ -298,6 +311,10 @@ <h4> Stats Display </h4>
          stats = {levelNb: data.levels.length};
          updateLevelInfo();
        });
+       hls.on(Hls.Events.AUDIO_TRACKS_UPDATED,function(event,data) {
+          $("#HlsStatus").text(data.audioTracks.length + " audio tracks found");
+          updateAudioTrackInfo();
+       });
        hls.on(Hls.Events.LEVEL_LOADED,function(event,data) {
           events.isLive = data.details.live;
           var event = {
@@ -314,9 +331,25 @@ <h4> Stats Display </h4>
           events.load.push(event);
           refreshCanvas();
        });
+       hls.on(Hls.Events.AUDIO_TRACK_LOADED,function(event,data) {
+          events.isLive = data.details.live;
+          var event = {
+            type : "audio track",
+            id : data.id,
+            start : data.details.startSN,
+            end : data.details.endSN,
+            time : data.stats.trequest - events.t0,
+            latency : data.stats.tfirst - data.stats.trequest,
+            load : data.stats.tload - data.stats.tfirst,
+            parsing : data.stats.tparsed - data.stats.tload,
+            duration : data.stats.tload - data.stats.tfirst
+          };
+          events.load.push(event);
+          refreshCanvas();
+       });
        hls.on(Hls.Events.FRAG_BUFFERED,function(event,data) {
           var event = {
-            type : "fragment",
+            type : data.frag.type + " fragment",
             id : data.frag.level,
             id2 : data.frag.sn,
             time : data.stats.trequest - events.t0,
@@ -928,6 +961,26 @@ <h4> Stats Display </h4>
     }
   }
 
+function updateAudioTrackInfo() {
+    var button_template = '<button type="button" class="btn btn-sm ';
+    var button_enabled  = 'btn-primary" ';
+    var button_disabled = 'btn-success" ';
+    var html1 = '';
+    var audioTrackId = hls.audioTrack, len = hls.audioTracks.length;
+
+    for (var i=0; i < len; i++) {
+      html1 += button_template;
+      if(audioTrackId === i) {
+        html1 += button_enabled;
+      } else {
+        html1 += button_disabled;
+      }
+      html1 += 'onclick="hls.audioTrack=' + i + '">' + hls.audioTracks[i].name + '</button>';
+    }
+    $("#audioTrackControl").html(html1);
+  }
+
+
   function level2label(index) {
     if(hls && hls.levels.length-1 >= index) {
       var level = hls.levels[index];

diff --git a/design.md b/design.md
@@ -21,11 +21,8 @@ design idea is pretty simple :
         - retrieve "not buffered" media position greater then current playback position. this is performed by comparing video.buffered and video.currentTime.
           - if there are holes in video.buffered, smaller than config.maxBufferHole, they will be ignored.
         - retrieve URL of fragment matching with this media position, and appropriate quality level
+        - trigger KEY_LOADING event (only if fragment is encrypted)
         - trigger FRAG_LOADING event
-        - **monitor fragment loading speed** (by monitoring data received from FRAG_LOAD_PROGRESS event)
-         - "expected time of fragment load completion" is computed using "fragment loading instant bandwidth".
-         - this time is compared to the "expected time of buffer starvation".
-         - if we have less than 2 fragments buffered and if "expected time of fragment load completion" is bigger than "expected time of buffer starvation" and also bigger than duration needed to load fragment at next quality level (determined by auto quality switch algorithm), current fragment loading is aborted, stream-controller will **trigger an emergency switch down**.
         - **trigger fragment demuxing** on FRAG_LOADED
         - trigger BUFFER_RESET on MANIFEST_PARSED or startLoad()        
         - trigger BUFFER_CODECS on FRAG_PARSING_INIT_SEGMENT
@@ -51,8 +48,12 @@ design idea is pretty simple :
     - a timer is armed to periodically refresh active live playlist.
 
   - [src/controller/abr-controller.js][]
-    - in charge of determining auto quality level.
-    - auto quality switch algorithm is pretty naive and simple ATM and similar to the one that could be found in google [StageFright](https://android.googlesource.com/platform/frameworks/av/+/master/media/libstagefright/httplive/LiveSession.cpp)
+    - in charge of determining auto quality level and monitoring fragment loading speed
+      - **fragment loading speed is monitored every 100ms, based on inputs received from FRAG_LOAD_PROGRESS event
+       - "expected time of fragment load completion" is computed using "fragment loading instant bandwidth".
+       - this time is compared to the "expected time of buffer starvation".
+       - if we have less than 2 fragments buffered and if "expected time of fragment load completion" is bigger than "expected time of buffer starvation" and also bigger than duration needed to load fragment at next quality level (determined by auto quality switch algorithm), current fragment loading is aborted, stream-controller will **trigger an emergency switch down**.
+    - auto quality switch algorithm is pretty naive and simple ATM and similar to the one that could be found in google [StageFright](https://android.googlesource.com/platform/frameworks/av/+/master/media/libstagefright/httplive/LiveSession.cpp)   
   - [src/controller/cap-level-controller.js][]
     - in charge of determining best quality level to actual size (dimensions: width and height) of the player 
   - [src/controller/timeline-controller.js][]

diff --git a/src/controller/abr-controller.js b/src/controller/abr-controller.js
@@ -31,12 +31,14 @@ class AbrController extends EventHandler {
   }
 
   onFragLoading(data) {
-    if (!this.timer) {
-      this.timer = setInterval(this.onCheck, 100);
-    }
     let frag = data.frag;
-    frag.trequest = performance.now();
-    this.fragCurrent = frag;
+    if (frag.type === 'main') {
+      if (!this.timer) {
+        this.timer = setInterval(this.onCheck, 100);
+      }
+      frag.trequest = performance.now();
+      this.fragCurrent = frag;
+    }
   }
 
   abandonRulesCheck() {
@@ -107,20 +109,23 @@ class AbrController extends EventHandler {
   }
 
   onFragLoaded(data) {
-    var stats = data.stats;
-    // only update stats on first frag loading
-    // if same frag is loaded multiple times, it might be in browser cache, and loaded quickly
-    // and leading to wrong bw estimation
-    if (stats.aborted === undefined && data.frag.loadCounter === 1) {
-      this.bwEstimator.sample(performance.now() - stats.trequest,stats.loaded);
-    }
+    let frag = data.frag;
+    if (frag.type === 'main') {
+      var stats = data.stats;
+      // only update stats on first frag loading
+      // if same frag is loaded multiple times, it might be in browser cache, and loaded quickly
+      // and leading to wrong bw estimation
+      if (stats.aborted === undefined && frag.loadCounter === 1) {
+        this.bwEstimator.sample(performance.now() - stats.trequest,stats.loaded);
+      }
 
-    // stop monitoring bw once frag loaded
-    this.clearTimer();
-    // store level id after successful fragment load
-    this.lastLoadedFragLevel = data.frag.level;
-    // reset forced auto level value so that next level will be selected
-    this._nextAutoLevel = -1;
+      // stop monitoring bw once frag loaded
+      this.clearTimer();
+      // store level id after successful fragment load
+      this.lastLoadedFragLevel = frag.level;
+      // reset forced auto level value so that next level will be selected
+      this._nextAutoLevel = -1;
+    }
   }
 
   onError(data) {