Calculate fragment start/end based on the intersection of audio/video…

… PTS (video-dev#2004)

* Calculate frag start/end based on the intersection of audio/video PTS

* Set last frag endTime to the max end PTS as specified in the MSE spec

* Remove maxBufferHole restriction for the starting segment

src/controller/level-helper.js

@@ -53,9 +53,23 @@ export function updatePTS (fragments, fromIdx, toIdx) {
   }
 }
 
+/**
+ * Updates the fragment object with the PTS/DTS calculated during muxing. These values are the true start and duration
+ * time of the fragment, which can differ from what is stated in the manifest. After updating the fragment, we recompute
+ * the start times of all other fragments within the level. This function is usually called once for each elementary
+ * stream - audio and video
+ * @param details - The details of the level of which the fragment belongs
+ * @param frag - The details of the muxed fragment
+ * @param startPTS - The start PTS (presentation timestamp) of the fragment
+ * @param endPTS - The end PTS of the fragment
+ * @param startDTS - The start DTS (decode timestamp) of the fragment
+ * @param endDTS - The end DTS of the fragment
+ * @returns {number}
+ */
 export function updateFragPTSDTS (details, frag, startPTS, endPTS, startDTS, endDTS) {
   // update frag PTS/DTS
   let maxStartPTS = startPTS;
+  // frag.startPTS will already be set in the case that a fragment contains both audio and video
   if (Number.isFinite(frag.startPTS)) {
     // delta PTS between audio and video
     let deltaPTS = Math.abs(frag.startPTS - startPTS);
@@ -65,14 +79,58 @@ export function updateFragPTSDTS (details, frag, startPTS, endPTS, startDTS, end
       frag.deltaPTS = Math.max(deltaPTS, frag.deltaPTS);
     }
 
+    /**
+     Hls.js allocates a sourceBuffer for both audio and video. The HTMLMediaElement (aka video tag) reports its length
+     as the intersection of these two sourceBuffers. Therefore, the start time of the fragment is the largest of the
+     audio/video start PTS values, and the minimum of the audio/video end PTS values.
+
+     A fragment with no overlap between audio/video will have an MSE buffered length of 0 because there is no
+     intersection between its two (audio/video) sourceBuffers. The actual length of the segment depends on where
+     other segments buffer. For example:
+
+     audio:    [           |----------|] (10, 20)
+     video:    [|---------|            ] (0, 9)
+     reported: [                       ] (0)
+
+     audio:    [                       |----------|] (20, 30)
+     video:    [           |----------|            ] (10, 19)
+     reported: [           |----------|            ] (10, 19)
+
+     on EOS:
+     reported: [           |-----------------------] (10, 30)
+
+     https://www.w3.org/TR/media-source/#htmlmediaelement-extensions for more info
+     **/
+
     maxStartPTS = Math.max(startPTS, frag.startPTS);
-    startPTS = Math.min(startPTS, frag.startPTS);
-    endPTS = Math.max(endPTS, frag.endPTS);
-    startDTS = Math.min(startDTS, frag.startDTS);
-    endDTS = Math.max(endDTS, frag.endDTS);
+    if (startPTS > frag.endPTS || endPTS < frag.startPTS) {
+      logger.warn(`Audio/video tracks have no PTS intersection. This may cause unrecoverable playback stalls. 
+      frag: ${frag.startPTS},${frag.endPTS}, new values: ${startPTS},${endPTS}`);
+
+      // Set the fragment start/end to encompass its entire PTS range. If this causes loop loading, Hls.js will force the
+      // next fragment to load in _findFragment.
+      startPTS = Math.min(startPTS, frag.startPTS);
+      endPTS = Math.max(endPTS, frag.endPTS);
+      startDTS = Math.min(startDTS, frag.startDTS);
+      endDTS = Math.max(endDTS, frag.endDTS);
+    } else {
+      startPTS = Math.max(startPTS, frag.startPTS);
+      startDTS = Math.max(startDTS, frag.startDTS);
+
+      // According to the spec: "If readyState is "ended", then set the end time on the last range in source ranges to highest end time."
+      // Therefore we should set the last frag's end time to the max end PTS; otherwise, it may fail to be selected for buffering
+      if (details && !details.live && frag.sn === details.endSN) {
+        endPTS = Math.max(endPTS, frag.endPTS);
+        endDTS = Math.max(endDTS, frag.endDTS);
+      } else {
+        endPTS = Math.min(endPTS, frag.endPTS);
+        endDTS = Math.min(endDTS, frag.endDTS);
+      }
+    }
   }
 
   const drift = startPTS - frag.start;
+  // According to MSE, buffer lengths are calculated via PTS (instead of DTS)
   frag.start = frag.startPTS = startPTS;
   frag.maxStartPTS = maxStartPTS;
   frag.endPTS = endPTS;
@@ -85,23 +143,21 @@ export function updateFragPTSDTS (details, frag, startPTS, endPTS, startDTS, end
   if (!details || sn < details.startSN || sn > details.endSN) {
     return 0;
   }
-
-  let fragIdx, fragments, i;
-  fragIdx = sn - details.startSN;
-  fragments = details.fragments;
+  let fragIdx = sn - details.startSN;
+  let fragments = details.fragments;
   // update frag reference in fragments array
   // rationale is that fragments array might not contain this frag object.
   // this will happen if playlist has been refreshed between frag loading and call to updateFragPTSDTS()
   // if we don't update frag, we won't be able to propagate PTS info on the playlist
   // resulting in invalid sliding computation
   fragments[fragIdx] = frag;
   // adjust fragment PTS/duration from seqnum-1 to frag 0
-  for (i = fragIdx; i > 0; i--) {
+  for (let i = fragIdx; i > 0; i--) {
     updatePTS(fragments, i, i - 1);
   }
 
   // adjust fragment PTS/duration from seqnum to last frag
-  for (i = fragIdx; i < fragments.length - 1; i++) {
+  for (let i = fragIdx; i < fragments.length - 1; i++) {
     updatePTS(fragments, i, i + 1);
   }
 

src/controller/stream-controller.js

@@ -1367,11 +1367,7 @@ class StreamController extends TaskLoop {
     if (!this.loadedmetadata && buffered.length) {
       this.loadedmetadata = true;
       // Need to check what the SourceBuffer reports as start time for the first fragment appended.
-      // If within the threshold of maxBufferHole, adjust this.startPosition for _seekToStartPos().
-      var firstbufferedPosition = buffered.start(0);
-      if (Math.abs(this.startPosition - firstbufferedPosition) < this.config.maxBufferHole) {
-        this.startPosition = firstbufferedPosition;
-      }
+      this.startPosition = buffered.start(0);
       this._seekToStartPos();
     } else if (this.immediateSwitch) {
       this.immediateLevelSwitchEnd();

src/remux/mp4-remuxer.js

@@ -261,7 +261,7 @@ class MP4Remuxer {
     firstDTS = Math.max(sample.dts, 0);
     firstPTS = Math.max(sample.pts, 0);
 
-    // check timestamp continuity accross consecutive fragments (this is to remove inter-fragment gap/hole)
+    // check timestamp continuity across consecutive fragments (this is to remove inter-fragment gap/hole)
     let delta = Math.round((firstDTS - nextAvcDts) / 90);
     // if fragment are contiguous, detect hole/overlapping between fragments
     if (contiguous) {

tests/test-streams.js

@@ -171,5 +171,13 @@ module.exports = {
   pdtOneValue: {
     url: 'https://playertest.longtailvideo.com/adaptive/aviion/manifest.m3u8',
     description: 'One PDT, no discontinuities'
+  },
+  differentPTSDTSWithSmallSubsequentSegment: {
+    url: 'https://9secfail-tepnrytnng.now.sh/index.m3u8',
+    description: 'Audio/video has different PTS; the following segment is very small (0.04s) and tests buffer intersection'
+  },
+  noTrackIntersection: {
+    url: 'https://s3.amazonaws.com/bob.jwplayer.com/%7Ealex/123633/new_master.m3u8',
+    description: 'Audio/video track PTS values do not intersect; 10 second start gap'
   }
 };

tests/unit/controller/level-helper.js

@@ -0,0 +1,126 @@
+import * as LevelHelper from '../../../src/controller/level-helper';
+const assert = require('assert');
+
+describe('level-helper', function () {
+  describe('updateFragPTSDTS', function () {
+    let details;
+    let frag;
+    beforeEach(function () {
+      details = {
+        fragments: []
+      };
+      frag = {
+        sn: 0
+      };
+    });
+
+    describe('PTS/DTS updating', function () {
+      function checkFragProperties (frag, startPTS, endPTS, startDTS, endDTS, maxStartPTS) {
+        assert.strictEqual(frag.start, startPTS);
+        assert.strictEqual(frag.startPTS, startPTS);
+        assert.strictEqual(frag.maxStartPTS, maxStartPTS);
+        assert.strictEqual(frag.endPTS, endPTS);
+        assert.strictEqual(frag.startDTS, startDTS);
+        assert.strictEqual(frag.endDTS, endDTS);
+        assert.strictEqual(frag.duration, endPTS - startPTS);
+      }
+
+      it('updates frag properties based on the provided PTS/DTS', function () {
+        const startPTS = 2;
+        const endPTS = 12;
+        const startDTS = 1;
+        const endDTS = 11;
+
+        LevelHelper.updateFragPTSDTS(null, frag, startPTS, endPTS, startDTS, endDTS);
+        checkFragProperties(frag, 2, 12, 1, 11, 2);
+      });
+
+      it('updates frag properties based on the intersection of existing frag PTS/DTS and provided frag PTS/DTS', function () {
+        const startPTS = 2;
+        const endPTS = 12;
+        const startDTS = 1;
+        const endDTS = 11;
+
+        frag.startPTS = 3;
+        frag.endPTS = 13;
+        frag.startDTS = 2;
+        frag.endDTS = 12;
+
+        LevelHelper.updateFragPTSDTS(null, frag, startPTS, endPTS, startDTS, endDTS);
+        checkFragProperties(frag, 3, 12, 2, 11, 3);
+        assert.strictEqual(frag.deltaPTS, 1);
+
+        frag.startPTS = 0;
+        frag.endPTS = 10;
+        frag.startDTS = 0;
+        frag.endDTS = 10;
+
+        LevelHelper.updateFragPTSDTS(null, frag, startPTS, endPTS, startDTS, endDTS);
+        checkFragProperties(frag, 2, 10, 1, 10, 2);
+        assert.strictEqual(frag.deltaPTS, 2);
+      });
+
+      it('chooses the min start and max end if the a/v PTS values do not intersect', function () {
+        const startPTS = 14;
+        const endPTS = 24;
+        const startDTS = 1;
+        const endDTS = 11;
+
+        frag.startPTS = 3;
+        frag.endPTS = 13;
+        frag.startDTS = 2;
+        frag.endDTS = 12;
+
+        LevelHelper.updateFragPTSDTS(null, frag, startPTS, endPTS, startDTS, endDTS);
+        checkFragProperties(frag, 3, 24, 1, 12, 14);
+        assert.strictEqual(frag.deltaPTS, 11);
+      });
+
+      it('sets the end PTS/DTS to the max end PTS/DTS if the fragment is the last of a non-live stream, and has a/v intersection', function () {
+        const startPTS = 2;
+        const endPTS = 12;
+        const startDTS = 1;
+        const endDTS = 11;
+
+        frag.startPTS = 3;
+        frag.endPTS = 13;
+        frag.startDTS = 2;
+        frag.endDTS = 12;
+
+        details.endSN = 5;
+        details.live = false;
+        frag.sn = 5;
+
+        LevelHelper.updateFragPTSDTS(details, frag, startPTS, endPTS, startDTS, endDTS);
+        checkFragProperties(frag, 3, 13, 2, 12, 3);
+        assert.strictEqual(frag.deltaPTS, 1);
+      });
+    });
+
+    describe('drift calculation', function () {
+      let startPTS;
+      let endPTS;
+      let startDTS;
+      let endDTS;
+      beforeEach(function () {
+        startPTS = 2;
+        endPTS = 12;
+        startDTS = 1;
+        endDTS = 11;
+        details.startSN = 0;
+        details.endSN = 10;
+      });
+
+      it('returns a drift of 0 if the fragment is out of the sequence range of its level', function () {
+        frag.sn = 50;
+        assert.strictEqual(LevelHelper.updateFragPTSDTS(details, frag, startPTS, endPTS, startDTS, endDTS), 0);
+      });
+
+      it('returns the drift between startPTS and fragStart if the fragment is within the sequence range', function () {
+        frag.sn = 0;
+        frag.start = 0;
+        assert.strictEqual(LevelHelper.updateFragPTSDTS(details, frag, startPTS, endPTS, startDTS, endDTS), 2);
+      });
+    });
+  });
+});

tests/unit/utils/discontinuities.js

@@ -1,4 +1,4 @@
-import { shouldAlignOnDiscontinuities, findDiscontinuousReferenceFrag, adjustPts, alignDiscontinuities, alignPDT } from '../../../src/utils/discontinuities';
+import { shouldAlignOnDiscontinuities, findDiscontinuousReferenceFrag, adjustPts, alignPDT } from '../../../src/utils/discontinuities';
 
 const assert = require('assert');
 
@@ -34,7 +34,7 @@ const mockFrags = [
   }
 ];
 
-describe('level-helper', function () {
+describe('discontinuities', function () {
   it('adjusts level fragments with overlapping CC range using a reference fragment', function () {
     const details = {
       fragments: mockFrags.slice(0),