Skip to content

Commit

Permalink
mp4-remuxer: don't inject empty audio frames when live playlist slidi…
Browse files Browse the repository at this point in the history 
…ng is unknown

related to https://github.com/dailymotion/hls.js/issues/734
  • Loading branch information
@mangui
mangui committed Oct 10, 2016
1 parent a651408 commit dcf67a1
Show file tree
Hide file tree
Showing 8 changed files with 74 additions and 62 deletions.
4 changes: 3 additions & 1 deletion src/controller/audio-stream-controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,9 @@ class AudioStreamController extends EventHandler {
this.demuxer = new Demuxer(this.hls,'audio');
}
logger.log(`Demuxing ${sn} of [${details.startSN} ,${details.endSN}],track ${trackId}`);
this.demuxer.push(data.payload, audioCodec, null, start, fragCurrent.cc, trackId, sn, duration, fragCurrent.decryptdata);
// time Offset is accurate if level PTS is known, or if playlist is not sliding (not live)
let accurateTimeOffset = details.PTSKnown || !details.live;
this.demuxer.push(data.payload, audioCodec, null, start, fragCurrent.cc, trackId, sn, duration, fragCurrent.decryptdata,accurateTimeOffset);
}
this.fragLoadError = 0;
}
Expand Down
4 changes: 3 additions & 1 deletion src/controller/stream-controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -926,7 +926,9 @@ class StreamController extends EventHandler {
if (!demuxer) {
demuxer = this.demuxer = new Demuxer(this.hls,'main');
}
demuxer.push(data.payload, audioCodec, currentLevel.videoCodec, start, fragCurrent.cc, level, sn, duration, fragCurrent.decryptdata);
// time Offset is accurate if level PTS is known, or if playlist is not sliding (not live)
let accurateTimeOffset = details.PTSKnown || !details.live;
demuxer.push(data.payload, audioCodec, currentLevel.videoCodec, start, fragCurrent.cc, level, sn, duration, fragCurrent.decryptdata, accurateTimeOffset);
}
}
this.fragLoadError = 0;
Expand Down
4 changes: 2 additions & 2 deletions src/demux/aacdemuxer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import ID3 from '../demux/id3';


// feed incoming data to the front of the parsing pipeline
push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration) {
push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration,accurateTimeOffset) {
var track,
id3 = new ID3(data),
pts = 90*id3.timeStamp,
Expand Down Expand Up @@ -108,7 +108,7 @@ import ID3 from '../demux/id3';
break;
}
}
this.remuxer.remux(level, sn , this._aacTrack,{samples : []}, {samples : [ { pts: pts, dts : pts, unit : id3.payload} ]}, { samples: [] }, timeOffset, contiguous);
this.remuxer.remux(level, sn , this._aacTrack,{samples : []}, {samples : [ { pts: pts, dts : pts, unit : id3.payload} ]}, { samples: [] }, timeOffset, contiguous,accurateTimeOffset);
}

destroy() {
Expand Down
4 changes: 2 additions & 2 deletions src/demux/demuxer-inline.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class DemuxerInline {
}
}

push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration) {
push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration,accurateTimeOffset) {
var demuxer = this.demuxer;
if (!demuxer) {
let hls = this.hls,
Expand All @@ -45,7 +45,7 @@ class DemuxerInline {
}
this.demuxer = demuxer;
}
demuxer.push(data,audioCodec,videoCodec,timeOffset,cc,level,sn,duration);
demuxer.push(data,audioCodec,videoCodec,timeOffset,cc,level,sn,duration,accurateTimeOffset);
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/demux/demuxer-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ var DemuxerWorker = function (self) {
}
break;
case 'demux':
self.demuxer.push(new Uint8Array(data.data), data.audioCodec, data.videoCodec, data.timeOffset, data.cc, data.level, data.sn, data.duration);
self.demuxer.push(new Uint8Array(data.data), data.audioCodec, data.videoCodec, data.timeOffset, data.cc, data.level, data.sn, data.duration,data.accurateTimeOffset);
break;
default:
break;
Expand Down
12 changes: 6 additions & 6 deletions src/demux/demuxer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,31 +53,31 @@ class Demuxer {
}
}

pushDecrypted(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration) {
pushDecrypted(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration,accurateTimeOffset) {
let w = this.w;
if (w) {
// post fragment payload as transferable objects (no copy)
w.postMessage({cmd: 'demux', data: data, audioCodec: audioCodec, videoCodec: videoCodec, timeOffset: timeOffset, cc: cc, level: level, sn : sn, duration: duration}, [data]);
w.postMessage({cmd: 'demux', data: data, audioCodec: audioCodec, videoCodec: videoCodec, timeOffset: timeOffset, cc: cc, level: level, sn : sn, duration: duration, accurateTimeOffset : accurateTimeOffset}, [data]);
} else {
let demuxer = this.demuxer;
if (demuxer) {
demuxer.push(new Uint8Array(data), audioCodec, videoCodec, timeOffset, cc, level, sn, duration);
demuxer.push(new Uint8Array(data), audioCodec, videoCodec, timeOffset, cc, level, sn, duration,accurateTimeOffset);
}
}
}

push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration, decryptdata) {
push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration, decryptdata,accurateTimeOffset) {
if ((data.byteLength > 0) && (decryptdata != null) && (decryptdata.key != null) && (decryptdata.method === 'AES-128')) {
if (this.decrypter == null) {
this.decrypter = new Decrypter(this.hls);
}

var localthis = this;
this.decrypter.decrypt(data, decryptdata.key, decryptdata.iv, function(decryptedData){
localthis.pushDecrypted(decryptedData, audioCodec, videoCodec, timeOffset, cc, level, sn, duration);
localthis.pushDecrypted(decryptedData, audioCodec, videoCodec, timeOffset, cc, level, sn, duration,accurateTimeOffset);
});
} else {
this.pushDecrypted(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration);
this.pushDecrypted(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration,accurateTimeOffset);
}
}

Expand Down
5 changes: 3 additions & 2 deletions src/demux/tsdemuxer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
}

// feed incoming data to the front of the parsing pipeline
push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration) {
push(data, audioCodec, videoCodec, timeOffset, cc, level, sn, duration,accurateTimeOffset) {
var start, len = data.length, stt, pid, atf, offset,pes,
codecsOnly = this.remuxer.passthrough,
unknownPIDs = false;
Expand All @@ -65,6 +65,7 @@
this.videoCodec = videoCodec;
this._duration = duration;
this.contiguous = false;
this.accurateTimeOffset = accurateTimeOffset;
if (cc !== this.lastCC) {
logger.log('discontinuity detected');
this.insertDiscontinuity();
Expand Down Expand Up @@ -252,7 +253,7 @@
};},{len : 0, nbNalu : 0});
avcTrack.len = trackData.len;
avcTrack.nbNalu = trackData.nbNalu;
this.remuxer.remux(level, sn, this._aacTrack, this._avcTrack, this._id3Track, this._txtTrack, timeOffset, this.contiguous, data);
this.remuxer.remux(level, sn, this._aacTrack, this._avcTrack, this._id3Track, this._txtTrack, timeOffset, this.contiguous, this.accurateTimeOffset, data);
}

destroy() {
Expand Down
101 changes: 54 additions & 47 deletions src/remux/mp4-remuxer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class MP4Remuxer {
this.ISGenerated = false;
}

remux(level,sn,audioTrack,videoTrack,id3Track,textTrack,timeOffset, contiguous) {
remux(level,sn,audioTrack,videoTrack,id3Track,textTrack,timeOffset, contiguous,accurateTimeOffset) {
this.level = level;
this.sn = sn;
// generate Init Segment if needed
Expand All @@ -49,7 +49,7 @@ class MP4Remuxer {
// calculated in remuxAudio.
//logger.log('nb AAC samples:' + audioTrack.samples.length);
if (audioTrack.samples.length) {
let audioData = this.remuxAudio(audioTrack,timeOffset,contiguous);
let audioData = this.remuxAudio(audioTrack,timeOffset,contiguous,accurateTimeOffset);
//logger.log('nb AVC samples:' + videoTrack.samples.length);
if (videoTrack.samples.length) {
let audioTrackLength;
Expand Down Expand Up @@ -190,6 +190,7 @@ class MP4Remuxer {
// PTS is coded on 33bits, and can loop from -2^32 to 2^32
// PTSNormalize will make PTS/DTS value monotonic, we use last known DTS value as reference value
let nextAvcDts;
// contiguous fragments are consecutive fragments from same quality level (same level, new SN = old SN + 1)
if (contiguous) {
// if parsed fragment is contiguous with last one, let's use last DTS value as reference
nextAvcDts = this.nextAvcDts;
Expand Down Expand Up @@ -219,7 +220,7 @@ class MP4Remuxer {
// offset PTS as well, ensure that PTS is smaller or equal than new DTS
firstPTS = Math.max(firstPTS - delta, nextAvcDts);
inputSamples[0].pts = firstPTS + this._initDTS;
logger.log(`Video/PTS/DTS adjusted: ${firstPTS}/${firstDTS},delta:${delta}`);
logger.log(`Video/PTS/DTS adjusted: ${Math.round(firstPTS/90)}/${Math.round(firstDTS/90)},delta:${delta} ms`);
}
}
nextDTS = firstDTS;
Expand Down Expand Up @@ -370,7 +371,7 @@ class MP4Remuxer {
return data;
}

remuxAudio(track, timeOffset, contiguous) {
remuxAudio(track, timeOffset, contiguous,accurateTimeOffset) {
let pesTimeScale = this.PES_TIMESCALE,
mp4timeScale = track.timescale,
pes2mp4ScaleFactor = pesTimeScale/mp4timeScale,
Expand Down Expand Up @@ -409,57 +410,62 @@ class MP4Remuxer {
// frame.
const pesFrameDuration = expectedSampleDuration * pes2mp4ScaleFactor;
let nextPtsNorm = nextAacPts;
for (var i = 0; i < samples0.length; ) {
// First, let's see how far off this frame is from where we expect it to be
var sample = samples0[i],
ptsNorm = this._PTSNormalize(sample.pts - this._initDTS, nextAacPts),
delta = ptsNorm - nextPtsNorm;

// If we're overlapping by more than a duration, drop this sample
if (delta <= -pesFrameDuration) {
logger.warn(`Dropping 1 audio frame @ ${Math.round(nextPtsNorm/90)/1000}s due to ${Math.round(Math.abs(delta / 90))} ms overlap.`);
samples0.splice(i, 1);
track.len -= sample.unit.length;
// Don't touch nextPtsNorm or i
}
// Otherwise, if we're more than a frame away from where we should be, insert missing frames
else if (delta >= pesFrameDuration) {
var missing = Math.round(delta / pesFrameDuration);
logger.warn(`Injecting ${missing} audio frame @ ${Math.round(nextPtsNorm/90)/1000}s due to ${Math.round(delta / 90)} ms gap.`);
for (var j = 0; j < missing; j++) {
newStamp = nextPtsNorm + this._initDTS;
newStamp = Math.max(newStamp, this._initDTS);
fillFrame = AAC.getSilentFrame(track.channelCount);
if (!fillFrame) {
logger.log('Unable to get silent frame for given audio codec; duplicating last frame instead.');
fillFrame = sample.unit.slice(0);

// only inject/drop audio frames in case time offset is accurate
if (accurateTimeOffset) {
for (var i = 0; i < samples0.length; ) {
// First, let's see how far off this frame is from where we expect it to be
var sample = samples0[i],
ptsNorm = this._PTSNormalize(sample.pts - this._initDTS, nextAacPts),
delta = ptsNorm - nextPtsNorm;

// If we're overlapping by more than a duration, drop this sample
if (delta <= -pesFrameDuration) {
logger.warn(`Dropping 1 audio frame @ ${Math.round(nextPtsNorm/90)/1000}s due to ${Math.round(Math.abs(delta / 90))} ms overlap.`);
samples0.splice(i, 1);
track.len -= sample.unit.length;
// Don't touch nextPtsNorm or i
}
// Otherwise, if we're more than a frame away from where we should be, insert missing frames
else if (delta >= pesFrameDuration) {
var missing = Math.round(delta / pesFrameDuration);
logger.warn(`Injecting ${missing} audio frame @ ${Math.round(nextPtsNorm/90)/1000}s due to ${Math.round(delta / 90)} ms gap.`);
for (var j = 0; j < missing; j++) {
newStamp = nextPtsNorm + this._initDTS;
newStamp = Math.max(newStamp, this._initDTS);
fillFrame = AAC.getSilentFrame(track.channelCount);
if (!fillFrame) {
logger.log('Unable to get silent frame for given audio codec; duplicating last frame instead.');
fillFrame = sample.unit.slice(0);
}
samples0.splice(i, 0, {unit: fillFrame, pts: newStamp, dts: newStamp});
track.len += fillFrame.length;
nextPtsNorm += pesFrameDuration;
i += 1;
}
samples0.splice(i, 0, {unit: fillFrame, pts: newStamp, dts: newStamp});
track.len += fillFrame.length;

// Adjust sample to next expected pts
sample.pts = sample.dts = nextPtsNorm + this._initDTS;
nextPtsNorm += pesFrameDuration;
i += 1;
}

// Adjust sample to next expected pts
sample.pts = sample.dts = nextPtsNorm + this._initDTS;
nextPtsNorm += pesFrameDuration;
i += 1;
}
// Otherwise, we're within half a frame duration, so just adjust pts
else {
if (Math.abs(delta) > (0.1 * pesFrameDuration)) {
//logger.log(`Invalid frame delta ${Math.round(ptsNorm - nextPtsNorm + pesFrameDuration)} at PTS ${Math.round(ptsNorm / 90)} (should be ${Math.round(pesFrameDuration)}).`);
}
nextPtsNorm += pesFrameDuration;
if (i === 0) {
sample.pts = sample.dts = this._initDTS + nextAacPts;
} else {
sample.pts = sample.dts = samples0[i - 1].pts + pesFrameDuration;
// Otherwise, we're within half a frame duration, so just adjust pts
else {
if (Math.abs(delta) > (0.1 * pesFrameDuration)) {
//logger.log(`Invalid frame delta ${Math.round(ptsNorm - nextPtsNorm + pesFrameDuration)} at PTS ${Math.round(ptsNorm / 90)} (should be ${Math.round(pesFrameDuration)}).`);
}
nextPtsNorm += pesFrameDuration;
if (i === 0) {
sample.pts = sample.dts = this._initDTS + nextAacPts;
} else {
sample.pts = sample.dts = samples0[i - 1].pts + pesFrameDuration;
}
i += 1;
}
i += 1;
}
}


while (samples0.length) {
aacSample = samples0.shift();
unit = aacSample.unit;
Expand All @@ -477,6 +483,7 @@ class MP4Remuxer {
let delta = Math.round(1000 * (ptsnorm - nextAacPts) / pesTimeScale),
numMissingFrames = 0;
// if fragment are contiguous, detect hole/overlapping between fragments
// contiguous fragments are consecutive fragments from same quality level (same level, new SN = old SN + 1)
if (contiguous) {
// log delta
if (delta) {
Expand Down

0 comments on commit dcf67a1

Please sign in to comment.