Skip to content

Commit

Permalink
Subtitle decryption support
Browse files Browse the repository at this point in the history
  • Loading branch information
taylornewton committed Oct 2, 2017
1 parent bc24f55 commit 2715239
Show file tree
Hide file tree
Showing 5 changed files with 190 additions and 57 deletions.
2 changes: 1 addition & 1 deletion doc/API.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1169,7 +1169,7 @@ Full list of Events is available below:
- `Hls.Events.FRAG_LOADED` - fired when a fragment loading is completed
- data: { frag : fragment object, payload : fragment payload, stats : { trequest, tfirst, tload, length}}
- `Hls.Events.FRAG_DECRYPTED` - fired when a fragment decryption is completed
- data: { id : demuxer id, frag : fragment object, stats : { tstart, tdecrypt}}
- data: { id : demuxer id, frag : fragment object, payload : fragment payload, stats : { tstart, tdecrypt}}
- `Hls.Events.FRAG_PARSING_INIT_SEGMENT` - fired when Init Segment has been extracted from fragment
- data: { id: demuxer id, frag : fragment object, moov : moov MP4 box, codecs : codecs found while parsing fragment }
- `Hls.Events.FRAG_PARSING_USERDATA` - fired when parsing sei text is completed
Expand Down
10 changes: 10 additions & 0 deletions doc/design.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,16 @@ design idea is pretty simple :
if playhead is stuck for more than `config.highBufferWatchdogPeriod` second in a buffered area, hls.js will nudge currentTime until playback recovers (it will retry every seconds, and report a fatal error after config.maxNudgeRetry retries)
- convert non-fatal `FRAG_LOAD_ERROR`/`FRAG_LOAD_TIMEOUT`/`KEY_LOAD_ERROR`/`KEY_LOAD_TIMEOUT` error into fatal error when media position is not buffered and max load retry has been reached
- stream controller actions are scheduled by a tick timer (invoked every 100ms) and actions are controlled by a state machine.
- [src/controller/subtitle-stream-controller.js][]
- subtitle stream controller is in charge of processing subtitle track fragments
- subtitle stream controller takes the following actions:
- once a SUBTITLE_TRACK_LOADED is received, the controller will begin processing the subtitle fragments
- trigger KEY_LOADING event if fragment is encrypted
- trigger FRAG_LOADING event
- invoke decrypter.decrypt method on FRAG_LOADED if frag is encrypted
- trigger FRAG_DECRYPTED event once encrypted fragment is decrypted
- [src/controller/subtitle-track-controller.js][]
- subtitle track controller handles subtitle track loading and switching
- [src/controller/timeline-controller.js][]
- Manages pulling CEA-708 caption data from the fragments, running them through the cea-608-parser, and handing them off to a display class, which defaults to src/utils/cues.js
- [src/crypt/aes.js][]
Expand Down
145 changes: 122 additions & 23 deletions src/controller/subtitle-stream-controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,23 @@
import Event from '../events';
import EventHandler from '../event-handler';
import {logger} from '../utils/logger';
import Decrypter from '../crypt/decrypter'

const State = {
STOPPED : 'STOPPED',
IDLE : 'IDLE',
KEY_LOADING : 'KEY_LOADING',
FRAG_LOADING : 'FRAG_LOADING'
};

class SubtitleStreamController extends EventHandler {

constructor(hls) {
super(hls,
Event.MEDIA_ATTACHED,
Event.ERROR,
Event.KEY_LOADED,
Event.FRAG_LOADED,
Event.SUBTITLE_TRACKS_UPDATED,
Event.SUBTITLE_TRACK_SWITCH,
Event.SUBTITLE_TRACK_LOADED,
Expand All @@ -19,11 +30,15 @@ class SubtitleStreamController extends EventHandler {
this.vttFragSNsProcessed = {};
this.vttFragQueues = undefined;
this.currentlyProcessing = null;
this.state = State.STOPPED;
this.currentTrackId = -1;
this.ticks = 0;
this.decrypter = new Decrypter(hls.observer, hls.config);
}

destroy() {
EventHandler.prototype.destroy.call(this);
this.state = State.STOPPED;
}

// Remove all queued items and create a new, empty queue for each track.
Expand All @@ -38,7 +53,9 @@ class SubtitleStreamController extends EventHandler {
nextFrag() {
if(this.currentlyProcessing === null && this.currentTrackId > -1 && this.vttFragQueues[this.currentTrackId].length) {
let frag = this.currentlyProcessing = this.vttFragQueues[this.currentTrackId].shift();
this.hls.trigger(Event.FRAG_LOADING, {frag});
this.fragCurrent = frag;
this.hls.trigger(Event.FRAG_LOADING, {frag: frag});
this.state = State.FRAG_LOADING;
}
}

Expand All @@ -48,9 +65,14 @@ class SubtitleStreamController extends EventHandler {
this.vttFragSNsProcessed[data.frag.trackId].push(data.frag.sn);
}
this.currentlyProcessing = null;
this.state = State.IDLE;
this.nextFrag();
}

onMediaAttached() {
this.state = State.IDLE;
}

// If something goes wrong, procede to next frag, if we were processing one.
onError(data) {
let frag = data.frag;
Expand All @@ -64,6 +86,68 @@ class SubtitleStreamController extends EventHandler {
}
}

tick() {
this.ticks++;
if (this.ticks === 1) {
this.doTick();
if (this.ticks > 1) {
setTimeout(() => { this.tick() }, 1);
}
this.ticks = 0;
}
}

doTick() {
switch(this.state) {
case State.IDLE:
const tracks = this.tracks;
let trackId = this.currentTrackId;

const processedFragSNs = this.vttFragSNsProcessed[trackId],
fragQueue = this.vttFragQueues[trackId],
currentFragSN = !!this.currentlyProcessing ? this.currentlyProcessing.sn : -1;

const alreadyProcessed = function(frag) {
return processedFragSNs.indexOf(frag.sn) > -1;
};

const alreadyInQueue = function(frag) {
return fragQueue.some(fragInQueue => {return fragInQueue.sn === frag.sn;});
};

// exit if tracks don't exist
if (!tracks) {
break;
}
var trackDetails;

if (trackId < tracks.length) {
trackDetails = tracks[trackId].details;
}

if (typeof trackDetails === 'undefined') {
break;
}

// Add all fragments that haven't been, aren't currently being and aren't waiting to be processed, to queue.
trackDetails.fragments.forEach(frag => {
if(!(alreadyProcessed(frag) || frag.sn === currentFragSN || alreadyInQueue(frag))) {
// Load key if subtitles are encrypted
if ((frag.decryptdata && frag.decryptdata.uri != null) && (frag.decryptdata.key == null)) {
logger.log(`Loading key for ${frag.sn}`);
this.state = State.KEY_LOADING;
hls.trigger(Event.KEY_LOADING, {frag: frag});
} else {
// Frags don't know their subtitle track ID, so let's just add that...
frag.trackId = trackId;
fragQueue.push(frag);
this.nextFrag();
}
}
});
}
}

// Got all new subtitle tracks.
onSubtitleTracksUpdated(data) {
logger.log('subtitle tracks updated');
Expand All @@ -82,29 +166,44 @@ class SubtitleStreamController extends EventHandler {

// Got a new set of subtitle fragments.
onSubtitleTrackLoaded(data) {
const processedFragSNs = this.vttFragSNsProcessed[data.id],
fragQueue = this.vttFragQueues[data.id],
currentFragSN = !!this.currentlyProcessing ? this.currentlyProcessing.sn : -1;

const alreadyProcessed = function(frag) {
return processedFragSNs.indexOf(frag.sn) > -1;
};

const alreadyInQueue = function(frag) {
return fragQueue.some(fragInQueue => {return fragInQueue.sn === frag.sn;});
};

// Add all fragments that haven't been, aren't currently being and aren't waiting to be processed, to queue.
data.details.fragments.forEach(frag => {
if(!(alreadyProcessed(frag) || frag.sn === currentFragSN || alreadyInQueue(frag))) {
// Frags don't know their subtitle track ID, so let's just add that...
frag.trackId = data.id;
fragQueue.push(frag);
}
});
this.tick();
}

this.nextFrag();
onKeyLoaded() {
if (this.state === State.KEY_LOADING) {
this.state = State.IDLE;
this.tick();
}
}

onFragLoaded(data) {
var fragCurrent = this.fragCurrent,
decryptData = data.frag.decryptdata;
let fragLoaded = data.frag;
if (this.state === State.FRAG_LOADING &&
fragCurrent &&
data.frag.type === 'subtitle' &&
fragCurrent.sn === data.frag.sn) {
// check to see if the payload needs to be decrypted
if ((data.payload.byteLength > 0) && (decryptData != null) && (decryptData.key != null) && (decryptData.method === 'AES-128')) {
var startTime;
try {
startTime = performance.now();
} catch (error) {
startTime = Date.now();
}
// decrypt the subtitles
this.decrypter.decrypt(data.payload, decryptData.key.buffer, decryptData.iv.buffer, function(decryptedData) {
var endTime;
try {
endTime = performance.now();
} catch (error) {
endTime = Date.now();
}
hls.trigger(Event.FRAG_DECRYPTED, { frag: fragLoaded, payload : decryptedData, stats: { tstart: startTime, tdecrypt: endTime } });
});
}
}
}
}
export default SubtitleStreamController;

88 changes: 56 additions & 32 deletions src/controller/timeline-controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class TimelineController extends EventHandler {
super(hls, Event.MEDIA_ATTACHING,
Event.MEDIA_DETACHING,
Event.FRAG_PARSING_USERDATA,
Event.FRAG_DECRYPTED,
Event.MANIFEST_LOADING,
Event.MANIFEST_LOADED,
Event.FRAG_LOADED,
Expand Down Expand Up @@ -268,39 +269,12 @@ class TimelineController extends EventHandler {
this.unparsedVttFrags.push(data);
return;
}
let vttCCs = this.vttCCs;
if (!vttCCs[frag.cc]) {
vttCCs[frag.cc] = { start: frag.start, prevCC: this.prevCC, new: true };
this.prevCC = frag.cc;

var decryptData = frag.decryptdata;
// If the subtitles are not encrypted, parse VTTs now. Otherwise, we need to wait.
if ((decryptData == null) || (decryptData.key == null) || (decryptData.method !== 'AES-128')) {
this._parseVTTs(frag, payload);
}
let textTracks = this.textTracks,
hls = this.hls;

// Parse the WebVTT file contents.
WebVTTParser.parse(payload, this.initPTS, vttCCs, frag.cc, function (cues) {
const currentTrack = textTracks[frag.trackId];
// Add cues and trigger event with success true.
cues.forEach(cue => {
// Sometimes there are cue overlaps on segmented vtts so the same
// cue can appear more than once in different vtt files.
// This avoid showing duplicated cues with same timecode and text.
if (!currentTrack.cues.getCueById(cue.id)) {
try {
currentTrack.addCue(cue);
} catch (err) {
const textTrackCue = new window.TextTrackCue(cue.startTime, cue.endTime, cue.text);
textTrackCue.id = cue.id;
currentTrack.addCue(textTrackCue);
}
}
});
hls.trigger(Event.SUBTITLE_FRAG_PROCESSED, {success: true, frag: frag});
},
function (e) {
// Something went wrong while parsing. Trigger event with success false.
logger.log(`Failed to parse VTT cue: ${e}`);
hls.trigger(Event.SUBTITLE_FRAG_PROCESSED, {success: false, frag: frag});
});
}
else {
// In case there is no payload, finish unsuccessfully.
Expand All @@ -309,6 +283,56 @@ class TimelineController extends EventHandler {
}
}

_parseVTTs(frag, payload) {
let vttCCs = this.vttCCs;
if (!vttCCs[frag.cc]) {
vttCCs[frag.cc] = { start: frag.start, prevCC: this.prevCC, new: true };
this.prevCC = frag.cc;
}
let textTracks = this.textTracks,
hls = this.hls;

// Parse the WebVTT file contents.
WebVTTParser.parse(payload, this.initPTS, vttCCs, frag.cc, function (cues) {
const currentTrack = textTracks[frag.trackId];
// Add cues and trigger event with success true.
cues.forEach(cue => {
// Sometimes there are cue overlaps on segmented vtts so the same
// cue can appear more than once in different vtt files.
// This avoid showing duplicated cues with same timecode and text.
if (!currentTrack.cues.getCueById(cue.id)) {
try {
currentTrack.addCue(cue);
} catch (err) {
const textTrackCue = new window.TextTrackCue(cue.startTime, cue.endTime, cue.text);
textTrackCue.id = cue.id;
currentTrack.addCue(textTrackCue);
}
}
});
hls.trigger(Event.SUBTITLE_FRAG_PROCESSED, {success: true, frag: frag});
},
function (e) {
// Something went wrong while parsing. Trigger event with success false.
logger.log(`Failed to parse VTT cue: ${e}`);
hls.trigger(Event.SUBTITLE_FRAG_PROCESSED, {success: false, frag: frag});
});
}

onFragDecrypted(data) {
var decryptedData = data.payload,
frag = data.frag;

if (frag.type === 'subtitle') {
if (typeof this.initPTS === 'undefined') {
this.unparsedVttFrags.push(data);
return;
}

this._parseVTTs(frag, decryptedData);
}
}

onFragParsingUserdata(data) {
// push all of the CEA-708 messages into the interpreter
// immediately. It will create the proper timestamps based on our PTS value
Expand Down
2 changes: 1 addition & 1 deletion src/events.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ export default {
FRAG_LOAD_EMERGENCY_ABORTED: 'hlsFragLoadEmergencyAborted',
// fired when a fragment loading is completed - data: { frag : fragment object, payload : fragment payload, stats : { trequest, tfirst, tload, length } }
FRAG_LOADED: 'hlsFragLoaded',
// fired when a fragment has finished decrypting - data: { id : demuxer id, frag: fragment object, stats : { tstart, tdecrypt } }
// fired when a fragment has finished decrypting - data: { id : demuxer id, frag: fragment object, payload : fragment payload, stats : { tstart, tdecrypt } }
FRAG_DECRYPTED: 'hlsFragDecrypted',
// fired when Init Segment has been extracted from fragment - data: { id : demuxer id, frag: fragment object, moov : moov MP4 box, codecs : codecs found while parsing fragment }
FRAG_PARSING_INIT_SEGMENT: 'hlsFragParsingInitSegment',
Expand Down

0 comments on commit 2715239

Please sign in to comment.