Add support for 608/708 caption channels 3 and 4, and non-native capt…

…ions track handling

Fixes video-dev#2329

docs/API.md

@@ -64,6 +64,11 @@
   - [`captionsTextTrack1LanguageCode`](#captionstexttrack1languagecode)
   - [`captionsTextTrack2Label`](#captionstexttrack2label)
   - [`captionsTextTrack2LanguageCode`](#captionstexttrack2languagecode)
+  - [`captionsTextTrack3Label`](#captionsTextTrack3Label)
+  - [`captionsTextTrack3LanguageCode`](#captionsTextTrack3LanguageCode)
+  - [`captionsTextTrack4Label`](#captionsTextTrack4Label)
+  - [`captionsTextTrack4LanguageCode`](#captionsTextTrack4LanguageCode)
+  - [`renderNatively`](#renderNatively)
   - [`stretchShortVideoTrack`](#stretchshortvideotrack)
   - [`maxAudioFramesDrift`](#maxaudioframesdrift)
   - [`forceKeyFrameOnDiscontinuity`](#forcekeyframeondiscontinuity)
@@ -879,6 +884,47 @@ RFC 3066 language code for the text track generated for CEA-708 captions track 2
 
 parameter should be a string
 
+### `captionsTextTrack3Label`
+
+(default: `Unknown CC`)
+
+Label for the text track generated for CEA-708 captions track 3. This is how it will appear in the browser's native menu for subtitles and captions.
+
+parameter should be a string
+
+### `captionsTextTrack3LanguageCode`
+
+(default: ``)
+
+RFC 3066 language code for the text track generated for CEA-708 captions track 3.
+
+parameter should be a string
+
+### `captionsTextTrack4Label`
+
+(default: `Unknown CC`)
+
+Label for the text track generated for CEA-708 captions track 4. This is how it will appear in the browser's native menu for subtitles and captions.
+
+parameter should be a string
+
+### `captionsTextTrack4LanguageCode`
+
+(default: ``)
+
+RFC 3066 language code for the text track generated for CEA-708 captions track 4.
+
+parameter should be a string
+
+### `renderNatively`
+
+(default: `true`)
+
+Whether or not render captions natively using the HTMLMediaElement's TextTracks. Disable native captions rendering
+when you want to handle rending of track and track cues using `NON_NATIVE_TEXT_TRACKS_FOUND` and `CUES_PARSED` events.
+
+parameter should be a boolean
+
 ### `stretchShortVideoTrack`
 
 (default: `false`)

src/config.ts

@@ -118,6 +118,11 @@ type TimelineControllerConfig = {
   captionsTextTrack1LanguageCode: string,
   captionsTextTrack2Label: string,
   captionsTextTrack2LanguageCode: string,
+  captionsTextTrack3Label: string,
+  captionsTextTrack3LanguageCode: string,
+  captionsTextTrack4Label: string,
+  captionsTextTrack4LanguageCode: string,
+  renderNatively: boolean,
 };
 
 type TSDemuxerConfig = {
@@ -259,6 +264,11 @@ function timelineConfig (): TimelineControllerConfig {
     captionsTextTrack1Label: 'English', // used by timeline-controller
     captionsTextTrack1LanguageCode: 'en', // used by timeline-controller
     captionsTextTrack2Label: 'Spanish', // used by timeline-controller
-    captionsTextTrack2LanguageCode: 'es' // used by timeline-controller
+    captionsTextTrack2LanguageCode: 'es', // used by timeline-controller
+    captionsTextTrack3Label: 'Unknown CC', // used by timeline-controller
+    captionsTextTrack3LanguageCode: '', // used by timeline-controller
+    captionsTextTrack4Label: 'Unknown CC', // used by timeline-controller
+    captionsTextTrack4LanguageCode: '', // used by timeline-controller
+    renderNatively: true
   };
 }

src/controller/timeline-controller.ts

@@ -48,13 +48,23 @@ class TimelineController extends EventHandler {
       textTrack2: {
         label: this.config.captionsTextTrack2Label,
         languageCode: this.config.captionsTextTrack2LanguageCode
+      },
+      textTrack3: {
+        label: this.config.captionsTextTrack3Label,
+        languageCode: this.config.captionsTextTrack3LanguageCode
+      },
+      textTrack4: {
+        label: this.config.captionsTextTrack4Label,
+        languageCode: this.config.captionsTextTrack4LanguageCode
       }
     };
 
     if (this.config.enableCEA708Captions) {
       const channel1 = new OutputFilter(this, 'textTrack1');
       const channel2 = new OutputFilter(this, 'textTrack2');
-      this.cea608Parser = new Cea608Parser(0, channel1, channel2);
+      const channel3 = new OutputFilter(this, 'textTrack3');
+      const channel4 = new OutputFilter(this, 'textTrack4');
+      this.cea608Parser = new Cea608Parser(channel1, channel2, channel3, channel4);
     }
   }
 
@@ -78,7 +88,18 @@ class TimelineController extends EventHandler {
       ranges.push([startTime, endTime]);
     }
 
-    this.Cues.newCue(this.captionsTracks[trackName], startTime, endTime, screen);
+    if (this.config.renderNatively) {
+      this.Cues.newCue(this.captionsTracks[trackName], startTime, endTime, screen);
+    } else {
+      const fakeTrack = {
+        cues: [],
+        addCue (cue) {
+          this.cues.pusj(cue);
+        }
+      };
+      this.Cues.newCue(fakeTrack, startTime, endTime, screen);
+      this.hls.trigger(Event.CUES_PARSED, { type: 'captions', cues: fakeTrack.cues, track: trackName });
+    }
   }
 
   // Triggered when an initial PTS is found; used for synchronisation of WebVTT.
@@ -114,11 +135,11 @@ class TimelineController extends EventHandler {
 
   createCaptionsTrack (trackName: string) {
     const { captionsProperties, captionsTracks, media } = this;
-    const { label, languageCode } = captionsProperties[trackName];
     if (!captionsTracks[trackName]) {
       // Enable reuse of existing text track.
       const existingTrack = this.getExistingTrack(trackName);
       if (!existingTrack) {
+        const { label, languageCode } = captionsProperties[trackName];
         const textTrack = this.createTextTrack('captions', label, languageCode);
         if (textTrack) {
           // Set a special property on the track so we know it's managed by Hls.js
@@ -185,44 +206,76 @@ class TimelineController extends EventHandler {
     }
   }
 
-  onManifestLoaded (data: { subtitles: Array<any> }) {
+  onManifestLoaded (data: { subtitles: Array<any>, captions: Array<any> }) {
     this.textTracks = [];
     this.unparsedVttFrags = this.unparsedVttFrags || [];
     this.initPTS = [];
     this.cueRanges = [];
 
     if (this.config.enableWebVTT) {
+      const tracks = data.subtitles || [];
+      const sameTracks = this.tracks && tracks && this.tracks.length === tracks.length;
       this.tracks = data.subtitles || [];
-      const inUseTracks = this.media ? this.media.textTracks : [];
 
-      this.tracks.forEach((track, index) => {
-        let textTrack;
-        if (index < inUseTracks.length) {
-          let inUseTrack: TextTrack | null = null;
+      if (this.config.renderNatively) {
+        const inUseTracks = this.media ? this.media.textTracks : [];
+
+        this.tracks.forEach((track, index) => {
+          let textTrack;
+          if (index < inUseTracks.length) {
+            let inUseTrack: TextTrack | null = null;
 
-          for (let i = 0; i < inUseTracks.length; i++) {
-            if (canReuseVttTextTrack(inUseTracks[i], track)) {
-              inUseTrack = inUseTracks[i];
-              break;
+            for (let i = 0; i < inUseTracks.length; i++) {
+              if (canReuseVttTextTrack(inUseTracks[i], track)) {
+                inUseTrack = inUseTracks[i];
+                break;
+              }
             }
+
+            // Reuse tracks with the same label, but do not reuse 608/708 tracks
+            if (inUseTrack) {
+              textTrack = inUseTrack;
+            }
+          }
+          if (!textTrack) {
+            textTrack = this.createTextTrack('subtitles', track.name, track.lang);
           }
 
-          // Reuse tracks with the same label, but do not reuse 608/708 tracks
-          if (inUseTrack) {
-            textTrack = inUseTrack;
+          if (track.default) {
+            textTrack.mode = this.hls.subtitleDisplay ? 'showing' : 'hidden';
+          } else {
+            textTrack.mode = 'disabled';
           }
-        }
-        if (!textTrack) {
-          textTrack = this.createTextTrack('subtitles', track.name, track.lang);
-        }
 
-        if (track.default) {
-          textTrack.mode = this.hls.subtitleDisplay ? 'showing' : 'hidden';
-        } else {
-          textTrack.mode = 'disabled';
+          this.textTracks.push(textTrack);
+        });
+      } else if (!sameTracks && this.tracks && this.tracks.length) {
+        // Create a list of tracks for the provider to consume
+        const tracksList = this.tracks.map((track) => {
+          return {
+            label: track.name,
+            kind: track.type.toLowerCase(),
+            default: track.default
+          };
+        });
+        this.hls.trigger(Event.NON_NATIVE_TEXT_TRACKS_FOUND, { tracks: tracksList });
+      }
+    }
+
+    if (this.config.enableCEA708Captions && data.captions) {
+      data.captions.forEach(captionsTrack => {
+        const instreamIdMatch = /(?:CC|SERVICE)([1-4])/.exec(captionsTrack.instreamId as string);
+
+        if (!instreamIdMatch) {
+          return;
         }
 
-        this.textTracks.push(textTrack);
+        const trackName = `textTrack${instreamIdMatch[1]}`;
+        this.captionsProperties[trackName].label = captionsTrack.name;
+
+        if (captionsTrack.lang) { // optional attribute
+          this.captionsProperties[trackName].languageCode = captionsTrack.lang;
+        }
       });
     }
   }
@@ -281,26 +334,30 @@ class TimelineController extends EventHandler {
         hls.trigger(Event.SUBTITLE_FRAG_PROCESSED, { success: false, frag: frag });
         return;
       }
-      // Add cues and trigger event with success true.
-      cues.forEach(cue => {
-        // Sometimes there are cue overlaps on segmented vtts so the same
-        // cue can appear more than once in different vtt files.
-        // This avoid showing duplicated cues with same timecode and text.
-        if (!currentTrack.cues.getCueById(cue.id)) {
-          try {
-            currentTrack.addCue(cue);
-            if (!currentTrack.cues.getCueById(cue.id)) {
-              throw new Error(`addCue is failed for: ${cue}`);
+      if (this.config.renderNatively) {
+        // Add cues and trigger event with success true.
+        cues.forEach(cue => {
+          // Sometimes there are cue overlaps on segmented vtts so the same
+          // cue can appear more than once in different vtt files.
+          // This avoid showing duplicated cues with same timecode and text.
+          if (!currentTrack.cues.getCueById(cue.id)) {
+            try {
+              currentTrack.addCue(cue);
+              if (!currentTrack.cues.getCueById(cue.id)) {
+                throw new Error(`addCue is failed for: ${cue}`);
+              }
+            } catch (err) {
+              logger.debug(`Failed occurred on adding cues: ${err}`);
+              const textTrackCue = new (window as any).TextTrackCue(cue.startTime, cue.endTime, cue.text);
+              textTrackCue.id = cue.id;
+              currentTrack.addCue(textTrackCue);
             }
-          } catch (err) {
-            logger.debug(`Failed occurred on adding cues: ${err}`);
-            const textTrackCue = new (window as any).TextTrackCue(cue.startTime, cue.endTime, cue.text);
-            textTrackCue.id = cue.id;
-            currentTrack.addCue(textTrackCue);
           }
-        }
+        });
+      } else {
+        let trackId = this.tracks[frag.level].default ? 'default' : 'subtitles' + frag.level;
+        hls.trigger(Event.CUES_PARSED, { type: 'subtitles', cues: cues, track: trackId });
       }
-      );
       hls.trigger(Event.SUBTITLE_FRAG_PROCESSED, { success: true, frag: frag });
     },
     function (e) {
@@ -329,36 +386,37 @@ class TimelineController extends EventHandler {
 
     // If the event contains captions (found in the bytes property), push all bytes into the parser immediately
     // It will create the proper timestamps based on the PTS value
+    const cea608Parser = this.cea608Parser as Cea608Parser;
     for (let i = 0; i < data.samples.length; i++) {
       const ccBytes = data.samples[i].bytes;
       if (ccBytes) {
         const ccdatas = this.extractCea608Data(ccBytes);
-        this.cea608Parser.addData(data.samples[i].pts, ccdatas);
+        cea608Parser.addData(data.samples[i].pts, ccdatas[0], 1);
+        cea608Parser.addData(data.samples[i].pts, ccdatas[1], 3);
       }
     }
   }
 
-  extractCea608Data (byteArray: Uint8Array): Array<number> {
-    let count = byteArray[0] & 31;
+  extractCea608Data (byteArray: Uint8Array): number[][] {
+    const count = byteArray[0] & 31;
     let position = 2;
-    let tmpByte, ccbyte1, ccbyte2, ccValid, ccType;
-    let actualCCBytes: number[] = [];
+    const actualCCBytes: number[][] = [[], []];
 
     for (let j = 0; j < count; j++) {
-      tmpByte = byteArray[position++];
-      ccbyte1 = 0x7F & byteArray[position++];
-      ccbyte2 = 0x7F & byteArray[position++];
-      ccValid = (4 & tmpByte) !== 0;
-      ccType = 3 & tmpByte;
+      const tmpByte = byteArray[position++];
+      const ccbyte1 = 0x7F & byteArray[position++];
+      const ccbyte2 = 0x7F & byteArray[position++];
+      const ccValid = (4 & tmpByte) !== 0;
+      const ccType = 3 & tmpByte;
 
       if (ccbyte1 === 0 && ccbyte2 === 0) {
         continue;
       }
 
       if (ccValid) {
-        if (ccType === 0) { // || ccType === 1
-          actualCCBytes.push(ccbyte1);
-          actualCCBytes.push(ccbyte2);
+        if (ccType === 0 || ccType === 1) {
+          actualCCBytes[ccType].push(ccbyte1);
+          actualCCBytes[ccType].push(ccbyte2);
         }
       }
     }

src/events.js

@@ -65,6 +65,10 @@ const HlsEvents = {
   SUBTITLE_TRACK_LOADED: 'hlsSubtitleTrackLoaded',
   // fired when a subtitle fragment has been processed - data: { success : boolean, frag : the processed frag }
   SUBTITLE_FRAG_PROCESSED: 'hlsSubtitleFragProcessed',
+  // fired when a set of VTTCues to be managed externally has been parsed - data: { type: string, track: string, cues: [ VTTCue ] }
+  CUES_PARSED: 'hlsCuesParsed',
+  // fired when a text track to be managed externally is found - data: { tracks: [ { label: string, kind: string, default: boolean } ] }
+  NON_NATIVE_TEXT_TRACKS_FOUND: 'hlsNonNativeTextTracksFound',
   // fired when the first timestamp is found - data: { id : demuxer id, initPTS: initPTS, frag : fragment object }
   INIT_PTS_FOUND: 'hlsInitPtsFound',
   // fired when a fragment loading starts - data: { frag : fragment object }