Error Handling controller and LoadPolicies (video-dev#5241)

* Implement ErrorController …
Add Error Handling Integration Tests
Handle missed probe transmux probe with with FRAG_PARSING_ERROR
Handle encrypted init segment and subtitle decryption failure with FRAG_DECRYPT_ERROR
Remove delay when retrying after timeout
Trigger FRAG_LOADING after request is made
ERROR event data type must include an error property
Add stats to network loader error callbacks to allow error handling to use request timing
Remove use of console.warn from mp4-tools
Add missing TypeScript type exports

* Remove console.assert statements
Resolves video-dev#5218
Closes video-dev#5221

* Remove unused frag buffered monitor

* Add Load Policies and deprecate frag/level/manifest timeout and retry config options

* Switch on FRAG_PARSING_ERROR or continue trying fragLoadPolicy.default.errorRetry.maxNumRetry (6) times
Fixes video-dev#5011

* Switch on FRAG_PARSING_ERROR or continue trying fragLoadPolicy.default.errorRetry.maxNumRetry (6) times 2/2
Fixes video-dev#5011

* Do not reload level after calling stopLoad() and startLoad()
Warn on currentFrag context reference change and use currentFrag to maintain correct stats ref
Fixes video-dev#5230

* Implement ErrorActions and Pathway Switching

* Fix part timeout handling
Cleanup frag stats handoff

* Update retry rules to require http status or timeout error (deprioritizes retry on parse errors)

* Retry audio and subtitle playlist loading after failure on level switch when no alternate is available

src/config.ts

@@ -12,6 +12,7 @@ import EMEController, {
 } from './controller/eme-controller';
 import CMCDController from './controller/cmcd-controller';
 import ContentSteeringController from './controller/content-steering-controller';
+import ErrorController from './controller/error-controller';
 import XhrLoader from './utils/xhr-loader';
 import FetchLoader, { fetchSupported } from './utils/fetch-loader';
 import Cues from './utils/cues';
@@ -118,9 +119,10 @@ export interface FragmentLoaderConstructor {
   new (confg: HlsConfig): Loader<FragmentLoaderContext>;
 }
 
+/**
+ * @deprecated use fragLoadPolicy.default
+ */
 export type FragmentLoaderConfig = {
-  fLoader?: FragmentLoaderConstructor;
-
   fragLoadingTimeOut: number;
   fragLoadingMaxRetry: number;
   fragLoadingRetryDelay: number;
@@ -146,9 +148,10 @@ export interface PlaylistLoaderConstructor {
   new (confg: HlsConfig): Loader<PlaylistLoaderContext>;
 }
 
+/**
+ * @deprecated use manifestLoadPolicy.default and playlistLoadPolicy.default
+ */
 export type PlaylistLoaderConfig = {
-  pLoader?: PlaylistLoaderConstructor;
-
   manifestLoadingTimeOut: number;
   manifestLoadingMaxRetry: number;
   manifestLoadingRetryDelay: number;
@@ -160,6 +163,33 @@ export type PlaylistLoaderConfig = {
   levelLoadingMaxRetryTimeout: number;
 };
 
+export type HlsLoadPolicies = {
+  fragLoadPolicy: LoadPolicy;
+  keyLoadPolicy: LoadPolicy;
+  certLoadPolicy: LoadPolicy;
+  playlistLoadPolicy: LoadPolicy;
+  manifestLoadPolicy: LoadPolicy;
+  steeringManifestLoadPolicy: LoadPolicy;
+};
+
+export type LoadPolicy = {
+  default: LoaderConfig;
+};
+
+export type LoaderConfig = {
+  maxTimeToFirstByteMs: number; // Max time to first byte
+  maxLoadTimeMs: number; // Max time for load completion
+  timeoutRetry: RetryConfig | null;
+  errorRetry: RetryConfig | null;
+};
+
+export type RetryConfig = {
+  maxNumRetry: number; // Maximum number of retries
+  retryDelayMs: number; // Retry delay = 2^retryCount * retryDelayMs (exponential) or retryCount * retryDelayMs (linear)
+  maxRetryDelayMs: number; // Maximum delay between retries
+  backoff?: 'exponential' | 'linear'; // used to determine retry backoff duration (see retryDelayMs)
+};
+
 export type StreamControllerConfig = {
   autoStartLoad: boolean;
   startPosition: number;
@@ -218,6 +248,8 @@ export type HlsConfig = {
   minAutoBitrate: number;
   ignoreDevicePixelRatio: boolean;
   loader: { new (confg: HlsConfig): Loader<LoaderContext> };
+  fLoader?: FragmentLoaderConstructor;
+  pLoader?: PlaylistLoaderConstructor;
   fetchSetup?: (context: LoaderContext, initParams: any) => Request;
   xhrSetup?: (xhr: XMLHttpRequest, url: string) => void;
 
@@ -239,6 +271,7 @@ export type HlsConfig = {
   abrController: typeof AbrController;
   bufferController: typeof BufferController;
   capLevelController: typeof CapLevelController;
+  errorController: typeof ErrorController;
   fpsController: typeof FPSController;
   progressive: boolean;
   lowLatencyMode: boolean;
@@ -247,15 +280,23 @@ export type HlsConfig = {
   CapLevelControllerConfig &
   EMEControllerConfig &
   FPSControllerConfig &
-  FragmentLoaderConfig &
   LevelControllerConfig &
   MP4RemuxerConfig &
-  PlaylistLoaderConfig &
   StreamControllerConfig &
   LatencyControllerConfig &
   MetadataControllerConfig &
   TimelineControllerConfig &
-  TSDemuxerConfig;
+  TSDemuxerConfig &
+  HlsLoadPolicies &
+  FragmentLoaderConfig &
+  PlaylistLoaderConfig;
+
+const defaultLoadPolicy: LoaderConfig = {
+  maxTimeToFirstByteMs: 8000,
+  maxLoadTimeMs: 20000,
+  timeoutRetry: null,
+  errorRetry: null,
+};
 
 /**
  * @ignore
@@ -293,19 +334,7 @@ export const hlsDefaultConfig: HlsConfig = {
   maxMaxBufferLength: 600, // used by stream-controller
   enableWorker: true, // used by demuxer
   enableSoftwareAES: true, // used by decrypter
-  manifestLoadingTimeOut: 10000, // used by playlist-loader
-  manifestLoadingMaxRetry: 1, // used by playlist-loader
-  manifestLoadingRetryDelay: 1000, // used by playlist-loader
-  manifestLoadingMaxRetryTimeout: 64000, // used by playlist-loader
   startLevel: undefined, // used by level-controller
-  levelLoadingTimeOut: 10000, // used by playlist-loader
-  levelLoadingMaxRetry: 4, // used by playlist-loader
-  levelLoadingRetryDelay: 1000, // used by playlist-loader
-  levelLoadingMaxRetryTimeout: 64000, // used by playlist-loader
-  fragLoadingTimeOut: 20000, // used by fragment-loader
-  fragLoadingMaxRetry: 6, // used by fragment-loader
-  fragLoadingRetryDelay: 1000, // used by fragment-loader
-  fragLoadingMaxRetryTimeout: 64000, // used by fragment-loader
   startFragPrefetch: false, // used by stream-controller
   fpsDroppedMonitoringPeriod: 5000, // used by fps-controller
   fpsDroppedMonitoringThreshold: 0.2, // used by fps-controller
@@ -320,6 +349,7 @@ export const hlsDefaultConfig: HlsConfig = {
   abrController: AbrController,
   bufferController: BufferController,
   capLevelController: CapLevelController,
+  errorController: ErrorController,
   fpsController: FPSController,
   stretchShortVideoTrack: false, // used by mp4-remuxer
   maxAudioFramesDrift: 1, // used by mp4-remuxer
@@ -350,6 +380,109 @@ export const hlsDefaultConfig: HlsConfig = {
   enableEmsgMetadataCues: true,
   enableID3MetadataCues: true,
 
+  certLoadPolicy: {
+    default: defaultLoadPolicy,
+  },
+  keyLoadPolicy: {
+    default: {
+      maxTimeToFirstByteMs: 8000,
+      maxLoadTimeMs: 20000,
+      timeoutRetry: {
+        maxNumRetry: 1,
+        retryDelayMs: 1000,
+        maxRetryDelayMs: 20000,
+        backoff: 'linear',
+      },
+      errorRetry: {
+        maxNumRetry: 8,
+        retryDelayMs: 1000,
+        maxRetryDelayMs: 20000,
+        backoff: 'linear',
+      },
+    },
+  },
+  manifestLoadPolicy: {
+    default: {
+      maxTimeToFirstByteMs: 10000,
+      maxLoadTimeMs: 20000,
+      timeoutRetry: {
+        maxNumRetry: 2,
+        retryDelayMs: 0,
+        maxRetryDelayMs: 0,
+      },
+      errorRetry: {
+        maxNumRetry: 1,
+        retryDelayMs: 1000,
+        maxRetryDelayMs: 8000,
+      },
+    },
+  },
+  playlistLoadPolicy: {
+    default: {
+      maxTimeToFirstByteMs: 10000,
+      maxLoadTimeMs: 20000,
+      timeoutRetry: {
+        maxNumRetry: 2,
+        retryDelayMs: 0,
+        maxRetryDelayMs: 0,
+      },
+      errorRetry: {
+        maxNumRetry: 2,
+        retryDelayMs: 1000,
+        maxRetryDelayMs: 8000,
+      },
+    },
+  },
+  fragLoadPolicy: {
+    default: {
+      maxTimeToFirstByteMs: 10000,
+      maxLoadTimeMs: 120000,
+      timeoutRetry: {
+        maxNumRetry: 4,
+        retryDelayMs: 0,
+        maxRetryDelayMs: 0,
+      },
+      errorRetry: {
+        maxNumRetry: 6,
+        retryDelayMs: 1000,
+        maxRetryDelayMs: 8000,
+      },
+    },
+  },
+  steeringManifestLoadPolicy: {
+    default: __USE_CONTENT_STEERING__
+      ? {
+          maxTimeToFirstByteMs: 10000,
+          maxLoadTimeMs: 20000,
+          timeoutRetry: {
+            maxNumRetry: 2,
+            retryDelayMs: 0,
+            maxRetryDelayMs: 0,
+          },
+          errorRetry: {
+            maxNumRetry: 1,
+            retryDelayMs: 1000,
+            maxRetryDelayMs: 8000,
+          },
+        }
+      : defaultLoadPolicy,
+  },
+
+  // These default settings are deprecated in favor of the above policies
+  // and are maintained for backwards compatibility
+  manifestLoadingTimeOut: 10000,
+  manifestLoadingMaxRetry: 1,
+  manifestLoadingRetryDelay: 1000,
+  manifestLoadingMaxRetryTimeout: 64000,
+  levelLoadingTimeOut: 10000,
+  levelLoadingMaxRetry: 4,
+  levelLoadingRetryDelay: 1000,
+  levelLoadingMaxRetryTimeout: 64000,
+  fragLoadingTimeOut: 20000,
+  fragLoadingMaxRetry: 6,
+  fragLoadingRetryDelay: 1000,
+  fragLoadingMaxRetryTimeout: 64000,
+
   // Dynamic Modules
   ...timelineConfig(),
   subtitleStreamController: __USE_SUBTITLES__
@@ -424,7 +557,75 @@ export function mergeConfig(
     );
   }
 
-  return Object.assign({}, defaultConfig, userConfig);
+  const defaultsCopy = deepCpy(defaultConfig);
+
+  // Backwards compatibility with deprecated config values
+  const deprecatedSettingTypes = ['manifest', 'level', 'frag'];
+  const deprecatedSettings = [
+    'TimeOut',
+    'MaxRetry',
+    'RetryDelay',
+    'MaxRetryTimeout',
+  ];
+  deprecatedSettingTypes.forEach((type) => {
+    const policyName = `${type === 'level' ? 'playlist' : type}LoadPolicy`;
+    const policyNotSet = userConfig[policyName] === undefined;
+    const report: string[] = [];
+    deprecatedSettings.forEach((setting) => {
+      const deprecatedSetting = `${type}Loading${setting}`;
+      const value = userConfig[deprecatedSetting];
+      if (value !== undefined && policyNotSet) {
+        report.push(deprecatedSetting);
+        const settings: LoaderConfig = defaultsCopy[policyName].default;
+        userConfig[policyName] = { default: settings };
+        switch (setting) {
+          case 'TimeOut':
+            settings.maxLoadTimeMs = value;
+            settings.maxTimeToFirstByteMs = value;
+            break;
+          case 'MaxRetry':
+            settings.errorRetry!.maxNumRetry = value;
+            settings.timeoutRetry!.maxNumRetry = value;
+            break;
+          case 'RetryDelay':
+            settings.errorRetry!.retryDelayMs = value;
+            settings.timeoutRetry!.retryDelayMs = value;
+            break;
+          case 'MaxRetryTimeout':
+            settings.errorRetry!.maxRetryDelayMs = value;
+            settings.timeoutRetry!.maxRetryDelayMs = value;
+            break;
+        }
+      }
+    });
+    if (report.length) {
+      logger.warn(
+        `hls.js config: "${report.join(
+          '", "'
+        )}" setting(s) are deprecated, use "${policyName}": ${JSON.stringify(
+          userConfig[policyName]
+        )}`
+      );
+    }
+  });
+
+  return {
+    ...defaultsCopy,
+    ...userConfig,
+  };
+}
+
+function deepCpy(obj: any): any {
+  if (obj && typeof obj === 'object') {
+    if (Array.isArray(obj)) {
+      return obj.map(deepCpy);
+    }
+    return Object.keys(obj).reduce((result, key) => {
+      result[key] = deepCpy(obj[key]);
+      return result;
+    }, {});
+  }
+  return obj;
 }
 
 /**