Skip to content

Commit

Permalink
Merge pull request video-dev#3404 from video-dev/bugfix/level-recover…
Browse files Browse the repository at this point in the history 
…y-frag-retry-interference

Allow fragment loading error retries to run before level recovery
  • Loading branch information
@robwalch
robwalch authored Jan 25, 2021
2 parents 18ec11a + cd4c841 commit cc499ec
Show file tree
Hide file tree
Showing 12 changed files with 142 additions and 144 deletions.
25 changes: 12 additions & 13 deletions src/controller/abr-controller.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,11 @@
/*
* simple ABR Controller
* - compute next level based on last fragment bw heuristics
* - implement an abandon rules triggered if we have less than 2 frag buffered and if computed bw shows that we risk buffer stalling
*/

import EwmaBandWidthEstimator from '../utils/ewma-bandwidth-estimator';
import { Events } from '../events';
import { BufferHelper } from '../utils/buffer-helper';
import { ErrorDetails } from '../errors';
import { PlaylistLevelType } from '../types/loader';
import { logger } from '../utils/logger';
import EwmaBandWidthEstimator from '../utils/ewma-bandwidth-estimator';

import type { Bufferable } from '../utils/buffer-helper';
// eslint-disable-next-line import/no-duplicates
import type Fragment from '../loader/fragment';
// eslint-disable-next-line import/no-duplicates
import type { Part } from '../loader/fragment';
import type { LoaderStats } from '../types/loader';
import type Hls from '../hls';
Expand Down Expand Up @@ -76,7 +68,7 @@ class AbrController implements ComponentAPI {

protected onFragLoading(event: Events.FRAG_LOADING, data: FragLoadingData) {
const frag = data.frag;
if (frag.type === 'main') {
if (frag.type === PlaylistLevelType.MAIN) {
if (!this.timer) {
this.fragCurrent = frag;
this.partCurrent = data.part ?? null;
Expand Down Expand Up @@ -214,7 +206,10 @@ class AbrController implements ComponentAPI {
event: Events.FRAG_LOADED,
{ frag, part }: FragLoadedData
) {
if (frag.type === 'main' && Number.isFinite(frag.sn as number)) {
if (
frag.type === PlaylistLevelType.MAIN &&
Number.isFinite(frag.sn as number)
) {
const stats = part ? part.stats : frag.stats;
const duration = part ? part.duration : frag.duration;
// stop monitoring bw once frag loaded
Expand Down Expand Up @@ -257,7 +252,11 @@ class AbrController implements ComponentAPI {
return;
}
// Only count non-alt-audio frags which were actually buffered in our BW calculations
if (frag.type !== 'main' || frag.sn === 'initSegment' || frag.bitrateTest) {
if (
frag.type !== PlaylistLevelType.MAIN ||
frag.sn === 'initSegment' ||
frag.bitrateTest
) {
return;
}
// Use the difference between parsing and request instead of buffering and request to compute fragLoadingProcessing;
Expand Down
68 changes: 31 additions & 37 deletions src/controller/audio-stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import type {
FragParsingUserdataData,
FragBufferedData,
} from '../types/events';
import type { ErrorData } from '../types/events';

const TICK_INTERVAL = 100; // how often to tick in ms

Expand Down Expand Up @@ -556,7 +557,7 @@ class AudioStreamController
super._handleFragmentLoadComplete(fragLoadedData);
}

onBufferReset() {
onBufferReset(/* event: Events.BUFFER_RESET */) {
// reset reference to sourcebuffers
this.mediaBuffer = this.videoBuffer = null;
this.loadedmetadata = false;
Expand All @@ -574,7 +575,7 @@ class AudioStreamController

onFragBuffered(event: Events.FRAG_BUFFERED, data: FragBufferedData) {
const { frag, part } = data;
if (frag && frag.type !== 'audio') {
if (frag.type !== PlaylistLevelType.AUDIO) {
return;
}
if (this.fragContextChanged(frag)) {
Expand All @@ -597,57 +598,57 @@ class AudioStreamController
this.fragBufferedComplete(frag, part);
}

onError(data) {
const frag = data.frag;
// don't handle frag error not related to audio fragment
if (frag && frag.type !== 'audio') {
return;
}

private onError(event: Events.ERROR, data: ErrorData) {
switch (data.details) {
case ErrorDetails.FRAG_LOAD_ERROR:
case ErrorDetails.FRAG_LOAD_TIMEOUT: {
const frag = data.frag;
// don't handle frag error not related to audio fragment
if (frag && frag.type !== 'audio') {
break;
}

case ErrorDetails.FRAG_LOAD_TIMEOUT:
case ErrorDetails.KEY_LOAD_ERROR:
case ErrorDetails.KEY_LOAD_TIMEOUT:
if (!data.fatal) {
let loadError = this.fragLoadError;
if (loadError) {
loadError++;
} else {
loadError = 1;
const frag = data.frag;
// don't handle frag error not related to audio fragment
if (!frag || frag.type !== PlaylistLevelType.AUDIO) {
return;
}

const fragCurrent = this.fragCurrent;
console.assert(
fragCurrent &&
frag.sn === fragCurrent.sn &&
frag.level === fragCurrent.level &&
frag.urlId === fragCurrent.urlId, // FIXME: audio-group id
'Frag load error must match current frag to retry'
);
const config = this.config;
if (loadError <= config.fragLoadingMaxRetry) {
this.fragLoadError = loadError;
if (this.fragLoadError + 1 <= this.config.fragLoadingMaxRetry) {
// exponential backoff capped to config.fragLoadingMaxRetryTimeout
const delay = Math.min(
Math.pow(2, loadError - 1) * config.fragLoadingRetryDelay,
Math.pow(2, this.fragLoadError) * config.fragLoadingRetryDelay,
config.fragLoadingMaxRetryTimeout
);
this.warn(`Frag loading failed, retry in ${delay} ms`);
this.retryDate = performance.now() + delay;
// retry loading state
this.fragLoadError++;
this.state = State.FRAG_LOADING_WAITING_RETRY;
} else if (data.levelRetry) {
// Reset current fragment since audio track audio is essential and may not have a fail-over track
this.fragCurrent = null;
// Fragment errors that result in a level switch or redundant fail-over
// should reset the audio stream controller state to idle
this.fragLoadError = 0;
this.state = State.IDLE;
} else {
logger.error(
`${data.details} reaches max retry, redispatch as fatal ...`
);
// switch error to fatal
data.fatal = true;
this.hls.stopLoad();
this.state = State.ERROR;
}
}
break;
}
case ErrorDetails.AUDIO_TRACK_LOAD_ERROR:
case ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT:
case ErrorDetails.KEY_LOAD_ERROR:
case ErrorDetails.KEY_LOAD_TIMEOUT:
// when in ERROR state, don't switch back to IDLE state in case a non-fatal error is received
if (this.state !== State.ERROR && this.state !== State.STOPPED) {
// if fatal error, stop processing, otherwise move to IDLE to retry loading
Expand All @@ -671,14 +672,7 @@ class AudioStreamController
BufferHelper.isBuffered(media, currentTime + 0.5);
// reduce max buf len if current position is buffered
if (mediaBuffered) {
const config = this.config;
if (config.maxMaxBufferLength >= config.maxBufferLength) {
// reduce max buffer length as it might be too high. we do this to avoid loop flushing ...
config.maxMaxBufferLength /= 2;
this.warn(
`Reduce max buffer length to ${config.maxMaxBufferLength}s`
);
}
this.reduceMaxBufferLength();
this.state = State.IDLE;
} else {
// current position is not buffered, but browser is still complaining about buffer full error
Expand Down
16 changes: 14 additions & 2 deletions src/controller/base-stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,11 +298,11 @@ export default class BaseStreamController
targetBufferTime,
progressCallback
).then((data) => {
this.fragLoadError = 0;
if (!data) {
// if we're here we probably needed to backtrack or are waiting for more parts
return;
}
this.fragLoadError = 0;
if (this.fragContextChanged(frag)) {
if (
this.state === State.FRAG_LOADING ||
Expand Down Expand Up @@ -674,6 +674,18 @@ export default class BaseStreamController
}
}

protected reduceMaxBufferLength(threshold?: number) {
const config = this.config;
const minLength = threshold || config.maxBufferLength;
if (config.maxMaxBufferLength >= minLength) {
// reduce max buffer length as it might be too high. we do this to avoid loop flushing ...
config.maxMaxBufferLength /= 2;
this.warn(`Reduce max buffer length to ${config.maxMaxBufferLength}s`);
return true;
}
return false;
}

protected getNextFragment(
pos: number,
levelDetails: LevelDetails
Expand Down Expand Up @@ -1080,7 +1092,7 @@ export default class BaseStreamController
const previousState = this._state;
if (previousState !== nextState) {
this._state = nextState;
// this.log(`${previousState}->${nextState}`);
this.log(`${previousState}->${nextState}`);
}
}

Expand Down
76 changes: 31 additions & 45 deletions src/controller/level-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ import { Events } from '../events';
import { ErrorTypes, ErrorDetails } from '../errors';
import { isCodecSupportedInMp4 } from '../utils/codecs';
import { addGroupId, assignTrackIdsByGroup } from './level-helper';
import Fragment from '../loader/fragment';
import BasePlaylistController from './base-playlist-controller';
import { PlaylistContextType } from '../types/loader';
import { PlaylistContextType, PlaylistLevelType } from '../types/loader';
import type Hls from '../hls';
import type { HlsUrlParameters, LevelParsed } from '../types/level';
import type { MediaPlaylist } from '../types/media-playlist';
Expand Down Expand Up @@ -317,7 +316,6 @@ export default class LevelController extends BasePlaylistController {
}

let levelError = false;
let fragmentError = false;
let levelSwitch = true;
let levelIndex;

Expand All @@ -327,11 +325,18 @@ export default class LevelController extends BasePlaylistController {
case ErrorDetails.FRAG_LOAD_TIMEOUT:
case ErrorDetails.KEY_LOAD_ERROR:
case ErrorDetails.KEY_LOAD_TIMEOUT:
// FIXME: What distinguishes these fragment events from level or track fragments?
// We shouldn't recover a level if the fragment or key is for a media track
console.assert(data.frag, 'Event has a fragment defined.');
levelIndex = (data.frag as Fragment).level;
fragmentError = true;
if (data.frag) {
const level = this._levels[data.frag.level];
// Set levelIndex when we're out of fragment retries
if (level) {
level.fragmentError++;
if (level.fragmentError > this.hls.config.fragLoadingMaxRetry) {
levelIndex = data.frag.level;
}
} else {
levelIndex = data.frag.level;
}
}
break;
case ErrorDetails.LEVEL_LOAD_ERROR:
case ErrorDetails.LEVEL_LOAD_TIMEOUT:
Expand All @@ -352,17 +357,7 @@ export default class LevelController extends BasePlaylistController {
}

if (levelIndex !== undefined) {
if (!this._levels[levelIndex]) {
data.fatal = true;
return;
}
this.recoverLevel(
data,
levelIndex,
levelError,
fragmentError,
levelSwitch
);
this.recoverLevel(data, levelIndex, levelError, levelSwitch);
}
}

Expand All @@ -374,14 +369,12 @@ export default class LevelController extends BasePlaylistController {
errorEvent: ErrorData,
levelIndex: number,
levelError: boolean,
fragmentError: boolean,
levelSwitch: boolean
): void {
const { details: errorDetails } = errorEvent;
const level = this._levels[levelIndex];

level.loadError++;
level.fragmentError = fragmentError;

if (levelError) {
const retrying = this.retryLoadingOrFail(errorEvent);
Expand All @@ -394,30 +387,23 @@ export default class LevelController extends BasePlaylistController {
}
}

// Try any redundant streams if available for both errors: level and fragment
// If level.loadError reaches redundantLevels it means that we tried them all, no hope => let's switch down
if (levelSwitch && (levelError || fragmentError)) {
if (levelSwitch) {
const redundantLevels = level.url.length;

// Try redundant fail-over until level.loadError reaches redundantLevels
if (redundantLevels > 1 && level.loadError < redundantLevels) {
errorEvent.levelRetry = true;
this.redundantFailover(levelIndex);
} else {
// Search for available level
if (this.manualLevelIndex === -1) {
// When lowest level has been reached, let's start hunt from the top
const nextLevel =
levelIndex === 0 ? this._levels.length - 1 : levelIndex - 1;
if (this.currentLevelIndex !== nextLevel) {
fragmentError = false;
this.warn(`${errorDetails}: switch to ${nextLevel}`);
this.hls.nextAutoLevel = this.currentLevelIndex = nextLevel;
}
}
if (fragmentError) {
// Allow fragment retry as long as configuration allows.
// reset this._level so that another call to set level() will trigger again a frag load
this.warn(`${errorDetails}: reload a fragment`);
this.currentLevelIndex = -1;
} else if (this.manualLevelIndex === -1) {
// Search for available level in auto level selection mode, cycling from highest to lowest bitrate
const nextLevel =
levelIndex === 0 ? this._levels.length - 1 : levelIndex - 1;
if (
this.currentLevelIndex !== nextLevel &&
this._levels[nextLevel].loadError === 0
) {
this.warn(`${errorDetails}: switch to ${nextLevel}`);
errorEvent.levelRetry = true;
this.hls.nextAutoLevel = nextLevel;
}
}
}
Expand All @@ -439,10 +425,10 @@ export default class LevelController extends BasePlaylistController {

// reset errors on the successful load of a fragment
protected onFragLoaded(event: Events.FRAG_LOADED, { frag }: FragLoadedData) {
if (frag !== undefined && frag.type === 'main') {
if (frag !== undefined && frag.type === PlaylistLevelType.MAIN) {
const level = this._levels[frag.level];
if (level !== undefined) {
level.fragmentError = false;
level.fragmentError = 0;
level.loadError = 0;
}
}
Expand All @@ -463,7 +449,7 @@ export default class LevelController extends BasePlaylistController {
// only process level loaded events matching with expected level
if (level === this.currentLevelIndex) {
// reset level load error counter on successful level loaded only if there is no issues with fragments
if (!curLevel.fragmentError) {
if (curLevel.fragmentError === 0) {
curLevel.loadError = 0;
this.retryCount = 0;
}
Expand Down
Loading

0 comments on commit cc499ec

Please sign in to comment.