feat(all): transcoding improvements (immich-app#2171)

* test: rename some fixtures and add text for vertical video conversion * feat: transcode video asset when audio or container don't match target * chore: add niceness to the ffmpeg command to allow other processes to be prioritised * chore: change video conversion queue to one concurrency * feat: add transcode disabled preset to completely turn off transcoding * linter * Change log level and remove unused await * opps forgot to save * better logging --------- Co-authored-by: Alex Tran <[email protected]>
fliiiix · Apr 6, 2023 · a5a6beb · a5a6beb
1 parent 6f1d0a3
commit a5a6beb
Show file tree

Hide file tree

Showing 11 changed files with 190 additions and 50 deletions.
diff --git a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart
diff --git a/server/apps/microservices/src/processors.ts b/server/apps/microservices/src/processors.ts
@@ -163,7 +163,7 @@ export class VideoTranscodeProcessor {
     await this.mediaService.handleQueueVideoConversion(job.data);
   }
 
-  @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
+  @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 1 })
   async onVideoConversion(job: Job<IAssetJob>) {
     await this.mediaService.handleVideoConversion(job.data);
   }

diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json
@@ -4679,7 +4679,8 @@
             "enum": [
               "all",
               "optimal",
-              "required"
+              "required",
+              "disabled"
             ]
           }
         },

diff --git a/server/libs/domain/src/media/media.repository.ts b/server/libs/domain/src/media/media.repository.ts
@@ -14,8 +14,21 @@ export interface VideoStreamInfo {
   frameCount: number;
 }
 
+export interface AudioStreamInfo {
+  codecName?: string;
+  codecType?: string;
+}
+
+export interface VideoFormat {
+  formatName?: string;
+  formatLongName?: string;
+  duration: number;
+}
+
 export interface VideoInfo {
-  streams: VideoStreamInfo[];
+  format: VideoFormat;
+  videoStreams: VideoStreamInfo[];
+  audioStreams: AudioStreamInfo[];
 }
 
 export interface IMediaRepository {

diff --git a/server/libs/domain/src/media/media.service.spec.ts b/server/libs/domain/src/media/media.service.spec.ts
@@ -222,7 +222,7 @@ describe(MediaService.name, () => {
     });
 
     it('should transcode the longest stream', async () => {
-      mediaMock.probe.mockResolvedValue(probeStub.multiple);
+      mediaMock.probe.mockResolvedValue(probeStub.multipleVideoStreams);
 
       await sut.handleVideoConversion({ asset: assetEntityStub.video });
 
@@ -237,7 +237,7 @@ describe(MediaService.name, () => {
     });
 
     it('should skip a video without any streams', async () => {
-      mediaMock.probe.mockResolvedValue(probeStub.empty);
+      mediaMock.probe.mockResolvedValue(probeStub.noVideoStreams);
       await sut.handleVideoConversion({ asset: assetEntityStub.video });
       expect(mediaMock.transcode).not.toHaveBeenCalled();
     });
@@ -249,7 +249,7 @@ describe(MediaService.name, () => {
     });
 
     it('should transcode when set to all', async () => {
-      mediaMock.probe.mockResolvedValue(probeStub.multiple);
+      mediaMock.probe.mockResolvedValue(probeStub.multipleVideoStreams);
       configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'all' }]);
       await sut.handleVideoConversion({ asset: assetEntityStub.video });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
@@ -260,7 +260,40 @@ describe(MediaService.name, () => {
     });
 
     it('should transcode when optimal and too big', async () => {
-      mediaMock.probe.mockResolvedValue(probeStub.tooBig);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
+      await sut.handleVideoConversion({ asset: assetEntityStub.video });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        ['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=-2:720'],
+      );
+    });
+
+    it('should transcode with alternate scaling video is vertical', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
+      await sut.handleVideoConversion({ asset: assetEntityStub.video });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        ['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=720:-2'],
+      );
+    });
+
+    it('should transcode when audio doesnt match target', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.audioStreamMp3);
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
+      await sut.handleVideoConversion({ asset: assetEntityStub.video });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/asset-id.mp4',
+        ['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=-2:720'],
+      );
+    });
+
+    it('should transcode when container doesnt match target', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
       configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]);
       await sut.handleVideoConversion({ asset: assetEntityStub.video });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
@@ -271,7 +304,7 @@ describe(MediaService.name, () => {
     });
 
     it('should not transcode an invalid transcode value', async () => {
-      mediaMock.probe.mockResolvedValue(probeStub.tooBig);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
       configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'invalid' }]);
       await sut.handleVideoConversion({ asset: assetEntityStub.video });
       expect(mediaMock.transcode).not.toHaveBeenCalled();

diff --git a/server/libs/domain/src/media/media.service.ts b/server/libs/domain/src/media/media.service.ts
@@ -7,7 +7,7 @@ import { IAssetJob, IBaseJob, IJobRepository, JobName } from '../job';
 import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
 import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config';
 import { SystemConfigCore } from '../system-config/system-config.core';
-import { IMediaRepository, VideoStreamInfo } from './media.repository';
+import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from './media.repository';
 
 @Injectable()
 export class MediaService {
@@ -127,65 +127,83 @@ export class MediaService {
       const output = join(outputFolder, `${asset.id}.mp4`);
       this.storageRepository.mkdirSync(outputFolder);
 
-      const { streams } = await this.mediaRepository.probe(input);
-      const stream = await this.getLongestStream(streams);
-      if (!stream) {
+      const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input);
+      const mainVideoStream = this.getMainVideoStream(videoStreams);
+      const mainAudioStream = this.getMainAudioStream(audioStreams);
+      const containerExtension = format.formatName;
+      if (!mainVideoStream || !mainAudioStream || !containerExtension) {
         return;
       }
 
       const { ffmpeg: config } = await this.configCore.getConfig();
 
-      const required = this.isTranscodeRequired(stream, config);
+      const required = this.isTranscodeRequired(mainVideoStream, mainAudioStream, containerExtension, config);
       if (!required) {
         return;
       }
 
-      const options = this.getFfmpegOptions(stream, config);
+      const options = this.getFfmpegOptions(mainVideoStream, config);
+
+      this.logger.log(`Start encoding video ${asset.id} ${options}`);
       await this.mediaRepository.transcode(input, output, options);
 
-      this.logger.log(`Converting Success ${asset.id}`);
+      this.logger.log(`Encoding success ${asset.id}`);
 
       await this.assetRepository.save({ id: asset.id, encodedVideoPath: output });
     } catch (error: any) {
       this.logger.error(`Failed to handle video conversion for asset: ${asset.id}`, error.stack);
     }
   }
 
-  private getLongestStream(streams: VideoStreamInfo[]): VideoStreamInfo | null {
-    return streams
-      .filter((stream) => stream.codecType === 'video')
-      .sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0];
+  private getMainVideoStream(streams: VideoStreamInfo[]): VideoStreamInfo | null {
+    return streams.sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0];
+  }
+
+  private getMainAudioStream(streams: AudioStreamInfo[]): AudioStreamInfo | null {
+    return streams[0];
   }
 
-  private isTranscodeRequired(stream: VideoStreamInfo, ffmpegConfig: SystemConfigFFmpegDto): boolean {
-    if (!stream.height || !stream.width) {
+  private isTranscodeRequired(
+    videoStream: VideoStreamInfo,
+    audioStream: AudioStreamInfo,
+    containerExtension: string,
+    ffmpegConfig: SystemConfigFFmpegDto,
+  ): boolean {
+    if (!videoStream.height || !videoStream.width) {
       this.logger.error('Skipping transcode, height or width undefined for video stream');
       return false;
     }
 
-    const isTargetVideoCodec = stream.codecName === ffmpegConfig.targetVideoCodec;
+    const isTargetVideoCodec = videoStream.codecName === ffmpegConfig.targetVideoCodec;
+    const isTargetAudioCodec = audioStream.codecName === ffmpegConfig.targetAudioCodec;
+    const isTargetContainer = ['mov,mp4,m4a,3gp,3g2,mj2', 'mp4', 'mov'].includes(containerExtension);
+
+    this.logger.debug(audioStream.codecName, audioStream.codecType, containerExtension);
+
+    const allTargetsMatching = isTargetVideoCodec && isTargetAudioCodec && isTargetContainer;
 
     const targetResolution = Number.parseInt(ffmpegConfig.targetResolution);
-    const isLargerThanTargetResolution = Math.min(stream.height, stream.width) > targetResolution;
+    const isLargerThanTargetResolution = Math.min(videoStream.height, videoStream.width) > targetResolution;
 
     switch (ffmpegConfig.transcode) {
+      case TranscodePreset.DISABLED:
+        return false;
+
       case TranscodePreset.ALL:
         return true;
 
       case TranscodePreset.REQUIRED:
-        return !isTargetVideoCodec;
+        return !allTargetsMatching;
 
       case TranscodePreset.OPTIMAL:
-        return !isTargetVideoCodec || isLargerThanTargetResolution;
+        return !allTargetsMatching || isLargerThanTargetResolution;
 
       default:
         return false;
     }
   }
 
   private getFfmpegOptions(stream: VideoStreamInfo, ffmpeg: SystemConfigFFmpegDto) {
-    // TODO: If video or audio are already the correct format, don't re-encode, copy the stream
-
     const options = [
       `-crf ${ffmpeg.crf}`,
       `-preset ${ffmpeg.preset}`,

diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts
@@ -13,12 +13,15 @@ import {
 import {
   AlbumResponseDto,
   AssetResponseDto,
+  AudioStreamInfo,
   AuthUserDto,
   ExifResponseDto,
   mapUser,
   SearchResult,
   SharedLinkResponseDto,
+  VideoFormat,
   VideoInfo,
+  VideoStreamInfo,
 } from '../src';
 
 const today = new Date();
@@ -706,10 +709,29 @@ export const searchStub = {
   }),
 };
 
+const probeStubDefaultFormat: VideoFormat = {
+  formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
+  formatLongName: 'QuickTime / MOV',
+  duration: 0,
+};
+
+const probeStubDefaultVideoStream: VideoStreamInfo[] = [
+  { height: 1080, width: 1920, codecName: 'h265', codecType: 'video', frameCount: 100, rotation: 0 },
+];
+
+const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ codecName: 'aac', codecType: 'audio' }];
+
+const probeStubDefault: VideoInfo = {
+  format: probeStubDefaultFormat,
+  videoStreams: probeStubDefaultVideoStream,
+  audioStreams: probeStubDefaultAudioStream,
+};
+
 export const probeStub = {
-  empty: { streams: [] },
-  multiple: Object.freeze<VideoInfo>({
-    streams: [
+  noVideoStreams: Object.freeze<VideoInfo>({ ...probeStubDefault, videoStreams: [] }),
+  multipleVideoStreams: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    videoStreams: [
       {
         height: 1080,
         width: 400,
@@ -729,7 +751,8 @@ export const probeStub = {
     ],
   }),
   noHeight: Object.freeze<VideoInfo>({
-    streams: [
+    ...probeStubDefault,
+    videoStreams: [
       {
         height: 0,
         width: 400,
@@ -740,16 +763,42 @@ export const probeStub = {
       },
     ],
   }),
-  tooBig: Object.freeze<VideoInfo>({
-    streams: [
+  videoStream2160p: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    videoStreams: [
       {
-        height: 10000,
-        width: 10000,
+        height: 2160,
+        width: 3840,
         codecName: 'h264',
         codecType: 'video',
         frameCount: 100,
         rotation: 0,
       },
     ],
   }),
+  videoStreamVertical2160p: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    videoStreams: [
+      {
+        height: 2160,
+        width: 3840,
+        codecName: 'h264',
+        codecType: 'video',
+        frameCount: 100,
+        rotation: 90,
+      },
+    ],
+  }),
+  audioStreamMp3: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    audioStreams: [{ codecType: 'audio', codecName: 'aac' }],
+  }),
+  matroskaContainer: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    format: {
+      formatName: 'matroska,webm',
+      formatLongName: 'Matroska / WebM',
+      duration: 0,
+    },
+  }),
 };
diff --git a/server/libs/infra/src/entities/system-config.entity.ts b/server/libs/infra/src/entities/system-config.entity.ts
@@ -37,6 +37,7 @@ export enum TranscodePreset {
   ALL = 'all',
   OPTIMAL = 'optimal',
   REQUIRED = 'required',
+  DISABLED = 'disabled',
 }
 
 export interface SystemConfig {