feat(server): vaapi hardware decoding (immich-app#13561)

* add hw decoding for vaapi

* add tests

* update docs

docs/docs/features/hardware-transcoding.md

@@ -23,7 +23,7 @@ You do not need to redo any transcoding jobs after enabling hardware acceleratio
 - Raspberry Pi is currently not supported.
 - Two-pass mode is only supported for NVENC. Other APIs will ignore this setting.
 - By default, only encoding is currently hardware accelerated. This means the CPU is still used for software decoding and tone-mapping.
-  - NVENC and RKMPP can be fully accelerated by enabling hardware decoding in the video transcoding settings.
+  - You can benefit from end-to-end acceleration by enabling hardware decoding in the video transcoding settings.
 - Hardware dependent
   - Codec support varies, but H.264 and HEVC are usually supported.
     - Notably, NVIDIA and AMD GPUs do not support VP9 encoding.
@@ -66,7 +66,7 @@ For RKMPP to work:
 
 3. Redeploy the `immich-server` container with these updated settings.
 4. In the Admin page under `Video transcoding settings`, change the hardware acceleration setting to the appropriate option and save.
-5. (Optional) If using a compatible backend, you may enable hardware decoding for optimal performance.
+5. (Optional) Enable hardware decoding for optimal performance.
 
 #### Single Compose File
 

server/src/services/media.service.spec.ts

@@ -1619,7 +1619,7 @@ describe(MediaService.name, () => {
             '-refs 5',
             '-g 256',
             '-v verbose',
-            '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
+            '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq',
             '-preset 7',
             '-global_quality:v 23',
             '-maxrate 10000k',
@@ -1803,7 +1803,7 @@ describe(MediaService.name, () => {
             '-strict unofficial',
             '-g 256',
             '-v verbose',
-            '-vf format=nv12,hwupload,scale_vaapi=-2:720',
+            '-vf format=nv12,hwupload,scale_vaapi=-2:720:mode=hq:out_range=pc',
             '-compression_level 7',
             '-rc_mode 1',
           ]),
@@ -1946,6 +1946,79 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should use hardware decoding for vaapi if enabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining([
+            '-hwaccel vaapi',
+            '-hwaccel_output_format vaapi',
+            '-noautorotate',
+            '-threads 1',
+          ]),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining('scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12'),
+          ]),
+          twoPass: false,
+        }),
+      );
+    });
+
+    it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1,format=vaapi',
+            ),
+          ]),
+          twoPass: false,
+        }),
+      );
+    });
+
+    it('should use preferred device for vaapi when hardware decoding', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true, preferredHwDevice: 'renderD129' },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_device /dev/dri/renderD129']),
+          outputOptions: expect.any(Array),
+          twoPass: false,
+        }),
+      );
+    });
+
     it('should fallback to sw transcoding if hw transcoding fails', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);

server/src/utils/media.ts

@@ -52,7 +52,9 @@ export class BaseConfig implements VideoCodecSWConfig {
         break;
       }
       case TranscodeHWAccel.VAAPI: {
-        handler = new VAAPIConfig(config, devices);
+        handler = config.accelDecode
+          ? new VaapiHwDecodeConfig(config, devices)
+          : new VaapiSwDecodeConfig(config, devices);
         break;
       }
       case TranscodeHWAccel.RKMPP: {
@@ -688,7 +690,7 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
     const options = this.getToneMapping(videoStream);
     options.push('format=nv12', 'hwupload=extra_hw_frames=64');
     if (this.shouldScale(videoStream)) {
-      options.push(`scale_qsv=${this.getScaling(videoStream)}`);
+      options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq`);
     }
     return options;
   }
@@ -811,7 +813,7 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
   }
 }
 
-export class VAAPIConfig extends BaseHWConfig {
+export class VaapiSwDecodeConfig extends BaseHWConfig {
   getBaseInputOptions() {
     if (this.devices.length === 0) {
       throw new Error('No VAAPI device found');
@@ -829,7 +831,7 @@ export class VAAPIConfig extends BaseHWConfig {
     const options = this.getToneMapping(videoStream);
     options.push('format=nv12', 'hwupload');
     if (this.shouldScale(videoStream)) {
-      options.push(`scale_vaapi=${this.getScaling(videoStream)}`);
+      options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`);
     }
 
     return options;
@@ -878,6 +880,76 @@ export class VAAPIConfig extends BaseHWConfig {
   }
 }
 
+export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw new Error('No VAAPI device found');
+    }
+
+    const options = [
+      '-hwaccel vaapi',
+      '-hwaccel_output_format vaapi',
+      '-noautorotate',
+      ...this.getInputThreadOptions(),
+    ];
+    const hwDevice = this.getPreferredHardwareDevice();
+    if (hwDevice) {
+      options.push(`-hwaccel_device ${hwDevice}`);
+    }
+
+    return options;
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    const options = [];
+    if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) {
+      let scaling = `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`;
+      if (!this.shouldToneMap(videoStream)) {
+        scaling += ':format=nv12';
+      }
+      options.push(scaling);
+    }
+
+    options.push(...this.getToneMapping(videoStream));
+    return options;
+  }
+
+  getToneMapping(videoStream: VideoStreamInfo): string[] {
+    if (!this.shouldToneMap(videoStream)) {
+      return [];
+    }
+
+    const colors = this.getColors();
+    const tonemapOptions = [
+      'desat=0',
+      'format=nv12',
+      `matrix=${colors.matrix}`,
+      `primaries=${colors.primaries}`,
+      'range=pc',
+      `tonemap=${this.config.tonemap}`,
+      `transfer=${colors.transfer}`,
+    ];
+
+    return [
+      'hwmap=derive_device=opencl',
+      `tonemap_opencl=${tonemapOptions.join(':')}`,
+      'hwmap=derive_device=vaapi:reverse=1,format=vaapi',
+    ];
+  }
+
+  getInputThreadOptions() {
+    return [`-threads 1`];
+  }
+
+  getColors() {
+    return {
+      primaries: 'bt709',
+      transfer: 'bt709',
+      matrix: 'bt709',
+    };
+  }
+}
+
 export class RkmppSwDecodeConfig extends BaseHWConfig {
   constructor(
     protected config: SystemConfigFFmpegDto,

web/src/lib/i18n/en.json

@@ -274,7 +274,7 @@
     "transcoding_hardware_acceleration": "Hardware Acceleration",
     "transcoding_hardware_acceleration_description": "Experimental; much faster, but will have lower quality at the same bitrate",
     "transcoding_hardware_decoding": "Hardware decoding",
-    "transcoding_hardware_decoding_setting_description": "Applies only to NVENC, QSV and RKMPP. Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.",
+    "transcoding_hardware_decoding_setting_description": "Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.",
     "transcoding_hevc_codec": "HEVC codec",
     "transcoding_max_b_frames": "Maximum B-frames",
     "transcoding_max_b_frames_description": "Higher values improve compression efficiency, but slow down encoding. May not be compatible with hardware acceleration on older devices. 0 disables B-frames, while -1 sets this value automatically.",