test(server): full backend end-to-end testing with microservices (imm…

…ich-app#4225)

* feat: asset e2e with job option

* feat: checkout test assets

* feat: library e2e tests

* fix: use node 21 in e2e

* fix: tests

* fix: use normalized external path

* feat: more external path tests

* chore: use parametrized tests

* chore: remove unused test code

* chore: refactor test asset path

* feat: centralize test app creation

* fix: correct error message for missing assets

* feat: test file formats

* fix: don't compare checksum

* feat: build libvips

* fix: install meson

* fix: use immich test asset repo

* feat: test nikon raw files

* fix: set Z timezone

* feat: test offline library files

* feat: richer metadata tests

* feat: e2e tests in docker

* feat: e2e test with arm64 docker

* fix: manual docker compose run

* fix: remove metadata processor import

* fix: run e2e tests in test.yml

* fix: checkout e2e assets

* fix: typo

* fix: checkout files in app directory

* fix: increase e2e memory

* fix: rm submodules

* fix: revert action name

* test: mark file offline when external path changes

* feat: rename env var to TEST_ENV

* docs: new test procedures

* feat: can run docker e2e tests manually if needed

* chore: use new node 20.8 for e2e

* chore: bump exiftool-vendored

* feat: simplify test launching

* fix: rename env vars to use immich_ prefix

* feat: asset folder is submodule

* chore: cleanup after 20.8 upgrade

* fix: don't log postgres in e2e

* fix: better warning about not running all tests

---------

Co-authored-by: Jonathan Jogenfors <[email protected]>

.github/workflows/test.yml

@@ -13,20 +13,15 @@ jobs:
   e2e-tests:
     name: Run end-to-end test suites
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ./server
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
-      - name: Run npm install
-        run: npm ci
+        with:
+          submodules: "recursive"
 
       - name: Run e2e tests
-        run: npm run test:e2e
-        if: ${{ !cancelled() }}
+        run: docker-compose -f ./docker/docker-compose.test.yml -p immich-test-e2e up --renew-anon-volumes --abort-on-container-exit --exit-code-from immich-server-test --remove-orphans --build
 
   doc-tests:
     name: Run documentation checks

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "mobile/.isar"]
 	path = mobile/.isar
 	url = https://github.com/isar/isar
+[submodule "server/test/assets"]
+	path = server/test/assets
+	url = https://github.com/immich-app/test-assets

Makefile

@@ -20,7 +20,7 @@ pull-stage:
 	docker-compose -f ./docker/docker-compose.staging.yml pull
 
 test-e2e:
-	docker-compose -f ./docker/docker-compose.test.yml --env-file ./docker/.env.test -p immich-test-e2e up  --renew-anon-volumes --abort-on-container-exit --exit-code-from immich-server-test --remove-orphans --build
+	docker-compose -f ./docker/docker-compose.test.yml -p immich-test-e2e up  --renew-anon-volumes --abort-on-container-exit --exit-code-from immich-server-test --remove-orphans --build
 
 prod:
 	docker-compose -f ./docker/docker-compose.prod.yml up --build -V --remove-orphans

docker/docker-compose.test.yml

@@ -1,5 +1,7 @@
 version: "3.8"
 
+# Compose file for dockerized end-to-end testing of the backend
+
 services:
   immich-server-test:
     image: immich-server-test
@@ -8,39 +10,31 @@ services:
       dockerfile: Dockerfile
       target: builder
     command: npm run test:e2e
-    expose:
-      - "3000"
     volumes:
       - ../server:/usr/src/app
       - /usr/src/app/node_modules
-    env_file:
-      - .env.test
     environment:
-      - NODE_ENV=development
-      - TYPESENSE_ENABLED=false
+      - DB_HOSTNAME=immich-database-test
+      - DB_USERNAME=postgres
+      - DB_PASSWORD=postgres
+      - DB_DATABASE_NAME=e2e_test
+      - IMMICH_RUN_ALL_TESTS=true
     depends_on:
-      - immich-redis-test
       - immich-database-test
     networks:
       - immich-test-network
-  immich-redis-test:
-    container_name: immich-redis-test
-    image: redis:6.2-alpine@sha256:70a7a5b641117670beae0d80658430853896b5ef269ccf00d1827427e3263fa3
-    networks:
-      - immich-test-network
+
   immich-database-test:
     container_name: immich-database-test
     image: postgres:14-alpine@sha256:28407a9961e76f2d285dc6991e8e48893503cc3836a4755bbc2d40bcc272a441
-    env_file:
-      - .env.test
     environment:
-      POSTGRES_PASSWORD: ${DB_PASSWORD}
-      POSTGRES_USER: ${DB_USERNAME}
-      POSTGRES_DB: ${DB_DATABASE_NAME}
-    volumes:
-      - /var/lib/postgresql/data
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_DB: e2e_test
     networks:
       - immich-test-network
+    logging:
+      driver: none
 
 networks:
   immich-test-network:

docs/docs/developer/testing.md

@@ -0,0 +1,17 @@
+# Testing
+
+## Server
+
+### Unit tests
+
+Unit are run by calling `npm run test` from the `server` directory.
+
+### End to end tests
+
+The backend has an end-to-end test suite that can be called with `npm run test:e2e` from the `server` directory. This will set up a dummy database inside a temporary container and run the tests against it. Setup and teardown is automatically taken care of. That test, however, can not set up all prerequisites to parse file formats, as that is very complex and error-prone. As such, this test excludes some test cases like HEIC file imports. The test suite will also print a friendly warning to remind you that not all tests are being run.
+
+Note that there is a bug in nodejs <20.8 that causes segmentation faults when running these tests. If you run into segfaults, ensure you are using at least version 20.8.
+
+To perform a full e2e test, you need to run e2e tests inside docker. The easiest way to do that is to run `make test-e2e` in the root directory. This will build and start a docker-compose consisting of the server, microservices, and a postgres database. It will then perfom the tests and exit.
+
+If you manually install the dependencies (see the DOCKERFILE) on your development machine, you can also run the full e2e tests manually by setting the `IMMICH_RUN_ALL_TESTS` environment value to true, i.e. `IMMICH_RUN_ALL_TESTS=true npm run test:e2e`.

server/package.json

@@ -26,7 +26,7 @@
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "jest --config test/e2e/jest-e2e.json --runInBand",
+    "test:e2e": "NODE_OPTIONS='--experimental-vm-modules --max_old_space_size=4096' jest --config test/e2e/jest-e2e.json --runInBand --forceExit",
     "typeorm": "typeorm",
     "typeorm:migrations:create": "typeorm migration:create",
     "typeorm:migrations:generate": "typeorm migration:generate -d ./dist/infra/database.config.js",
@@ -126,7 +126,8 @@
     "ts-loader": "^9.4.4",
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "^5.2.2",
+    "utimes": "^5.2.1"
   },
   "jest": {
     "clearMocks": true,

server/src/domain/job/job.repository.ts

@@ -107,11 +107,12 @@ export type JobItem =
   | { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob };
 
 export type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
+export type JobItemHandler = (item: JobItem) => Promise<void>;
 
 export const IJobRepository = 'IJobRepository';
 
 export interface IJobRepository {
-  addHandler(queueName: QueueName, concurrency: number, handler: (job: JobItem) => Promise<void>): void;
+  addHandler(queueName: QueueName, concurrency: number, handler: JobItemHandler): void;
   setConcurrency(queueName: QueueName, concurrency: number): void;
   queue(item: JobItem): Promise<void>;
   pause(name: QueueName): Promise<void>;

server/src/domain/library/library.service.spec.ts

@@ -1172,7 +1172,7 @@ describe(LibraryService.name, () => {
     });
   });
 
-  describe('handleEmptyTrash', () => {
+  describe('handleRemoveOfflineFiles', () => {
     it('can queue trash deletion jobs', async () => {
       assetMock.getWith.mockResolvedValue({ items: [assetStub.image1], hasNextPage: false });
       assetMock.getById.mockResolvedValue(assetStub.image1);

server/src/domain/library/library.service.ts

@@ -363,6 +363,8 @@ export class LibraryService {
       return false;
     }
 
+    const normalizedExternalPath = path.normalize(user.externalPath);
+
     this.logger.verbose(`Refreshing library: ${job.id}`);
     const crawledAssetPaths = (
       await this.storageRepository.crawl({
@@ -373,7 +375,7 @@ export class LibraryService {
       .map(path.normalize)
       .filter((assetPath) =>
         // Filter out paths that are not within the user's external path
-        assetPath.match(new RegExp(`^${user.externalPath}`)),
+        assetPath.match(new RegExp(`^${normalizedExternalPath}`)),
       );
 
     this.logger.debug(`Found ${crawledAssetPaths.length} assets when crawling import paths ${library.importPaths}`);

server/src/immich/api-v1/asset/asset.service.ts

@@ -119,7 +119,7 @@ export class AssetService {
       }
 
       this.logger.error(`Error uploading file ${error}`, error?.stack);
-      throw new BadRequestException(`Error uploading file`, `${error}`);
+      throw error;
     }
   }
 

server/src/infra/infra.config.ts

@@ -5,6 +5,10 @@ import { RedisOptions } from 'ioredis';
 import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
 
 function parseRedisConfig(): RedisOptions {
+  if (process.env.IMMICH_TEST_ENV == 'true') {
+    return {};
+  }
+
   const redisUrl = process.env.REDIS_URL;
   if (redisUrl && redisUrl.startsWith('ioredis://')) {
     try {

server/src/infra/infra.module.ts

@@ -80,16 +80,24 @@ const providers: Provider[] = [
   { provide: IUserTokenRepository, useClass: UserTokenRepository },
 ];
 
+const imports = [
+  ConfigModule.forRoot(immichAppConfig),
+  TypeOrmModule.forRoot(databaseConfig),
+  TypeOrmModule.forFeature(databaseEntities),
+];
+
+const moduleExports = [...providers];
+
+if (process.env.IMMICH_TEST_ENV !== 'true') {
+  imports.push(BullModule.forRoot(bullConfig));
+  imports.push(BullModule.registerQueue(...bullQueues));
+  moduleExports.push(BullModule);
+}
+
 @Global()
 @Module({
-  imports: [
-    ConfigModule.forRoot(immichAppConfig),
-    TypeOrmModule.forRoot(databaseConfig),
-    TypeOrmModule.forFeature(databaseEntities),
-    BullModule.forRoot(bullConfig),
-    BullModule.registerQueue(...bullQueues),
-  ],
+  imports,
   providers: [...providers],
-  exports: [...providers, BullModule],
+  exports: moduleExports,
 })
 export class InfraModule {}

server/test/api/asset-api.ts

@@ -7,13 +7,18 @@ import request from 'supertest';
 type UploadDto = Partial<CreateAssetDto> & { content?: Buffer };
 
 export const assetApi = {
-  get: async (server: any, accessToken: string, id: string) => {
+  get: async (server: any, accessToken: string, id: string): Promise<AssetResponseDto> => {
     const { body, status } = await request(server)
       .get(`/asset/assetById/${id}`)
       .set('Authorization', `Bearer ${accessToken}`);
     expect(status).toBe(200);
     return body as AssetResponseDto;
   },
+  getAllAssets: async (server: any, accessToken: string) => {
+    const { body, status } = await request(server).get(`/asset/`).set('Authorization', `Bearer ${accessToken}`);
+    expect(status).toBe(200);
+    return body as AssetResponseDto[];
+  },
   upload: async (server: any, accessToken: string, id: string, dto: UploadDto = {}) => {
     const { content, isFavorite = false, isArchived = false } = dto;
     const { body, status } = await request(server)