WIP refactor container and queuing system (immich-app#206)

* refactor microservices to machine-learning * Update tGithub issue template with correct task syntax * Added microservices container * Communicate between service based on queue system * added dependency * Fixed problem with having to import BullQueue into the individual service * Added todo * refactor server into monorepo with microservices * refactor database and entity to library * added simple migration * Move migrations and database config to library * Migration works in library * Cosmetic change in logging message * added user dto * Fixed issue with testing not able to find the shared library * Clean up library mapping path * Added webp generator to microservices * Update Github Action build latest * Fixed issue NPM cannot install due to conflict witl Bull Queue * format project with prettier * Modified docker-compose file * Add GH Action for Staging build: * Fixed GH action job name * Modified GH Action to only build & push latest when pushing to main * Added Test 2e2 Github Action * Added Test 2e2 Github Action * Implemented microservice to extract exif * Added cronjob to scan and generate webp thumbnail at midnight * Refactor to ireduce hit time to database when running microservices * Added error handling to asset services that handle read file from disk * Added video transcoding queue to process one video at a time * Fixed loading spinner on web while loading covering the info panel * Add mechanism to show new release announcement to web and mobile app (immich-app#209) * Added changelog page * Fixed issues based on PR comments * Fixed issue with video transcoding run on the server * Change entry point content for backward combatibility when starting up server * Added announcement box * Added error handling to failed silently when the app version checking is not able to make the request to GITHUB * Added new version announcement overlay * Update message * Added messages * Added logic to check and show announcement * Add method to handle saving new version * Added button to dimiss the acknowledge message * Up version for deployment to the app store
zierka · Jun 11, 2022 · a822017 · a822017
1 parent 397f8c7
commit a822017
Show file tree

Hide file tree

Showing 192 changed files with 1,827 additions and 2,121 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -16,10 +16,10 @@ Note: Please search to see if an issue already exists for the bug you encountere
 A clear and concise description of what the bug is.
 
 **Task List**
-[ ] I have read thoroughly the README setup and installation instructions.
-[ ] If my setup is different, I have included my docker-compose file.
-[ ] I have included my redacted `.env` file.
-[ ] I have included information on my machine, and environment.
+- [ ] I have read thoroughly the README setup and installation instructions.
+- [ ] If my setup is different, I have included my docker-compose file.
+- [ ] I have included my redacted `.env` file.
+- [ ] I have included information on my machine, and environment.
 
 **To Reproduce**
 Steps to reproduce the behavior:

diff --git a/.github/workflows/build_push_docker_latest.yml b/.github/workflows/build_push_docker_latest.yml
@@ -4,17 +4,16 @@ on:
   workflow_dispatch:
   push:
     branches: [main]
-  pull_request:
-    branches: [main]
 
 jobs:
-  build_and_push_server_latest:
+  # This image include both the server and microservices - the two containers can be slitted into separated
+  # service with its coressponding entry file.
+  build_and_push_server_monorepo_latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          # ref: "main" # branch
           fetch-depth: 0
 
       - name: Set up QEMU
@@ -27,23 +26,22 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push Immich
+      - name: Build and push Immich Mono Repo
         uses: docker/[email protected]
         with:
           context: ./server
           file: ./server/Dockerfile
           platforms: linux/arm/v7,linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: |
             altran1502/immich-server:latest
 
-  build_and_push_microservice_latest:
+  build_and_push_machine_learning_latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          # ref: "main" # branch
           fetch-depth: 0
 
       - name: Set up QEMU
@@ -56,23 +54,22 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and Push Microservices
+      - name: Build and Push Machine Learning
         uses: docker/[email protected]
         with:
-          context: ./microservices
-          file: ./microservices/Dockerfile
+          context: ./machine-learning
+          file: ./machine-learning/Dockerfile
           platforms: linux/arm/v7,linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: |
-            altran1502/immich-microservices:latest
+            altran1502/immich-machine-learning:latest
 
   build_and_push_web_latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          # ref: "main" # branch
           fetch-depth: 0
       - name: Set up QEMU
         uses: docker/[email protected]
@@ -91,6 +88,6 @@ jobs:
           file: ./web/Dockerfile
           platforms: linux/arm/v7,linux/amd64,linux/arm64
           target: prod
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: |
             altran1502/immich-web:latest
diff --git a/.github/workflows/build_push_docker_staging.yml b/.github/workflows/build_push_docker_staging.yml
@@ -0,0 +1,95 @@
+name: Build and Push Docker Image - Staging
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  # This image include both the server and microservices - the two containers can be slitted into separated
+  # service with its coressponding entry file.
+  build_and_push_server_monorepo_staging:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/[email protected]
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/[email protected]
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push Immich Mono Repo
+        uses: docker/[email protected]
+        with:
+          context: ./server
+          file: ./server/Dockerfile
+          platforms: linux/arm/v7,linux/amd64,linux/arm64
+          push: ${{ github.event_name == 'pull_request' }}
+          tags: |
+            altran1502/immich-server:staging
+
+  build_and_push_machine_learning_staging:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/[email protected]
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/[email protected]
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and Push Machine Learning
+        uses: docker/[email protected]
+        with:
+          context: ./machine-learning
+          file: ./machine-learning/Dockerfile
+          platforms: linux/arm/v7,linux/amd64
+          push: ${{ github.event_name == 'pull_request' }}
+          tags: |
+            altran1502/immich-machine-learning:staging
+
+  build_and_push_web_staging:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/[email protected]
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/[email protected]
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and Push Web
+        uses: docker/[email protected]
+        with:
+          context: ./web
+          file: ./web/Dockerfile
+          platforms: linux/arm/v7,linux/amd64,linux/arm64
+          target: prod
+          push: ${{ github.event_name == 'pull_request' }}
+          tags: |
+            altran1502/immich-web:staging
diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml
@@ -6,7 +6,7 @@ services:
     build:
       context: ../server
       dockerfile: Dockerfile
-    command: npm run start:dev
+    command: npm run start:dev immich
     expose:
       - "3000"
     volumes:
@@ -23,16 +23,35 @@ services:
     networks:
       - immich-network
 
-  immich-microservices:
-    image: immich-microservices-dev:1.9.0
+  immich-machine-learning:
+    image: immich-machine-learning-dev:1.9.0
     build:
-      context: ../microservices
+      context: ../machine-learning
       dockerfile: Dockerfile
     command: npm run start:dev
     expose:
       - "3001"
     volumes:
-      - ../microservices:/usr/src/app
+      - ../machine-learning:/usr/src/app
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+      - /usr/src/app/node_modules
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=development
+    depends_on:
+      - database
+    networks:
+      - immich-network
+
+  immich-microservices:
+    image: immich-microservices:1.9.0
+    build:
+      context: ../server
+      dockerfile: Dockerfile
+    command: npm run start:dev microservices
+    volumes:
+      - ../server:/usr/src/app
       - ${UPLOAD_LOCATION}:/usr/src/app/upload
       - /usr/src/app/node_modules
     env_file:

diff --git a/docker/docker-compose.staging.yml b/docker/docker-compose.staging.yml
@@ -2,11 +2,8 @@ version: "3.8"
 
 services:
   immich-server:
-    image: immich-server-staging:latest
-    build:
-      context: ../server
-      dockerfile: Dockerfile
-    entrypoint: ["/bin/sh", "./entrypoint.sh"]
+    image: altran1502/immich-server:staging
+    entrypoint: ["/bin/sh", "./start-server.sh"]
     expose:
       - "3000"
     volumes:
@@ -23,10 +20,23 @@ services:
     restart: always
 
   immich-microservices:
-    image: immich-microservices-staging:latest
-    build:
-      context: ../microservices
-      dockerfile: Dockerfile
+    image: altran1502/immich-server:staging
+    entrypoint: ["/bin/sh", "./start-microservices.sh"]
+    volumes:
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=production
+    depends_on:
+      - redis
+      - database
+    networks:
+      - immich-network
+    restart: always
+
+  immich-machine-learning:
+    image: altran1502/immich-machine-learning:staging
     entrypoint: ["/bin/sh", "./entrypoint.sh"]
     expose:
       - "3001"
@@ -43,12 +53,8 @@ services:
     restart: always
 
   immich-web:
-    image: immich-web-staging:latest
+    image: altran1502/immich-web:staging
     entrypoint: ["/bin/sh", "./entrypoint.sh"]
-    build:
-      context: ../web
-      dockerfile: Dockerfile
-      target: prod
     env_file:
       - .env
     ports:
@@ -57,14 +63,12 @@ services:
       - immich-network
     restart: always
 
-
   redis:
     container_name: immich_redis
     image: redis:6.2
     networks:
       - immich-network
     restart: always
-
 
   database:
     container_name: immich_postgres
@@ -82,6 +86,7 @@ services:
       - 5432:5432
     networks:
       - immich-network
+    restart: always
 
   nginx:
     container_name: proxy_nginx
@@ -102,4 +107,4 @@ services:
 networks:
   immich-network:
 volumes:
-  pgdata:
+  pgdata:
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -3,7 +3,7 @@ version: "3.8"
 services:
   immich-server:
     image: altran1502/immich-server:latest
-    entrypoint: ["/bin/sh", "./entrypoint.sh"]
+    entrypoint: ["/bin/sh", "./start-server.sh"]
     expose:
       - "3000"
     volumes:
@@ -20,7 +20,23 @@ services:
     restart: always
 
   immich-microservices:
-    image: altran1502/immich-microservices:latest
+    image: altran1502/immich-server:latest
+    entrypoint: ["/bin/sh", "./start-microservices.sh"]
+    volumes:
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=production
+    depends_on:
+      - redis
+      - database
+    networks:
+      - immich-network
+    restart: always
+
+  immich-machine-learning:
+    image: altran1502/immich-machine-learning:latest
     entrypoint: ["/bin/sh", "./entrypoint.sh"]
     expose:
       - "3001"
@@ -47,14 +63,12 @@ services:
       - immich-network
     restart: always
 
-
   redis:
     container_name: immich_redis
     image: redis:6.2
     networks:
       - immich-network
     restart: always
-
 
   database:
     container_name: immich_postgres
@@ -73,7 +87,7 @@ services:
     networks:
       - immich-network
     restart: always
-    
+
   nginx:
     container_name: proxy_nginx
     image: nginx:latest
@@ -93,4 +107,4 @@ services:
 networks:
   immich-network:
 volumes:
-  pgdata:
+  pgdata:
diff --git a/microservices/.dockerignore → machine-learning/.dockerignore b/microservices/.dockerignore → machine-learning/.dockerignore
diff --git a/microservices/.eslintrc.js → machine-learning/.eslintrc.js b/microservices/.eslintrc.js → machine-learning/.eslintrc.js