MySQL support (HumanSignal#1385)

* Adding support for MySQL

* Updating MySQL version to 8 in docker-compose

* Using docker-compose overrides for MySQL

* Conditionally setting db_index if DJANGO_DB is not mysql

.gitignore

@@ -28,7 +28,9 @@ deploy/license.txt
 deploy/logs/
 deploy/redis-data/
 deploy/dockerfiles/postgres-data
+deploy/dockerfiles/mysql-data
 postgres-data
+mysql-data
 
 label_studio/core/media
 label_studio/core/downloads

Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /label-studio
 
 ENV TZ=Europe/Berlin
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-RUN apt-get update && apt-get install -y build-essential postgresql-client python3.8 python3-pip python3.8-dev uwsgi  git libxml2-dev libxslt-dev zlib1g-dev uwsgi
+RUN apt-get update && apt-get install -y build-essential postgresql-client libmysqlclient-dev mysql-client python3.8 python3-pip python3.8-dev uwsgi  git libxml2-dev libxslt-dev zlib1g-dev uwsgi
 
 RUN chgrp -R 0 /var/log /var/cache /var/run /run /tmp /etc/uwsgi && \
     chmod -R g+rwX /var/log /var/cache /var/run /run /tmp /etc/uwsgi

deploy/docker-entrypoint.d/20-wait-for-postgres.sh → deploy/docker-entrypoint.d/20-wait-for-db.sh

@@ -8,6 +8,13 @@ if [ -n "${POSTGRE_HOST:-}" ]; then
     sleep 1
   done
   echo >&3 "=> Postgres is up."
+elif [ -n "${MYSQL_HOST:-}" ]; then
+  echo >&3 "=> Waiting for MySQL..."
+  while ! mysqladmin ping -h"$MYSQL_HOST" -p"${MYSQL_PORT:-3306}" --silent; do
+      echo >&3 "==> MySQL is unavailable - sleeping..."
+      sleep 1
+  done
+  echo >&3 "=> MySQL is up."
 else
-  echo >&3 "=> Skipping wait for postgres."
+  echo >&3 "=> Skipping wait for database."
 fi

deploy/docker-entrypoint.d/30-run-db-migrations.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e
 
-if [ -n "${POSTGRE_HOST:-}" ]; then
+if [ -n "${POSTGRE_HOST:-}" ] || [ -n "${MYSQL_HOST:-}" ]; then
   echo >&3 "=> Do database migrations..."
   python3 /label-studio/label_studio/manage.py migrate >&3
   echo >&3 "=> Migrations completed."

deploy/requirements.txt

@@ -46,3 +46,4 @@ boxing>=0.1.4
 redis>=3.5.0
 sentry-sdk>=1.1.0
 label-studio-converter==0.0.31rc1
+mysqlclient>=2.0.3

docker-compose.mysql.yml

@@ -0,0 +1,22 @@
+version: '3.3'
+
+services:
+  app:
+    environment:
+      - DJANGO_DB=mysql
+      - MYSQL_NAME=labelstudio
+      - MYSQL_USER=root
+      - MYSQL_PASSWORD=password
+      - MYSQL_PORT=3306
+      - MYSQL_HOST=db
+      - POSTGRE_HOST=           # override POSTGRE vars to empty values
+
+  db:
+    image: mysql:8
+    hostname: db
+    restart: always
+    environment:
+      - MYSQL_ROOT_PASSWORD=password
+      - MYSQL_DATABASE=labelstudio
+    volumes:
+      - ${MYSQL_DATA_DIR:-./mysql-data}:/var/lib/mysql

label_studio/core/settings/base.py

@@ -62,6 +62,7 @@
 
 # Databases
 # https://docs.djangoproject.com/en/2.1/ref/settings/#databases
+DJANGO_DB_MYSQL = 'mysql'
 DJANGO_DB_SQLITE = 'sqlite'
 DJANGO_DB = 'default'
 DATABASE_NAME_DEFAULT = os.path.join(BASE_DATA_DIR, 'label_studio.sqlite3')
@@ -75,6 +76,14 @@
         'HOST': get_env('POSTGRE_HOST', 'localhost'),
         'PORT': int(get_env('POSTGRE_PORT', '5432')),
     },
+    DJANGO_DB_MYSQL: {
+        'ENGINE': 'django.db.backends.mysql',
+        'USER': get_env('MYSQL_USER', 'root'),
+        'PASSWORD': get_env('MYSQL_PASSWORD', ''),
+        'NAME': get_env('MYSQL_NAME', 'labelstudio'),
+        'HOST': get_env('MYSQL_HOST', 'localhost'),
+        'PORT': int(get_env('MYSQL_PORT', '3306')),
+    },
     DJANGO_DB_SQLITE: {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': DATABASE_NAME,

label_studio/tasks/migrations/0001_squashed_0041_taskcompletionhistory_was_cancelled.py

@@ -64,7 +64,7 @@ class Migration(migrations.Migration):
                 ('created_at', models.DateTimeField(auto_now_add=True, help_text='Creation time', verbose_name='created at')),
                 ('updated_at', models.DateTimeField(auto_now=True, help_text='Last update time', verbose_name='updated at')),
                 ('completed_by', models.ForeignKey(help_text='User who made this completion', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='completions', to=settings.AUTH_USER_MODEL)),
-                ('task', models.ForeignKey(help_text='Corresponding task for this completion', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='completions', to='tasks.task')),
+                ('task', models.ForeignKey(help_text='Corresponding task for this completion', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='completions', to='tasks.task', db_index=settings.DJANGO_DB != settings.DJANGO_DB_MYSQL)),
                 ('prediction', models.JSONField(default=dict, help_text='Prediction viewed at the time of completion', null=True, verbose_name='prediction')),
                 ('prediction_equal_score', models.FloatField(default=0.0, help_text='Comparison result with prediction viewed at the time of completion', verbose_name='prediction_equal_score')),
                 ('updates_history', models.JSONField(default=list, help_text='Updates history by experts', null=True, verbose_name='update history')),