migrate from SQLite to PostgreSQL + refactoring

.env.template

@@ -0,0 +1,12 @@
+# telegram-bot
+BOT_TOKEN=#Your BOT_TOKEN here
+DB_HOST=#Your DB_HOST here
+DB_PORT=#Your DB_PORT here
+DB_USER=#Your DB_USER here
+DB_PASSWORD=#Your DB_PASSWORD here
+DB_NAME=#Your DB_NAME here
+
+# db
+POSTGRES_USER=#Your POSTGRES_USER here
+POSTGRES_PASSWORD=#Your POSTGRES_PASSWORD here
+POSTGRES_DB=#Your POSTGRES_DB here

.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+venv/
+
+.env
+.env.test
+.env.prod
+.DS_Store
+db.db
+r

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "Python-Shared-Library"]
+	path = lib
+	url = https://github.com/Deimvis/Python-Shared-Library.git

Dockerfile

@@ -1,11 +1,15 @@
-FROM python:3.7
+FROM python:3.11.2
 
-WORKDIR /bot/world_bot
-COPY apps/ apps/
-COPY config.py .
-COPY main.py .
+WORKDIR /bot
 
 COPY requirements.txt .
-RUN pip install -r requirements.txt
+RUN pip install --upgrade pip && pip install -r requirements.txt
+
+COPY src /bot/src
+COPY lib /bot/lib
+COPY scripts /bot/scripts
+COPY main.py .
+
+ENV PYTHONUNBUFFERED=1
 
-ENTRYPOINT ["python", "main.py"]
+ENTRYPOINT ["./scripts/run"]

deploy/bundle/docker-compose.yaml

@@ -0,0 +1,27 @@
+version: '3'
+
+services:
+
+  telegram-bot:
+    platform: linux/x86_64
+    image: deimvis/world_bot:latest
+    container_name: world-bot
+    env_file:
+      - .env.prod
+    depends_on:
+      - db
+
+  db:
+    platform: linux/x86_64
+    image: postgres:15.2
+    container_name: world-bot-db
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    ports:
+      - 5434:5432
+    env_file:
+      - .env.prod
+
+
+volumes:
+  postgres_data:

deploy/bundle/down

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sudo docker-compose -f /home/dbrusenin/world_bot/docker-compose.yaml down $@

deploy/bundle/init_db

@@ -0,0 +1,3 @@
+#!/usr/bin/env
+
+docker-compose run --rm --entrypoint /bot/scripts/init_db telegram-bot

deploy/bundle/pull

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sudo docker-compose -f /home/dbrusenin/world_bot/docker-compose.yaml pull $@

deploy/bundle/up

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sudo docker-compose -f /home/dbrusenin/world_bot/docker-compose.yaml up -d $@

deploy/deploy

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -ex
+
+TARGET_HOST=${1:-ya_cloud_world_bot}
+REMOTE_WORKDIR=${2:-/home/dbrusenin/world_bot}
+
+docker-compose -f docker-compose.prod.yaml build
+docker-compose -f docker-compose.prod.yaml push
+
+./deploy/make_deploy_docker_compose docker-compose.prod.yaml deploy/bundle/docker-compose.yaml
+cp .env.prod deploy/bundle/.env.prod
+
+scp -r deploy/bundle/. dbrusenin@$TARGET_HOST:$REMOTE_WORKDIR
+
+ssh $TARGET_HOST "$REMOTE_WORKDIR/pull"
+ssh $TARGET_HOST "$REMOTE_WORKDIR/down"
+ssh $TARGET_HOST "$REMOTE_WORKDIR/up"

deploy/make_deploy_docker_compose

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import re
+import sys
+import logging
+
+
+def process_line(line: str) -> str | None:
+    if re.fullmatch(r'^\s*build:.*\n$', line):
+        logging.info(f'Remove line: {line.strip()}')
+        return None
+    return line
+
+
+def main():
+    original_docker_compose = sys.argv[1]
+    deploy_docker_compose = sys.argv[2]
+    with open(deploy_docker_compose, 'w') as f_out:
+        with open(original_docker_compose, 'r') as f_in:
+            for line in f_in:
+                out_line = process_line(line)
+                if out_line is not None:
+                    f_out.write(out_line)
+
+
+if __name__ == '__main__':
+    main()

devtools/pip_install

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import re
+import sys
+import subprocess as sp
+
+logging.basicConfig(level=logging.INFO)
+
+
+REQUIREMENTS_FILE_PATH = 'requirements.txt'
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Install python package via pip and update requirements.txt')
+    parser.add_argument('packages', type=str, nargs='+', help='Package names')
+    return parser.parse_args()
+
+
+def get_current_packages():
+    with open(REQUIREMENTS_FILE_PATH, 'r') as f:
+        packages = [line for line in f if len(line) >  0]
+    return packages
+
+
+def update_packages(package_names):
+    installed_packages = sp.check_output([sys.executable, '-m', 'pip', 'freeze']).decode('utf-8').split('\n')
+    packages = []
+    for name in package_names:
+        package = None
+        for p in installed_packages:
+            if p.lower().startswith(name.lower()):
+                package = p.replace('==', '~=')
+        assert package is not None, f'Package {name} isn\'t found among installed'
+        packages.append(package)
+    with open(REQUIREMENTS_FILE_PATH, 'w') as f:
+        for package in sorted(packages):
+            f.write(package + '\n')
+            logging.info(package)
+
+
+def main():
+    args = parse_args()
+    packages = get_current_packages()
+    package_names = set()
+    for package in packages:
+        match_result = re.fullmatch(r'^(?P<package_name>^.+?)(?=(=|~|>)=).+\s?$', package)
+        assert match_result is not None, 'Couldn\'t parse package string from requirements.txt'
+        package_names.add(match_result.group('package_name'))
+    for new_package_name in args.packages:
+        if new_package_name in package_names:
+            logging.warning(f'Package is already installed ({new_package_name})')
+            continue
+        package_names.add(new_package_name)
+        sp.check_call([sys.executable, '-m', 'pip', 'install', new_package_name])
+    update_packages(package_names)
+
+if __name__ == '__main__':
+    main()

devtools/sqlite2postgres

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# usage:   ./migrate.sh  PATH_TO_SQLITE_DATABASE         PG_HOST         PG_DBNAME   PG_USER
+# example: ./migrate.sh      $(pwd)/user.db        host.docker.internal    user      deimvis
+
+SQLITE_DBPATH=${1:-$(pwd)/db.db}
+PG_HOST=${2:-host.docker.internal}  # use host.docker.internal instead of localhost
+PG_DBNAME=${3:-mypostgresdatabase}
+PG_USER=${4:-postgres}
+
+docker run --rm -it -v ${SQLITE_DBPATH}:/db.db dimitri/pgloader:latest pgloader sqlite:///db.db pgsql://${PG_USER}@${PG_HOST}/${PG_DBNAME}
+

docker-compose.prod.yaml

@@ -0,0 +1,28 @@
+version: '3'
+
+services:
+
+  telegram-bot:
+    platform: linux/x86_64
+    image: deimvis/world_bot:latest
+    build: .
+    container_name: world-bot
+    env_file:
+      - .env.prod
+    depends_on:
+      - db
+
+  db:
+    platform: linux/x86_64
+    image: postgres:15.2
+    container_name: world-bot-db
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    ports:
+      - 5434:5432
+    env_file:
+      - .env.prod
+
+
+volumes:
+  postgres_data:

docker-compose.test.yaml

@@ -0,0 +1,25 @@
+version: '3'
+
+services:
+
+  telegram-bot:
+    build: .
+    container_name: world-bot
+    env_file:
+      - .env.test
+    depends_on:
+      - world-bot-db
+
+  db:
+    image: postgres:15.2
+    container_name: world-bot-db
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    ports:
+      - 5434:5432
+    env_file:
+      - .env.test
+
+
+volumes:
+  postgres_data:

main.py

@@ -1,9 +1,10 @@
+import os
+import logging
 import re
 import threading
 import telebot
 from telebot import types
-from config import BOT_TOKEN, DB_PATH
-from apps.quotes.api import (
+from src.apps.quotes.api import (
     send_quotes_menu,
     send_great_quote,
     send_quotes_subscription_menu,
@@ -14,16 +15,31 @@
     quotes_subscription_type_menu,
     quotes_subscription_everyday_value_menu
 )
-from apps.quotes.utils import (
+from src.apps.quotes.utils import (
     DayOfWeekRUEN,
     get_build_quotes_subscription_status,
     get_remove_quotes_subscription_status
 )
-from apps.quotes.scheduler import QuotesSubscription
-from apps.quotes.sqlite import QuotesSQLiteDatabase
-
-
-bot = telebot.TeleBot(BOT_TOKEN)
+from src.apps.quotes.scheduler import QuotesSubscription
+
+
+if os.getenv('DEBUG'):
+    logging.basicConfig(
+        level=logging.DEBUG,
+        format='%(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s',
+        datefmt="%d/%b/%Y %H:%M:%S",
+    )
+    logging.getLogger('urllib3').setLevel(logging.WARNING)
+    logging.getLogger('yt').setLevel(logging.WARNING)
+else:
+    logging.basicConfig(
+        level=logging.INFO,
+        format='[%(asctime)s] %(levelname)s [%(name)s] %(message)s',
+        datefmt="%d/%b/%Y %H:%M:%S",
+    )
+
+
+bot = telebot.TeleBot(os.getenv('BOT_TOKEN'))
 build_quotes_subscription = dict()
 remove_quotes_subscription_stage = dict()
 
@@ -83,8 +99,15 @@ def handle_bad_interaction(chat_id):
 
 
 def update_user_info(message):
-    db = QuotesSQLiteDatabase(DB_PATH)
-    db.update_user(message.chat.id, message.chat.username, message.chat.first_name, message.chat.last_name)
+    from src.apps.core.db import USER_TABLE
+    if not USER_TABLE.has(where={'chat_id': message.chat.id}):
+        USER_TABLE.insert([{
+            'chat_id': message.chat.id,
+            'username': message.chat.username,
+            'first_name': message.chat.first_name,
+            'last_name': message.chat.last_name
+        }])
+        USER_TABLE.commit()
 
 
 @bot.callback_query_handler(func=lambda call: call.data in ['quotes_menu'])

requirements.txt

@@ -1,4 +1,7 @@
+pydantic~=2.3.0
+pytz~=2021.1
+pyTelegramBotAPI~=3.8.3
+psycopg2-binary~=2.9.7
 requests~=2.26.0
 wheel~=0.37.0
-pyTelegramBotAPI~=3.8.3
-pytz~=2021.1
+wrapt~=1.15.0