publish raw alpha version & update workflow

.github/workflows/ci.yml

@@ -12,24 +12,38 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
+    services:
+      clickhouse:
+        image: yandex/clickhouse-server:latest
+        ports:
+          - 9000:9000
+          - 8123:8123
+        options: --health-cmd='curl -f http://localhost:8123 || exit 1' --health-interval=10s --health-timeout=5s --health-retries=5
+
     steps:
-    - uses: actions/checkout@v2
+    - name: Check out code
+      uses: actions/checkout@v2
 
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
+        python-version: '3.x'
+
+    - name: Install Poetry
+      run: pip install poetry
 
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install poetry
-        poetry install
+      run: poetry install --with dev
 
-    - name: Lint with Ruff
+    - name: Wait for ClickHouse service to be healthy
       run: |
-        poetry run ruff check clickhouserag/
+        while ! curl -sSf http://localhost:8123 > /dev/null; do
+          echo "Waiting for ClickHouse to be up..."
+          sleep 5
+        done
 
-    - name: Test with pytest
-      run: |
-        poetry run pytest
+    - name: Run ruff
+      run: poetry run ruff .
+
+    - name: Run tests
+      run: poetry run pytest

.github/workflows/publish.yml

@@ -0,0 +1,56 @@
+name: Publish Python Package
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  test-and-publish:
+    runs-on: ubuntu-latest
+    environment: pypi-publish
+
+    services:
+      clickhouse:
+        image: yandex/clickhouse-server:latest
+        ports:
+          - 9000:9000
+          - 8123:8123
+        options: --health-cmd='curl -f http://localhost:8123 || exit 1' --health-interval=10s --health-timeout=5s --health-retries=5
+
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - name: Install Poetry
+      run: pip install poetry
+
+    - name: Install dependencies
+      run: poetry install --with dev
+
+    - name: Wait for ClickHouse service to be healthy
+      run: |
+        while ! curl -sSf http://localhost:8123 > /dev/null; do
+          echo "Waiting for ClickHouse to be up..."
+          sleep 5
+        done
+
+    - name: Run ruff
+      run: poetry run ruff .
+
+    - name: Run tests
+      run: poetry run pytest
+
+    - name: Build package
+      run: poetry build
+
+    - name: Publish package to PyPI
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+      run: poetry publish --username $TWINE_USERNAME --password $TWINE_PASSWORD

clickhouserag/config/clickhouse_config.py

@@ -0,0 +1 @@
+"""Clickhouse configuration module for ClickhouseRAG."""

clickhouserag/data_access/clickhouse_client.py

@@ -0,0 +1,85 @@
+"""Clickhouse client module for Clickhouse data access."""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from clickhouse_driver import Client
+
+from clickhouserag.data_access.abstract_client import ClickhouseClient
+
+
+class ClickhouseConnectClient(ClickhouseClient):
+    """Clickhouse client implementation using clickhouse-driver."""
+
+    def __init__(self, host: str, port: int, username: str, password: str, database: str):
+        """Initialize ClickhouseConnectClient.
+
+        Args:
+        ----
+            host (str): The host of the Clickhouse server.
+            port (int): The port of the Clickhouse server.
+            username (str): The username for Clickhouse authentication.
+            password (str): The password for Clickhouse authentication.
+            database (str): The database to connect to.
+
+        """
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.database = database
+        self.client = None
+        self.logger = logging.getLogger(__name__)
+
+    def connect(self) -> None:
+        """Connect to the Clickhouse database."""
+        try:
+            self.client = Client(host=self.host, port=self.port, user=self.username, password=self.password, database=self.database)
+        except Exception as e:
+            self.logger.error(f"Failed to connect to Clickhouse: {e}")
+            raise
+
+    def execute_query(self, query: str, params: Optional[Dict[str, Any]] = None) -> Any:
+        """Execute a query in the Clickhouse database."""
+        try:
+            if params:
+                return self.client.execute(query, params)
+            return self.client.execute(query)
+        except Exception as e:
+            self.logger.error(f"Failed to execute query: {e}")
+            raise
+
+    def fetch_one(self, query: str, params: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
+        """Fetch one result from the Clickhouse database."""
+        try:
+            result = self.execute_query(query, params)
+            return result[0] if result else None
+        except Exception as e:
+            self.logger.error(f"Failed to fetch one: {e}")
+            raise
+
+    def fetch_all(self, query: str, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
+        """Fetch all results from the Clickhouse database."""
+        try:
+            return self.execute_query(query, params)
+        except Exception as e:
+            self.logger.error(f"Failed to fetch all: {e}")
+            raise
+
+    def fetch_column(self, query: str, column: int = 0, params: Optional[Dict[str, Any]] = None) -> List[Any]:
+        """Fetch a specific column from the Clickhouse database."""
+        try:
+            result = self.execute_query(query, params)
+            return [row[column] for row in result]
+        except Exception as e:
+            self.logger.error(f"Failed to fetch column: {e}")
+            raise
+
+    def close(self) -> None:
+        """Close the connection to the Clickhouse database."""
+        try:
+            if self.client:
+                self.client.disconnect()
+        except Exception as e:
+            self.logger.error(f"Failed to close connection: {e}")
+            raise

clickhouserag/data_access/clickhouse_table.py

@@ -0,0 +1,54 @@
+"""Clickhouse table management module."""
+
+from typing import Any, Dict, List, Optional
+
+from clickhouserag.data_access.abstract_table import ClickhouseTable
+from clickhouserag.data_access.clickhouse_client import ClickhouseConnectClient
+
+
+class ClickhouseTableManager(ClickhouseTable):
+    """Clickhouse table manager implementation."""
+
+    def __init__(self, client: ClickhouseConnectClient, table_name: str) -> None:
+        """Initialize ClickhouseTableManager.
+
+        Args:
+        ----
+            client (ClickhouseConnectClient): The Clickhouse client.
+            table_name (str): The name of the table.
+
+        """
+        super().__init__(client, table_name)
+
+    def insert(self, values: List[Dict[str, Any]]) -> None:
+        """Insert values into the table."""
+        query = f"INSERT INTO {self.table_name} VALUES"
+        self.client.execute_query(query, values)
+
+    def update(self, values: Dict[str, Any], conditions: Dict[str, Any]) -> None:
+        """Update values in the table based on conditions."""
+        set_clause = ", ".join([f"{key} = %({key})s" for key in values.keys()])
+        condition_clause = " AND ".join([f"{key} = %({key})s" for key in conditions.keys()])
+        query = f"ALTER TABLE {self.table_name} UPDATE {set_clause} WHERE {condition_clause}"
+        self.client.execute_query(query, {**values, **conditions})
+
+    def delete(self, conditions: Dict[str, Any]) -> None:
+        """Delete values from the table based on conditions."""
+        condition_clause = " AND ".join([f"{key} = %(cond_{key})s" for key in conditions.keys()])
+        query = f"DELETE FROM {self.table_name} WHERE {condition_clause}"
+        params = {f"cond_{key}": value for key, value in conditions.items()}
+        self.client.execute_query(query, params)
+
+    def search(self, query: str, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
+        """Search the table based on a query."""
+        return self.client.fetch_all(query, params)
+
+    def fetch_all(self, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
+        """Fetch all values from the table."""
+        query = f"SELECT * FROM {self.table_name}"
+        return self.client.fetch_all(query, params)
+
+    def reset_table(self) -> None:
+        """Reset the table."""
+        query = f"TRUNCATE TABLE {self.table_name}"
+        self.client.execute_query(query)