Read dlt secrets from a file

karanrat-rattanawichai · Aug 28, 2024 · 0e67f79 · 0e67f79
1 parent 584d44c
commit 0e67f79
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 13 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -105,19 +105,7 @@ services:
       DAGSTER_DB_USERNAME: dagster
       DAGSTER_DB_PASSWORD: $DAGSTER_DB_PASSWORD
       DBT_PROFILES_DIR: /run/secrets/dbt
-      SOURCES__SQL_DATABASE__PAGILA__CREDENTIALS__DRIVERNAME: "postgresql+psycopg2"
-      SOURCES__SQL_DATABASE__PAGILA__CREDENTIALS__HOST: app-db
-      SOURCES__SQL_DATABASE__PAGILA__CREDENTIALS__PORT: 5432
-      SOURCES__SQL_DATABASE__PAGILA__CREDENTIALS__DATABASE: pagila
-      SOURCES__SQL_DATABASE__PAGILA__CREDENTIALS__USERNAME: $DAGSTER_APP_DB_USERNAME
-      SOURCES__SQL_DATABASE__PAGILA__CREDENTIALS__PASSWORD: $DAGSTER_APP_DB_PASSWORD
-      ANALYTICS_DWH__PAGILA__DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: analytics-dwh
-      ANALYTICS_DWH__PAGILA__DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: raw_pagila
-      ANALYTICS_DWH__PAGILA__DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: $DAGSTER_CLICKHOUSE_USERNAME
-      ANALYTICS_DWH__PAGILA__DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: $DAGSTER_CLICKHOUSE_PASSWORD
-      ANALYTICS_DWH__PAGILA__DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000
-      ANALYTICS_DWH__PAGILA__DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123
-      ANALYTICS_DWH__PAGILA__DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0
+      DLT_SECRETS_DIR: /run/secrets/dlt
       DESTINATION__FILESYSTEM__BUCKET_URL: "data/dlt"
       DESTINATION__FILESYSTEM__KWARGS: '{"auto_mkdir": true}'
     volumes:
@@ -126,6 +114,8 @@ services:
     secrets:
       - source: dagster-dbt-profiles
         target: /run/secrets/dbt/profiles.yml
+      - source: dagster-dlt-secrets
+        target: /run/secrets/dlt/secrets.toml
     depends_on:
       - dagster-db
       - app-db # Required for loading dlt source assets
@@ -175,3 +165,5 @@ secrets:
     file: ./compose/metabase-db/secrets/db_password.txt
   dagster-dbt-profiles:
     file: ./compose/dagster/secrets/dbt_profiles.yml
+  dagster-dlt-secrets:
+    file: ./compose/dagster/secrets/dlt_secrets.toml
diff --git a/sakila_etl/sakila_etl/dlt_project/sources.py b/sakila_etl/sakila_etl/dlt_project/sources.py
@@ -1,10 +1,24 @@
+import os
 from typing import Dict
 import dlt
+from dlt.common.configuration.providers import ProjectDocProvider, SECRETS_TOML
 
 from .connectors.sql_database import sql_database
 from .utils import TableConfig
 
 
+# Add extra secret lookup file if provided
+DLT_SECRETS_DIR = os.getenv("DLT_SECRETS_DIR")
+if DLT_SECRETS_DIR:
+    provider = ProjectDocProvider(
+        name="custom",
+        supports_secrets=True,
+        project_dir=DLT_SECRETS_DIR,
+        file_name=SECRETS_TOML,
+        add_global_config=False,
+    )
+    dlt.secrets.register_provider(provider)
+
 TABLE_CONFIGS: Dict[str, TableConfig] = {
     "actor": {
         "hints": {

diff --git a/scripts/init.sh b/scripts/init.sh
@@ -20,6 +20,8 @@ echo $APP_DB_PASSWORD > compose/app-db/secrets/db_password.txt
 echo -n "Provide analytics-dwh password: "
 read -s ANALYTICS_DWH_PASSWORD
 echo
+
+echo -n Generating compose/dagster/secrets/dbt_profiles.yml .
 mkdir -p compose/dagster/secrets/
 cat > compose/dagster/secrets/dbt_profiles.yml <<EOF
 sakila_dwh:
@@ -47,6 +49,29 @@ sakila_dwh:
       # use_lw_deletes: False
       # custom_settings: <empty>
 EOF
+echo ..done
+
+echo -n Generating compose/dagster/secrets/dlt_secrets.toml .
+mkdir -p compose/dagster/secrets/
+cat > compose/dagster/secrets/dlt_secrets.toml <<EOF
+[sources.sql_database.pagila.credentials]
+drivername = "postgresql+psycopg2"
+host = "app-db"
+port = "5432"
+database = "pagila"
+username = "postgres"
+password = "$APP_DB_PASSWORD"
+
+[analytics_dwh__pagila.destination.clickhouse.credentials]
+host = "analytics-dwh"
+database = "raw_pagila"
+username = "admin"
+password = "$ANALYTICS_DWH_PASSWORD"
+port = 9000
+http_port = 8123
+secure = 0
+EOF
+echo ..done
 
 echo -n Downloading metabase clickhouse driver.
 mkdir -p compose/metabase/plugins/