Read config from secret managers like AWS's, Azure's (#1022)

* Read config secret from AWS, Azure * Update GitHub Actions Co-authored-by: Muazzam Ali <[email protected]>
agentlang-ai · Sep 1, 2023 · 40499be · 40499be
1 parent 625e463
commit 40499be
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 9 deletions.
diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml
@@ -19,6 +19,11 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - name: Installing fractl-io dependancies
+        run: |
+          git clone https://${{ secrets.REPO_READ_WRITE_TOKEN }}@github.com/fractl-io/fractl-config-secrets-reader
+          cd fractl-config-secrets-reader
+          lein install
       - name: Install dependencies
         run: lein deps
       - name: Install Node JS ${{ matrix.node-version }}
@@ -35,4 +40,4 @@ jobs:
           kill -9 `lsof -i:8000 -t`
         env:
           API_HOST: 127.0.0.1:8000
-          CI: true
+          CI: true
diff --git a/.github/workflows/draft-new-release.yml b/.github/workflows/draft-new-release.yml
@@ -26,6 +26,13 @@ jobs:
         run: |
           git config user.name "GitHub actions"
           git config user.email [email protected]
+          
+      - name: Installing fractl-io dependancies
+        run: |
+          git clone https://${{ secrets.REPO_READ_WRITE_TOKEN }}@github.com/fractl-io/fractl-config-secrets-reader
+          cd fractl-config-secrets-reader
+          lein install
+
       - name: Install dependencies
         run: lein deps
 

diff --git a/.github/workflows/fractl-clj.yml b/.github/workflows/fractl-clj.yml
@@ -34,6 +34,11 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
+    - name: Installing fractl-io dependancies
+      run: |
+        git clone https://${{ secrets.REPO_READ_WRITE_TOKEN }}@github.com/fractl-io/fractl-config-secrets-reader
+        cd fractl-config-secrets-reader
+        lein install
     - name: Install dependencies
       run: lein deps
     - name: Run fractl test
@@ -45,4 +50,4 @@ jobs:
         POSTGRES_PORT: 5432
         POSTGRES_PASSWORD: postgres
     - name: Clean target
-      run: lein clean
+      run: lein clean
diff --git a/.github/workflows/fractl-cljs.yml b/.github/workflows/fractl-cljs.yml
@@ -17,11 +17,16 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - name: Installing fractl-io dependancies
+        run: |
+          git clone https://${{ secrets.REPO_READ_WRITE_TOKEN }}@github.com/fractl-io/fractl-config-secrets-reader
+          cd fractl-config-secrets-reader
+          lein install
       - name: Install dependencies
         run: lein deps
       - name: Run cljs test
         run: lein fig:ci
       - name: Run cljs reagent test
         run: lein fig:rci
       - name: Clean target
-        run: lein clean
+        run: lein clean
diff --git a/.github/workflows/publish-new-release.yml b/.github/workflows/publish-new-release.yml
@@ -14,7 +14,11 @@ jobs:
     if: github.event.pull_request.merged == true # only merged pull requests must trigger this job
     steps:
       - uses: actions/checkout@v2
-
+      - name: Installing fractl-io dependancies
+        run: |
+          git clone https://${{ secrets.REPO_READ_WRITE_TOKEN }}@github.com/fractl-io/fractl-config-secrets-reader
+          cd fractl-config-secrets-reader
+          lein install
       - name: Install dependencies
         run: lein deps
       - name: Generate pom.xml needed for mvn deploy
@@ -66,4 +70,4 @@ jobs:
         run: |
           rm -r /home/runner/work/fractl/fractl/lib/fractl-${{env.RELEASE_VERSION}}.jar
           rm -r /home/runner/work/fractl/fractl/lib/fractl-${{env.RELEASE_VERSION}}-standalone.jar
-          lein clean
+          lein clean
diff --git a/project.clj b/project.clj
@@ -40,7 +40,8 @@
                  [org.clojure/algo.generic "0.1.3"]
                  [metosin/ring-swagger "0.26.2"]
                  [cheshire "5.11.0"]
-                 [org.liquibase/liquibase-core "4.23.0"]]
+                 [org.liquibase/liquibase-core "4.23.0"]
+                 [fractl-io/fractl-config-secrets-reader "0.1.0"]]
 
   :java-source-paths ["src/java"]
 

diff --git a/src/fractl/core.clj b/src/fractl/core.clj
@@ -26,7 +26,8 @@
             [fractl.rbac.core :as rbac]
             [fractl.gpt.core :as gpt]
             [fractl.swagger.doc :as doc]
-            [fractl.swagger.docindex :as docindex])
+            [fractl.swagger.docindex :as docindex]
+            [fractl-config-secrets-reader.core :as fractl-secret-reader])
   (:import [java.util Properties]
            [java.net URL]
            [java.io File]
@@ -262,8 +263,12 @@
   (let [config (or (config-data-key options) (load-config options))]
     (when-let [extn (:script-extn config)]
       (u/set-script-extn! extn))
-    (let [[model _ :as m] (maybe-read-model (find-model-to-read args config))]
-      [m (merge (:config model) config)])))
+    (let [[model _ :as m] (maybe-read-model (find-model-to-read args config))
+          config (merge (:config model) config)]
+      (try
+        [m (fractl-secret-reader/read-secret-config config)]
+        (catch Exception e
+          (u/throw-ex (str "error reading secret config " e)))))))
 
 (defn- read-model-from-resource [component-root]
   (let [^String s (slurp