feat: add updater job

.github/workflows/updater.yml

@@ -0,0 +1,95 @@
+# name: Run merger and save each hash
+
+on:
+    workflow_dispatch:
+
+jobs:
+  generate_hash_letters:
+    runs-on: ubuntu-latest
+    outputs:
+        string_list: ${{ steps.generate.outputs.string_list }}
+    steps:
+        - uses: actions/checkout@v4
+
+        - name: Set up Python
+          uses: actions/setup-python@v5
+          with:
+            python-version: '3.9'  # Specify your Python version
+        - name: Install dependencies
+          run: |
+            pip install -e .
+
+        - name: Generate Strings
+          id: generate
+          run: |
+            # Run your script here to produce a list of strings
+            message=$(python src/parselmouth/updater_producer.py)
+            echo $message
+            echo "string_list=$message" >> $GITHUB_OUTPUT
+          env:
+            R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }}
+            R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }}
+            R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }}
+            R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }}
+
+
+  updater_of_records:
+    runs-on: ubuntu-latest
+    needs: generate_hash_letters
+    strategy:
+        matrix:
+            subdirs: ${{fromJson(needs.generate_hash_letters.outputs.string_list)}}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'  # Specify your Python version
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+
+      - name: Run dumper
+        run: python src/parselmouth/updater.py ${{ matrix.subdirs }}
+        env:
+            R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }}
+            R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }}
+            R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }}
+            R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }}
+
+      - name: Upload partial index
+        uses: actions/upload-artifact@v4
+        with:
+            name: partial_index
+            path: output
+
+  updater_of_index:
+    runs-on: ubuntu-latest
+    needs: updater_of_records    
+    steps:
+        - uses: actions/checkout@v4
+
+        - name: Set up Python
+          uses: actions/setup-python@v5
+          with:
+            python-version: '3.9'  # Specify your Python version
+        - name: Install dependencies
+          run: |
+                python -m pip install --upgrade pip
+                pip install -e .
+        - name: Get partial index artifacts
+          uses: actions/download-artifact@v4
+          with:
+            name: partial_index
+
+        - name: Run dumper
+          run: python src/parselmouth/updater_merger.py
+          env:
+            R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }}
+            R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }}
+            R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }}
+            R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }}

pyproject.toml

@@ -23,4 +23,5 @@ dependencies = [
     "requests~=2.31.0",
     "conda-forge-metadata~=0.6.2",
     "boto3~=1.34.51",
+    "python-dotenv~=1.0.1",
 ]