manual optional tests

.github/workflows/lmql-optional-tests.yml

@@ -0,0 +1,28 @@
+name: Run Tests
+
+# optional tests only run manually
+on: workflow_dispatch
+
+
+jobs:
+    optional-tests:
+      runs-on: lmql-ci
+      steps:
+          - uses: actions/checkout@v3
+          - name: Setup Fresh Virtual Environment
+            run: |
+              pip install --upgrade pip
+              python3.10 -m venv env
+              export PATH=$PATH:/home/docker/.local/bin
+              source env/bin/activate
+              echo "VIRTUAL ENV:" $VIRTUAL_ENV
+          # cache dependencies
+          - name: Cache Dependencies
+            uses: actions/cache@v3
+            with:
+              path: env/lib/python3.10/site-packages
+              key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }}
+          - name: Install Dependencies
+            run: source env/bin/activate && pip install -e '.[hf,hf-accel,tests,llama]' && pip install langchain
+          - name: Run Optional Tests
+            run: source env/bin/activate && python src/lmql/tests/all.py only optional --failearly

.github/workflows/lmql-tests.yml

@@ -3,6 +3,8 @@ name: Run Tests
 on:
   push:
     branches: [ tests ]
+  pull_request:
+    branches: [ main ]
 
 jobs:
     tests:
@@ -23,8 +25,6 @@ jobs:
               path: env/lib/python3.10/site-packages
               key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }}
           - name: Install Dependencies
-            run: source env/bin/activate && pip install -e '.[hf,hf-accel,tests,llama]' && pip install langchain
-          - name: Run Tests
-            env:
-              OPENAI_API_KEY: ${{ secrets.LMQL_CI_OPENAI_KEY }}
-            run: source env/bin/activate && python src/lmql/tests/all.py langchain --failearly
+            run: source env/bin/activate && pip install -e '.[hf,hf-accel,tests,llama]'
+          - name: Run Default Tests
+            run: source env/bin/activate && python src/lmql/tests/all.py --failearly

src/lmql/tests/all.py

@@ -51,9 +51,11 @@ def run_tests(directory):
     # default is the explicit name for .
     if "default" in sys.argv:
         targets.append(THIS_DIR)
+
+    include_all_optional = "optional" in sys.argv
 
     optional_targets = os.listdir(os.path.join(THIS_DIR, "optional"))
-    optional_targets = [t for t in optional_targets if os.path.isdir(os.path.join(THIS_DIR, "optional", t)) and t in sys.argv]
+    optional_targets = [t for t in optional_targets if os.path.isdir(os.path.join(THIS_DIR, "optional", t)) and (t in sys.argv or include_all_optional)]
     optional_targets = [os.path.join(THIS_DIR, "optional", t) for t in optional_targets]
 
     targets = sorted(set(targets + optional_targets))