Fix benchmark ci (Significant-Gravitas#5478)

Fix benchmark CI Signed-off-by: Merwane Hamadi <[email protected]>
colkassad · Oct 2, 2023 · a30cbcc · a30cbcc
1 parent 163ab75
commit a30cbcc
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 8 deletions.
diff --git a/.github/workflows/benchmark-ci.yml b/.github/workflows/benchmark-ci.yml
@@ -97,10 +97,10 @@ jobs:
           curl -sSL https://install.python-poetry.org | python -
 
       - name: Run regression tests
-        working-directory: ./autogpts/${{ matrix.agent-name }}/
         run: |
-          sh run &
-          sleep 20
+          ./run agent start ${{ matrix.agent-name }}
+          sleep 10
+          cd autogpts/${{ matrix.agent-name }}
           set +e # Ignore non-zero exit codes and continue execution
           echo "Running the following command: poetry run agbenchmark --maintain --mock"
 
@@ -123,8 +123,6 @@ jobs:
 
           echo "Running the following command: poetry run agbenchmark --test=WriteFile"
           poetry run agbenchmark --test=WriteFile
-          sh run_benchmark serve &
-          sleep 10
           cd ../../benchmark
           poetry install
           echo "Adding the BUILD_SKILL_TREE environment variable. This will attempt to add new elements in the skill tree. If new elements are added, the CI fails because they should have been pushed"

diff --git a/benchmark/agbenchmark/__main__.py b/benchmark/agbenchmark/__main__.py
@@ -9,12 +9,13 @@
 import click
 import pytest
 import toml
+from dotenv import load_dotenv
 from helicone.lock import HeliconeLockManager
 
 from agbenchmark.app import app
 from agbenchmark.reports.ReportManager import SingletonReportManager
 from agbenchmark.utils.data_types import AgentBenchmarkConfig
-from dotenv import load_dotenv
+
 load_dotenv()
 
 BENCHMARK_START_TIME_DT = datetime.now(timezone.utc)

diff --git a/benchmark/tests/test_benchmark_workflow.py b/benchmark/tests/test_benchmark_workflow.py
@@ -12,14 +12,14 @@
     "eval_id, input_text, expected_artifact_length, test_name, should_be_successful",
     [
         (
-            "f219f3d3-a41b-45a9-a3d0-389832086ee8",
+            "021c695a-6cc4-46c2-b93a-f3a9b0f4d123",
             "Write the word 'Washington' to a .txt file",
             0,
             "WriteFile",
             True,
         ),
         (
-            "021c695a-6cc4-46c2-b93a-f3a9b0f4d123",
+            "f219f3d3-a41b-45a9-a3d0-389832086ee8",
             "Read the file called file_to_read.txt and write its content to a file called output.txt",
             1,
             "ReadFile",