refactor(benchmark): Disable Helicone integrations

We want to upgrade the OpenAI library, but `helicone` does not support `openai@^1.0.0`, so we're disabling the Helicone integration for now.
lordjoe · Jan 16, 2024 · 056163e · 056163e
1 parent 797c5bb
commit 056163e
Show file tree

Hide file tree

Showing 5 changed files with 123 additions and 160 deletions.
diff --git a/benchmark/agbenchmark/__main__.py b/benchmark/agbenchmark/__main__.py
@@ -14,15 +14,15 @@
 
 load_dotenv()
 
-try:
-    if os.getenv("HELICONE_API_KEY"):
-        import helicone  # noqa
+# try:
+#     if os.getenv("HELICONE_API_KEY"):
+#         import helicone  # noqa
 
-        helicone_enabled = True
-    else:
-        helicone_enabled = False
-except ImportError:
-    helicone_enabled = False
+#         helicone_enabled = True
+#     else:
+#         helicone_enabled = False
+# except ImportError:
+#     helicone_enabled = False
 
 
 class InvalidInvocationError(ValueError):
@@ -35,12 +35,12 @@ class InvalidInvocationError(ValueError):
 BENCHMARK_START_TIME = BENCHMARK_START_TIME_DT.strftime("%Y-%m-%dT%H:%M:%S+00:00")
 
 
-if helicone_enabled:
-    from helicone.lock import HeliconeLockManager
+# if helicone_enabled:
+#     from helicone.lock import HeliconeLockManager
 
-    HeliconeLockManager.write_custom_property(
-        "benchmark_start_time", BENCHMARK_START_TIME
-    )
+#     HeliconeLockManager.write_custom_property(
+#         "benchmark_start_time", BENCHMARK_START_TIME
+#     )
 
 
 @click.group(cls=DefaultGroup, default_if_no_args=True)

diff --git a/benchmark/agbenchmark/reports/reports.py b/benchmark/agbenchmark/reports/reports.py
@@ -10,9 +10,10 @@
 from agbenchmark.config import AgentBenchmarkConfig
 from agbenchmark.reports.ReportManager import SingletonReportManager
 from agbenchmark.utils.data_types import ChallengeData, DifficultyLevel
-from agbenchmark.utils.get_data_from_helicone import get_data_from_helicone
 from agbenchmark.utils.utils import calculate_success_percentage
 
+# from agbenchmark.utils.get_data_from_helicone import get_data_from_helicone
+
 logger = logging.getLogger(__name__)
 
 
@@ -128,10 +129,10 @@ def finalize_reports(
     if info_details and test_name:
         if run_time is not None:
             cost = None
-            if "--mock" not in sys.argv and os.environ.get("HELICONE_API_KEY"):
-                logger.debug("Getting cost from Helicone")
-                cost = get_data_from_helicone(test_name)
-                logger.debug(f"Cost: {cost}")
+            # if "--mock" not in sys.argv and os.environ.get("HELICONE_API_KEY"):
+            #     logger.debug("Getting cost from Helicone")
+            #     cost = get_data_from_helicone(test_name)
+            #     logger.debug(f"Cost: {cost}")
 
             info_details["metrics"]["cost"] = cost
 

diff --git a/benchmark/agbenchmark/utils/challenge.py b/benchmark/agbenchmark/utils/challenge.py
@@ -64,10 +64,10 @@ async def test_method(
         # skip optional categories
         self.skip_optional_categories(config)
 
-        if os.environ.get("HELICONE_API_KEY"):
-            from helicone.lock import HeliconeLockManager
+        # if os.environ.get("HELICONE_API_KEY"):
+        #     from helicone.lock import HeliconeLockManager
 
-            HeliconeLockManager.write_custom_property("challenge", self.data.name)
+        #     HeliconeLockManager.write_custom_property("challenge", self.data.name)
 
         timeout = self.data.cutoff or 60