Fix Monitor (lm-sys#1762)

AIGuy207 · Jun 22, 2023 · 861edd0 · 861edd0
1 parent be39e0c
commit 861edd0
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 34 deletions.
diff --git a/docs/commands/webserver.md b/docs/commands/webserver.md
@@ -26,7 +26,7 @@ cd fastchat_logs/server0
 export OPENAI_API_KEY=
 export ANTHROPIC_API_KEY=
 
-python3 -m fastchat.serve.gradio_web_server_multi --controller http://localhost:21001 --moderate --concurrency 10 --add-chatgpt --add-claude --add-bard --anony-only --elo ~/elo_results_20230508.pkl
+python3 -m fastchat.serve.gradio_web_server_multi --controller http://localhost:21001 --concurrency 10 --add-chatgpt --add-claude --add-palm --anony-only --elo ~/elo_results/elo_results_20230619.pkl --leaderboard-table-file ~/elo_results/leaderboard_table_20230619.csv
 
 python3 backup_logs.py
 ```

diff --git a/fastchat/model/model_registry.py b/fastchat/model/model_registry.py
@@ -69,7 +69,7 @@ def get_model_info(name: str) -> ModelInfo:
     "A finetuned LLaMA model on assistant style data by Nomic AI",
 )
 register_model_info(
-    ["guanaco-33b"],
+    ["guanaco-33b", "guanaco-65b"],
     "Guanaco",
     "https://github.com/artidoro/qlora",
     "a model fine-tuned with QLoRA by UW",
@@ -123,10 +123,10 @@ def get_model_info(name: str) -> ModelInfo:
     "Stability AI language models",
 )
 register_model_info(
-    ["mpt-7b-chat"],
+    ["mpt-7b-chat", "mpt-30b-chat"],
     "MPT-Chat",
-    "https://www.mosaicml.com/blog/mpt-7b",
-    "a chatbot fine-tuned from MPT-7B by MosaicML",
+    "https://www.mosaicml.com/blog/mpt-30b",
+    "a chatbot fine-tuned from MPT by MosaicML",
 )
 register_model_info(
     ["fastchat-t5-3b", "fastchat-t5-3b-v1.0"],

diff --git a/fastchat/serve/monitor/monitor.py b/fastchat/serve/monitor/monitor.py
@@ -42,6 +42,15 @@ def make_leaderboard_md(elo_results):
     return leaderboard_md
 
 
+def make_leaderboard_md_live(elo_results):
+    leaderboard_md = f"""
+# Leaderboard
+Last updated: {elo_results["last_updated_datetime"]}
+{elo_results["leaderboard_table"]}
+"""
+    return leaderboard_md
+
+
 def update_elo_components(max_num_files, elo_results_file):
     log_files = get_log_files(max_num_files)
 
@@ -50,7 +59,7 @@ def update_elo_components(max_num_files, elo_results_file):
         battles = clean_battle_data(log_files)
         elo_results = report_elo_analysis_results(battles)
 
-        leader_component_values[0] = make_leaderboard_md(elo_results)
+        leader_component_values[0] = make_leaderboard_md_live(elo_results)
         leader_component_values[1] = elo_results["win_fraction_heatmap"]
         leader_component_values[2] = elo_results["battle_count_heatmap"]
         leader_component_values[3] = elo_results["average_win_rate_bar"]
@@ -168,36 +177,38 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file):
         md = "Loading ..."
         p1 = p2 = p3 = p4 = None
 
-    leader_component_values[:] = [md, p1, p2, p3, p4]
-
     md_1 = gr.Markdown(md, elem_id="leaderboard_markdown")
 
-    data = load_leaderboard_table_csv(leaderboard_table_file)
-    headers = [
-        "Model",
-        "Arena Elo rating",
-        "MT-bench (score)",
-        "MT-bench (win rate %)",
-        "MMLU",
-    ]
-    values = []
-    for item in data:
-        row = []
-        for key in headers:
-            value = item[key]
-            row.append(value)
-        values.append(row)
-    values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9)
-
-    headers[1] = "⭐ " + headers[1]
-    headers[2] = "📈 " + headers[2]
-
-    gr.Dataframe(
-        headers=headers,
-        datatype=["markdown", "number", "number", "number", "number"],
-        value=values,
-        elem_id="leaderboard_dataframe",
-    )
+    if leaderboard_table_file:
+        data = load_leaderboard_table_csv(leaderboard_table_file)
+        headers = [
+            "Model",
+            "Arena Elo rating",
+            "MT-bench (score)",
+            "MT-bench (win rate %)",
+            "MMLU",
+        ]
+        values = []
+        for item in data:
+            row = []
+            for key in headers:
+                value = item[key]
+                row.append(value)
+            values.append(row)
+        values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9)
+
+        headers[1] = "⭐ " + headers[1]
+        headers[2] = "📈 " + headers[2]
+
+        gr.Dataframe(
+            headers=headers,
+            datatype=["markdown", "number", "number", "number", "number"],
+            value=values,
+            elem_id="leaderboard_dataframe",
+        )
+        gr.Markdown("If you want to see more models, please help us [add them](https://github.com/lm-sys/FastChat/blob/main/docs/arena.md#how-to-add-a-new-model).")
+    else:
+        pass
 
     gr.Markdown(
         f"""## More Statistics for Chatbot Arena\n
@@ -206,6 +217,8 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file):
 """
     )
 
+    leader_component_values[:] = [md, p1, p2, p3, p4]
+
     with gr.Row():
         with gr.Column():
             gr.Markdown(