Skip to content

Commit

Permalink
fix: change summary info of log folder (microsoft#552)
Browse files Browse the repository at this point in the history 
* update stat data in log summary

* show logic change

* add workspace None judgement

* fix CI
  • Loading branch information
@XianBW
XianBW authored Feb 5, 2025
1 parent 177096d commit 0eb258d
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 26 deletions.
37 changes: 27 additions & 10 deletions rdagent/log/mle_summary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
de.prepare()


def extract_mle_json(log_content):
def extract_mle_json(log_content: str) -> dict | None:
match = re.search(r"\{.*\}", log_content, re.DOTALL)
if match:
return json.loads(match.group(0))
Expand Down Expand Up @@ -54,9 +54,14 @@ def summarize_folder(log_folder: Path):
continue
loop_num = 0
made_submission_num = 0
valid_submission_num = 0
above_median_num = 0
get_medal_num = 0
bronze_num = 0
silver_num = 0
gold_num = 0
test_scores = {}
valid_scores = {}
medal = "None"
success_loop_num = 0

for msg in FileStorage(log_trace_path).iter_msg(): # messages in log trace
Expand All @@ -80,14 +85,21 @@ def summarize_folder(log_folder: Path):
f"mle_score.txt in {grade_output_path} not found, genarate it first!"
)
grade_output = extract_mle_json(grade_output_path.read_text())
if grade_output and grade_output["score"] is not None:
test_scores[loop_num - 1] = grade_output["score"]
if grade_output:
if grade_output["score"] is not None:
test_scores[loop_num - 1] = grade_output["score"]
if grade_output["valid_submission"]:
valid_submission_num += 1
if grade_output["above_median"]:
above_median_num += 1
if grade_output["any_medal"]:
medal = (
"gold"
if grade_output["gold_medal"]
else "silver" if grade_output["silver_medal"] else "bronze"
)
get_medal_num += 1
if grade_output["bronze_medal"]:
bronze_num += 1
if grade_output["silver_medal"]:
silver_num += 1
if grade_output["gold_medal"]:
gold_num += 1

if "feedback" in msg.tag and "evolving" not in msg.tag:
if bool(msg.content):
Expand All @@ -97,9 +109,14 @@ def summarize_folder(log_folder: Path):
{
"loop_num": loop_num,
"made_submission_num": made_submission_num,
"valid_submission_num": valid_submission_num,
"above_median_num": above_median_num,
"get_medal_num": get_medal_num,
"bronze_num": bronze_num,
"silver_num": silver_num,
"gold_num": gold_num,
"test_scores": test_scores,
"valid_scores": valid_scores,
"medal": medal,
"success_loop_num": success_loop_num,
}
)
Expand Down
60 changes: 44 additions & 16 deletions rdagent/log/ui/dsapp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,14 +144,18 @@ def evolving_win(data):
evo_id = 0

if evo_id in data:
st.subheader("codes")
workspace_win(data[evo_id]["evolving code"][0])
fb = data[evo_id]["evolving feedback"][0]
st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌"), anchor="c_feedback")
f1, f2, f3 = st.tabs(["execution", "return_checking", "code"])
f1.code(fb.execution, wrap_lines=True)
f2.code(fb.return_checking, wrap_lines=True)
f3.code(fb.code, wrap_lines=True)
if data[evo_id]["evolving code"][0] is not None:
st.subheader("codes")
workspace_win(data[evo_id]["evolving code"][0])
fb = data[evo_id]["evolving feedback"][0]
st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌"), anchor="c_feedback")
f1, f2, f3 = st.tabs(["execution", "return_checking", "code"])
f1.code(fb.execution, wrap_lines=True)
f2.code(fb.return_checking, wrap_lines=True)
f3.code(fb.code, wrap_lines=True)
else:
st.write("data[evo_id]['evolving code'][0] is None.")
st.write(data[evo_id])
else:
st.markdown("No evolving.")

Expand Down Expand Up @@ -268,20 +272,44 @@ def all_summarize_win():
summary = pd.read_pickle(state.log_folder / "summary.pkl")
summary = {k: v for k, v in summary.items() if "competition" in v}
base_df = pd.DataFrame(
columns=["Competition", "Total Loops", "Made Submission", "Successful Final Decision", "Medal"],
columns=[
"Competition",
"Total Loops",
"Successful Final Decision",
"Made Submission",
"Valid Submission",
"Above Median",
"Bronze",
"Silver",
"Gold",
"Any Medal",
],
index=summary.keys(),
)
for k, v in summary.items():
loop_num = v["loop_num"]
base_df.loc[k, "Competition"] = v["competition"]
base_df.loc[k, "Total Loops"] = loop_num
base_df.loc[k, "Made Submission"] = (
f"{v['made_submission_num']} ({round(v['made_submission_num'] / loop_num * 100, 2) if loop_num != 0 else 0}%)"
)
base_df.loc[k, "Successful Final Decision"] = (
f"{v['success_loop_num']} ({round(v['success_loop_num'] / loop_num * 100, 2) if loop_num != 0 else 0}%)"
)
base_df.loc[k, "Medal"] = v["medal"]
if loop_num == 0:
base_df.loc[k] = "N/A"
else:
base_df.loc[k, "Successful Final Decision"] = (
f"{v['success_loop_num']} ({round(v['success_loop_num'] / loop_num * 100, 2)}%)"
)
base_df.loc[k, "Made Submission"] = (
f"{v['made_submission_num']} ({round(v['made_submission_num'] / loop_num * 100, 2)}%)"
)
base_df.loc[k, "Valid Submission"] = (
f"{v['valid_submission_num']} ({round(v['valid_submission_num'] / loop_num * 100, 2)}%)"
)
base_df.loc[k, "Above Median"] = (
f"{v['above_median_num']} ({round(v['above_median_num'] / loop_num * 100, 2)}%)"
)
base_df.loc[k, "Bronze"] = f"{v['bronze_num']} ({round(v['bronze_num'] / loop_num * 100, 2)}%)"
base_df.loc[k, "Silver"] = f"{v['silver_num']} ({round(v['silver_num'] / loop_num * 100, 2)}%)"
base_df.loc[k, "Gold"] = f"{v['gold_num']} ({round(v['gold_num'] / loop_num * 100, 2)}%)"
base_df.loc[k, "Any Medal"] = f"{v['get_medal_num']} ({round(v['get_medal_num'] / loop_num * 100, 2)}%)"

st.dataframe(base_df)
# write curve
for k, v in summary.items():
Expand Down

0 comments on commit 0eb258d

Please sign in to comment.