Skip to content

Commit

Permalink
optimizing the score not to save unused statistics
Browse files Browse the repository at this point in the history
  • Loading branch information
@BDonnot
BDonnot committed Jul 21, 2021
1 parent ce4d78b commit 930e853
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 101 deletions.
123 changes: 44 additions & 79 deletions grid2op/tests/test_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,21 +108,16 @@ def test_can_compute(self):
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

# delete them
stats_0 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN)
stats_1 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)
stats_2 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)
stats_0.clear_all()
stats_1.clear_all()
stats_2.clear_all()
scores.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
# assert not os.path.exists(os.path.join(env.get_path_env(),
# EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

def test_donothing_0(self):
"""test that do nothing has a score of 0.00"""
Expand All @@ -135,25 +130,20 @@ def test_donothing_0(self):
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

my_agent = DoNothingAgent(env.action_space)
my_scores, *_ = scores.get(my_agent)
assert np.max(np.abs(my_scores)) <= self.tol_one

# delete them
stats_0 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN)
stats_1 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)
stats_2 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)
stats_0.clear_all()
stats_1.clear_all()
stats_2.clear_all()
scores.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
# assert not os.path.exists(os.path.join(env.get_path_env(),
# EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

def test_modif_max_step_decrease(self):
"""
Expand All @@ -169,32 +159,27 @@ def test_modif_max_step_decrease(self):
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

my_agent = DoNothingAgent(env.action_space)
my_scores, *_ = scores.get(my_agent)
assert np.max(np.abs(my_scores)) <= self.tol_one, "error for the first do nothing"

scores2 = ScoreL2RPN2020(env, nb_scenario=2, verbose=0, max_step=10)
assert not scores2._recomputed_dn
assert not scores2._recomputed_no_ov
assert not scores2._recomputed_no_ov_rp
my_agent = DoNothingAgent(env.action_space)
my_scores2, *_ = scores2.get(my_agent)
assert np.max(np.abs(my_scores2)) <= self.tol_one, "error for the second do nothing"

# delete them
stats_0 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN)
stats_1 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)
stats_2 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)
stats_0.clear_all()
stats_1.clear_all()
stats_2.clear_all()
scores.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
# assert not os.path.exists(os.path.join(env.get_path_env(),
# EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

def test_modif_max_step_increase(self):
"""test that i can modify the max step (and that if I increase it it does trigger a recomputation)"""
Expand All @@ -207,29 +192,24 @@ def test_modif_max_step_increase(self):
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

my_agent = DoNothingAgent(env.action_space)
my_scores, *_ = scores.get(my_agent)
assert np.max(np.abs(my_scores)) <= self.tol_one, "error for the first do nothing"

scores2 = ScoreL2RPN2020(env, nb_scenario=2, verbose=0, max_step=10)
assert scores2._recomputed_dn
assert scores2._recomputed_no_ov
assert scores2._recomputed_no_ov_rp

# delete them
stats_0 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN)
stats_1 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)
stats_2 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)
stats_0.clear_all()
stats_1.clear_all()
stats_2.clear_all()
scores.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
# assert not os.path.exists(os.path.join(env.get_path_env(),
# EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

def test_modif_nb_scenario(self):
"""
Expand All @@ -245,33 +225,29 @@ def test_modif_nb_scenario(self):
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

my_agent = DoNothingAgent(env.action_space)
my_scores, *_ = scores.get(my_agent)
assert np.max(np.abs(my_scores)) <= self.tol_one, "error for the first do nothing"

scores2 = ScoreL2RPN2020(env, nb_scenario=4, verbose=0, max_step=5)
assert scores2._recomputed_dn
assert scores2._recomputed_no_ov
assert scores2._recomputed_no_ov_rp

scores2 = ScoreL2RPN2020(env, nb_scenario=3, verbose=0, max_step=5)
assert not scores2._recomputed_dn
assert not scores2._recomputed_no_ov
assert not scores2._recomputed_no_ov_rp

# delete them
stats_0 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN)
stats_1 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)
stats_2 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)
stats_0.clear_all()
stats_1.clear_all()
stats_2.clear_all()
# delete them
scores.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
# assert not os.path.exists(os.path.join(env.get_path_env(),
# EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

def test_reco_noov_80(self):
"""test that do nothing has a score of 80.0 if it is run with "no overflow disconnection" """
Expand All @@ -281,14 +257,13 @@ def test_reco_noov_80(self):
# I cannot decrease the max step: it must be above the number of steps the do nothing does
scores = ScoreL2RPN2020(env, nb_scenario=2, verbose=0, max_step=130)
assert scores._recomputed_dn
assert scores._recomputed_no_ov
assert scores._recomputed_no_ov_rp

# the statistics have been properly computed
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))

my_agent = DoNothingAgent(env.action_space)
my_scores, *_ = scores.get(my_agent)
Expand All @@ -299,25 +274,20 @@ def test_reco_noov_80(self):
with make("rte_case5_example", test=True, param=param) as env:
scores2 = ScoreL2RPN2020(env, nb_scenario=2, verbose=0, max_step=130)
assert not scores2._recomputed_dn
assert not scores2._recomputed_no_ov
assert not scores2._recomputed_no_ov_rp
my_agent = RecoPowerlineAgent(env.action_space)
my_scores, *_ = scores2.get(my_agent)
assert np.max(np.abs(np.array(my_scores) - 80.0)) <= self.tol_one

# delete them
stats_0 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN)
stats_1 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)
stats_2 = EpisodeStatistics(env, ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)
stats_0.clear_all()
stats_1.clear_all()
stats_2.clear_all()
# delete them
scores.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
# assert not os.path.exists(os.path.join(env.get_path_env(),
# EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))


class TestICAPSSCORE(HelperTests):
Expand All @@ -334,8 +304,8 @@ def test_can_compute(self):
env_seeds=[1, 2], # with these seeds do nothing goes till the end
agent_seeds=[3, 4])
my_agent = DoNothingAgent(env.action_space)
scores, n_played, total_ts = scores.get(my_agent)
for (ep_score, op_score, alarm_score) in scores:
scores_this, n_played, total_ts = scores.get(my_agent)
for (ep_score, op_score, alarm_score) in scores_this:
assert np.abs(ep_score - 30.) <= self.tol_one, f"wrong score for the episode: {ep_score} vs 30."
assert np.abs(op_score - 0.) <= self.tol_one, f"wrong score for the operationnal cost: " \
f"{op_score} vs 0."
Expand All @@ -346,21 +316,16 @@ def test_can_compute(self):
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreICAPS2021.NAME_DN)))
assert os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreICAPS2021.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreICAPS2021.NAME_RP_NO_OVERFLOW)))

# delete them
stats_0 = EpisodeStatistics(env, ScoreICAPS2021.NAME_DN)
stats_1 = EpisodeStatistics(env, ScoreICAPS2021.NAME_DN_NO_OVERWLOW)
stats_2 = EpisodeStatistics(env, ScoreICAPS2021.NAME_RP_NO_OVERWLOW)
stats_0.clear_all()
stats_1.clear_all()
stats_2.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreICAPS2021.NAME_DN)))
scores.clear_all()
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreICAPS2021.NAME_DN_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN)))
# assert not os.path.exists(os.path.join(env.get_path_env(),
# EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_DN_NO_OVERWLOW)))
assert not os.path.exists(os.path.join(env.get_path_env(),
EpisodeStatistics.get_name_dir(ScoreICAPS2021.NAME_RP_NO_OVERWLOW)))
EpisodeStatistics.get_name_dir(ScoreL2RPN2020.NAME_RP_NO_OVERFLOW)))


if __name__ == "__main__":
Expand Down
2 changes: 1 addition & 1 deletion grid2op/utils/icaps_2021_scores.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class ScoreICAPS2021(ScoreL2RPN2020):
"""

NAME_DN = "icaps2021_dn"
NAME_DN_NO_OVERWLOW = "icaps2021_no_overflow"
# NAME_DN_NO_OVERFLOW = "icaps2021_no_overflow"
NAME_RP_NO_OVERWLOW = "icaps2021_no_overflow_reco"

def __init__(self,
Expand Down
Loading

0 comments on commit 930e853

Please sign in to comment.