forked from mila-iqia/babyai
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add rl dataeff ttest and fps scripts
- Loading branch information
Showing
2 changed files
with
161 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
#!/usr/bin/env python3 | ||
|
||
import os | ||
import re | ||
import babyai.plotting as bp | ||
import pandas | ||
import argparse | ||
import json | ||
import numpy | ||
from scipy import stats | ||
|
||
# dictionary of level to window used | ||
levels = { | ||
'GoToRedBallGrey': 10, | ||
'GoToRedBall': 100, | ||
'GoToLocal': 100, | ||
'PickupLoc': 100, | ||
# 'PutNextLocal': 100, | ||
# 'GoTo': 100, | ||
} | ||
|
||
model1 = 'expert_filmcnn' | ||
model2 = 'expert_filmcnn_endpool' | ||
model1 = 'expert_filmcnn_endpool' | ||
model2 = 'expert_filmcnn_endpool_res/lr-0.0001' | ||
model1 = 'expert_filmcnn_endpool_res/lr-0.0001' | ||
model2 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001' | ||
model1 = 'expert_filmcnn_endpool_res/lr-0.0001' | ||
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.0001' | ||
model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001' | ||
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.0001' | ||
model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001' | ||
model2 = 'expert_filmcnn_endpool_res_bow/lr-0.0001' | ||
|
||
model1 = 'expert_filmcnn_endpool_res/lr-0.00005' | ||
model2 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005' | ||
model1 = 'expert_filmcnn_endpool_res/lr-0.00005' | ||
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.00005' | ||
|
||
model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005' | ||
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.00005' | ||
model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005' | ||
model2 = 'expert_filmcnn_endpool_res_bow/lr-0.00005' | ||
|
||
|
||
path1 = [f'../beluga/logs/rl/{level}/{model1}' for level in levels] | ||
path2 = [f'../beluga/logs/rl/{level}/{model2}' for level in levels] | ||
windows = [levels[i] for i in levels] | ||
|
||
|
||
def dataeff(df_model, window): | ||
smoothed_sr = df_model['success_rate'].rolling(window, center=True).mean() | ||
if smoothed_sr.max() < 0.99: | ||
# print('not done, success rate is only {}% so far'.format(100 * smoothed_sr.max())) | ||
return int(1e9) | ||
return df_model[smoothed_sr >= 0.99].iloc[0].episodes | ||
|
||
def get_dataeff(path, window): | ||
df = pandas.concat(bp.load_logs(path), sort=True) | ||
models = df['model'].unique() | ||
data = [] | ||
for model in models: | ||
x = df[df['model'] == model] | ||
eff = float(dataeff(x, window)) | ||
# print(model, eff) | ||
if eff != 1e9: | ||
data.append(eff) | ||
return numpy.array(data) | ||
|
||
data1 = [get_dataeff(path, window) for path, window in zip(path1, windows)] | ||
data2 = [get_dataeff(path, window) for path, window in zip(path2, windows)] | ||
|
||
'https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_ind.html' | ||
ttests = [stats.ttest_ind(d1, d2, equal_var=False) for d1, d2 in zip(data1, data2)] | ||
for ttest in ttests: | ||
print(f'{ttest.statistic}\t{ttest.pvalue}') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
#!/usr/bin/env python3 | ||
|
||
import os | ||
import re | ||
import babyai.plotting as bp | ||
import pandas | ||
import argparse | ||
import json | ||
import numpy | ||
from scipy import stats | ||
|
||
levels = [ | ||
'GoToRedBallGrey', | ||
'GoToRedBall', | ||
'GoToLocal', | ||
'PickupLoc', | ||
'PutNextLocal', | ||
# 'GoTo' | ||
] | ||
|
||
models = [ | ||
'expert_filmcnn', | ||
'expert_filmcnn_endpool', | ||
'expert_filmcnn_endpool_res/lr-0.0001', | ||
'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001', | ||
'expert_filmcnn_endpool_res_pixels/lr-0.0001', | ||
'expert_filmcnn_endpool_res_bow/lr-0.0001' | ||
] | ||
|
||
models = [ | ||
'expert_filmcnn_endpool_res/lr-0.00005', | ||
'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005', | ||
'expert_filmcnn_endpool_res_pixels/lr-0.00005', | ||
'expert_filmcnn_endpool_res_bow/lr-0.00005' | ||
] | ||
|
||
|
||
def dataeff(df_model, window): | ||
smoothed_sr = df_model['success_rate'].rolling(window, center=True).mean() | ||
if smoothed_sr.max() < 0.99: | ||
# print('not done, success rate is only {}% so far'.format(100 * smoothed_sr.max())) | ||
return int(1e9) | ||
return df_model[smoothed_sr >= 0.99].iloc[0].episodes | ||
|
||
def get_fps(path): | ||
df = pandas.concat(bp.load_logs(path), sort=True) | ||
data = df['FPS'] | ||
data = data.tolist() | ||
return data | ||
|
||
# # per level average | ||
# for level in levels: | ||
# row = '' | ||
# for model in models: | ||
# path = f'../beluga/logs/rl/{level}/{model}' | ||
# data = numpy.array(get_fps(path)) | ||
# row += f'\t{data.mean()}\t{data.std()}' | ||
# print(row) | ||
|
||
levels = [ | ||
'GoToRedBallGrey', | ||
'GoToRedBall', | ||
'GoToLocal', | ||
'PickupLoc', | ||
'PutNextLocal', | ||
'GoTo' | ||
] | ||
|
||
models = [ | ||
'expert_filmcnn', | ||
'expert_filmcnn_endpool', | ||
'expert_filmcnn_endpool_res', | ||
'expert_filmcnn_endpool_res_not_conv_bow', | ||
'expert_filmcnn_endpool_res_pixels', | ||
'expert_filmcnn_endpool_res_bow' | ||
] | ||
|
||
row = '' | ||
for model in models: | ||
data = [] | ||
for level in levels: | ||
data += get_fps(f'../beluga/logs/rl/{level}/{model}') | ||
data = numpy.array(data) | ||
row += f'\t{data.mean()}\t{data.std()}' | ||
print(row) |