Skip to content

Commit

Permalink
add rl dataeff ttest and fps scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
dyth committed Mar 20, 2020
1 parent a3d86cc commit d0b75b0
Show file tree
Hide file tree
Showing 2 changed files with 161 additions and 0 deletions.
76 changes: 76 additions & 0 deletions scripts/rl_dataeff_ttest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/usr/bin/env python3

import os
import re
import babyai.plotting as bp
import pandas
import argparse
import json
import numpy
from scipy import stats

# dictionary of level to window used
levels = {
'GoToRedBallGrey': 10,
'GoToRedBall': 100,
'GoToLocal': 100,
'PickupLoc': 100,
# 'PutNextLocal': 100,
# 'GoTo': 100,
}

model1 = 'expert_filmcnn'
model2 = 'expert_filmcnn_endpool'
model1 = 'expert_filmcnn_endpool'
model2 = 'expert_filmcnn_endpool_res/lr-0.0001'
model1 = 'expert_filmcnn_endpool_res/lr-0.0001'
model2 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001'
model1 = 'expert_filmcnn_endpool_res/lr-0.0001'
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.0001'
model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001'
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.0001'
model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001'
model2 = 'expert_filmcnn_endpool_res_bow/lr-0.0001'

model1 = 'expert_filmcnn_endpool_res/lr-0.00005'
model2 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005'
model1 = 'expert_filmcnn_endpool_res/lr-0.00005'
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.00005'

model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005'
model2 = 'expert_filmcnn_endpool_res_pixels/lr-0.00005'
model1 = 'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005'
model2 = 'expert_filmcnn_endpool_res_bow/lr-0.00005'


path1 = [f'../beluga/logs/rl/{level}/{model1}' for level in levels]
path2 = [f'../beluga/logs/rl/{level}/{model2}' for level in levels]
windows = [levels[i] for i in levels]


def dataeff(df_model, window):
smoothed_sr = df_model['success_rate'].rolling(window, center=True).mean()
if smoothed_sr.max() < 0.99:
# print('not done, success rate is only {}% so far'.format(100 * smoothed_sr.max()))
return int(1e9)
return df_model[smoothed_sr >= 0.99].iloc[0].episodes

def get_dataeff(path, window):
df = pandas.concat(bp.load_logs(path), sort=True)
models = df['model'].unique()
data = []
for model in models:
x = df[df['model'] == model]
eff = float(dataeff(x, window))
# print(model, eff)
if eff != 1e9:
data.append(eff)
return numpy.array(data)

data1 = [get_dataeff(path, window) for path, window in zip(path1, windows)]
data2 = [get_dataeff(path, window) for path, window in zip(path2, windows)]

'https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_ind.html'
ttests = [stats.ttest_ind(d1, d2, equal_var=False) for d1, d2 in zip(data1, data2)]
for ttest in ttests:
print(f'{ttest.statistic}\t{ttest.pvalue}')
85 changes: 85 additions & 0 deletions scripts/rl_fps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/usr/bin/env python3

import os
import re
import babyai.plotting as bp
import pandas
import argparse
import json
import numpy
from scipy import stats

levels = [
'GoToRedBallGrey',
'GoToRedBall',
'GoToLocal',
'PickupLoc',
'PutNextLocal',
# 'GoTo'
]

models = [
'expert_filmcnn',
'expert_filmcnn_endpool',
'expert_filmcnn_endpool_res/lr-0.0001',
'expert_filmcnn_endpool_res_not_conv_bow/lr-0.0001',
'expert_filmcnn_endpool_res_pixels/lr-0.0001',
'expert_filmcnn_endpool_res_bow/lr-0.0001'
]

models = [
'expert_filmcnn_endpool_res/lr-0.00005',
'expert_filmcnn_endpool_res_not_conv_bow/lr-0.00005',
'expert_filmcnn_endpool_res_pixels/lr-0.00005',
'expert_filmcnn_endpool_res_bow/lr-0.00005'
]


def dataeff(df_model, window):
smoothed_sr = df_model['success_rate'].rolling(window, center=True).mean()
if smoothed_sr.max() < 0.99:
# print('not done, success rate is only {}% so far'.format(100 * smoothed_sr.max()))
return int(1e9)
return df_model[smoothed_sr >= 0.99].iloc[0].episodes

def get_fps(path):
df = pandas.concat(bp.load_logs(path), sort=True)
data = df['FPS']
data = data.tolist()
return data

# # per level average
# for level in levels:
# row = ''
# for model in models:
# path = f'../beluga/logs/rl/{level}/{model}'
# data = numpy.array(get_fps(path))
# row += f'\t{data.mean()}\t{data.std()}'
# print(row)

levels = [
'GoToRedBallGrey',
'GoToRedBall',
'GoToLocal',
'PickupLoc',
'PutNextLocal',
'GoTo'
]

models = [
'expert_filmcnn',
'expert_filmcnn_endpool',
'expert_filmcnn_endpool_res',
'expert_filmcnn_endpool_res_not_conv_bow',
'expert_filmcnn_endpool_res_pixels',
'expert_filmcnn_endpool_res_bow'
]

row = ''
for model in models:
data = []
for level in levels:
data += get_fps(f'../beluga/logs/rl/{level}/{model}')
data = numpy.array(data)
row += f'\t{data.mean()}\t{data.std()}'
print(row)

0 comments on commit d0b75b0

Please sign in to comment.