all scripts dyth used for paper before refactoring

scripts/il_performance.py

@@ -7,7 +7,6 @@
 import numpy as np
 
 from babyai import plotting
-from babyai.plotting import model_num_samples
 
 
 parser = argparse.ArgumentParser("Analyze data efficiency of imitation learning")
@@ -20,93 +19,49 @@
 
 
 
-def best_within_normal_time_mutilated(df, regex, patience, limit='epochs', window=1, normal_time=None, summary_path=None):
+def best_in_training(df, regex, patience, limit='epochs', window=1, normal_time=None, summary_path=None):
     """
-    Compute the best success rate that is achieved in all runs within the normal time.
-
-    The normal time is defined as `patience * T`, where `T` is the time it takes for the run
-    with the most demonstrations to converge. `window` is the size of the sliding window that is
-    used for smoothing.
-
-    Returns a dataframe with the best success rate for the runs that match `regex`.
+    Return the best success rate
 
+    First smooth the success rate with a sliding window of size 'window'.
+    Return an array with best result for each runs that matches `regex`.
     """
     models = [model for model in df['model'].unique() if re.match(regex, model)]
-    num_samples = [model_num_samples(model) for model in models]
-    print(len(num_samples))
+    num_samples = range(len(models))
     # sort models according to the number of samples
     models, num_samples = zip(*sorted(list(zip(models, num_samples)), key=lambda tupl: tupl[1]))
 
     maxes = []
     for model, num in zip(models, num_samples):
         df_model = df[df['model'] == model]
         success_rate = df_model['validation_success_rate'].rolling(window, center=True).mean()
-        # print(success_rate.tolist())
         success_rate = success_rate[np.logical_not(np.isnan(success_rate))]
         maxes.append(max(success_rate))
-    print(maxes)
     return np.array(maxes)
 
 
 
-
-
-# levels = ['GoTo']
-# archs = ['expert_filmcnn', 'expert_filmcnn_endpool_res', 'expert_filmcnn_endpool_res_not_conv_bow']
-# samples = [10, 100]
-#
-# all_results = []
-# for level in levels:
-#     arch_results = []
-#     for arch in archs:
-#         results = ''
-#         path = f'../beluga/logs/il/GoTo/{arch}/'
-#         print(path)
-#
-#         df_logs = pandas.concat(plotting.load_logs(path), sort=True)
-#
-#         maxes = best_within_normal_time_mutilated(
-#             df_logs, args.regex,
-#             patience=args.patience, window=args.window, limit=args.limit,
-#             summary_path=None)
-#         results += f"{np.mean(maxes)}\t"
-#         arch_results.append(results)
-#     all_results.append(arch_results)
-#
-# demos = '\t'
-# for sample in samples:
-#     demos += f'\t{sample}K'
-#
-# print(demos)
-# for level, ar in zip(levels, all_results):
-#     for arch, ar in zip(archs, ar):
-#         print(f'{level}\t{arch}\t' + ar)
-
-
-
-levels = ['GoToRedBallGrey', 'GoToRedBall', 'GoToLocal', 'PutNextLocal', 'PickupLoc']
-levels = ['GoTo']
+levels = ['GoToRedBallGrey', 'GoToRedBall', 'GoToLocal', 'PutNextLocal', 'PickupLoc']#, 'GoTo']
 archs = ['expert_filmcnn', 'expert_filmcnn_endpool_res', 'expert_filmcnn_endpool_res_not_conv_bow']
-samples = [5, 10, 25, 50, 100, 250, 500]
 samples = [5, 10, 50, 100, 500]
-samples = [10, 100]
+# samples = [10, 100]
 
 all_results = []
 for level in levels:
     arch_results = []
     for sample in samples:
         results = ''
         for arch in archs:
-            path = f'../beluga/logs/il/{level}/{arch}/{sample}000/'
+            path = f'../beluga/il/{level}/{arch}/{sample}000/'
             print(path)
 
             df_logs = pandas.concat(plotting.load_logs(path), sort=True)
-            maxes = best_within_normal_time_mutilated(
+            maxes = best_in_training(
                 df_logs, args.regex,
                 patience=args.patience, window=args.window, limit=args.limit,
                 summary_path=None)
-            # results += f"{maxes.mean()}\t{maxes.std()}\t"
-            results += f" & {100.*maxes.mean():.1f} $\pm$ {100.*maxes.std():.1f}"
+            results += f"{maxes.mean()}\t{maxes.std()}\t"
+            # results += f" & {100.*maxes.mean():.1f} $\pm$ {100.*maxes.std():.1f}"
         arch_results.append(results)
     all_results.append(arch_results)
 

scripts/il_performance_ttest.py

@@ -33,16 +33,14 @@ def best_within_normal_time_mutilated(df, regex, patience, limit='epochs', windo
 
     """
     models = [model for model in df['model'].unique() if re.match(regex, model)]
-    num_samples = [model_num_samples(model) for model in models]
-    print(len(num_samples))
+    num_samples = range(len(models))
     # sort models according to the number of samples
     models, num_samples = zip(*sorted(list(zip(models, num_samples)), key=lambda tupl: tupl[1]))
 
     maxes = []
     for model, num in zip(models, num_samples):
         df_model = df[df['model'] == model]
         success_rate = df_model['validation_success_rate'].rolling(window, center=True).mean()
-        # print(success_rate.tolist())
         success_rate = success_rate[np.logical_not(np.isnan(success_rate))]
         maxes.append(max(success_rate))
     return maxes

scripts/rl_dataeff.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 import os
 import re
 import babyai.plotting as bp
@@ -16,22 +15,26 @@
 args = parser.parse_args()
 
 def dataeff(df_model, window):
-  smoothed_sr = df_model['success_rate'].rolling(window, center=True).mean()
-  if smoothed_sr.max() < 0.99:
-    print('not done, success rate is only {}% so far'.format(100 * smoothed_sr.max()))
-    return int(1e9)
-  return df_model[smoothed_sr >= 0.99].iloc[0].episodes
+    smoothed_sr = df_model['success_rate'].rolling(window, center=True).mean()
+    if smoothed_sr.max() < 0.99:
+        print('not done, success rate is only {}% so far'.format(100 * smoothed_sr.max()))
+        return int(1e9)
+    return df_model[smoothed_sr >= 0.99].iloc[0].episodes
+
+def get_fps(df):
+    data = df['FPS']
+    data = data.tolist()
+    return numpy.array(data)
 
 # if os.path.exists(args.report):
 #     raise ValueError("report directory already exists")
 # os.mkdir(args.report)
 
+print(args.regex)
 df = pandas.concat(bp.load_logs(args.path), sort=True)
+fps = get_fps(df)
 models = df['model'].unique()
-
-print(args.regex)
 models = [model for model in df['model'].unique() if re.match(args.regex, model)]
-# models = [model for model in df['model'].unique() if 'lr' not in model]
 
 data = []
 for model in models:
@@ -40,13 +43,12 @@ def dataeff(df_model, window):
     print(model, eff)
     if eff != 1e9:
         data.append(eff)
-print(len(data))
 data = numpy.array(data)
 
 Z = 2.576
-result = {'mean': data.mean(), 'std': data.std(),
-          'min': data.mean() - Z * data.std(), 'max': data.mean() + Z * data.std()}
+result = {'samples': len(data), 'mean': data.mean(), 'std': data.std(),
+          'min': data.mean() - Z * data.std(), 'max': data.mean() + Z * data.std(),
+          'fps_mean': fps.mean(), 'fps_std': fps.std()}
 # with open(os.path.join(args.report, 'result.json'), 'w') as dst:
 #     json.dump(result, dst)
 print(result)
-print(f'{data.mean()}\t{data.std()}')