forked from ellisk42/ec
-
Notifications
You must be signed in to change notification settings - Fork 0
/
official_figures
167 lines (118 loc) · 41.9 KB
/
official_figures
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
Put your plotting/figure-creating command lines here
### Comparing ensemble vs. no ensemble: run these commands under /om2/user/ellisk/ec
python bin/graphs.py --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/list_ensemble*` `grep -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*` `grep -h 'it=20.*graph=True.pickle' jobs/list_no_dsl_ellisk*` `grep -h 'it=20.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_ellisk_2019-01-28T1*` `grep -h 'it=14.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_no_auxiliary_ellisk_2019-01-29T10.*` --title "List ensemble" --export figures/list.png --colors green green green green green blue blue blue blue red red red orange orange orange orange orange purple purple purple --testingTimeout 600
python bin/graphs.py --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/text_ensemble*` `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019*` `grep -h 'it=20.*graph=True.pickle' jobs/text_no_dsl_ellisk_2019*` `grep -h 'it=20.*graph=True.pickle' jobs/text_no_dsl_no_ensemble_ellisk_2019-01-28T*` `grep -h 'it=13.*graph=True.pickle' jobs/text_no_dsl_no_ensemble_no_auxiliary_ellisk_2019-01-29T10.*` `grep -h 'it=7.*graph=True.pickle' jobs/text_no_generative*` --title "Text ensemble" --export figures/text.png --colors green green green green green blue blue blue blue blue blue red red red orange orange orange orange orange purple purple purple cyan cyan cyan --testingTimeout 600
### Text figures - time and hits split
python bin/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_hits.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
python bin/graphs.py -i 20 --testingTimeout 600 --export figures/text_time.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --title "Text editing" --onlyTime --averageColors -c
python bin/graphs.py -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_hits_ws.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
python bin/graphs.py -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_talk_memorize_hits_ws.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h it=20.*graph=True jobs/text_memorize_ellisk*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
python bin/graphs.py -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_memorize.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` orange `grep -h 'it=20.*graph=True.pickle' jobs/text_memorize_ellisk*` purple `grep -h it=20 jobs/text_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --noTime --averageColors -c
### List figures - time and hits split
python bin/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hits.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head -n 5` purple `grep -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/list_no_dsl_no_ensemble_ellisk*` --noTime
python bin/graphs.py -i 20 --testingTimeout 600 --export figures/list_time.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head -n 5` purple `grep -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/list_no_dsl_no_ensemble_ellisk*` --onlyTime
### List figures - hard testing tasks only
python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hard_hits_ws.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head -n 5` purple `grep -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*` cyan `grep -h it=20 jobs/baseline_ec_list*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 22 NS 15
python bin/graphs.py -i 20 --testingTimeout 600 --export figures/list_hard_time.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head -n 5` purple `grep -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*` cyan `grep -h it=20 jobs/baseline_ec_list*|sed 's/Exported checkpoint to//g'` --title "List processing" --onlyTime -c --averageColors
python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hard_hits_ws.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head -n 5` purple `grep -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*` --noTime 31 NS 15
python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_talk_memorize_hard_hits_ws.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head -n 5` purple `grep -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*` cyan `grep -h it=20.*graph jobs/list_memorize_ellisk*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 22 NS 15
python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_memorize.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head -n 5` orange `grep -h 'it=20.*graph=True.pickle' jobs/list_memorize_ellisk*` purple `grep -h 'it=20.*.pickle' jobs/list_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --noTime --averageColors -c
### LOGO figures - time and hits split
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'` --export figures/logo_hits.png --noTime -i 20 --x-label "" --title "LOGO Graphics" --baseline enum 25 NS 1.2
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'` --export figures/logo_time.png --onlyTime -i 20 --testingTimeout 600 --title "LOGO graphics" -c --averageColors
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'` --export figures/logo_hits_ws.png --noTime -i 20 --title "LOGO Graphics" --baseline enum 25 NS 1.2
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20.*graph jobs/logo_memorize_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/logo_memorize_talk_hits_ws.png --noTime -i 20 --title "LOGO Graphics" --baseline enum 25 NS 1.2
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep -h it=20.*graph jobs/logo_memorize_ellisk*` purple `grep -h it=20.*pickle jobs/logo_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/logo_memorize.png --noTime -i 20 --title "LOGO Graphics" --averageColors -c
### Rational figures - time and hits split
python bin/graphs.py --checkpoint teal `grep -h it=20.*graph jobs/rational_batch*` orange `grep -h it=20.*graph jobs/rational_no_dsl*` purple `grep -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'` --noTime --x-label "" --title "Symbolic Regression" --export figures/rational_hits.png --baseline enum 65 NS 69 -i 20b
python bin/graphs.py --checkpoint teal `grep -h it=20.*graph jobs/rational_batch*` orange `grep -h it=20.*graph jobs/rational_no_dsl*` purple `grep -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'` --onlyTime --export figures/rational_time.png --testingTimeout 600 -i 20 --title "Symbolic regression" -c --averageColors
python bin/graphs.py --checkpoint teal `grep -h it=20.*graph jobs/rational_batch*` orange `grep -h it=20.*graph jobs/rational_no_dsl*` purple `grep -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'` --noTime --title "Symbolic Regression" --export figures/rational_hits_ws.png --baseline enum 65 NS 69 -i 20
python bin/graphs.py --checkpoint teal `grep -h it=20.*graph jobs/rational_batch*` orange `grep -h it=20.*graph jobs/rational_no_dsl*` purple `grep -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20.*graph jobs/rational_memorize_*2020-04-14_*|sed 's/Exported checkpoint to//g'` --noTime --title "Symbolic Regression" --export figures/rational_memorize_talk_hits_ws.png --baseline enum 65 NS 69 -i 20
python bin/graphs.py --checkpoint teal `grep -h it=20.*graph jobs/rational_batch*` orange `grep -h it=20.*graph jobs/rational_memorize_*2020-04-14_*` purple `grep -h it=20.*pickle jobs/rational_memorize_no_recognition*|sed 's/Exported checkpoint to//g'` --noTime --title "Symbolic Regression" --export figures/rational_memorize.png -i 20 --averageColors -c
### Tower figures - time and hits split
# actual baseline results are both 9, separated for visibility
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_hits.png --noTime --x-label "" --title "Tower building" --baseline enum 8 NS 10 -i 20
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_time.png --onlyTime --testingTimeout 600 -i 20 --title "Tower building" --averageColors -c
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_hits_ws.png --noTime --title "Tower building" --baseline enum 8 NS 10 -i 20
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h it=20.*graph jobs/tower_memorize_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/tower_memorize_talk_hits_ws.png --noTime --title "Tower building" --baseline enum 8 NS 10 -i 20
python bin/graphs.py --checkpoints teal `grep -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep -h it=20.*graph jobs/tower_memorize_ellisk*` purple `grep -h it=20.* jobs/tower_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/tower_memorize.png --noTime --title "Tower building" --noTime --averageColors -c -i 20
### Regex figures - time and hits split
### Regex figures - likelihoods
python bin/graphs.py --checkpoints teal `grep -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --cutoff gt --noTime --export figures/regex_7m_gt.png
python bin/graphs.py --checkpoints teal `grep -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --cutoff bigram --noTime --export figures/regex_7m_bigram.png
python bin/graphs.py --checkpoints teal `grep -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --likelihood maximum --noTime --export figures/regex_7m_maximum.png -i 6
python bin/graphs.py --checkpoints teal `grep -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --likelihood task --noTime --export figures/regex_7m_marginal.png -i 6
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --cutoff gt --noTime --export figures/regex_2m_gt.png
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --cutoff bigram --noTime --export figures/regex_2m_bigram.png
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --likelihood maximum --noTime --export figures/regex_2m_maximum.png -i 10
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --likelihood task --noTime --export figures/regex_2m_marginal.png -i 10
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --cutoff gt --noTime --export figures/regex_2m_b10_gt.png
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --cutoff bigram --noTime --export figures/regex_2m_b10_bigram.png
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --likelihood maximum --noTime --export figures/regex_2m_b10_maximum.png -i 10
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --likelihood task --noTime --export figures/regex_2m_b10_marginal.png -i 10
python bin/graphs.py --checkpoints teal `grep -h it=9.*graph jobs/regex_b10_strConst_7m_no_mask_ellisk*` orange `grep -h it=5.*graph jobs/regex_b10_strConst_7m_no_mask_no_dsl_ellisk*` --cutoff gt --noTime --export figures/regex_7m_b10_no_mask_gt.png -i 10
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --cutoff gt --noTime --export figures/regex_7m_b10_gt.png
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --cutoff bigram --noTime --export figures/regex_7m_b10_bigram.png
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --likelihood maximum --noTime --export figures/regex_7m_b10_maximum.png -i 10
python bin/graphs.py --checkpoints teal `grep -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --likelihood task --noTime --export figures/regex_7m_b10_marginal.png -i 10
### Comparing masked vs full recognition model
python bin/graphs.py --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/list_no_generative_mask*` `grep -h 'it=20.*graph=True.pickle' jobs/list_no_generative_ellisk*` `grep -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T**` `grep -h it=20.*graph.*pickle jobs/list_mask_ellisk_2019-02-07T11.1*` --title "List, mask RM" --export figures/list_mask.png --names "no dsl (mask),no dsl,dc,dc (mask)" --colors green green green orange orange orange blue blue blue blue blue red red red --noTime
python bin/graphs.py --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head -n 5` `grep -h 'it=20.*graph=True.pickle' jobs/text_mask_ellisk_2019-02-07T11.*` `grep -h 'it=20.*graph.*pickle' jobs/text_no_generative_ellisk_2019-01-30T15.*` --export figures/text_mask.png --colors orange orange orange orange orange purple purple purple green green green --names "dc,dc+mask,dc-dsl"
### List w/ small amount of training data
python bin/graphs.py --testingTimeout 600 --title "List, 75/25 test/train" --export figures/list_small_train.png --checkpoints teal `grep -h 'it=10.*graph.*pickle' jobs/list_small_train_ellisk*` orange `grep -h 'it=10.*graph.*pickle' jobs/list_small_train_no_dsl_ellisk*` --names "dc,dc-dsl"
### List w/ tiny amount of training data
python bin/graphs.py --testingTimeout 600 --title "List, 85/15 test/train" --export figures/list_tiny_train.png --checkpoints teal `grep -h 'it=10.*graph.*pickle' jobs/list_tiny_train_ellisk*` orange `grep -h 'it=10.*graph.*pickle' jobs/list_tiny_train_no_dsl_ellisk*` --names "dc,dc-dsl"
### List w/ large amount of training data
python bin/graphs.py --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*` `grep -h 'it=20.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_ellisk_2019-01-28T1*` `grep -h 'it=14.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_no_auxiliary_ellisk_2019-01-29T10.*` --title "List, 50/50 test/train" --export figures/list_big_train.png --colors orange orange orange orange orange purple purple purple purple purple --testingTimeout 600
### Regular expressions
python bin/graphs.py --title "Regular expressions" --export figures/regex.png --checkpoints `grep -h it=20.*graph=True jobs/regex_batch_40_mask_ellisk_2019-02-06T14.*` `grep -h it=20.*graph jobs/regex_no_dsl_batch_40_mask*` `grep -h it=10.*graph jobs/regex_2h_mask*` --colors orange orange orange orange orange purple purple purple purple purple green --names "dc,dc (no dsl),dc (no batch)" --noTime --alpha 0.75
### Regex, for josh
python bin/graphs.py --title "Regular expressions" --export regex_baselines.png --checkpoints orange `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_mask_mnye_2019-02-12T1*` purple `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_bigram_mnye_2019-02-12T1*` green `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_unigram_mnye_2019-02-12T1*` blue `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_nodsl_mask_mnye_2019-02-12T1*` --noTime --alpha 0.75 #--names "mask (hybrid),bigram,unigram,nodsl (mask, hybrid)"
### Expertise figures
python ~/projects/ec/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "List processing" --export figures/list_no_recognition_solved.png --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head -n 5` `grep -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --colors teal teal teal teal orange orange orange --noTime
python ~/projects/ec/graphs.py --x-label "Iterations (learning time)" -i 20 --testingTimeout 600 --export figures/list_no_recognition_time.png --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head -n 5` `grep -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --colors teal teal teal teal orange orange orange --onlyTime
python ~/projects/ec/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_no_recognition_solved.png --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head -n 5` `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --colors teal teal teal teal teal orange orange orange --noTime
python ~/projects/ec/graphs.py --x-label "Iterations (learning time)" -i 20 --testingTimeout 600 --export figures/text_no_recognition_time.png --checkpoints `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head -n 5` `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --colors teal teal teal teal orange orange orange --onlyTime
python ~/projects/ec/graphs.py --x-label "" -i 12 --testingTimeout 600 --title "LOGO Graphics" --epochFrequency 2 --colors teal orange --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T00:44:56.828323/logo_aic=1.0_arity=3_ET=3600_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle --export figures/logo_no_recognition_solved.png --noTime
python ~/projects/ec/graphs.py --x-label "Iterations (learning time)" -i 12 --testingTimeout 600 --epochFrequency 2 --colors teal orange --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T00:44:56.828323/logo_aic=1.0_arity=3_ET=3600_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle --export figures/logo_no_recognition_time.png --onlyTime
### Text - we should add no DSL baseline to this
python bin/graphs.py -i 11 --testingTimeout 600 --title "Text editing" --export figures/text.png --checkpoints teal `grep -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head -n 5` orange `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` purple `grep -h it=20.*graph=True jobs/text_no_dsl_no_ensemble --colors teal teal teal teal teal orange orange orange --noTime
### Comparing full model to ablations
python bin/graphs.py -i 12 --testingTimeout 600 --title "LOGO Graphics" --epochFrequency 2 --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T00:44:56.828323/logo_aic=1.0_arity=3_ET=3600_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T21:58:32.845270/logo_aic=1000.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --export ../ecPaper/figures/logoLearningCurve_batch.png
python bin/graphs.py --testingTimeout 600 --title "Text editing" --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_ET=720_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1000.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --export ../ecPaper/figures/textLearningCurve_batch.png
python bin/graphs.py --testingTimeout 600 --title "List processing" --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/list_aic=1.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/list_aic=1.0_arity=3_ET=720_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/list_aic=1000.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --export ../ecPaper/figures/listLearningCurve_batch.png
python bin/graphs.py --testingTimeout 600 --title "Tower building" --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=300_RR=True_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-08T15:08:42.341376/tower_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-08T15:06:41.057543/tower_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --export ../ecPaper/figures/towerLearningCurve_batch.png
python bin/graphs.py --testingTimeout 600 --title "Tower building, new primitives" --showEpochs --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/towers/2019-01-05T19:28:09.239650/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=19_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-05T19:34:18.595339/tower_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-05T19:36:21.603299/tower_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=13_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --export /tmp/tower.png
python bin/graphs.py --testingTimeout 600 --title "Symbolic regression" --interval --checkpoints /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_STM=True_L=1.0_batch=10_TRR=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/rational_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=30.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle --export ../ecPaper/figures/rationalLearningCurve_batch.png -i 20 --epochFrequency 3
python bin/graphs.py --title "Generative Regexes" --checkpoints /home/ellisk/om/ec/experimentOutputs/regex/2019-01-11T21:52:14.607041/regex_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=20_MR=32_MF=10_baseline=False_pc=30.0_RT=1800_RR=True_RW=False_STM=True_L=1.0_batch=40_TRR=randomShuffle_K=10_topkNotMAP=True_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle ~/om/ec/experimentOutputs/regex/regex_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=20_MF=10_baseline=False_pc=30.0_RT=1800_RR=True_RW=False_storeTask=False_L=1.0_batch=40_taskReranker=randomShuffle_K=10_topkNotMAP=True_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --names "bigram (rank 32),bigram" --export ../ecPaper/figures/regexLearningCurve_batch.png
#### Comparing old and new recognition models
python bin/graphs.py --testingTimeout 600 --title "List, recognition model only" --interval --checkpoints /home/ellisk/om/ec/experimentOutputs/list_aic=1.0_arity=3_ET=720_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/list_aic=1000.0_arity=3_BO=False_CO=False_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle -i 20 --export ../ecPaper/figures/list_recognition_comparison.png
python bin/graphs.py --testingTimeout 600 --title "Text, recognition model only" --interval --checkpoints /home/ellisk/om/ec/experimentOutputs/text_aic=1000.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1000.0_arity=3_BO=False_CO=False_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle -i 20 --export ../ecPaper/figures/text_recognition_comparison.png
### Batching figures
python bin/graphs.py --testingTimeout 600 --title "LOGO, varying batch size" --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-17T20:58:48.581674/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T16:34:58.729116/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=30_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=20_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T03:32:41.459119/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1440_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-07T16:31:32.509029/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=9_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --names "no batch,40,20,10,10 (1h)" --export ../ecPaper/figures/logo_batch_size.png --noTime
python bin/graphs.py --testingTimeout 600 --showTraining --title "LOGO, varying batch size (training)" --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-17T20:58:48.581674/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T16:34:58.729116/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=30_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=20_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T03:32:41.459119/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1440_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-07T16:31:32.509029/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=9_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --names "no batch,40,20,10,10 (1h)" --export ../ecPaper/figures/logo_batch_size_training.png --noTime
python bin/graphs.py --testingTimeout 600 --title "Text, varying batch size" --checkpoints /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --names "no batch,40,10" --export ../ecPaper/figures/text_batch_size.png
# Visualizing program embeddings
python bin/taskRankGraphs.py --checkpoints /home/ellisk/om/ec/experimentOutputs/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=300_RR=True_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses
python bin/taskRankGraphs.py --checkpoints /home/ellisk/om/ec/experimentOutputs/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=300_RR=True_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses
python bin/taskRankGraphs.py --checkpoints experimentOutputs/logo/2018-12-17T20:58:48.581674/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses
# Cathy: try changing the tsne learning rate if you haven't already; recommend 350.0 for this one.
python bin/taskRankGraphs.py --checkpoints experimentOutputs/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_FTM=True_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses
Rational, Feature vectors.
python bin/taskRankGraphs.py --checkpoints /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle --labelWithImages 1 --metricsToCluster contextualLogProductions --tsneLearningRate 300
# Visualizing confusion matrices
./container.img python synergyStatistics.py /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_STM=True_L=1.0_batch=10_TRR=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/rational_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=14_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=30.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle
### ~deep program learning~
python bin/tower.py --primitive-graph `grep -h it=20.*graph jobs/tower_batch_40_3600_ellisk_2019-0*` # batching
python bin/tower.py --primitive-graph experimentOutputs/towers/2019-02-12T22:26:56.272150/tower_aic=1.0_arity=3_BO=True_CO=True_ES=1_ET=3600_HR=0.5_it=10_MF=5_pc=30.0_RS=5000_RT=3600_RR=False_RW=False_STM=True_L=1.5_TRR=default_K=2_topkNotMAP=False_graph=True.pickle # no batching
python bin/logo.py --primitive-graph `grep -h it=20.*graph jobs/logo_batch_50*` # batching
python bin/list.py --primitive-graph `grep -h it=20.*graph jobs/list_hard_test_ellisk*` # batching
python bin/list.py --primitive-graph experimentOutputs/list_aic=1.0_arity=4_ET=11640_expandFrontier=4.0_it=4_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_taskReranker=default_K=2_topkNotMAP=False_rec=False.pickle `grep -h it=20.*graph jobs/list_hard_test_ellisk*` /om2/user/zyzzyva/ec/experimentOutputs/list_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=10_baseline=False_pc=10.0_RT=3600_RW=False_storeTask=True_L=2.0_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle # both batching and not batching and then McCarthy
python bin/text.py --primitive-graph `grep -h it=20.*graph jobs/text_ellisk_2019-01-2*`
python bin/rational.py --primitive-graph `grep -h it=20.*graph jobs/rational_batch*`
python bin/regexes.py --primitive-graph `grep -h it=20.*graph jobs/regex_b40_str_red_12m_mnye_2019-03*`
python bin/scientificLaws.py --primitive-graph `grep -h it=.*graph jobs/scientific_unsolved_4h*`
python bin/list.py --primitive-graph experimentOutputs/list_aic=1.0_arity=4_ET=11640_expandFrontier=4.0_it=4_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_taskReranker=default_K=2_topkNotMAP=False_rec=False.pickle
# this will show you the correlation between depth and accuracy
python bin/analyzeDepth.py --recognition `grep -h it=20.*graph jobs/tower_batch_50_3600*` `grep -h it=20.*graph jobs/logo_batch_50*` `grep -h it=20.*graph jobs/list_hard_test_ellisk*` `grep -h it=20.*graph jobs/text_ellisk_2019-01-2*` `grep -h it=20.*graph jobs/rational_batch*` --generative `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` `grep -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` `grep -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` --legend MEAN
python bin/analyzeDepth.py --recognition `grep -h it=20.*graph jobs/tower_batch_50_3600*` `grep -h it=20.*graph jobs/logo_batch_50*` `grep -h it=20.*graph jobs/list_hard_test_ellisk*` `grep -h it=20.*graph jobs/text_ellisk_2019-01-2*` `grep -h it=20.*graph jobs/rational_batch*` --generative `grep -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` `grep -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` `grep -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` --every