From 37cc5c8c39aae302ad49d1626805b714529f36c7 Mon Sep 17 00:00:00 2001 From: bstadt Date: Tue, 28 Mar 2023 20:22:48 +0000 Subject: [PATCH] updated eval --- configs/eval/generate_llama.yaml | 14 -------------- eval_figures.py | 8 ++++++-- figs/perplexity_hist.png | Bin 0 -> 15322 bytes 3 files changed, 6 insertions(+), 16 deletions(-) delete mode 100644 configs/eval/generate_llama.yaml create mode 100644 figs/perplexity_hist.png diff --git a/configs/eval/generate_llama.yaml b/configs/eval/generate_llama.yaml deleted file mode 100644 index 15a114c2c94e..000000000000 --- a/configs/eval/generate_llama.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# model/tokenizer -model_name: "zpn/llama-7b" -tokenizer_name: "zpn/llama-7b" - - -max_new_tokens: 512 -temperature: 0.001 -prompt: | - #this code prints a string reversed - my_string = "hello how are you" - print(len(my_string)) - - - My code above does not work. Can you help me? diff --git a/eval_figures.py b/eval_figures.py index 0126bda46aed..e1b50bbec8ec 100644 --- a/eval_figures.py +++ b/eval_figures.py @@ -4,7 +4,7 @@ from matplotlib import pyplot as plt plt.figure() -for fpath in glob.glob('./eval_data/*multi*.pkl'): +for fpath in glob.glob('./eval_data/*.pkl'): parts = fpath.split('__') model_name = parts[1].replace('model-', '').replace('.pkl', '') lora_name = parts[2].replace('lora-', '').replace('.pkl', '') @@ -13,7 +13,11 @@ perplexities = data['perplexities'] perplexities = np.nan_to_num(perplexities, 100) perplexities = np.clip(perplexities, 0, 100) - plt.hist(perplexities, label='{}-{}'.format(model_name, lora_name), alpha=.5) + if 'nomic' in fpath: + label = 'GPT4all-lora' + else: + label = 'alpaca-lora' + plt.hist(perplexities, label=label, alpha=.5) plt.xlabel('Perplexity') plt.ylabel('Frequency') diff --git a/figs/perplexity_hist.png b/figs/perplexity_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..be3780b07bea2c62e34b20e68cf763bd3415de4f GIT binary patch literal 15322 zcmeHuXINC(*5(0PR7}@aj0gryAi;nV#f$|4O3tVVN(RZ9)>hkuiUdWHl8Yoc6v+e% z0ul<4ARr(~K*^bTx7~g3{bu@$^JDIhd7f^4q>8F@_Fj9fH=UcO6=arhtmB|4Y60J+5J^Y-wccaPG1pC4bJ=%FNQ%%=r99d&A2%#+DX) z1rG@B-nH?Pt*w=fxR8+f-`^l;dHJGH1IL0J_>hHGCpB#-YS}sRKTEW9lrcp~$k2`* zR&fmMYjV+1Rh^ytK6bc$+kwNap69*Kn>>s3d9Xi>cGWnt+h^66t=oz!2fbsal^7b& z{M0@b<;~k~py@6bWt+NtU(nB%jyp>}$R1bzkmK29wS08oUS3f59slYUu5-D;_ln#l zB~A5fI^$%Pxn+#%P1WrxN&X2}WZ;;|z+Y76Dwe-dRDtkMhwzVAf1_9^YVlDD?+{-6 z|6lmOS(3KyepY(&;NayA2VFH-SXh!jn)M6Lrzjovl4RS-VdLW`P87d-^*&3OwZ+nE z{M)(X$B$<*7E_cJ=TdIT=*f{@24iu?MrxKFzw7Njxon59@P|?+MR}H-Iej`bI(kP@ ze%rRqcwM@Abjjkyyc8>S`pu*1algk0^2jEh_~%WOWLrQ*uPQDsR?*Vx>EC0uB>GI) zbxA3yyY7VpwPe#(_0FApx^#m$Z%J~aE=R(0cX#(&zx{UK-`}4hm$hv zX=!O@ytcn4&nr#82ygG7H%Qo@ni}m>K6UC&l8{T@O4f~3s4TyLfUlt@T{|Pe)y=I* zcWDAkUS1wuGbM5q1C?tw{JG!-hTU~ZVJGga zX&rBTe(q@QWKZmBDfhYA8LVP`l7UP_?REZFdp#*nlWt$8i%xy>TZtz+OsA}@ELnMZ z<}FTs+pZ6r=FOWIW8GHRQ=1sVzIe?>T(^i>-Rb#Dx$X~F_0+7l$UMAr=g#bbys5xZ z-M5}BY(Bxk{o^`1I&t=W^~RlLLG^iV46O_cH&weZlPy@@mTk=*bGh{ymWeGbXXS$> zHItH(21iG0{Po>q<}F$kt(WVZ-5Kn{ODmV2%i(rvU(UI5<)z8t?w+csdp*q9C;{#C z=(xC3-l^wav5)jOX@C6q(W^OnZs6K(gTgy!E?fw=?{Dm`7jp5Hm6dh$mD}2~N|>T7 z2I~y&y{oEH*3PhC&WtrD8kGigPxdCN-{Y1_J^zNoZs1dswb`LVo?q=50sD-fiP?O4 zTU)En<2L-DrzW1C(Im)dO1OXjZ!{$hjYnYuJ9fOazU#}c-V-u69i`_!(J^Eq=02Ut z?byP?j8#`8+kW>hQ(_=TxwYA;Dcdf)!%w@tuP^?$-xl0me?S4NV)JzXTXTKM`pD&5 zPDCpNiOZfk<-=S0X!lVp2VFJR@7c3w!?@;{=g*(N?A<$$bddw1_>S3$au1WP4^Q!) zntAuK>WYmN6&0Crn#xriQZmQB=`(kD>|J~h+bBo^@A6@k>kVJ2YAs8IBlg`mrzNir#ee z`VK)sg{iRt>48ViBR$5$T@_yC_oQbt56+Go~f4~0JtHm2B zs`h^AlLMwjUK_H%7p_;;(fN9bN5VdCdS<4hvok72Im)EiXD4>P!BE$xCQeh_Z8B9! zNlDQ86i)9p`GD}cI*n_;{`$7LS)a#aI_dJ|%Yw1@aP{~>bCT}c{w4+|56{59l#h9C zE=7tHGjZk(>77AS@3x0T<09zliNUO_ti#xc?(y$!;x40K4Vm*VlG)K1ilc_pSzK0T zZ<0Sd*~`qdZW~%< zqD$5YAMzfqloRSstJSNjOEO5a@6+n5OFDY<=1qUMCleE$A+=r`B)A$Zzc**R9vf&* zWDMkK<5*Qs4df@r#qBD6yjR9Mp=-bB>{aao%(laqE=Bj}3@MZbOKD>>#*c@vO=|{; z+i|_z_cR9MF4qs(A zsd#qHz_hYGsxxi9h)Mebz1_QaYvnqp?Ki6nZDKg|mV3;m3=R(Rh0ldZjn)~6JAMo0 z_L#PI8!A08In_Lua532F1U@XgT`@z%qERCvA_A-Jmp3=-Le7@muB9)Sn|^e&yX!!j!ePwkGdAagRHm_HbKPoZt7hqFDJS`*FloP;rJC4a}5_v$Q)4)4#<+og!4CB$T%ylkdhB27TT<0D zkD^ZRtjQkFI4=>AnJ(RO;4C}3RVF5vxI=#+qePhFNB7c9Gw~I-?FvQPzjpn4Pd{Uz z#-Z8G;e~qC^$Lzfiyq<&$aUbm0>^aK5_DqH%^Qjv8ni>)Mjz)*^-GL&RUB7KVq;~+7)>BQOUxY&(DRtF%N;D{S0BzTyt6iJtl2}K^j}~sIm)SAfX=#vDO*xIJLy1LG$gmD~X8*jTO zO}A*uo*XO@>KGVEM75bW=jq04XVgT6xYN<^jlX|>Lk2c)H?EAXpPyG*Rb@9fGp7G7 z0DGPhu-}}Hd9N9z7@~&pDB?0|Hq))>ksTL+VQMI#m!p9>o!hmWZT`|~G~-mWk9&f| zY)+(^)~Y!>XLGxLJ4{nDHjY`h-z*yAvnwVl__C|NVToUi%sp=ZuldJsuOep&)74&c zWcKT`Io;^NxtRoxM=U>41@fc)O|kR29S_y&d*tGtnLtLg?DjkAuC6(4<(oEb%5J^B zKGkK+f~NGB&A!c<;V__PZJmVTcCF~mpXz6rH0R|4IAD6<>^c@rV2x2c4rb`o2M4Cn z6cx}u$eY6Afjh_y zTX*jGlp0kf8n>i`T*{DnN5~z`aqXfuhz&Xfztf;>9XfR?pE# zR5diZK3lRb1yE3K&-a2is_!VrFYniq=4kWfy_jV)=60HS!+!Zk!iO8u z&8z0~xQ2fHYSv^^E~QWeAk!V2m{$p~m*zTgIbJ_s@9O+z_DprOs(#E%%*Go0#mus~ zC}LW3YP(|aG;?UbMPt|a(7T7*(=L{ClOKBzW6s5*9td#NN;8Qd?Z9Ppy3_jEqep8d zCMWr}Y-zhv`|0_)s|k#LajBW%s#~{j6Zj2m-BLHu-5pbScfCaV{rmTa5^&Fn%_VM*Dq}Idp@}gsG>We3`Pf<=VA@6@T+faW^=67b?CfUzz+l}^rSQr4WA}0Z#{qhds)nel4=y(Y|1Kdgp({A78_LHPE^PNBW>{Q<8I&Ot(Q3EfT>opuXSha88bQhU5|v%T?uv_q{#=0o+%9wQ4TriW;@(L@a);9aKh z;0bo=7n8!ZugYF*E4;PH&>s%?^yw2ARzhk37_H7ztuktsYnF6DOtxcBpFUNXYHMq= z>#5!e+$B~IxElt(jTIP;w$DAQYd9b5IRZT6ju~CO6ohw|0N+UbrFQ_~GcFVm>v(poN0s?5cuc|XF z_0=dgaL+`vI*blFkd9Y5AQy-N5N!FusknpF-;UChdTZi?%Cs5w@#I2iTB(;F5ZZ(r zEWz;2bs4L9|9G#ik~C0_1p3VQ(7y3EI%O}vpjdNfMjJRaS5XCb>Em(&r(cW@v{uqP z2#&O=jy`j=k~!zPjvfUpCOg^f@SAlWpBNxgK{*&bVF7>=G3d*K-_BaRy5MKn{qa^Q z=4Ve%cBc#N_3$TRg32OpZf@peREy`W*H^KJ{~Zwa--m7gLcbCDR}(F8RNF5Cl#Orm z<|7z6vL{b|_>}7s*(L?z6d=kLe8Ilu{_w@Jz(bcojcVJ6%WQ>p2>LwbaHZYCkzkUj`TIClgSU} zoMu?E3Iv4ajTsz~pv&m_@boBVaZX&`A`Xs~D_4f&l+I&kR~vUIaW$~Lrk|4hW_$iu zpeZUdym)qu!58g~W1DQ*e6Zuo7rsrKUR12YFF`NaiGk}4N*9BhmKlE-{_C$lIiWG4 z;jYfL2jl=!3*3)Fr$^S$DV%9;iA4tgmMyO;)S#uI`RzP-P|tfM4^JYF@=kt! zz<2N4BPhtMP8d5fmaL8*LaC^#?x6rkRl!OxyK;+| z`0yGRc(TYUD%L*QZD0%Nv4mSvBT6Ai2@E>bgvkeTBwD8V*R5n$7Z<7%Uz|VxLy+UL zmI=pY8>tqpA{Q#Qm9!2jbKn1rLj4P!#s3Q9`%f=7CY|LuggCNGgdDiJF@cig1NPDU z48^>XcFAGznR$pp1qtZvTM0#q+z`AL!}3OG|W! zKA^S~OfyTeN%xT9N8mQ+;jX9HbecEe0S=#>WS$(b^c$}F+6?qaC!8DT&pSNhld~My zoL0KoCZFWGR}C zEDm9j7>Kq>Z9>qVy%;rHa2FDpuI$gDxJ=FrcqF#BUzquR&jZX=F=BYHMWZMr%V@|? zbjZU%G{c{73juD^yzy!LG>&D<9K?krB^#3l@;QNkGBfMGXN87_Dw#mmjy@$A!qS*- z{sfmBb0+M>JBy4TN<~|`*>PtGAw&isQ>`9vStY0wO-K}|U`~%|I`zav?ZZWr1_$wxuv3D+pf<>Q z1J3g96cQ@$#w?R8%A521NAGyS5rZyT;MDYUAuOz*9=G~Ats;O+JqeCA!ng^ ziksseRPQ#hw$#LHk3b3Ci4g%n_MxjHYy`@W@kkGo+?Ii1KeOjkECi zjF2O+x5;H7*P9p3op8i!tMxwI;Bk)@%AZz(l%bBs;yn6QlcgC_MN57A;PH(F?P+NR zB&Vb>p@`;=)MyjpV~{S03+?Xi26$|>?70Yd1dw}$+vV%-4N}f30O^xMA#-s=cJx}i z*XT}Jpjg<8tMhwN(ijU@F(Fiq7b%ndn76;zU;1t*=|SSMIb8QsdBLx^8X{{s5(qK)vMTJ%FAk zAOWg3=eg*R4bR)*8+^F8vxlXoeWhhz&038{mvIuUWGV4e;)m79|@3>C

BuN$6MIc$p}5W z603}%O-e4IpRD^FSzQ5vQl>6IOUr7V*~Ou}K@tuMA+CcrX9xZByN0`~yu-0*1ler( zu^(>$P2sDY23RKK5Q-TzM>G=5hB@$iTZX?;oM9V^)_-4lR~mhW6CLDwsb{(rwZfniCyzE@b`6tIG~Xqq!5+$GJ1e zLFbSC=cWL*5^={SpmmezfGQdq(P)%l_hJGZ=)s3NtehwxUbq*~Im$QaVvMgT$BxHm ze9CU0ot>Ft5Q;+@*HgJwt?VN7A(b&GMbo}AYK~9_qarzB}CGx(wKkOE){$G zbjW%In^9iWYO4}UmbnZtefh_GI&wZ9#?Mr8OgB@q(gKUc&^8z)_(}9AwxtDKp=sc=BICzq}!$Opeex^|{#3jnp zlX^J8SpyCH;8|7SP8UGAz|`6Be1R?6k^YDHn*-G{*1lUMEk=8L404q4nYrgE0|*N>Fj=$rqQ7g z5a6W7f-B4PEMYr)`0y{#`0e$k!10Ms`j2Zyg{og=Us6dlm!mh9c4POQq0E8J$3bMx z%I3m#v}I1OEkL!I!hmu9_F}%Ome!>5^59@u1%(zYvI?HP#<)5g;stJ(VjgF3f*@G+0tjU~S_3iD0EvhL-@*VB%(V8hnUZV%yCNE(R zIY;>(L48D#%NiRSd!<9>ED8}Qsr(P%h4v=9M_=HjHtN3QT<>&og@ZpXUM|a({Tp!# zBNc+^u%so@ieA6&D)@zsv!Cj?p=Ec10Ob4UDOZ}E0VHBSo&3Dp9Mq4h*@ z^hUvsA+`pzJ>KrfCl-}pJ^MUnUBT2_=n0Pl0_e~*5?-3bd0CWdw)Fp;@Uo;|5bNuI z8G{O+k_zpr$*xuxFsK5Ey!zSkMMT#FPl$i+4*8%u$H^utov5G~NZuDR8n)7)w<#1s z>k%8B$$+r}MlO*K9Z*dp7oVGeMqh^y>HC->LjVnIT!Qj}3FOErgNncx7XsCrUqYfG z#=JaODjSfFcP4e?>)#nmQa;QRLKa7@a z8{>ADOH3)F$^OYU#o#!&WcomwHCN^>!fh#L>@eT|1Rnd-)q0MHG23r>dDZ;7aK%b! zq&VYhCIrXyw^vdcs4%zhZ&nL-_rj>E0Y>o!n1Xz$hXL$7!c_P5@o5)@pSoKdTg$$T zI}WQ8gA+>BK}hd9zvIrjxZ?4JUUF;1E{9{+MGT8qs@rSt1Tp-bcTUS@Z3vtw?BdAZ zR<=6iJ({5~?T7o5>B{*HCv_O#-2-+@w3_m%B`ts~nSYFwU6ece0AS8j(6tDN#3v_> z&lC#5nNjy84R4qKyaF^-(k3woeTS(zKRIX%|y470g0_pw_AqN z$2c9MX5YUtRm3WM^UzsEneT6KDj)y;o(He?W_?|Is3dtGkdTv$i=6kZTetRIczb_H zjl}~^CxMT2x|4#7JlWV6TTM15>PxHJppE3kkaVR`p{xwrK zj@cw2)WT}2+@$-$x7X)$KL=gF=)WBhTC<#rI`w$-?_gH6o~)E@*J-G~#qgjpe$`<7 zPu(J9{(%V(m0|YrS^ji~kjcm72<0iXw-HFV#_znhB8U-$Y5Jqn$H?NsYGZ0x5HUCn zp~QJ+q*j$-Z!Y5kGw73J+Y(|w!k@nZvJsn_y3f4s`Qh~kENl%3yYNlanqTnr=neQ> z7Eo=hOv<6$Vz?>0K&&8koS~6XcJ?W})1Od&Uqhe(LRuhmU;dN_=>o?ajz?XL6x3@7 zyQ_1Jpl<-rYzTpd!WXB9-&BwB10Y()UHz@~RT_v7NV`g%owBkr=v8|i*ahF$db}+` zNr3IFnwp;%w-g;!`xay*I5?QxsZm+rPT-4cH*Um+haUs|zl>;^kRrt5XSMP)+8NxZ zfrR6?vs*n@pnz_v3s|x^^fzkiyH7+YH-sL(TAgaFe0PI{+L<$F$Yj~GXHPZ5!IGu~ zqyftxLWf61ud|CwNkaoe$&|1+Gv*$O{imOzVL&OE!|hPQfD4m?ypD#X1fdjm(+JEn ze&{5irLhvS?AWl!CTl67BL2jj&Ctb3q6N|tq~>Zk-A%fHl7}MpZll~MyQ36B++1Dv zT`c2*L0=>_4@I9zOVjxY_|o$>N!I+=oZSDE*Z3*0&;b`vN#E9N}Ps5UyC#R_KRI&s@*xJl6n@}TNfJjP$~kkL}K zdy9-$wG5A2I2hI?ypNDiWK=o|>DUOqkSNBY=v}dD@s}q9!A`)lAf|Ua(#<0cx^D`f~;sn%^Lnw)#%`**BI`>a%LSX(gX2g8KMv2fu7UJ`-8w=Z0{ z(93|hrErRN?A<$br4|7hHGsP+RL*H8x@=^roLAJp5;7PbU=3j^4M| zQyRSC5i$RW8R17XP5QG&%Te0yf;oncwm0H*fei<1KhW?1&osPl!r|yy?xg9@2 zT{^re6^uBADU6(02!b%)h-~l2Yk2V@KgRv{@8A1(tff37SmotT(-<;oQw09Gg@J9&AJ{qoB%&ePwEkhv-6IExF2 zS8bp~AurLC;6Vn}S5;P$5ksr|%QL2{v$L`9e}in%FMr%ZulhY)Tt%Rm4LNnIRY-d^ z7e3D^jB=U1`hTV-V$yFP@+`44En~%PaO&ARuuz_Sch(-l+yCEYLVpY)b-|K##( zxBe8P=WhIdX(SlJd{?jHpl;P69+AmlEf9VG&+Wv5h0W@wy4rxX!$Nhl@Xo1zK^BT{ z-VZJGuM#90Kd}~U{R0Lzjy&y8K)LXj=w6Q;8 zDXoa?6~t?7TACUBeXl1R<1??WFX?D&3j;7A$+dpMfkKtmO0kU-{Qyv%sKTh_D6m#l z#GG)})nL0ZO4ul!FpZ7B(#yKm769sWJ&@t9AdMYxlh_3a>N&&iY|@1ma42H8J^ORa z2~AB+SsKkc70UaLO`8rse*Ac~WK?FR2HLV$IjE<1II2%J=KK5n+a5ExYI;oZuf1 zT>_U<3-Y3GeNkF*9pxy3pxHi?>K#R$V3$EK_2JOLS>>!($FaxIv;Ed`n3xM)Y2SJ7 zq?ihqCA<)2u(y3(=POnQmkP21tcbV|y=q$v*$c$&dZJI3SvEcMzV7%&Jv}`VV8Qg`GDo*Wp5!#566^Va z`l59T%P84O)g7Blg7t1bv2tC{{2*4IfBB@iu-C=Olj6c7=Nq7xR zJ{E=VcWBsQ>`Eyh` zE(zFs(F-#04jvvJIF|~*=k^XXRV@<}x<%fE5qigtUAs;pgho_?s@-rzNSlYLPU0(Y zoI=LDt#lsmRE~TLQ}^|6%f0p&W}Tn@_G%eP50JO@_QoRfXWo>}OlnTvMGWTU%a;$1 zj1YnbE48=qR%|eYFcN4$uOlyD9yDaEG=>pvFC6_YM+4HWlw9*=gvrmkpSz4SoXJ!~Xfx$B}-r zHMtn*!T4+HtPaAtq3G0;{qoq!>gkCg2o>9ybX7T=E)e3&1|6y;M795SMEu2Vu69HT zh5TuZzgIVUh_way_=J<_Xsmi-4P4?h_ZernGIgr@Po1Az$!8%F5r=*+e$e07R}-?T z8uo5|EL{(<&V-f>5%?ltyiaJZy)G+imFRrru_1rl8RE?BOKom?ck}&TbNoM37ODl{wV!R+?84l#c zrNMYDT7?Be2Pz6#+u57|1+6o;@Y0`h2@RbrxuH*?P3L!ecXye3%O5Vz_GC;IJ|Q9h z->2@~_@@9#hlsjnzuq<0i?>$`rIWJ;AH5ertaQj5X7lFH*Tz=&3&s~a z4_C;Xdf_#gMGD>Z$Y^Eo-6=YR+N zqN1WkATetpq?5KvKYLo-*=smFWXa4cP#K07;>udVO;{TdUgWPWk+`pL2U zxxT{USU(mvb`9u8DFC?^2t}1owp?8nhqtO_&6AO*3DWi(a?R6NiYC+tU0F;>z<~y@Q1#$kf%aEt5PuF*T(PNKgZu6o*@9v^J>d z?k|~qV_FL0aJujNKO?n22k?IoAkP1QCr$3I+5PNgftf`c_)5@}3>@qqnLh}m zBymDCSePw$xuvvF2+GKhR43?4xPftd4Y}v_P2w2|Vs?Pzs3|CTkp~qZ>=7uZjDwCy z1bJ>DIy}7nnTQTO5wVjd$F{rcBpsi(jg#mn9y}r#Gst9OqP$#cERZ~&@~G()q6x2- zNyj02-w@Lby%8+6`|g2H(T`sIhwW~vcEAa8zap2)sh2m%gYtLNevumc2&j$C}HH%h&BrJq2lB;;WYE5y?xq5pG1ro z3FU-JBQ8vOH+c>Udil(i=2v)J2l3g#;bEU93xE6vmKau;k0h#>0SKUNv)kCa=H~p( zH-4sgce5>MS(lD%DN3Did>l;Ta*U7+8*zFo!*5n4p|m)bEU|MP9UN3b+_Riv1a{AL z?zA_W5Tdow(4u>?dlPKLM@xkARWvm}CXSB4IE3V87$D`64jZC7b>K3IHCI9@#)C3z z@HNIA#lGmxrJVZnDZ%+He=87)g~}Unz%rPO*tKhkT$PJNF-Z<2Cnpn!D5}jvO#2JU z_(|#LlS!dnyAU%XQWtrm0;^ija7aW_^YRbzV3;x*ibh3^?gHfdml49U1b@)Qc79kC8q z;z=J)ezhIQPyp^C`d{l*YX+T~Jm~XU