From b8d51811e041267df0eabe27f8bddf7b139f6eb7 Mon Sep 17 00:00:00 2001 From: ltcptgeneral <35508619+ltcptgeneral@users.noreply.github.com> Date: Mon, 13 Apr 2020 19:58:04 +0000 Subject: [PATCH] testing release 1.2 of analysis.py --- .../__pycache__/analysis.cpython-38.pyc | Bin 24239 -> 28553 bytes .../analysis-amd64/analysis/analysis.py | 217 +++++++++++++++++- 2 files changed, 213 insertions(+), 4 deletions(-) diff --git a/analysis-master/analysis-amd64/analysis/__pycache__/analysis.cpython-38.pyc b/analysis-master/analysis-amd64/analysis/__pycache__/analysis.cpython-38.pyc index dad67484f706facc586e30849757a7ea78a95eea..b2bf81759d5a4cf5f6be74d3208efa0288a16e97 100644 GIT binary patch delta 10703 zcmb_i33y!9b)Gl7cG;3_NwzF`k>!zXS(Z21h}f1v#zr2z$O|u+;c4!Z{P1ktHR(wwKl#$T@YWWE{xZyb@6(&KHi`<#22ZH;)~VA@kX^V-lR6g zo7Lv{5_O5>Z;AQhOVt*|wb#X#vDR}g)~YYN)1$VsHs(3!Qrq-0ww$fFpssgnS5kgCbp=kr_mQ9^`qK&Lawye#!Y+{>b%es8a7PeKktk1V> zW4*GaOSZtyqiiRuKBthS8}e<(SU;>yi2rL9D9h#e-1!$h%-YDd@*)rP3n%Nmck zLh4l6#;%^O?ym0cZcTYb814@7yr*K@86(Epq8evyMm&||X|wH&kv`RCMvc^Do0ed0 z=INNO@kDtWeKvN{-zE~VU7T|dKH2EGusq|og2`0ch#OP7TZdCc*6t5qXqMNB=WM(D557B<$AyL=ojv`f{GqX znXnYiQihA9ys$3T3!;$Q*64aF&-nOW`p2&W>?5$0sHHF`AA0r+zJBlqC-V(x9t1eN zC`k!Em~*wut$I?iN#1~--2@@U@-uyuW}Ekms;=D!95PjJT1%XoSLo*oLPLukJ|*wd z8s>{hx_G3kyZ;(0SU#>LMs@xkj4OS;@L)1FnMlSBEjFNQ=`pUGyqS6_K5D1+NN(Zs zB@olLVNlsA;u{t#`^8-wdWwAg#SM+h!hWKZOgkRVj@R@R<|45##;ZR*{-8jT%*HGlvh2q*&-sAZ`WZ6^u71Yi595OfFq!5}|L+p9%9 zxcPm`W>LSTw@9tyTOwD>S@hWMUP^3gkoQF7u9)#yUfQxFoI|z=C=4pci;JIbIZ;(4 zLEN;pv0r=gPEqzM$nz!+EVPt*=Y^8XU)bV3+Ve7ITJZ^@`&$C>U z24ftM0D#<~s_h?<-u97Ia(J51Xq>*Br-})sO=+>ESM4jZiKx`tXfiP}W>Un9knY@= zD5>ZKkzN)jx33gG>g}54n9%kmB^@&H6g*`Bb$R$!@$^;ew%VSscfdAqO_g9pN5H|lZ8j^#_cSZXis)MDVC|jDINR;H=LA;9`ygR@JNG zm}}San?-Nm+jTNHwNRJpzy1Jv_%X5j>g6Y;|KCsT{7%3*z+Hg50lyCT4ZwN8J%HZ? ze1O36pVoOokICJp2BLa`4qMZ)j61~_u6{5BK~}XveOg*)kyP@G&ZW4J*WO9Xh(Prm z+_PJ)8Zpw5G(}T$gbu4vnY}tbhym{hJOB^?soig(Bo(_CCArY-)rM*nL8sd#UVYEQ zb^TPHaa(0+-AqSJIs^H6anH^*ZPF_%sne>ZVU0w@CP>AMG;gJ5@!HO&u47c-7XTFm zp(?4B<&SB@dd%b>M#sZ|-vL|{t9NZ}z6nJcgb<*dnu+mUQU8NzyhFUUt6AACF7KK+ z4*mE;1fihq6X?W00%#*p>#q;zvXwnCjq;xngPb@^u?4ofJ_;_W)d$3jyI1eUd{K%M zDdam!$_ku;ytK(b2I@xuj{+V86ts7^RP^;fG2qnmk3sqb0E>bDA>iYH8NeR_Fmwrv z#D+ah;eztzpvo$L9BfkgcPKtCbc&^UH-UiRFTCVg1ZPf;p47K<5kRj!_2pn3@-@GJB zyOD^`qif|JdQOSyaSEkZq-;)47yk?CWVtvWLGe+*V}R1Tqh)`e+#O;5DI%hfEVs&^ zPpfrNn2F-1q42=m#O6s1^?AUb1O5_FYHuu{j)T}6LD!St++V9G_lir`ceW$0 zTjfzLdPH!JZav2RE%RDT%r*|Db%3zuR^(LMDHaWIwb+%9Z9n4KZ>Aj~oFH$dpi6ejXV% zw4&_89`Nv?Z4D&{6dygbM0ttC_lma<9b7Aikj|CT-M0R+VU7{;!`+=FWx@#0;r@XK z92R_44)M#wC+r~#24si$2k1Fxi1#1ad}`hy0!NI|c+y}GIfnTfi7D}*@gk(Z1o$cd zRF1*qu$^MrP`mQ7xMrxUelzvc4aYJ3^V-4jUUBzOYwaurkBQF??QfFQ?m})v5BTF4 z;9k*qw53+&KMp=|?a|woJ6Uk>^M3?5slIV^$JMLR*a47{>^YQP0K5!GO!YEwuLn`a-%--2YBAYMet0=xqFHi1R_YbbsPP^=pNF4|t*|n_SFu{Gyu#dc*eJ78Z2KJrx*OKd!~b^vi*iU7-7+L)66^3Dn?7 zB$?39Xp?jUx04#I08Ij=Z$%+e9C~ld3T3Kgww_&DA~~*Q1w`HAt=`5=8wr7ikC2Ab z%A@YH?rFEw(QWGp6=$3(&X9`S){MVF%*E=Z=*HGeCODnpZ~(s%@O?mGKjYA~0ybqP z@hmya^iq?{U9C(t&e1t5(RPwmMpwnD{i}eecVJN&kJsz^^8-$`e|Go@HZ3G<)HQN#U0Wip(rCp;cTXJAH+F zPbq@`8#qc?9LzC2U$D4-nKIQh$K)L06-*KT;cOkN+F>x6PM3fcC+a({gC1q?%VMfwtlBO0<*~fSTjYcYE2$2iW;_>1|l3Ww=yeaZueH>4=W>kym zkx{OtPDLql)2UVnO+SacUjY6G@V|ibfKqG46CaI!M>>(lr+C$@Ox4fvBGH6A*^vGw zQK9k{B;Fw_L*b#2FU&8aMY=W|DxZD-@B?)IE8yP&KL-34;Aa4r;&SykcCy^lh^hQ_ zf~TQhH5KUE-52zg%Wt`lzm9A^gP6ICzUByl?ot)zQr)bTdFXAcqI&2pte*v_z7m33+1LiyrQIH^TUgFHFZF0=T`|vx@|mgHwUDwGwJGlyJ0r1BYG)Rrr$P3 zp%a(bGScOZ=N?)pArD?H8mY8QA2Mp0t%;FfS&QvAc0MV@7EsFG1$j0-MX{DHGTtn* zc*Q&x(>OVXC0Xy^Hp$?&L=L_HaK0 zl=4G%-Vt)Jo}+ISywEfMyywYfASj-CL=r!JuP}4rZaKLQqp~wgi7EdP$#Vn z8R%T_pY_xEfLzSaGLB>{Ob@3>YWXmMRiVWwu}F}o$LU=Xk+_N z(+>p9bSR*D__(QhqHLrfmseqO>E|r=3}&n^zhWNbou!Eth-vX*rbP;KOGY-x+aHGP zXjG@PU>-}=+KeQm^dUzX^7DFFIm_M#v=xlb+ica$8p&6%pVGj&fqWKuhF(6MNz$Dr zZ3sIDepjAKaGVvEX@2^74Vh82(lfLRB0bpytdP|dcuUz_Nj%x4DyEkg45kThnKR)| z+8Oug!ym@yTRXFb%%z=(>-i6?$FtZT`kJczFoisFGlwrXjOO78} z;h_P$8TZN=Sw!4^5R@MR{5Htx&EGe3En)j@TtDhyvUy~Ygk?Q45DwMaED7+xBbLy1 z@%BwUnQmgB=c$Ain=}p6{%MLqZK^LZ79ZC5qFkAN;iNH;0w>{{uwGeR}`^ delta 6585 zcmb7IYfv1=5x%`U;D9?o4-g>4LoYZU2M7reMzRC~WP?r!PFUW=J}oy3thw96vkL-Q z(qW5A#fo3j#EI?Lj%CMju&%)hu zoKn)M_|((W-P6-E)6=uJel9%uypXfsrafH%qbCSPu9%D0yeUW6ox&b z2mW6j_V9K|I4zPR<_fkR8(Bk2=WL`@&O7c9myt5!n6rt?6u9T&tE zan3rD#yKnFob@D~bMoVy4J3ne3P@}tae={AFi<6_f`O`z+r>g?d5IfZZfF%jtD1PA z<$+c)H&O#b*OEGJWObb5BlVn966Z9KM$TCi=WHTPoKtFYVqj=9*}_?6aiM0?!a3!f zgYyiK?IdSTfSFdrdHcyulFNB(!Y-8eI>oe!sG{n$8dw*P^6TqNAGO09<5u z*!w|t+#L0?9o3CiU(X$>{*S#p;jF_DrVT+dgyBV2?x_p6qL9~a`gSGIXVT~u@E>hQ z*alz-AwwWmJ?z=e+I@W+t;(Iq-hp6=rzv{YS5&wKNUva|6M2-%GHr&Y=+Kl=w%50H zz+y*q>XJH^+7ReGv#@$LyLXi(q7Zv!V#aVqCv+tO@ndp*Lw_qM7->{eM`hXwG~I-w zF7J=6I~O^gdA2>S@cAtaXA=ox{)c-U?fW9F#EW^uc!%)3I_~f zM$D!-8CW{lo*0paO^X&`rlC++juSnP=?hkLeq%k-*kzSt|7?7%JRUQz-LU%_%r%HJ zVPhJbzP|igFhfOzuc5&Vx}WWCDiem-SX1R9i}y9%Z?^MMxS8$TytcRpNYP0pUG}Dn zS-LDmv>=gmm>WB~d0pN?l(Y``)aKTnVq~sHC_%_Ta3N$OB&0c2io5}US8$EefpFPf z=`I%?VEYEwu)ZzFgevydmexg{<}?R)^3`|n6*4j+vQCu{tO(r{c2=)(6(}v0;%z5%`1jwpfr)NMV0#6_X~k zcWXgWDyLM?|BB_ctz)OR`jT`$-MTtu5TeZ8R_jYO0klS=i(Q)ceWa~#QL@0icSfKM zi-o!W*ky+l%hRmQL91C_du_8haq|$=pw>hRRqV*HsMaK_LekJlN>$9kZi{MdvZ~>j zl4wfRI#69^QLRJO7K_7>!y}5CQr8E%1r}W&>h>k+O4BKI>p^$5MHk!WT>xKYj@LzG zN!4m<=pg%h`uUU3(DO0!K-GhdV90mFye~{or^pH%sIj@tS9H_zM zP&6_T)fBv}MMr<=s_LQuQmeLnm3a+u!`-4W!$M-(E;wyailCKnjO0G(G{3A z&)AJj9WM2t2Kk&GWA!_#Vi*M@mnY*OBn3HyqZxTHh@u8f;fb)K!zBWjj99Qs4k?(O z1JLTkib5Gt!0}!j@)m?_0I!E{Vk12)4a;GT-hmx=BHV>=62Qp9Oc~VK8#`)>_aT=j zBc3j-uIKCwRh+=?DS-2M7K}{TH{;Pm+PLCBpN3sq*r}b<*Wm=|?EqevnR-qkg8>+s zs;rET4M(XauI%+M&=+@qnRy1kjRv^yH#4bYqcF?9*|ElZH*&v&@Lhy^5E2ux`u=Xm z6Wvy?9!Ak0AmBcs-$!@^;UR?62(ti7{HG7HmpY65i5O?0)Vz~@sK%YVMDx^~0IS+n zk)PV{>vuKA;xVW9;Q-$Q@cZfgNMqeW9{>PHIRk0FKBg@1Hx-~nOUjA4D4U0nh!m%I zn!6V|+VS*eIe~K7voh(u?B)$CS#8%84>EpC9qW68{K!XAmxRfrV0bs{7o6L%ltfNe8^@ zmv*WjySkm94nKVibRb*vqw*7=1Af@=@fMw1!V37ip~Or&5kJO}pF;R4!p{&=osDJC z(T``tWji_SqRCzKFYGcv_qHMs!R-VoFHbAmpDjAs#^L--{Ezh-`g)UOfZW76k~u+8tDgy;M~ zD=e_5wj*iAys%3MatfslcPHG%(MT`9Kc{i7@CRvtzQ~inM|;{zFwV3Eg>$7zJsKqN z*poweh0-}z>kq|#jsm{{SeP2W4f*_bVb|qN3^h{D3p!>O9JJXF7Y`2Y@xjAcWjTBE zJK|2#m(T#;2;KB$q<;(G@20;)`W2+<3rHqMgUNf#gje~jkM?G=hxdL_!RNszCYWoM z(81tQg+1QiRF~2kbQ|e?{Mq%51*ajZufZ##m38bp*v4)1@bD$)qd52|_$G(S#7L*X zVMSV0#+ZYCy04}2qJw579)`DB27ZGNd}N@%h7Zh_gpX-AXT}*zruVObhU$xslZyYTK4{ zqF+RrR}tPu;FZd|NWF)UQ07!A`($W+I~i9pG8Z@5RL*ES%)MV32N}z)_FEsay=L>S=Hpoa2SeK)}LK4m|O-FCszRPqj6rP zo_YFl7+f%x&z~Vsi2oqqHV6Fmz@JtEpxbT`9?ofeJ?M+71i z*#PZDwoyBLZgu!+KFIKIZtxYV3uyRO3ExfOE6YeQs>+9?8TbU)g!~K$6O@3EI%B9} z9s!^Xx1>g=qSA9xi#B!vU8gqol7`#Yf>_mO@Mny*_-VB4EbDf~w@gh2!yVG>~q0Yl>t_<6&d z;SY41Y`g}6f+*l5PGIX>2)Cg)JZFBt!10?;44xL|l|X`DqhJi4Avi#{-vE5u{GG)Mkb4WzjQ_{Z9! He!=xWXv(Fc diff --git a/analysis-master/analysis-amd64/analysis/analysis.py b/analysis-master/analysis-amd64/analysis/analysis.py index 944dd0c7..533a31af 100644 --- a/analysis-master/analysis-amd64/analysis/analysis.py +++ b/analysis-master/analysis-amd64/analysis/analysis.py @@ -7,10 +7,16 @@ # current benchmark of optimization: 1.33 times faster # setup: -__version__ = "1.1.13.009" +__version__ = "1.2.0.000" # changelog should be viewed using print(analysis.__changelog__) __changelog__ = """changelog: + 1.2.0.000: + - cleaned up wild card imports with scipy and sklearn + - added CorrelationTests class + - added StatisticalTests class + - added several correlation tests to CorrelationTests + - added several statistical tests to StatisticalTests 1.1.13.009: - moved elo, glicko2, trueskill functions under class Metrics 1.1.13.008: @@ -288,9 +294,9 @@ import numba from numba import jit import numpy as np import scipy -from scipy import * +from scipy import optimize, stats import sklearn -from sklearn import * +from sklearn import preprocessing, pipeline, linear_model, metrics, cluster, decomposition, tree, neighbors, naive_bayes, svm, model_selection, ensemble from analysis import trueskill as Trueskill class error(ValueError): @@ -697,4 +703,207 @@ def random_forest_regressor(data, outputs, test_size, n_estimators="warn", crite kernel.fit(data_train, outputs_train) predictions = kernel.predict(data_test) - return kernel, RegressionMetrics(predictions, outputs_test) \ No newline at end of file + return kernel, RegressionMetrics(predictions, outputs_test) + +class CorrelationTests: + + def anova_oneway(*args): #expects arrays of samples + + results = scipy.stats.f_oneway(*args) + return {"F-value": results[0], "p-value": results[1]} + + def pearson(x, y): + + results = scipy.stats.pearsonr(x, y) + return {"r-value": results[0], "p-value": results[1]} + + def spearman(a, b = None, axis = 0, nan_policy = 'propagate'): + + results = scipy.stats.spearmanr(a, b = b, axis = axis, nan_policy = nan_policy) + return {"r-value": results[0], "p-value": results[1]} + + def point_biserial(x,y): + + results = scipy.stats.pointbiserialr(x, y) + return {"r-value": results[0], "p-value": results[1]} + + def kendall(x, y, initial_lexsort = None, nan_policy = 'propagate', method = 'auto'): + + results = scipy.stats.kendalltau(x, y, initial_lexsort = initial_lexsort, nan_policy = nan_policy, method = method) + return {"tau": results[0], "p-value": results[1]} + + def kendall_weighted(x, y, rank = True, weigher = None, additive = True): + + results = scipy.stats.weightedtau(x, y, rank = rank, weigher = weigher, additive = additive) + return {"tau": results[0], "p-value": results[1]} + + def mgc(x, y, compute_distance = None, reps = 1000, workers = 1, is_twosamp = False, random_state = None): + + results = scipy.stats.multiscale_graphcorr(x, y, compute_distance = compute_distance, reps = reps, workers = workers, is_twosamp = is_twosamp, random_state = random_state) + return {"k-value": results[0], "p-value": results[1], "data": results[2]} # unsure if MGC test returns a k value + +class StatisticalTests: + + def ttest_onesample(a, popmean, axis = 0, nan_policy = 'propagate'): + + results = scipy.stats.ttest_1samp(a, popmean, axis = axis, nan_policy = nan_policy) + return {"t-value": results[0], "p-value": results[1]} + + def ttest_independent(a, b, equal = True, nan_policy = 'propagate'): + + results = scipt.stats.ttest_ind(a, b, equal_var = equal, nan_policy = nan_policy) + return {"t-value": results[0], "p-value": results[1]} + + def ttest_statistic(o1, o2, equal = True): + + results = scipy.stats.ttest_ind_from_stats(o1["mean"], o1["std"], o1["nobs"], o2["mean"], o2["std"], o2["nobs"], equal_var = equal) + return {"t-value": results[0], "p-value": results[1]} + + def ttest_related(a, b, axis = 0, nan_policy='propagate'): + + results = scipy.stats.ttest_rel(a, b, axis = axis, nan_policy = nan_policy) + return {"t-value": results[0], "p-value": results[1]} + + def ks_fitness(rvs, cdf, args = (), N = 20, alternative = 'two-sided', mode = 'approx'): + + results = scipy.stats.kstest(rvs, cdf, args = args, N = N, alternative = alternative, mode = mode) + return {"ks-value": results[0], "p-value": results[1]} + + def chisquare(f_obs, f_exp = None, ddof = None, axis = 0): + + results = scipy.stats.chisquare(f_obs, f_exp = f_exp, ddof = ddof, axis = axis) + return {"chisquared-value": results[0], "p-value": results[1]} + + def powerdivergence(f_obs, f_exp = None, ddof = None, axis = 0, lambda_ = None): + + results = scipy.stats.power_divergence(f_obs, f_exp = f_exp, ddof = ddof, axis = axis, lambda_ = lambda_) + return {"powerdivergence-value": results[0], "p-value": results[1]} + + def ks_twosample(x, y, alternative = 'two_sided', mode = 'auto'): + + results = scipy.stats.ks_2samp(x, y, alternative = alternative, mode = mode) + return {"ks-value": results[0], "p-value": results[1]} + + def es_twosample(x, y, t = (0.4, 0.8)): + + results = scipy.stats.epps_singleton_2samp(x, y, t = t) + return {"es-value": results[0], "p-value": results[1]} + + def mw_rank(x, y, use_continuity = True, alternative = None): + + results = scipy.stats.mannwhitneyu(x, y, use_continuity = use_continuity, alternative = alternative) + return {"u-value": results[0], "p-value": results[1]} + + def mw_tiecorrection(rank_values): + + results = scipy.stats.tiecorrect(rank_values) + return {"correction-factor": results} + + def rankdata(a, method = 'average'): + + results = scipy.stats.rankdata(a, method = method) + return results + + def wilcoxon_ranksum(a, b): # this seems to be superceded by Mann Whitney Wilcoxon U Test + + results = scipy.stats.ranksums(a, b) + return {"u-value": results[0], "p-value": results[1]} + + def wilcoxon_signedrank(x, y = None, method = 'wilcox', correction = False, alternative = 'two-sided'): + + results = scipy.stats.wilcoxon(x, y = y, method = method, correction = correction, alternative = alternative) + return {"t-value": results[0], "p-value": results[1]} + + def kw_htest(*args, nan_policy = 'propagate'): + + results = scipy.stats.kruskal(*args, nan_policy = nan_policy) + return {"h-value": results[0], "p-value": results[1]} + + def friedman_chisquare(*args): + + results = scipy.stats.friedmanchisquare(*args) + return {"chisquared-value": results[0], "p-value": results[1]} + + def bm-wtest(x, y, alternative = 'two-sided', distribution = 't', nan_policy = 'propagate'): + + results = scipy.stats.brunnermunzel(x, y, alternative = alternative, distribution = distribution, nan_policy = nan_policy) + return {"w-value": results[0], "p-value": results[1]} + + def combine_pvalues(pvalues, method = 'fisher', weights = None): + + results = scipy.stats.combine_pvalues(pvalues, method = method, weights = weights) + return {"combined-statistic": results[0], "p-value": results[1]} + + def jb_fitness(x): + + results = scipy.stats.jarque_bera(x) + return {"jb-value": results[0], "p-value": results[1]} + + def ab_equality(x, y): + + results = scipy.stats.ansari(x, y) + return {"ab-value": results[0], "p-value": results[1]} + + def bartlett_variance(*args): + + results = scipy.stats.bartlett(*args) + return {"t-value": results[0], "p-value": results[1]} + + def levene_variance(*args, center = 'median', proportiontocut = 0.05): + + results = scipy.stats.levene(*args center = center, proportiontocut = proportiontocut) + return {"w-value": results[0], "p-value": results[1]} + + def sw_normality(x): + + results = scipy.stats.shapiro(x) + return {"w-value": results[0], "p-value": results[1]} + + def shapiro(x): + + return "destroyed by facts and logic" + + def ad_onesample(x, dist = 'norm'): + + results = scipy.stats.anderson(x, dist = dist): + return {"d-value": results[0], "critical-values": results[1], "significance-value": results[2]} + + def ad_ksample(samples, midrank = True): + + results = scipy.stats.anderson_ksamp(samples, midrank = midrank) + return {"d-value": results[0], "critical-values": results[1], "significance-value": results[2]} + + def binomial(x, n = None, p = 0.5, alternative = 'two-sided'): + + results = scipy.stats.binom_test(x, n = n, p = p, alternative = alternative) + return {"p-value": results} + + def fk_variance(*args, center = 'median', proportiontocut = 0.05): + + results = scipy.stats.fligner(*args center = center, proportiontocut = proportiontocut) + return {"h-value": results[0], "p-value": results[1]} # unknown if the statistic is an h value + + def mood_mediantest(*args, ties = 'below', correction = True, lambda_ = 1, nan_policy = 'propagate'): + + results = scipy.stats.median_test(*args, ties = ties, correction = correction, lambda_ = lambda_, nan_policy = nan_policy)* + return {"chisquared-value": results[0], "p-value": results[1], "m-value": results[2], "table": results[3]} + + def mood_equalscale(x, y, axis = 0): + + results = scipy.stats.mood(x, y, axis = axis) + return {"z-score": results[0], "p-value": results[1]} + + def skewtest(a, axis = 0, nan_policy = 'propogate'): + + results = scipy.stats.skewtest(a, axis = axis, nan_policy = nan_policy) + return {"z-score": results[0], "p-value": results[1]} + + def kurtosistest(a, axis = 0, nan_policy = 'propogate'): + + results = scipy.stats.kurtosistest(a, axis = axis, nan_policy = nan_policy) + return {"z-score": results[0], "p-value": results[1]} + + def normaltest(a, axis = 0, nan_policy = 'propogate'): + + results = scipy.stats.normaltest(a, axis = axis, nan_policy = nan_policy) + return {"z-score": results[0], "p-value": results[1]} \ No newline at end of file