A Concise Course in Statistical Inference Brief Contents

All of Statistics: A Concise Course in Statistical Inference Brief Contents 1. Introduction…………………………………………………………11 Part I Probability 2. Probability…………………………………………………………...21 3. Random Variables…………………………….…………………….37 4. Expectation…………………………………………………………..69 5. Equalities…………………………………………………………….85 6. Convergence of Random Variables………………………………...89 Part II Statistical Inference 7. Models, Statistical Inference and Learning………………………105 8. Estimating the CDF and Statistical Functionals…………………117 9. The Bootstrap………………………………………………………129 10. Parametric Inference……………………………………………..145 11. Hypothesis Testing and p-values…………………………………179 12. Bayesian Inference………………………………………………..205 13. Statistical Decision Theory……………………………………….227 Part III Statistical Models and Methods

14. Linear Regression………………………………………………...245 15. Multivariate Models……………………………………………...269 16. Inference about Independence…………………………………..279 17. Undirected Graphs and Conditional Independence……………297 18. Loglinear Models…………………………………………………309 19. Causal Inference………………………………………………….327 20. Directed Graphs………………………………………………….343 21. Nonparametric curve Estimation……………………………….359 22. Smoothing Using Orthogonal Functions..………………………393 23. Classification……………………………………………………..425 24. Stochastic Processes………………………………………………473 25. Simulation Methods………………………………………………505 Appendix Fundamental Concepts in Inference


  

    !#"%$&%')(*,+.-0/12!34+5-6(*879:"?/9@#"%=A/B"%CD3E!9:%FG=+5G7,9:!/B"H/+.$5+5(JIK"%G= -@(*"H(*+5-@(*+5CL-M'N!9O-P(*F=,LG#(*-+.G-P(D"H(:+.-@(:+.CQ-LRSCL!T7F,(:L9U-:CQ+.QGCLVWQ-:7XLCQ+Y"%$5$5I=B"H(D" T+.G[Z +.G>"%G=8T>"%CD+.G,\$5]"%9@G+.G,E^_RBT>"H(*,LT>"H(*+5CL-LR"%G=K9@L$."H(*Q=`=+5-:CQ+.7$5+.GQ-La)+.-b/X[!3 CLQG#(*-La&’2(D"H(:+.-P(*+.CQ+Y"%G,-j(*%F!#( (*B"i(vCL%T>7F[(*L9>-:CQ+.LG#(:+.-@(:-he“L9: 9@L+5GE;%LG#(*+5G(:KebQL$”a–•r!T>7,F,(*Q9h-:CQ+.QGE(:+.-P(*(*%F!#(b(*B"H(b-P(D"H(:+.-P(*+.CL"%$j(:L!9PIŒ=+5=Go—˜(n"%77$qIh(*U(*,L+.97,9:!/$5LT-La +.G%-U"%9:ŒCDB"HG!+.G,a’2(*"H(*+5-@(:+.CL+."%G-tG"H(:+.C]"H$j-@(*"H(*+5-@(:+.CL-QaSm-@+.GO'W"%GC_Iv(*2!$5-s$.+53%zGQF[Z ž!ž


ž  

    ! 9*"H$1G_(*-QR/12!-P(*+.G,M"%G=Œ-@F77X!9@(r;!LC_(*!9T>"%CD+5GL-)eb+5(:!F,(F,G=L9@-@(*"%G=+5GO/B"%-@+.C -@(*"H(*+5-@(:+.CL-+5-$.+53%z=!+5GO/9:"%+.G?-:F,9:!Q9@I?/XQ'N!9@z32G,eb+5GebL9@ C]"%Gu-P(*F=QG#(*-Œ$.L"%9:G /B"%-@+.C479@!/B"%/,+.$.+q(JIc"%G= -@(D"i(*+.-P(*+5CL-hyEF+5CD3E$5$ I# wm"H(:+.C]"H$š-@(*"H(*+5-@(*+5C CL%F9:-@`-@71QG=-h(:[ TMFCD (*+5T>?!Gc(*Q=+.!F,-h"%G,= FG,+.G-@7+.9@+.G (*%7+.CQ-4VNCL!FG#(:+.G TQ(*,[=-QR(Jer=+5T>QG-:+5!GB"%$1+.G#(*Q!9*"%$5-Q(:C%a ^>"H((:tg[71QG-:\%'šCQVW/X2%(*-P(*9*"H77+.G,RoCLF,9@;!UQ-@(*+5Th"i(*+.%GoRj!9:"%7+5C]"%$T>2=Q$.-œ_(*CHa ^aU’[?km-@Q( !F,(b(:>9@L=Q-:+.%G`!F,9nF,G=L9@!9*"H=FB"H(:t%G!9:-nCL%F9:-@t!G879@!/B"%/+5$.+q(JIv"%G,=KT>"H(*Z T>"H(*+5C]"%$‘-P(D"H(:+.-@(:+.CQ-Lan+.-m/12!38"%9:%-:t'N9@!T (*B"i(6CL%F9:-@%a +nL9:t+.-œ"h-:FTT>"%9@I?%' (*,\T>"%+5Gv'N]"H(:F9:Q-nH'(:+.-/X[%3Xa ž abœ/12!3U+.-)-:F+q(D"%/,$.m'N%9“%G!9:-rFG=Q9:!9:"%=FB"H(:L-s+5GvT>"H(*oR2-@(*"H(*+5-@(:+.CL-“"%G= CQ!T>7,F,(*Q9“-@CL+.QGCL6"%-“e“L$5$j"%-“!9*"H=FB"H(:œ-@(:F=QGE(:-+.G>CL!T7F,(:L9r-@CL+5LGCQ6"%G= %(:L9yEFB"HGE(:+5(*"H(*+q;!zpBL$5=-La abkvCQg,"%T>7,$.%RrG!G7B"H9*"%TQ(:9:+.C?9@L!9@L-:-@+.!GjR/12%(*-P(*9:"%77+5GRr=,LG[Z -@+5(JIvQ-@(:+.T>"H(*+5!G?"%G=8!9*"H7+.CL"%$jT2=L$5-La , abkOB"];%4!T+5(:(:L= (*!7+5CL->+.G 7,9:!/B"H/+.$5+5(JI(*"H(v=G%(v7,$Y"]I "CLQG#(*9*"H$œ9:%$. +5G-@(*"H(*+5-@(*+5C]"%$)+.G['NL9:QGCLHaA %9d_g,"%T7$5%R&CL%FG#(*+.G, T>_(*2=-M"H9:h;2+.9P(*FB"H$.$5I "%/,-:LG#(La - abkJG !LG,L9*"H$”R)kt(*9PI(*A"];!!+5= /XL$."%/X!9:+5G4(*Q=+.!F,-CL"%$.CQF$Y"i(*+.%G-M+.Gc'W"];!!9U%' QT>7"%-:+5ŽL+.G,OCL!GCQL7,(:-La . abk1CL_(*9:+5CS+.G,'NQ9:LG,CL)/1_'N!9:)7B"%9:"%T>_(*9@+.C&+.G['NL9:QGCLHax+5-+5-‘(* !7,71!-@+5(:&%',T!-P(-P(D"H(:+.-P(*+.CQ-‘/12!3E-‡/F,(‰kj/XL$5+._;!S+5(‡+5-o(*)9:+5!#(‘es"]Iz(*n=b+5(La /"%9*"%TQ(:9:+5C\T2=L$5-m"H9:dF,G9:L"%$.+5-@(*+5C\"%G=K7XL="%!!!+5C]"%$5$5IhFGGB"H(:F9*"H$”a\0V +ne“œ3EG(: (*+q(*$5%a‰sCL%F9:-@+.-š=QTh"HG=+.G,6/F,(&k‡B"];%ser!9@3%Q=>B"H9:=M(*œTh"%3Hr(*,sTh"Z (*Q9:+."%$#"%-‰+.G#(*F,+5(*+q;!S"%-‰7X!-:-@+./$5&-:m(:B"H(‡(*,“T>"H(*Q9:+."%$%+5-‡;%L9PItF,G=L9@-@(*"%G=B"%/,$. =Q-:7+q(*z(*z'W"%-P(n7B"%CLHaS›œG#IEes"]I!RX-:$5"%G 71!+5G#(*L= !F,(LRs9:+5!!9U"%G= CL$Y"H9:+5(JI"%9:8G%(>-@I2G!G#I#Z T!F-Laœkn"79@!/B"%/,+.$.+q(JId(:L!9PI!RE(*b'N!9@Th"%$ $Y"%G,!FB"%!zH'xFG,CLL9P(D"%+5G#(JIŒeb+.CD8+.-s(:0/B"%-:+5-b%'x-P(D"H(:+.-P(*+.CL"%$o+5G,'NL9@LGCQ%aS\/B"%-:+5C 79:%/$.QT(*"H(ber0-@(:F=,I?+5G87,9:!/B"H/+.$5+5(JI>+.- %     !"#$&% !('*)+-,,+.0/213 !"451% !('0%6!(#7+,8'9:51;'0= )-'*?4+,#@

)-:LCQ!G=\7B"%9P(‡%'[(:S/12!3m+.-‡"%/X!F,(‡-P(D"H(:+.-@(:+.CL"%$%+5G,'NL9@LGCQ“"HG=t+q(*-‘CQ$.!-@&CL!F-@+.G-QR =B"H(*"0T+.G,+.Gz"HG=UTh"%CD,+.G$5]"%9@G+.G,ab/B"H-:+.C7,9:!/$5LT %'X-@(D"i(*+.-P(*+5C]"%$[+.G,'NQ9:LG,CL +.-s(:0+.G#;!Q9:-:\%'79:%/B"%/+5$.+5(JI %   A51B'0"%F,(:!9:->"%G=c(*71%+.G#(9:]"H=L9@-(:%(:L9UF-@Q'NF$ 9:_'NL9@LGCQL-Laukter!F,$.= L-:7XLCQ+Y"%$5$5I$5+.3H?(:TLG#(:+.!G(*,`/X[%32-/#I M6
9:2%("%G= ’[CDQ9@;2+.-@ V  ^m"%G,=
9@+.TT>_(:(m"%G=A’E(*+.9@Ž]"%3HL9UV ž 6 ^b'N9:%T eb,+.CD kn"%=B"%7[(*L= T>"%G#Iv_g,"%T7$5L-n"%G=8_g[CLQ9:CQ+.-:Q-La

ž
.

#7,C#7)-, N L 6   )+#7'* !(F 






2’ (D"H(:+.-P(*+.CQ+Y"%G,-b"%G=KCQ!T7F,(*Q9b-:CL+5LG#(*+5-@(:-m%' (:LG8F-:t=+ 1L9@LG#(n$Y"%G,!FB"%!z'N!9(: -*"%TK(:+.G,a +mQ9:4+5-v"=+5CQ(*+5!GB"%9PI(*B"i(v(*K9:]"H=L9vT>"]Ices"%G#(Œ(:9:_(*F9@G(* (*9@!F!,!F,(b(*0CQ!F9:-@%a 

#7,C#7)-,

L-P(*+.T>"H(:+.!G CL$."%-:-@+5pBCL"H(*+5!G CL$5F-@(:L9:+5G =B"H(*" CL"H(*Q-m"HG=?CL!G,p=LGCQd+5GE(:L9P;%"H$.eb+q(*?!FB"%9:"%G#(*QL-b!G?'N9:LyEFQGCQI8/XLB"];2+.%9 FG,+5'N!9@T/X!FG=,-m!G?79@!/B"%/,+.$.+q(JIhH'xQ9:9:%9:





       "!

%$

ž2

 

    !

'#7'* FI?H6'


-6$

@9 ]"%$oGEFTd/1Q9:+.G['  xV 1^ +5G,pBTMFT %x(:0$Y"%9@!L-P(bG2F,TM/1Q9h-:F,CD`(*"H( šV 1^“'N%9b"%$.$  (:+.G3>%'‰(*+5-b"%-(*0T+.G+5TMFT %'

-:F,7  šV 1^ -@F79:QTMFT %š(*0-@Th"%$5$.Q-@(GEFTM/XL9 v-@FCD`(:B"H(  x V 1^r'N!9b"%$5$  (:+.G3>%'‰(*+5-b"%-(*0T>"ig[+.TMFT %'

V 1 " ž ^ V #" ^ %$&$&$' , ž )+ * ! ) -, )2# ( ,/. -0 , 0 =+? 
"%TTh"t'NF,GCQ(:+.!G768 ;: 3)V54x^ 9 !F[(*CL%T> @ A -:"%T>7,$.6-@7B"%CLhVW-@Q(n%'!F,(:CL!TL-*^  _;!LG#(MVW-@F/-:_(œH' A ^ B xV @ ^ +5G=+.CL"H(*%9r'NFGC_(*+.%GDC ž +5' @ E "%G=  H(*Q9@eb+.-@ F“VG6^ 79@!/B"%/,+.$.+q(JI%'x_;!QGEH(  I I GEFTM/XL9b%'7X!+.G#(*-+5G?-:Q(  JLK CQFTMF$."H(*+q;!z=+5-@(*9@+./F[(*+.%Gh'NFGC_(*+.%G

MK 79@!/B"%/,+.$.+q(JI=LG-@+5(JIVN!9Th"H-:-D^“'NF,GCQ(:+.!G ONPJ B"%-b=+5-@(:9:+./,F,(*+5!GQJ ONR

B"%-b=QG-:+q(J7I

4S "%G= B"];!0(*,\-:"%T\=,+.-@(:9:+5/F,(*+5!G TNPJ -:"%T>7,$.6%'x-:+5ŽL  'N9:!TUJ V -P(D"%G="%9:=`wm!9@Th"H$j79@!/B"%/+5$.+q(JIh=QG-:+q(JI W -P(D"%G="%9:=`wm!9@Th"H$j=+5-@(*9@+./F[(*+.%Gv'NFGCQ0 (:+.!G F7,71QY9 4yEFB"HGE(:+.$5z%'[ZV  ž ^r+”a %\a W V ž " 4š^ X : ? ]6V A^ 4^6_ JhV` 1^ g,7XLC_(*Q=`;H"%$.F,VWT]"%G ^rH'x9:"%G=!T;%"H9:+Y"H/$. ]6VGa,V A^@^ 4 6 aV` 1^ ? JhV` 1^ g,7XLC_(*Q=`;H"%$.F,VWT]"%G ^rH'a,V A^ bUV A^ ;H"%9@+Y"%GCQz%'9*"%G=,!T ;H"%9:+."%/$5 ' V M^ CQ;H"%9@+Y"%G,CLz/XQ(Je“LLG "HG= =B"i(D" -:"%T>7,$.6-@+.ŽL  c CQ!G#;!L9@!LG,CLd+5G?79:!/"%/+.$5+5(JI "ed CQ!G#;!L9@!LG,CLd+5G?=+.-P(*9:+5/F,(:+.!G f g"eid h CQ!G#;!L9@!LG,CLd+5G?y2F"%=9*"i(*+.CzT]"HG TjPZV k 2lDm ^ V " k‡o^ npl f ZV  ž ^ 



































  



























  









 "





ž
FI?H'


-6$

'# '0 '0#$IT>"\'NF,GCQ(:+.!GŒ+5-s+.-s=,QpBGQ=?/EI#3“VG4š^ 46\9 8  : 0 < 0>= ? 'N!9 4   a&kl' 4 ž (*QG3)V54x^ 4 V54 " ž ^3)V54 " ž ^aSkl'  +.-n"%G8+.G#(*Q!L9s(:LG 3)V  ^ 4 V Q" ž ^  a ’[!Tz-:7XLCQ+Y"%$1;%"H$.FQ-m"H9:&%3)V ž ^ 4 ž "%G= 3)V ž n ^ 4 &a 






  
   





   


     

  !#"%$&"('*)*',+.-/'*01+325467$8+32(467$8+3':9;$&)#?%$8>#4A@B&DCE?%$&=H?(=594I!+J$8'*=Z+3 +3254/$8=%$&):-[0!'*0\8@]9I#6N5?(+34I$&):>##Q',+32560K1^254O0_+J$&_+3'*=(>`Na#':=#=$˜!4;$&)%=E?56¦"d4 \Ï ¼ Òx+3/4IU&4!-4­U#4=5VE?5=5N5!4S5':9I+J$8"5)*4P0Q4CE?54I=594/&@ +3&0Q0Q4I0 2T25&0Q4H):'*6':+Q'*=5>RN(Q#Nd#!+Q'*#=`+Q4=5S50A+3i$˜9I#=50!+3$&=m':=E+Q4!N5Q4­+J$8+Q'*#=50i)*4;$8S +3m+ 2ž 0!9J25[#)*0`&@c':=(@B4!4=59I4&ë +32(4/@B!4CE?54I=E+Q'*0_+H$8=5S|+Q254 Z$;-#4I0Q' $8=³0!9J25G&)*0KL4/S54I@B4IcS('*0Q9I?50Q0!'*#=|?5=2Z$;-[0&@ #!S54!'*=5>/+3254I0Q4R&" !49­+30Z':0
® Ï Ð  Ò Ï ÐÛ#Ò Ü Û  K `%#ž9I#=  #" !4I9I+30 @BQ&6 4K `×&\4­~($&6N5):4&V(',@ 2ž4O2%$;U#4˜$i9)*$&0Q0\&@xÛ Ú Nd4#N5):4O$&=5S 2ž4 2\$&=#4I=54Q$&)EŒ„ׇ+QQ?54+Q2%$8+ \Ï ¼ L + / MÒO® \Ï ¼ L +`Ò  \Ï ¼ L MÒ­Kb^2(47!?5)*4I0M&@\N5Q&"%$&"5':)*':+.-}$&N5N():-m+Q³4­U#4=#':U&4= +32%$¨+A-&#?b25$U&4P0!Na8+30A'*0=5&+  K $.=+Q25'*09;$80Q4&VE+32(4cS('SPa4Q4I=594c"d4I+ 2ž44= ZÏ ¼ L +`Ò$&=5S \Ï +,L ¼ Ò1':0D#" + S5G4I0Q=  +9J2%$8=5>#4 +3254/N5!#"%$&"('*)*',+.-b8@ ¼ K `×!#6 +Q254 S5.4 -5=5':+Q'*#=Ó&@˜9I#=5S5',+3'*&=%$&)HN(Q#"%$8"5'*):':+.- 2\4 9$&= 2T!':+Q4 ZÏ ¼ +`Ò® \Ï ¼ L +`Ò \Ï +`Òc$8=5S$&):0Q ZÏ ¼ +`Ò\® \Ï +'L ¼ Ò žÏ ¼ Ò­K  @F+34= Vd+Q2540!4i@B#Q6i?5) $&4/>#',U#4˜?50 $`9#=&':U#4I0 -#&?+Q254P&N5Na&!+3?(=5':+.-M+3&44IN7-&#?5S5[##0 2T',+39J2Ã+3/+3254T&+Q254?5=5&Na4I=54S $

2$







,





,



Ü'


   

S5[#IK