³ò èAc@s;dZdZddkZddkZddd„ƒYZdS(sHugo Liu s2.0iÿÿÿÿNtMontyTokenizerc‡Bs†eZd„Zd„Zd„Zdd„Zhdd<dd<d d <d d <d d<dd<dd<dd<dd<dd<dd<dd<dd<dd <d!d"<d#d$<d%d&<d'd(<d)d*<d+d,<d-d.<d/d0<d1d2<d3d4<d5d6<d7d8<d9d:<d;d<<d=d><d?d@<dAdB<dCdD<dEdF<dGdH<dIdJ<dKdL<dMdN<dOdP<dQdR<dSdT<dUdV<dWdX<dYdZ<d[d\<d]d^<d_d`<dadbt s(\ ?){2,20}s s&(\<(sentence|paragraph)_break\/\> *)+$t( tcommon_abbrev_and_acroRRtcompiletlentsearchtinserttlowertjointsub(RRtargsstinputtdirnamet contents_arrtmore1t info_cleaned((sF/afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyTokenizer.pytsplit_sentencess,    icCs¯|i}ti}ti}d|d}dddddddd d d d d dddddddddddg}ddddg}|iddƒ}x(|D] }|i|d|dƒ}qŸW|iƒ} x4tt| ƒƒD] } | | } d| jo4| | i dƒi ƒ| | i dƒjoqân| i ƒ|joqân|d| ƒ} | oqân|d| ƒ} | o;| i dƒdjo| | d dd| | t[.]?)$tperiodiÿÿÿÿs (?P)(?Ps )(?P) iitwordtbegintends's s 's s'd s 'd s'll s will s 'll s i s I (R RR tstringt uppercasetreplaceRtrangeR tindextupperRtgrouptstripRtcontractions_unwoundtcontractions_separatedtkeyststartR6(Rtsentencetexpand_contractions_pt built_in_strtcdtthe_tokenizer_ptj_arrtb_dictt hostnamesst buffer_strRtbsRt filename_strtdomain_cleanedt _montylinguat_montylingua_arrtid_arr((sF/afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyTokenizer.pyttokenize*sv   K  :&      (sai n'tsain'tsare n'tsaren'tsis n'tsisn'tswas n'tswasn'tswere n'tsweren'tsdid n'tsdidn'tsdoes n'tsdoesn'tsdo n'tsdon'tshad n'tshadn'tshas n'tshasn'tshave n'tshaven'tsca n'tscan'ts could n'tscouldn'tsneed n'tsneedn'ts should n'ts shouldn'tssha n'tsshan'tswo n'tswon'ts would n'tswouldn'tsi 'msi'msyou 'resyou'reshe 'sshe'ssshe 'ssshe'ssit 'ssit'sswe 'reswe'resthey 'resthey'resi 'vesi'vesyou 'vesyou'veswe 'veswe'vesthey 'vesthey'veswho 'veswho'veswhat 'veswhat'veswhen 'veswhen'ves where 'veswhere'veswhy 'veswhy'veshow 'veshow'vesi 'dsi'dsyou 'dsyou'dshe 'dshe'dsshe 'dsshe'dswe 'dswe'dsthey 'dsthey'dsi 'llsi'llsyou 'llsyou'llshe 'llshe'llsshe 'llsshe'llswe 'llswe'llsthey 'llsthey'llsai notsmr.smrs.sms.ssr.sesq.sjr.sdr.ss.b.sph.d.sm.d.sm.eng.sm.f.a.sd.d.s.ssc.d.sb.s.sb.sc.sb.a.sa.b.sm.a.sc.p.a.sprof.scapt.scol.sgen.ssgt.slt.spriv.sft.snav.sa.f.su.s.a.f.s a.f.b.i.e.setc.se.g.sc.f.sp.s.sq.e.d.si.sii.siii.siv.sv.svi.svii.sviii.six.sx.sa.m.sp.m.smorn.seve.scorp.sinc.sco.sltd.sreg.su.p.s.su.s.p.s.sfedex.si.b.m.sa.o.l.sjan.sfeb.sfebr.smar.sapr.smay.sjun.sjul.saug.ssep.ssept.soct.snov.sdec.sala.sariz.sark.scalif.scolo.sconn.sdel.sd.c.sfla.sga.sill.sind.skans.sky.sla.smd.smass.smich.sminn.smiss.smo.snebr.snev.sn.h.sn.j.sn.m.sn.y.sn.c.sn.d.toklasore.spa.sp.r.sr.i.ss.c.ss.d.stenn.stex.svt.sva.sv.i.swash.sw.va.swis.swyo.sv.c.r.sv.h.s.sd.v.d.sv.c.d.sc.d.stele.stv.st.v.sp.c.sd.s.l.sa.s.a.p.sr.s.v.p.sn.y.c.sc.o.d.ss.u.v.( t__name__t __module__RRRRRR@R?R (((sF/afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyTokenizer.pyRsz    J                                                ((t __author__t __version__R7RR(((sF/afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyTokenizer.pyss