³ò
èAc        
   @   sÓ   d  Z  d Z d d k Z d d k Z d d k Z d d k Z e i ƒ  i o d d k Z d d k Z n d d k	 Z	 d d	 d „  ƒ  YZ
 e d j o; e
 ƒ  Z d Z x) e i ƒ  D] Z e i e d ƒ GHq° Wn d S(
   s   Hugo Liu <hugo@media.mit.edu>s   2.0iÿÿÿÿNt   MontyLexiconFastc           B   s3  e  Z e i ƒ  i Z d  Z d Z d Z d Z d Z	 e oL e
 i g  d ƒ Z e
 i g  d ƒ Z e
 i g  d ƒ Z e
 i g  d ƒ Z n= e i d ƒ Z e i d ƒ Z e i d ƒ Z e i d ƒ Z d „  Z d „  Z d „  Z d	 „  Z d
 „  Z d „  Z d „  Z d „  Z d d „ Z d „  Z d d d d „ Z RS(   i   s   LEXICON.MDFt   FASTLEXICONt    t   lt   Lc         C   sÖ   t  i  ƒ  |  _ t i ƒ  i |  i d ƒ d j o d GH|  i ƒ  n‰ |  i oh d GHt i ƒ  i |  i ƒ |  _ |  i d j o d |  i GHd GHn |  i |  i ƒ |  i	 ƒ  d GHn d GHt
 i d	 ƒ d  Sd
 GHd  S(   Ns   _1.MDFR   s    Fast Lexicon Found! Now Loading!s*   No Fast Lexicon Detected...Now Building...s   ERROR: could not find %ss'   in current dir, %MONTYLINGUA% or %PATH%s$   Finished building FASTLEXICON files!s0   No Fast Lexicon Detected. Standard Lexicon used.iÿÿÿÿs   Lexicon OK!(   t   MontyLexiconCustomt   lexicon_customt
   MontyUtilst	   find_filet   fast_lexicon_filenamet   load_fastlexicont   auto_load_lexicont   lexicon_filenamet   populate_lexicon_from_filet   make_fastlexicont   notifyt   append(   t   self(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyt   __init__    s$    #
	
	c         C   sÄ  t  |  i d d ƒ } | i |  i ƒ | i ƒ  t  |  i d d ƒ } | i |  i ƒ | i ƒ  t  |  i d d ƒ } |  i i | ƒ | i ƒ  t  |  i d d ƒ } |  i i | ƒ | i ƒ  t  |  i d d ƒ } |  i	 i | ƒ | i ƒ  t  |  i d d ƒ } |  i
 i | ƒ | i ƒ  t  |  i d	 d ƒ } | i t t |  i ƒ ƒ d
 ƒ | i t t |  i ƒ ƒ d
 ƒ | i t t |  i	 ƒ ƒ d
 ƒ | i t t |  i
 ƒ ƒ d
 ƒ | i ƒ  d  S(   Ns   _1.MDFt   ws   _2.MDFs   _3.MDFt   wbs   _4.MDFs   _5.MDFs   _6.MDFs   _7.MDFs   
(   t   openR	   t   writet   packed_wordst   closet
   packed_post   word_start_arrt   tofilet   word_end_arrt   pos_start_arrt   pos_end_arrt   strt   len(   R   t   res_arr(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyR   7   s2    





    
c         C   s+  t  i  ƒ  } | i |  i d ƒ } | i |  i d ƒ } | i |  i d ƒ } | i |  i d ƒ } | i |  i d ƒ } | i |  i d ƒ } | i |  i d ƒ } t | d ƒ }	 t d	 „  |	 i ƒ  i ƒ  ƒ \ }
 } } } |	 i ƒ  t | d ƒ }	 |	 i ƒ  |  _ |	 i ƒ  t | d ƒ }	 |	 i ƒ  |  _	 |	 i ƒ  t | d
 ƒ }	 |  i
 |	 |  i |
 |  i d d ƒ} |	 i ƒ  t | d
 ƒ }	 |  i
 |	 |  i | |  i d d ƒ|	 i ƒ  t | d
 ƒ }	 |  i
 |	 |  i | |  i d d ƒ|	 i ƒ  t | d
 ƒ }	 |  i
 |	 |  i | |  i d d ƒ|	 i ƒ  d  S(   Ns   _1.MDFs   _2.MDFs   _3.MDFs   _4.MDFs   _5.MDFs   _6.MDFs   _7.MDFt   rc         S   s
   t  |  ƒ S(    (   t   int(   t	   hostnames(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyt   <lambda>\   s    t   rbt	   java_codet   wst   wet   pst   pe(   R   R   R	   R   t   mapt   readt   splitR   R   R   t   array_fromfileR   t   java_pR   R   R   (   R   t   chown_pt   names_pt   res_arrkt   aliasst   dirname_dictt   output_cleanedt   pairs_cleanedt   c_pR!   t
   built_in_pt	   input_arrt   chgrp1t   cd_arrt   line1(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyR
   R   s:    *


$
"
"
"c         C   sB   | d | d j  o d Sn" | d | d j o d Sn d Sd  S(   Ni    iÿÿÿÿi   (    (   R   t   element1t   element2(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyt   compareq   s    c         C   s²  |  i  } |  i } |  i } |  i } |  i } |  i } |  i i | g  ƒ }	 t |	 ƒ d j o |	 Sn d }
 t | ƒ g d \ } } x&| |
 j o|
 | j  o| |
 d |
 } | | | | | !} | | j  o
 d } n | | j o
 d } n d } | d j o |
 | j o | Sqˆ | d j  o
 | } qˆ | d j o | |
 d j o
 | }
 qˆ | d j o
 | }
 qˆ | d j o# | | | | | !} | i	 ƒ  Sqˆ | Sqˆ W| S(   Ni    i   iÿÿÿÿi   (
   R   R   R   R   R   R   R   t   getR    R.   (   R   t   wordt   defaultt   the_tokenizer1t   gawkt   built_in_dictt   cleaned_arrt   hostnamet
   domain_strt   chrootst   alias1t   alias_pt	   chgrp_strt   cron_cleanedot   cron_pt   pathname_cleanedt   awk_arr(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyRA   w   s<    						


	c         C   s3   |  i  | g  ƒ } | g  j o d Sn	 | d Sd  S(   NR   i    (   RA   (   R   RB   t   names_pt(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyt   primary_posœ   s    c         C   s   |  i  | g  ƒ } | S(   N(   RA   (   R   RB   RR   (    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyt   all_pos¤   s    c         C   s   | |  i  | g  ƒ j S(   N(   RA   (   R   RB   t   pos(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyt   has_pos¨   s    i    c         C   sž   |  i  } | o | | d  ƒ d  j Snt | i ƒ  } | | d  ƒ d  j pK | | i ƒ  d  ƒ d  j p/ | | i ƒ  d  ƒ d  j p | | d  ƒ d  j } | Sd  S(   N(   RA   t   Nonet
   capitalizet   lowert   upper(   R   RB   t   case_sensitivityt	   a_cleanedt	   b_cleanedRP   (    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyt   is_word«   s    	cc      	   C   s  g  } y~ t  | d ƒ } | i ƒ  } xR | oJ | i d ƒ } | |  } | | d } | i | | f ƒ | i ƒ  } q' W| i ƒ  Wn d GHt i d ƒ n X| i |  i ƒ d } d }	 g  }
 g  } g  } g  } d } d } d } xŒ | D]„ \ } } | d 7} | d d j o	 | GHn |
 i | ƒ | t	 | ƒ 7} | i | ƒ | i | ƒ | t	 | ƒ 7} | i | ƒ qí W| GHd i
 t d	 „  | ƒ ƒ } d i
 t d
 „  | ƒ ƒ }	 | |  _ |	 |  _ |  i i |
 ƒ |  i i | ƒ |  i i | ƒ |  i i | ƒ d  S(   NR"   t    i   s   Error parsing Lexicon!iÿÿÿÿR   i    i † c         S   s   |  d  S(   i    (    (   R$   (    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyR%   Ý   s    c         S   s   |  d  S(   i   (    (   R$   (    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyR%   Þ   s    (   R   t   readlinet   findR   R   t   syst   exitt   sortR@   R    t   joinR,   R   R   R   t   fromlistR   R   R   (   R   t   filenamet   line_strR!   t
   alias_dictt	   line_dictt   env_dictRQ   t	   cleaned_pt	   popd_dictt   cmp_arrt
   groups_arrt   inputst   res_dictt   the_parser_dictt	   chmod_strt   alias_dicts(    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyR   µ   sZ    

 
			t   littlec   
      C   s  d } | oñ | d j o
 d } n d } t  i | t | ƒ d | i ƒ  ƒ }	 | d j o t i |	 d ƒ |  _ q| d j o t i |	 d ƒ |  _ q| d	 j o t i |	 d ƒ |  _ q| d
 j o t i |	 d ƒ |  _	 qd GHt
 i d ƒ n | i | | ƒ d  S(   Ni   t   bigt   >t   <R   R(   R   R)   R*   R+   s   error! java code invalid!iÿÿÿÿ(   t   structt   unpackR   R-   t   jarrayt   arrayR   R   R   R   Rb   Rc   t   fromfile(
   R   t   file_ptrt	   array_ptrt   lengthR0   R'   t   endian_ordert   historyt	   command_pR=   (    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyR/   ç   s"    
&(   t   __name__t
   __module__t   MontySettingst   JYTHON_PR0   R   R   R	   R   R   R{   R|   R   R   R   R   R   R   R
   R@   RA   RS   RT   RV   R^   R   R/   (    (    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pyR       s4   					%			
	2t   __main__s   aberration bird ate an applet   UNK(    (   t
   __author__t   __version__R   Rb   R†   R   R‡   R{   Ry   R|   R    R„   R   t   aR.   RB   RA   (    (    (    sH   /afs/cs.wisc.edu/p/zhu/06/nlp/montylingua-2.1/python/MontyLexiconFast.pys   <module>   s   ó	 