
B_Kc           @   s6   d  Z  d d k Z d d k Td e f d     YZ d S(   s]  ISRI Arabic Stemmer

The algorithm for this stemmer is described in:

Taghva, K., Elkoury, R., and Coombs, J. 2005. Arabic Stemming without a root dictionary.
Information Science Research Institute. University of Nevada, Las Vegas, USA.

The Information Science Research Institute’s (ISRI) Arabic stemmer shares many features
with the Khoja stemmer. However, the main difference is that ISRI stemmer does not use root
dictionary. Also, if a root is not found, ISRI stemmer returned normalized form, rather than
returning the original unmodified word.

Additional adjustments were made to improve the algorithm:

1- Adding 60 stop words.
2- Adding the pattern (تفاعيل) to ISRI pattern set.
3- The step 2 in the original algorithm was normalizing all hamza. This step is discarded because it
increases the word ambiguities and changes the original root.

iN(   t   *t   ISRIStemmerc           B   s   e  Z d  Z d   Z d   Z d   Z d   Z d   Z d   Z d   Z	 d   Z
 d	   Z d
   Z d   Z d   Z d   Z d   Z d   Z RS(   s&  
    ISRI Arabic stemmer based on algorithm: Arabic Stemming without a root dictionary.
    Information Science Research Institute. University of Nevada, Las Vegas, USA.

    A few minor modifications have been made to ISRI basic algorithm.
    See the source code of this module for more information.

    isri.stem(token) returns Arabic root for the given token.

    The ISRI Stemmer requires that all tokens have Unicode string types.
    If you use Python IDLE on Arabic Windows you have to decode text first
    using Arabic '1256' coding.
    c      1   C   sP  d |  _  d d d d g |  _ d d g |  _ d d	 d
 d d d d d d g	 |  _ d d d d d g |  _ d d d d d d d d d d d  d! d" d# d$ d% g |  _ d& d' d d( d d d g |  _ h  d) g d* <d g d+ <d d d g d, <d& g d- <|  _ h  d d g d* <d d d g d+ <d d d) g d, <d) d d g d- <d) d g d. <d d g d/ <d d) g d0 <|  _ t	 i
 d1  |  _ t	 i
 d2  |  _ t	 i
 d3  |  _ d4 d5 d6 d7 d8 d9 d: d; d< d= d> d? d@ dA dB dC dD dE dF dG dH dI dJ dK dL dM dN dO dP dQ dR dS dT dU dV dW dX dY dZ d[ d\ d] d^ d_ d` da db dc dd g1 |  _ d  S(e   Ns   defult noneu   كالu   بالu   وللu   والu   الu   للu   لu   بu   فu   سu   وu   يu   تu   نu   اu   تملu   هملu   تانu   تينu   كملu   ونu   اتu   انu   ينu   تنu   كمu   هنu   ناu   ياu   هاu   تمu   كنu   نيu   واu   ماu   همu   ةu   هu   كu   مi    i   i   i   i   i   i   u   [ً-ْ]u   [ءؤئ]u	   ^[آأإ]u   يكونu   وليسu   وكانu   كذلكu   التيu   وبينu
   عليهاu   مساءu   الذيu
   وكانتu   ولكنu
   والتيu   تكونu
   اليومu   اللذينu   عليهu   كانتu   لذلكu   أمامu   هناكu   منهاu
   مازالu
   لازالu   لايزالu   مايزالu   اصبحu   أصبحu   أمسىu   امسىu   أضحىu   اضحىu
   مابرحu
   مافتئu   ماانفكu   لاسيماu   ولايزالu   الحاليu
   اليهاu
   الذينu   فانهu
   والذيu   وهذاu   لهذاu   فكانu
   ستكونu   اليهu   يمكنu   بهذاu   الذى(   t   stmt   p3t   p2t   p1t   s3t   s2t   s1t   pr4t   pr53t   ret   compilet   re_short_vowelst   re_hamzat   re_intial_hamzat
   stop_words(   t   self(    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyt   __init__1   s    	$9?{c         C   s  | |  _  |  i d  |  i  |  i j o |  i  Sn |  i   |  i   |  i   |  i d  t |  i   d j o |  i  Sn t |  i   d j o |  i   |  i  Sn t |  i   d j o |  i   |  i	   |  i  Sn t |  i   d j o |  i
   |  i   |  i  Sn~ t |  i   d j og |  i   t |  i   d j o |  i   n t |  i   d j o |  i
   |  i   |  i  Sqn |  i  S(   s?   
        Stemming a word token using the ISRI stemmer.
        i   i   i   i   i   i   i   (   R   t   normR   t   pre32t   suf32t   wawt   lent   pro_w4t   pro_w53t   end_w5t   pro_w6t   end_w6t   suf1t   pre1(   R   t   token(    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyt   stemF   s8    	


!







c         C   s   | |  _  |  i  d j o& |  i i d |  i  |  _ |  i Sn |  i  d j o& |  i i d |  i  |  _ |  i SnR |  i  d j oA |  i i d |  i  |  _ |  i i d |  i  |  _ |  i Sn d S(   s   
        normalization:
        num=1  normalize diacritics
        num=2  normalize initial hamza
        num=3  both 1&2
        i   t    i   u   اi   N(   t   kR   t   subR   R   (   R   t   num(    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR   i   s    	c         C   s   t  |  i  d j o x |  i D] } |  i i |  o |  i d |  _ |  i Sq  t  |  i  d j oF xC |  i D]4 } |  i i |  o |  i d |  _ |  i Sqt qt Wq  q  Wn d S(   s9   remove length three and length two prefixes in this orderi   i   i   i   N(   R   R   R   t
   startswithR   (   R   t   pre3t   pre2(    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR   }   s    
 
 c         C   s   t  |  i  d j o x |  i D] } |  i i |  o |  i d  |  _ |  i Sq  t  |  i  d j oF xC |  i D]4 } |  i i |  o |  i d  |  _ |  i Sqt qt Wq  q  Wn d S(   s9   remove length three and length two suffixes in this orderi   ii   iN(   R   R   R   t   endswithR   (   R   t   suf3t   suf2(    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR      s    
 
 c         C   sF   t  |  i  d j |  i d  d j @o |  i d |  _ |  i Sn d S(   sI   remove connective ‘و’ if it precedes a word beginning with ‘و’ i   i   u   ووi   N(   R   R   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR      s    'c         C   s'  |  i  d |  i d j o |  i  d |  _  |  i  Sn |  i  d |  i d j o& |  i  d |  i  d |  _  |  i  Sn |  i  d |  i d j o& |  i  d  |  i  d |  _  |  i  Snl |  i  d |  i d j o |  i  d  |  _  |  i  Sn6 |  i   t |  i   d j o |  i   n |  i  Sd S(   s;   process length four patterns and extract length three rootsi    i   i   i   ii   N(   R   R	   R   R   R   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR      s     
c         C   s  |  i  d |  i d j |  i  d d j @o& |  i  d |  i  d |  _  |  i  Sns|  i  d |  i d j |  i  d d j @o) |  i  d d !|  i  d |  _  |  i  Sn|  i  d |  i d j |  i  d d j @o |  i  d d !|  _  |  i  Sn|  i  d |  i d j |  i  d d	 j @o& |  i  d |  i  d |  _  |  i  Sn|  i  d |  i d j |  i  d d j @o& |  i  d |  i  d |  _  |  i  Sn0|  i  d |  i d
 j |  i  d d j @o& |  i  d  |  i  d |  _  |  i  Sn|  i  d |  i d j |  i  d d j @o |  i  d |  _  |  i  Sn|  i  d d j |  i  d d j @o) |  i  d d !|  i  d |  _  |  i  SnI|  i  d d j |  i  d d j @o |  i  d  |  _  |  i  Sn	|  i  d d j |  i  d d	 j @o) |  i  d d !|  i  d |  _  |  i  Sn|  i  d d j |  i  d d j @o1 |  i  d |  i  d |  i  d |  _  |  i  Sne|  i  d d j |  i  d d j @o& |  i  d |  i  d |  _  |  i  Sn|  i  d d j |  i  d d j @o& |  i  d  |  i  d |  _  |  i  Sn |  i  d d j |  i  d d j @o) |  i  d |  i  d d !|  _  |  i  Sn |  i  d d j |  i  d d j @o& |  i  d  |  i  d |  _  |  i  Sn6 |  i   t |  i   d
 j o |  i   n |  i  Sd S(   s;   process length five patterns and extract length three rootsi   i    u   اi   i   u   مi   u   ةu   تi   i   u   نu   يu   وu   ئN(   R   R
   R   R   R   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR      sb    ,,,,,,,%%%%&%%%%
c         C   s   |  i  d |  i d j o |  i  d |  _  |  i  Snj |  i  d d j o |  i  d  |  _  |  i  Sn; |  i  d d j o& |  i  d  |  i  d |  _  |  i  Sn d S(	   s:   process length five patterns and extract length four rootsi    i   i   i   u   ةu   اi   N(   R   R
   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyt   pro_w54   s    c         C   s{   t  |  i  d j o |  i SnW t  |  i  d j o |  i   |  i Sn, t  |  i  d j o |  i   |  i Sn d S(   s!   ending step (word of length five)i   i   i   N(   R   R   R   R+   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR      s    

c         C   s"  |  i  i d  p |  i  i d  o |  i  d |  _  |  i  Sn|  i  d d j oQ |  i  d d j o= |  i  d d j o) |  i  d	 d !|  i  d
 |  _  |  i  Sny|  i  d d j oY |  i  d d j oE |  i  d
 d j o1 |  i  d	 |  i  d |  i  d |  _  |  i  Sn|  i  d d j oU |  i  d d j oA |  i  d |  i  d
 j o& |  i  d	 |  i  d
 |  _  |  i  Sn |  i  d d j oY |  i  d d j oE |  i  d
 d j o1 |  i  d	 |  i  d |  i  d |  _  |  i  Sn6 |  i   t |  i   d j o |  i   n |  i  Sd S(   s:   process length six patterns and extract length three rootsu   استu   مستi   i    u   مu   اi   u   ةi   i   i   u   تu   وu   يi   N(   R   R%   R   R   R   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR      s&    &<<&C<&
c         C   s}   |  i  d o |  i  d d j o) |  i  d d !|  i  d |  _  |  i  Sn/ |  i  i d  o |  i  d |  _  |  i  Sn d S(	   s9   process length six patterns and extract length four rootsi    i   u   اi   i   u   متi   N(   R   R%   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyt   pro_w64  s    "c         C   s   t  |  i  d j o |  i Sna t  |  i  d j o |  i   |  i   |  i Sn, t  |  i  d j o |  i   |  i Sn d S(   s    ending step (word of length six)i   i   i   N(   R   R   R   R   R,   (   R   (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR     s    


c         C   sF   x? |  i  D]4 } |  i i |  o |  i d  |  _ |  i Sq
 q
 Wd S(   s   normalize short sufixiN(   R   R   R(   (   R   t   sf1(    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR   *  s
    
 c         C   sF   x? |  i  D]4 } |  i i |  o |  i d |  _ |  i Sq
 q
 Wd S(   s   normalize short prefixi   N(   R   R   R%   (   R   t   sp1(    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR   1  s
    
 (   t   __name__t
   __module__t   __doc__R   R    R   R   R   R   R   R   R+   R   R   R,   R   R   R   (    (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pyR   "   s    		#						5							(   R1   R   t   apit   StemmerIR   (    (    (    s$   /p/zhu/06/nlp/nltk/nltk/stem/isri.pys   <module>   s   
