³ò
5ÒÇIc           @   sl   d  d g Z  d d k l Z d d k Td e f d „  ƒ  YZ d e f d „  ƒ  YZ d  e f d „  ƒ  YZ d	 S(
   t   Scannert   ScannerErroriÿÿÿÿ(   t   MarkedYAMLError(   t   *c           B   s   e  Z RS(    (   t   __name__t
   __module__(    (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR       s   t	   SimpleKeyc           B   s   e  Z d  „  Z RS(   c         C   s:   | |  _  | |  _ | |  _ | |  _ | |  _ | |  _ d  S(   N(   t   token_numbert   requiredt   indext   linet   columnt   mark(   t   selfR   R   R	   R
   R   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyt   __init__&   s    					(   R   R   R   (    (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR   #   s   c           B   sG  e  Z d  „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z	 d „  Z
 d	 „  Z d
 „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z  d „  Z! d  „  Z" d! „  Z# d" „  Z$ d# „  Z% d$ „  Z& d% „  Z' d& „  Z( d' „  Z) d( „  Z* d) „  Z+ d* „  Z, d+ „  Z- d, „  Z. d- „  Z/ d. „  Z0 d/ „  Z1 d0 „  Z2 d1 „  Z3 d2 „  Z4 d3 „  Z5 d4 „  Z6 d5 „  Z7 d6 „  Z8 d7 „  Z9 d8 „  Z: d9 „  Z; d: „  Z< d; „  Z= d< „  Z> d= „  Z? h  d> d? <d@ dA <dB dC <dD dE <dD dD <dF dG <dH dI <dJ dK <dL dM <dN dO <dP dP <dQ dQ <dR dR <dS dT <dU dV <dW dX <dY dZ <Z@ h  d[ d\ <d] d^ <d_ d` <ZA da „  ZB db „  ZC dc „  ZD dd „  ZE de „  ZF df „  ZG dg „  ZH dh „  ZI di „  ZJ RS(j   c         C   sV   t  |  _ d |  _ g  |  _ |  i ƒ  d |  _ d |  _ g  |  _ t |  _	 h  |  _
 d S(   s   Initialize the scanner.i    iÿÿÿÿN(   t   Falset   donet
   flow_levelt   tokenst   fetch_stream_startt   tokens_takent   indentt   indentst   Truet   allow_simple_keyt   possible_simple_keys(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR   0   s    			
				c         G   sp   x |  i  ƒ  o |  i ƒ  q W|  i oC | p t Sn x1 | D]% } t |  i d | ƒ o t Sq? q? Wn t S(   Ni    (   t   need_more_tokenst   fetch_more_tokensR   R   t
   isinstanceR   (   R   t   choicest   choice(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyt   check_tokenq   s    
 c         C   s<   x |  i  ƒ  o |  i ƒ  q W|  i o |  i d Sn d  S(   Ni    (   R   R   R   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyt
   peek_token}   s    
c         C   sP   x |  i  ƒ  o |  i ƒ  q W|  i o# |  i d 7_ |  i i d ƒ Sn d  S(   Ni   i    (   R   R   R   R   t   pop(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyt	   get_token„   s
    
c         C   sP   |  i  o t Sn |  i p t Sn |  i ƒ  |  i ƒ  |  i j o t Sn d  S(   N(   R   R   R   R   t   stale_possible_simple_keyst   next_possible_simple_keyR   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR   Ž   s    


c         C   sÜ  |  i  ƒ  |  i ƒ  |  i |  i ƒ |  i ƒ  } | d j o |  i ƒ  Sn | d j o |  i ƒ  o |  i ƒ  Sn | d j o |  i ƒ  o |  i	 ƒ  Sn | d j o |  i
 ƒ  o |  i ƒ  Sn | d j o |  i ƒ  Sn | d j o |  i ƒ  Sn | d j o |  i ƒ  Sn | d j o |  i ƒ  Sn | d	 j o |  i ƒ  Sn | d j o |  i ƒ  o |  i ƒ  Sn | d
 j o |  i ƒ  o |  i ƒ  Sn | d j o |  i ƒ  o |  i ƒ  Sn | d j o |  i ƒ  Sn | d j o |  i ƒ  Sn | d j o |  i ƒ  Sn | d j o |  i o |  i ƒ  Sn | d j o |  i o |  i ƒ  Sn | d j o |  i ƒ  Sn | d j o |  i ƒ  Sn |  i ƒ  o |  i  ƒ  Sn t! d d  d | i# d ƒ |  i$ ƒ  ƒ ‚ d  S(   Nu    u   %u   -u   .u   [u   {u   ]u   }u   ,u   ?u   :u   *u   &u   !u   |u   >u   'u   "s!   while scanning for the next tokens.   found character %r that cannot start any tokens   utf-8(%   t   scan_to_next_tokenR#   t   unwind_indentR   t   peekt   fetch_stream_endt   check_directivet   fetch_directivet   check_document_startt   fetch_document_startt   check_document_endt   fetch_document_endt   fetch_flow_sequence_startt   fetch_flow_mapping_startt   fetch_flow_sequence_endt   fetch_flow_mapping_endt   fetch_flow_entryt   check_block_entryt   fetch_block_entryt	   check_keyt	   fetch_keyt   check_valuet   fetch_valuet   fetch_aliast   fetch_anchort	   fetch_tagR   t   fetch_literalt   fetch_foldedt   fetch_singlet   fetch_doublet   check_plaint   fetch_plainR   t   Nonet   encodet   get_mark(   R   t   ch(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR   ™   s^    

		c         C   sU   d  } xH |  i D]= } |  i | } | d  j p | i | j  o | i } q q W| S(   N(   RC   R   R   (   R   t   min_token_numbert   levelt   key(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR$     s    
 c         C   s   xˆ |  i  i ƒ  D]w } |  i  | } | i |  i j p |  i | i d j o: | i o" t d | i d |  i ƒ  ƒ ‚ n |  i  | =q q Wd  S(   Ni   s   while scanning a simple keys   could not found expected ':'(   R   t   keysR
   R	   R   R   R   RE   (   R   RH   RI   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR#     s     
c         C   s¢   |  i  o |  i |  i j } |  i p | p t ‚ |  i o^ |  i ƒ  |  i t |  i ƒ } t	 | | |  i
 |  i |  i |  i ƒ  ƒ } | |  i |  i  <n d  S(   N(   R   R   R   R   t   AssertionErrort   remove_possible_simple_keyR   t   lenR   R   R	   R
   RE   R   (   R   R   R   RI   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyt   save_possible_simple_key$  s    

	!c         C   sd   |  i  |  i j oM |  i |  i  } | i o" t d | i d |  i ƒ  ƒ ‚ n |  i |  i  =n d  S(   Ns   while scanning a simple keys   could not found expected ':'(   R   R   R   R   R   RE   (   R   RI   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRL   9  s    
c         C   se   |  i  o d  Sn xL |  i | j o; |  i ƒ  } |  i i ƒ  |  _ |  i i t | | ƒ ƒ q Wd  S(   N(   R   R   RE   R   R!   R   t   appendt   BlockEndToken(   R   R   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR&   F  s    
c         C   s8   |  i  | j  o$ |  i i |  i  ƒ | |  _  t Sn t S(   N(   R   R   RO   R   R   (   R   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyt
   add_indent^  s
    	c         C   s2   |  i  ƒ  } |  i i t | | d |  i ƒƒ d  S(   Nt   encoding(   RE   R   RO   t   StreamStartTokenRR   (   R   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR   h  s    c         C   sQ   |  i  d ƒ t |  _ h  |  _ |  i ƒ  } |  i i t | | ƒ ƒ t |  _	 d  S(   Niÿÿÿÿ(
   R&   R   R   R   RE   R   RO   t   StreamEndTokenR   R   (   R   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR(   t  s    		c         C   s:   |  i  d ƒ |  i ƒ  t |  _ |  i i |  i ƒ  ƒ d  S(   Niÿÿÿÿ(   R&   RL   R   R   R   RO   t   scan_directive(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR*   †  s    
	c         C   s   |  i  t ƒ d  S(   N(   t   fetch_document_indicatort   DocumentStartToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR,   ’  s    c         C   s   |  i  t ƒ d  S(   N(   RV   t   DocumentEndToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR.   •  s    c         C   sb   |  i  d ƒ |  i ƒ  t |  _ |  i ƒ  } |  i d ƒ |  i ƒ  } |  i i | | | ƒ ƒ d  S(   Niÿÿÿÿi   (   R&   RL   R   R   RE   t   forwardR   RO   (   R   t
   TokenClasst
   start_markt   end_mark(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRV   ˜  s    
	c         C   s   |  i  t ƒ d  S(   N(   t   fetch_flow_collection_startt   FlowSequenceStartToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR/   ¨  s    c         C   s   |  i  t ƒ d  S(   N(   R]   t   FlowMappingStartToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR0   «  s    c         C   sa   |  i  ƒ  |  i d 7_ t |  _ |  i ƒ  } |  i ƒ  |  i ƒ  } |  i i | | | ƒ ƒ d  S(   Ni   (   RN   R   R   R   RE   RY   R   RO   (   R   RZ   R[   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR]   ®  s    
	
c         C   s   |  i  t ƒ d  S(   N(   t   fetch_flow_collection_endt   FlowSequenceEndToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR1   ¿  s    c         C   s   |  i  t ƒ d  S(   N(   R`   t   FlowMappingEndToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR2   Â  s    c         C   sa   |  i  ƒ  |  i d 8_ t |  _ |  i ƒ  } |  i ƒ  |  i ƒ  } |  i i | | | ƒ ƒ d  S(   Ni   (   RL   R   R   R   RE   RY   R   RO   (   R   RZ   R[   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR`   Å  s    
	
c         C   sR   t  |  _ |  i ƒ  |  i ƒ  } |  i ƒ  |  i ƒ  } |  i i t | | ƒ ƒ d  S(   N(   R   R   RL   RE   RY   R   RO   t   FlowEntryToken(   R   R[   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR3   Ö  s    	

c         C   sÅ   |  i  pi |  i p t d  d  d |  i ƒ  ƒ ‚ n |  i |  i ƒ o) |  i ƒ  } |  i i t	 | | ƒ ƒ qs n t
 |  _ |  i ƒ  |  i ƒ  } |  i ƒ  |  i ƒ  } |  i i t | | ƒ ƒ d  S(   Ns%   sequence entries are not allowed here(   R   R   R   RC   RE   RQ   R   R   RO   t   BlockSequenceStartTokenR   RL   RY   t   BlockEntryToken(   R   R   R[   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR5   ä  s    

	!	

c         C   sÉ   |  i  pi |  i p t d  d  d |  i ƒ  ƒ ‚ n |  i |  i ƒ o) |  i ƒ  } |  i i t	 | | ƒ ƒ qs n |  i  |  _ |  i
 ƒ  |  i ƒ  } |  i ƒ  |  i ƒ  } |  i i t | | ƒ ƒ d  S(   Ns!   mapping keys are not allowed here(   R   R   R   RC   RE   RQ   R   R   RO   t   BlockMappingStartTokenRL   RY   t   KeyToken(   R   R   R[   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR7     s    

	!

c         C   s‘  |  i  |  i j o§ |  i |  i  } |  i |  i  =|  i i | i |  i t | i | i ƒ ƒ |  i  pG |  i | i	 ƒ o0 |  i i | i |  i t
 | i | i ƒ ƒ q­ n t |  _ n™ |  i  p- |  i p t d  d  d |  i ƒ  ƒ ‚ qñ n |  i  p@ |  i |  i	 ƒ o) |  i ƒ  } |  i i t
 | | ƒ ƒ q;n |  i  |  _ |  i ƒ  |  i ƒ  } |  i ƒ  |  i ƒ  } |  i i t | | ƒ ƒ d  S(   Ns#   mapping values are not allowed here(   R   R   R   t   insertR   R   Rg   R   RQ   R   Rf   R   R   R   RC   RE   RO   RL   RY   t
   ValueToken(   R   RI   R   R[   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR9   !  s2    


	
!

c         C   s0   |  i  ƒ  t |  _ |  i i |  i t ƒ ƒ d  S(   N(   RN   R   R   R   RO   t   scan_anchort
   AliasToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR:   Y  s    
	c         C   s0   |  i  ƒ  t |  _ |  i i |  i t ƒ ƒ d  S(   N(   RN   R   R   R   RO   Rj   t   AnchorToken(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR;   d  s    
	c         C   s-   |  i  ƒ  t |  _ |  i i |  i ƒ  ƒ d  S(   N(   RN   R   R   R   RO   t   scan_tag(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR<   o  s    
	c         C   s   |  i  d d ƒ d  S(   Nt   stylet   |(   t   fetch_block_scalar(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR=   z  s    c         C   s   |  i  d d ƒ d  S(   NRn   t   >(   Rp   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR>   }  s    c         C   s0   t  |  _ |  i ƒ  |  i i |  i | ƒ ƒ d  S(   N(   R   R   RL   R   RO   t   scan_block_scalar(   R   Rn   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRp   €  s    	
c         C   s   |  i  d d ƒ d  S(   NRn   s   '(   t   fetch_flow_scalar(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR?   ‹  s    c         C   s   |  i  d d ƒ d  S(   NRn   t   "(   Rs   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR@   Ž  s    c         C   s0   |  i  ƒ  t |  _ |  i i |  i | ƒ ƒ d  S(   N(   RN   R   R   R   RO   t   scan_flow_scalar(   R   Rn   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRs   ‘  s    
	c         C   s-   |  i  ƒ  t |  _ |  i i |  i ƒ  ƒ d  S(   N(   RN   R   R   R   RO   t
   scan_plain(   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRB   œ  s    
	c         C   s   |  i  d j o t Sn d  S(   Ni    (   R   R   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR)   «  s    c         C   sL   |  i  d j o8 |  i d ƒ d j o |  i d ƒ d j o t SqH n d  S(   Ni    i   u   ---u     	
Â…â€¨â€©(   R   t   prefixR'   R   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR+   ²  s    c         C   sL   |  i  d j o8 |  i d ƒ d j o |  i d ƒ d j o t SqH n d  S(   Ni    i   u   ...u     	
Â…â€¨â€©(   R   Rw   R'   R   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR-   º  s    c         C   s   |  i  d ƒ d j S(   Ni   u     	
Â…â€¨â€©(   R'   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR4   Â  s    c         C   s)   |  i  o t Sn |  i d ƒ d j Sd  S(   Ni   u     	
Â…â€¨â€©(   R   R   R'   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR6   Ç  s    
c         C   s)   |  i  o t Sn |  i d ƒ d j Sd  S(   Ni   u     	
Â…â€¨â€©(   R   R   R'   (   R   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR8   Ñ  s    
c         C   sQ   |  i  ƒ  } | d j p8 |  i  d ƒ d j o" | d j p |  i o
 | d j S(   Nu      	
Â…â€¨â€©-?:,[]{}#&*!|>'"%@`i   u     	
Â…â€¨â€©u   -u   ?:(   R'   R   (   R   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRA   Û  s    c         C   sÙ   |  i  d j o! |  i ƒ  d j o |  i ƒ  n t } x› | p“ x" |  i ƒ  d j o |  i ƒ  qD W|  i ƒ  d j o) x& |  i ƒ  d j o |  i ƒ  q| Wn |  i ƒ  o |  i p t |  _ qÐ q: t } q: Wd  S(   Ni    u   ï»¿u    u   #u    
Â…â€¨â€©(   R	   R'   RY   R   t   scan_line_breakR   R   R   (   R   t   found(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR%   ð  s    #

c         C   sÔ   |  i  ƒ  } |  i ƒ  |  i | ƒ } d  } | d j o |  i | ƒ } |  i  ƒ  } n^ | d j o |  i | ƒ } |  i  ƒ  } n2 |  i  ƒ  } x" |  i ƒ  d j o |  i ƒ  q’ W|  i | ƒ t | | | | ƒ S(   Nu   YAMLu   TAGu    
Â…â€¨â€©(	   RE   RY   t   scan_directive_nameRC   t   scan_yaml_directive_valuet   scan_tag_directive_valueR'   t   scan_directive_ignored_linet   DirectiveToken(   R   R[   t   namet   valueR\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRU     s    
c         C   s5  d } |  i  | ƒ } x… d | j o
 d j n pI d | j o
 d j n p+ d | j o
 d j n p | d j o | d	 7} |  i  | ƒ } q W| p, t d
 | d | i d ƒ |  i ƒ  ƒ ‚ n |  i | ƒ } |  i | ƒ |  i  ƒ  } | d j o, t d
 | d | i d ƒ |  i ƒ  ƒ ‚ n | S(   Ni    u   0u   9u   At   Zu   at   zu   -_i   s   while scanning a directives6   expected alphabetic or numeric character, but found %rs   utf-8u     
Â…â€¨â€©(   R'   R   RD   RE   Rw   RY   (   R   R[   t   lengthRF   R€   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRz   &  s$    ]
	 	 c         C   sá   x" |  i  ƒ  d j o |  i ƒ  q W|  i | ƒ } |  i  ƒ  d j o2 t d | d |  i  ƒ  i d ƒ |  i ƒ  ƒ ‚ n |  i ƒ  |  i | ƒ } |  i  ƒ  d j o2 t d | d |  i  ƒ  i d ƒ |  i ƒ  ƒ ‚ n | | f S(   Nu    t   .s   while scanning a directives%   expected a digit or '.', but found %rs   utf-8u     
Â…â€¨â€©s%   expected a digit or ' ', but found %r(   R'   RY   t   scan_yaml_directive_numberR   RD   RE   (   R   R[   t   majort   minor(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR{   ;  s     	
	c         C   s»   |  i  ƒ  } d | j o
 d j n p, t d | d | i d ƒ |  i ƒ  ƒ ‚ n d } x6 d |  i  | ƒ j o
 d j n o | d 7} q_ Wt |  i | ƒ ƒ } |  i | ƒ | S(	   Nu   0t   9s   while scanning a directives   expected a digit, but found %rs   utf-8i    u   9i   (   R'   R   RD   RE   t   intRw   RY   (   R   R[   RF   Rƒ   R€   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR…   N  s    	*c         C   sr   x" |  i  ƒ  d j o |  i ƒ  q W|  i | ƒ } x" |  i  ƒ  d j o |  i ƒ  q7 W|  i | ƒ } | | f S(   Nu    (   R'   RY   t   scan_tag_directive_handlet   scan_tag_directive_prefix(   R   R[   t   handleRw   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR|   \  s    c         C   s[   |  i  d | ƒ } |  i ƒ  } | d j o, t d | d | i d ƒ |  i ƒ  ƒ ‚ n | S(   Nt	   directiveu    s   while scanning a directives   expected ' ', but found %rs   utf-8(   t   scan_tag_handleR'   R   RD   RE   (   R   R[   R€   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRŠ   f  s    	c         C   s[   |  i  d | ƒ } |  i ƒ  } | d j o, t d | d | i d ƒ |  i ƒ  ƒ ‚ n | S(   NR   u     
Â…â€¨â€©s   while scanning a directives   expected ' ', but found %rs   utf-8(   t   scan_tag_uriR'   R   RD   RE   (   R   R[   R€   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR‹   p  s    	c         C   s´   x" |  i  ƒ  d j o |  i ƒ  q W|  i  ƒ  d j o) x& |  i  ƒ  d j o |  i ƒ  q; Wn |  i  ƒ  } | d j o, t d | d | i d ƒ |  i ƒ  ƒ ‚ n |  i ƒ  d  S(   Nu    u   #u    
Â…â€¨â€©s   while scanning a directives0   expected a comment or a line break, but found %rs   utf-8(   R'   RY   R   RD   RE   Rx   (   R   R[   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR}   z  s    	 c   	      C   s”  |  i  ƒ  } |  i ƒ  } | d j o
 d } n d } |  i ƒ  d } |  i | ƒ } x… d | j o
 d j n pI d | j o
 d j n p+ d	 | j o
 d
 j n p | d j o | d 7} |  i | ƒ } qW W| p0 t d | | d | i d ƒ |  i  ƒ  ƒ ‚ n |  i | ƒ } |  i | ƒ |  i ƒ  } | d j o0 t d | | d | i d ƒ |  i  ƒ  ƒ ‚ n |  i  ƒ  } | | | | ƒ S(   NR   t   aliast   anchori    u   0u   9u   AR   u   aR‚   u   -_i   s   while scanning an %ss6   expected alphabetic or numeric character, but found %rs   utf-8u     	
Â…â€¨â€©?:,]}%@`(   RE   R'   RY   R   RD   Rw   (	   R   RZ   R[   t	   indicatorR   Rƒ   RF   R€   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRj   ˆ  s2    	

]
  c   	      C   sË  |  i  ƒ  } |  i d ƒ } | d j ox d  } |  i d ƒ |  i d | ƒ } |  i ƒ  d j o2 t d | d |  i ƒ  i d ƒ |  i  ƒ  ƒ ‚ n |  i ƒ  n¿ | d	 j o d  } d
 } |  i ƒ  n˜ d } t } xC | d j o5 | d
 j o t } Pn | d 7} |  i | ƒ } qÖ Wd
 } | o |  i	 d | ƒ } n d
 } |  i ƒ  |  i d | ƒ } |  i ƒ  } | d j o, t d | d | i d ƒ |  i  ƒ  ƒ ‚ n | | f } |  i  ƒ  } t
 | | | ƒ S(   Ni   u   <i   t   tagu   >s   while parsing a tags   expected '>', but found %rs   utf-8u     	
Â…â€¨â€©u   !u     
Â…â€¨â€©s   while scanning a tags   expected ' ', but found %r(   RE   R'   RC   RY   R   R   RD   R   R   RŽ   t   TagToken(	   R   R[   RF   RŒ   t   suffixRƒ   t
   use_handleR€   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRm   ¬  sJ    	

	c         C   ss  | d j o
 t  } n t } g  } |  i ƒ  } |  i ƒ  |  i | ƒ \ } } |  i | ƒ |  i d } | d j  o
 d } n | d  j o( |  i ƒ  \ } }	 }
 t	 | |	 ƒ } n$ | | d } |  i
 | ƒ \ } }
 d } x8|  i | j o'|  i ƒ  d j o| i | ƒ |  i ƒ  d j } d } x% |  i | ƒ d j o | d 7} q+W| i |  i | ƒ ƒ |  i | ƒ |  i ƒ  } |  i
 | ƒ \ } }
 |  i | j on |  i ƒ  d j o[ | oC | d j o6 | o/ |  i ƒ  d j o | p | i d	 ƒ qq| i | ƒ qà Pqà W| t j	 o | i | ƒ n | t  j o | i | ƒ n t d i | ƒ t | |
 | ƒ S(
   NRq   i   u    u    u    	i    u    
Â…â€¨â€©u   
u    (   R   R   RE   RY   t   scan_block_scalar_indicatorst   scan_block_scalar_ignored_lineR   RC   t   scan_block_scalar_indentationt   maxt   scan_block_scalar_breaksR   R'   t   extendRO   Rw   Rx   t   ScalarTokent   join(   R   Rn   t   foldedt   chunksR[   t   chompingt	   incrementt
   min_indentt   breakst
   max_indentR\   R   t
   line_breakt   leading_non_spaceRƒ   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRr   Ö  sP    


&#c         C   s•  d  } d  } |  i ƒ  } | d j oŠ | d j o
 t } n t } |  i ƒ  |  i ƒ  } | d j oF t | ƒ } | d j o t d | d |  i ƒ  ƒ ‚ n |  i ƒ  qFn˜ | d j oŠ t | ƒ } | d j o t d | d |  i ƒ  ƒ ‚ n |  i ƒ  |  i ƒ  } | d j o+ | d j o
 t } n t } |  i ƒ  qFn |  i ƒ  } | d j o, t d | d | i d	 ƒ |  i ƒ  ƒ ‚ n | | f S(
   Nu   +-t   +u
   0123456789i    s   while scanning a block scalars<   expected indentation indicator in the range 1-9, but found 0u     
Â…â€¨â€©s9   expected chomping or indentation indicators, but found %rs   utf-8(	   RC   R'   R   R   RY   R‰   R   RE   RD   (   R   R[   R¡   R¢   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR—   "  sF    

		

	 c         C   s´   x" |  i  ƒ  d j o |  i ƒ  q W|  i  ƒ  d j o) x& |  i  ƒ  d j o |  i ƒ  q; Wn |  i  ƒ  } | d j o, t d | d | i d ƒ |  i ƒ  ƒ ‚ n |  i ƒ  d  S(   Nu    u   #u    
Â…â€¨â€©s   while scanning a block scalars0   expected a comment or a line break, but found %rs   utf-8(   R'   RY   R   RD   RE   Rx   (   R   R[   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR˜   J  s    	 c         C   s   g  } d } |  i  ƒ  } xu |  i ƒ  d j oa |  i ƒ  d j o# | i |  i ƒ  ƒ |  i  ƒ  } q |  i ƒ  |  i | j o |  i } q q W| | | f S(   Ni    u    
Â…â€¨â€©u    (   RE   R'   RO   Rx   RY   R   (   R   R    R¥   R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR™   X  s    
c         C   sÀ   g  } |  i  ƒ  } x2 |  i | j  o! |  i ƒ  d j o |  i ƒ  q Wxl |  i ƒ  d j oX | i |  i ƒ  ƒ |  i  ƒ  } x2 |  i | j  o! |  i ƒ  d j o |  i ƒ  q WqJ W| | f S(   Nu    u
   
Â…â€¨â€©(   RE   R   R'   RY   RO   Rx   (   R   R   R    R\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR›   g  s    &&c         C   sà   | d j o
 t  } n t } g  } |  i ƒ  } |  i ƒ  } |  i ƒ  | i |  i | | ƒ ƒ xJ |  i ƒ  | j o6 | i |  i | | ƒ ƒ | i |  i | | ƒ ƒ qa W|  i ƒ  |  i ƒ  } t d i	 | ƒ t | | | ƒ S(   NRt   u    (
   R   R   RE   R'   RY   Rœ   t   scan_flow_scalar_non_spacest   scan_flow_scalar_spacesR   Rž   (   R   Rn   t   doubleR    R[   t   quoteR\   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRu   t  s    


u    u   0u   u   au   u   bu   	u   tu   
u   nu   u   vu   u   fu   u   ru   u   eu    u   "u   \u   Â…u   Nu   Â u   _u   â€¨u   Lu   â€©u   Pi   u   xi   u   ui   u   Uc         C   s™  g  } xŒt  o„d } x% |  i | ƒ d j o | d 7} q W| o' | i |  i | ƒ ƒ |  i | ƒ n |  i ƒ  } | oA | d j o4 |  i d ƒ d j o | i d ƒ |  i d ƒ q	 | o | d j p | o( | d j o | i | ƒ |  i ƒ  q	 | o€| d j os|  i ƒ  |  i ƒ  } | |  i j o" | i |  i | ƒ |  i ƒ  q| |  i j o» |  i | } |  i ƒ  xe t | ƒ D]W } |  i | ƒ d j o; t d	 | d
 | |  i | ƒ i	 d ƒ f |  i
 ƒ  ƒ ‚ q•q•Wt |  i | ƒ d ƒ } | i t | ƒ ƒ |  i | ƒ q| d j o' |  i ƒ  | i |  i | | ƒ ƒ qt d	 | d | i	 d ƒ |  i
 ƒ  ƒ ‚ q	 | Sq	 Wd  S(   Ni    u   '"\  	
Â…â€¨â€©i   u   'i   u   "\u   \u   0123456789ABCDEFabcdefs%   while scanning a double-quoted scalars?   expected escape sequence of %d hexdecimal numbers, but found %rs   utf-8i   u
   
Â…â€¨â€©s!   found unknown escape character %r(   R   R'   RO   Rw   RY   t   ESCAPE_REPLACEMENTSt   ESCAPE_CODESt   rangeR   RD   RE   R‰   t   unichrRx   Rœ   t   scan_flow_scalar_breaks(   R   R«   R[   R    Rƒ   RF   t   kt   code(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR©   ¦  sN    
+)

 	3
	#c   	      C   s  g  } d } x% |  i  | ƒ d j o | d 7} q W|  i | ƒ } |  i | ƒ |  i  ƒ  } | d j o t d | d |  i ƒ  ƒ ‚ n€ | d j oe |  i ƒ  } |  i | | ƒ } | d j o | i | ƒ n | p | i d	 ƒ n | i | ƒ n | i | ƒ | S(
   Ni    u    	i   u    s   while scanning a quoted scalars   found unexpected end of streamu
   
Â…â€¨â€©u   
u    (	   R'   Rw   RY   R   RE   Rx   R±   RO   Rœ   (	   R   R«   R[   R    Rƒ   t   whitespacesRF   R¦   R¤   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRª   Ñ  s(    	c         C   sÊ   g  } x½ t  oµ |  i d ƒ } | d j p | d j o5 |  i d ƒ d j o t d | d |  i ƒ  ƒ ‚ n x" |  i ƒ  d j o |  i ƒ  qq W|  i ƒ  d j o | i |  i ƒ  ƒ q	 | Sq	 Wd  S(	   Ni   u   ---u   ...u     	
Â…â€¨â€©s   while scanning a quoted scalars#   found unexpected document separatoru    	u
   
Â…â€¨â€©(   R   Rw   R'   R   RE   RY   RO   Rx   (   R   R«   R[   R    Rw   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR±   é  s    
	c         C   sû  g  } |  i  ƒ  } | } |  i d } g  } x±t o©d } |  i ƒ  d j o Pn x€ t ox |  i | ƒ } | d j pI |  i o' | d j o |  i | d ƒ d j p |  i o | d j o Pn | d 7} qV W|  i oV | d j oI |  i | d ƒ d j o/ |  i | ƒ t d | d	 |  i  ƒ  d
 ƒ ‚ n | d j o Pn t |  _ | i	 | ƒ | i
 |  i | ƒ ƒ |  i | ƒ |  i  ƒ  } |  i | | ƒ } | p. |  i ƒ  d j p |  i o |  i | j  o Pq. q. Wt d i | ƒ t | | ƒ S(   Ni   i    u   #u     	
Â…â€¨â€©u   :u   ,:?[]{}u     	
Â…â€¨â€©,[]{}s   while scanning a plain scalars   found unexpected ':'sG   Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.u    (   RE   R   R   R'   R   RY   R   R   R   Rœ   RO   Rw   t   scan_plain_spacesR   R   Rž   (   R   R    R[   R\   R   t   spacesRƒ   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRv   û  sF    

		
c   
      C   sÄ  g  } d } x% |  i  | ƒ d j o | d 7} q W|  i | ƒ } |  i | ƒ |  i  ƒ  } | d j o?|  i ƒ  } t |  _ |  i d ƒ } | d j p | d j o |  i  d ƒ d j o d  Sn g  }	 x“ |  i  ƒ  d	 j o |  i  ƒ  d
 j o |  i ƒ  qÎ |	 i |  i ƒ  ƒ |  i d ƒ } | d j p | d j o |  i  d ƒ d j o d  SqÎ qÎ W| d j o | i | ƒ n |	 p | i d ƒ n | i |	 ƒ n | o | i | ƒ n | S(   Ni    u    i   u
   
Â…â€¨â€©i   u   ---u   ...u     	
Â…â€¨â€©u    
Â…â€¨â€©t    u   
(   R'   Rw   RY   Rx   R   R   RO   Rœ   (
   R   R   R[   R    Rƒ   R´   RF   R¦   Rw   R¤   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRµ   *  s>    	c         C   sk  |  i  ƒ  } | d j o0 t d | | d | i d ƒ |  i ƒ  ƒ ‚ n d } |  i  | ƒ } | d j oà x… d | j o
 d j n pI d	 | j o
 d
 j n p+ d | j o
 d j n p | d j o | d 7} |  i  | ƒ } qn W| d j o= |  i | ƒ t d | | d | i d ƒ |  i ƒ  ƒ ‚ n | d 7} n |  i | ƒ } |  i | ƒ | S(   Nu   !s   while scanning a %ss   expected '!', but found %rs   utf-8i   u    u   0u   9u   AR   u   aR‚   u   -_(   R'   R   RD   RE   RY   Rw   (   R   R   R[   RF   Rƒ   R€   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRŽ   O  s*    ]
c         C   sn  g  } d } |  i  | ƒ } xØ d | j o
 d j n pI d | j o
 d j n p+ d | j o
 d j n p | d j op | d	 j oF | i |  i | ƒ ƒ |  i | ƒ d } | i |  i | | ƒ ƒ n | d
 7} |  i  | ƒ } q W| o- | i |  i | ƒ ƒ |  i | ƒ d } n | p0 t d | | d | i d ƒ |  i ƒ  ƒ ‚ n d i | ƒ S(   Ni    u   0u   9u   AR   u   aR‚   u   -;/?:@&=+$,_.!~*'()[]%u   %i   s   while parsing a %ss   expected URI, but found %rs   utf-8u    (	   R'   RO   Rw   RY   t   scan_uri_escapesR   RD   RE   Rž   (   R   R   R[   R    Rƒ   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR   i  s*    ]

c         C   s&  g  } |  i  ƒ  } xº |  i ƒ  d j o¦ |  i ƒ  xc t d ƒ D]U } |  i | ƒ d j o9 t d | | d |  i | ƒ i d ƒ |  i  ƒ  ƒ ‚ q? q? W| i t t |  i	 d ƒ d ƒ ƒ ƒ |  i d ƒ q Wy t
 d i | ƒ d ƒ } Wn4 t j
 o( } t d | | t | ƒ | ƒ ‚ n X| S(	   Nu   %i   u   0123456789ABCDEFabcdefs   while scanning a %ssB   expected URI escape sequence of 2 hexdecimal numbers, but found %rs   utf-8i   t    (   RE   R'   RY   R¯   R   RD   RO   t   chrR‰   Rw   t   unicodeRž   t   UnicodeDecodeErrort   str(   R   R   R[   t   bytesR   R²   R€   t   exc(    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR¸   ƒ  s"    
 -%$c         C   su   |  i  ƒ  } | d j o9 |  i d ƒ d j o |  i d ƒ n |  i ƒ  d Sn  | d j o |  i ƒ  | Sn d S(   Nu   
Â…i   u   
u   
u   â€¨â€©u    (   R'   Rw   RY   (   R   RF   (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyRx   –  s    	

(K   R   R   R   R   R    R"   R   R   R$   R#   RN   RL   R&   RQ   R   R(   R*   R,   R.   RV   R/   R0   R]   R1   R2   R`   R3   R5   R7   R9   R:   R;   R<   R=   R>   Rp   R?   R@   Rs   RB   R)   R+   R-   R4   R6   R8   RA   R%   RU   Rz   R{   R…   R|   RŠ   R‹   R}   Rj   Rm   Rr   R—   R˜   R™   R›   Ru   R­   R®   R©   Rª   R±   Rv   Rµ   RŽ   R   R¸   Rx   (    (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pyR    .   sº   	A			
		l						
														!		8															
	
		#					
	
	
		$	*	L	(																							+			/	%			N(   t   __all__t   errorR   R   R   t   objectR   R    (    (    (    s"   /p/zhu/06/nlp/nltk/yaml/scanner.pys   <module>   s
   
