³ò
.™™Ic        
   @   s·   d  d k  Z  d  d k Z d  d k Z d  d k Z d  d k l Z l Z d  d k Z e i d ƒ e i	 d ƒ Z
 e i	 d ƒ Z e i	 d ƒ Z e i	 d ƒ Z h  d d	 <Z d
 „  Z d S(   iÿÿÿÿN(   t   BeautifulSoupt   Commenti
   s   <a.*?>(.*?)</a>s   <[/]?strong>s   <[/]?b>s   <[/]?i>s.   Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)s
   User-agentc         C   s„  y@ t  i |  d  t ƒ } t  i ƒ  } | i | ƒ } | i ƒ  } Wn d Sn X| i d d ƒ } | i d d ƒ } | i d d ƒ } | i d d ƒ } t i	 d | ƒ } t
 i	 d | ƒ } t i	 d | ƒ } t i	 d | ƒ } y t | d	 t i ƒ} Wn d Sn X| i d
 d „  ƒ } g  } | D] } | | i ƒ  q%~ y | i d
 t ƒ }	 Wn | i d
 t ƒ }	 n Xd }
 x|	 D]þ } y | i i i d d ƒ } Wn d } n X| i d d ƒ i ƒ  i d d ƒ } d i g  } | D]$ } t | ƒ d j  o | | qåqå~ ƒ } | i d d ƒ } t | ƒ d j oB | d j o5 | d j o( | i d ƒ d j o |
 | d 7}
 q~q~W|
 S(   Ns1   An error occured. We could not retrieve the text.s   t    s   
s   <br>s   <br/>s   \1t    t   convertEntitiest   textc         S   s   t  |  t ƒ S(    (   t
   isinstanceR   (   R   (    (    s:   /afs/cs.wisc.edu/p/zhu/public/html/space2/TTP/save_text.pyt   <lambda>)   s    s   utf-8t   ignorei€   t   |s    | i    t   scriptt   styles	   </script>iÿÿÿÿs    .
(   t   urllib2t   Requestt   Nonet
   txtheaderst   build_openert   opent   readt   replacet   re_at   subt   re_bt   re_it	   re_strongR    t   ALL_ENTITIESt   findAllt   extractt   bodyt   Truet   parentt   namet   encodet   stript   joint   ordt   lent   find(   t   urlt   reqt   openert   handlet   paget   soupt   commentst   _[1]t   commentt   pageTextt   contentt   et   pR   t   _[2]t   c(    (    s:   /afs/cs.wisc.edu/p/zhu/public/html/space2/TTP/save_text.pyt   fetch   sR    % 
$A(   R   t   syst   ret   os.patht   osR    R   t   sockett   setdefaulttimeoutt   compileR   R   R   R   R   R5   (    (    (    s:   /afs/cs.wisc.edu/p/zhu/public/html/space2/TTP/save_text.pys   <module>   s   0