%PDF-
%PDF-
Mini Shell
Mini Shell
ó
Nêáac @ s" d d l Z d d l Z d d l Z d d l m Z d d l m Z m Z d d l Z y d d l Z Wn e
k
r d Z n Xd d# d „ ƒ YZ d e e j
f d „ ƒ YZ d e e j
f d „ ƒ YZ d
e e j
f d „ ƒ YZ d e f d
„ ƒ YZ d e e j
f d „ ƒ YZ d e e j
f d „ ƒ YZ d e e j
f d „ ƒ YZ d e e j
f d „ ƒ YZ d e e j
f d „ ƒ YZ d e e j
f d „ ƒ YZ d e f d „ ƒ YZ e j e d ƒ d e j
f d „ ƒ Yƒ Z d e j
f d „ ƒ YZ d! „ Z e d" k r e ƒ n d S($ iÿÿÿÿN( t support( t BaseHTTPRequestHandlert
HTTPServert
BaseRobotTestc B sD e Z d Z d Z g Z g Z d „ Z d „ Z d „ Z d „ Z RS( t t test_robotparserc C s; t j | j ƒ j ƒ } t j ƒ | _ | j j | ƒ d S( N( t StringIOt
robots_txtt readlinest robotparsert RobotFileParsert parsert parse( t selft lines( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt setUp s c C s2 t | t ƒ r% | \ } } | | f S| j | f S( N( t
isinstancet tuplet agent( R
t urlR ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt get_agent_and_url s
c C sI xB | j D]7 } | j | ƒ \ } } | j | j j | | ƒ ƒ q
Wd S( N( t goodR t
assertTrueR t can_fetch( R
R R ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt test_good_urls s c C sI xB | j D]7 } | j | ƒ \ } } | j | j j | | ƒ ƒ q
Wd S( N( t badR t assertFalseR R ( R
R R ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt
test_bad_urls# s (
t __name__t
__module__R R R R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR
s t UserAgentWildcardTestc B s) e Z d Z d d g Z d d d g Z RS( s• User-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /foo.html
t /s
/test.htmls /cyberworld/map/index.htmls /tmp/xxxs /foo.html( R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR ) s t RejectAllRobotsTestc B s# e Z d Z g Z d d d g Z RS( s( # go away
User-agent: *
Disallow: /
s /cyberworld/map/index.htmlR s /tmp/( R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR 4 s t UserAgentOrderingTestc B s e Z d Z d Z d g Z RS( sM User-agent: Googlebot
Disallow: /
User-agent: Googlebot-Mobile
Allow: /
t Googlebots /something.jpg( R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR! > s
t UserAgentGoogleMobileTestc B s e Z d Z RS( s Googlebot-Mobile( R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR# M s t GoogleURLOrderingTestc B s& e Z d Z d Z d g Z d g Z RS( sJ User-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
t googlebots /folder1/myfile.htmls /folder1/anotherfile.html( R R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR$ Q s t DisallowQueryStringTestc B s e Z d Z d g Z d g Z RS( s2 User-agent: *
Disallow: /some/path?name=value
s
/some/paths /some/path?name=value( R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR& ^ s t UseFirstUserAgentWildcardTestc B s e Z d Z d g Z d g Z RS( sN User-agent: *
Disallow: /some/path
User-agent: *
Disallow: /another/path
s
/another/paths
/some/path( R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR' h s t EmptyQueryStringTestc B s e Z d Z d g Z d g Z RS( s> User-agent: *
Allow: /some/path?
Disallow: /another/path?
s /some/path?s /another/path?( R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR( u s t DefaultEntryTestc B s# e Z d Z d d g Z d g Z RS( sO User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/
R s
/test.htmls /cyberworld/map/index.html( R R R R R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR) € s t StringFormattingTestc B s e Z d Z d Z d „ Z RS( sÆ User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow: /some/path
sX User-agent: cybermapper
Disallow: /some/path
User-agent: *
Disallow: /cyberworld/map/
c C s | j t | j ƒ | j ƒ d S( N( t assertEqualt strR t expected_output( R
( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt test_string_formatting s ( R R R R- R. ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR* ‹ s
t RobotHandlerc B s e Z d „ Z d „ Z RS( c C s | j d d ƒ d S( Ni“ s Forbidden access( t
send_error( R
( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt do_GET¦ s c G s d S( N( ( R
t formatt args( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt log_message© s ( R R R1 R4 ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR/ ¤ s s threading required for this testt PasswordProtectedSiteTestCasec B s, e Z d „ Z d „ Z e j d „ ƒ Z RS( c C sf t t j d f t ƒ | _ t j d d d | j j d i d d 6ƒ | _ t | j _
| j j ƒ d S( Ni t names HTTPServer servingt targett kwargsg{®Gáz„?t
poll_interval( R R t HOSTR/ t servert threadingt Threadt
serve_forevert tt Truet daemont start( R
( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR ° s c C s+ | j j ƒ | j j ƒ | j j ƒ d S( N( R; t shutdownR? t joint server_close( R
( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt tearDown½ s
c C su | j j } d t j d t | d ƒ } | d } t j ƒ } | j | ƒ | j ƒ | j | j
d | ƒ ƒ d S( Ns http://t :i s /robots.txtt *( R; t server_addressR R: R, R R
t set_urlt readR R ( R
t addrR t
robots_urlR ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt testPasswordProtectedSite s
( R R R RF R t reap_threadsRN ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR5 s
t NetworkTestCasec B sP e Z d Z d j e ƒ Z e d „ ƒ Z d „ Z d „ Z d „ Z d „ Z
RS( s http://www.pythontest.net/s {}elsewhere/robots.txtc C sL t j d ƒ t j | j ƒ ' t j | j ƒ | _ | j j ƒ Wd QXd S( Nt network( R t requirest transient_internett base_urlR R
R R RK ( t cls( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt
setUpClassÒ s
c C s2 d j | j | t j j | ƒ d s+ d n d ƒ S( Ns {}{}{}i R R ( R2 RT t ost patht splitext( R
RX ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyR Ù s c C sC | j | j j ƒ | j | j j ƒ | j | j j ƒ d ƒ d S( Ni ( R R t disallow_allt allow_allt
assertGreatert mtime( R
( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt
test_basicÞ s c C sÖ | j | j j d | j d ƒ ƒ ƒ | j | j j d | j ƒ ƒ | j | j j d | j d ƒ ƒ ƒ | j | j j d | j d ƒ ƒ ƒ | j | j j d | j d ƒ ƒ ƒ | j | j j d | j ƒ ƒ d S( NRH t elsewheret Nutcht briant webstats( R R R R R RT ( R
( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt test_can_fetchã s % %%%c C s\ t j | j d ƒ ƒ } | j ƒ | j | j ƒ | j | j ƒ | j | j ƒ d ƒ d S( Ns i-robot.txti (
R R
R RK R R[ R RZ R+ R] ( R
R ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt
test_read_404ë s
( R R RT R2 R t classmethodRV R R^ Rc Rd ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyRP Í s c
C s2 t j t t t t t t t t t
t t t
ƒ d S( N( R t run_unittestR R R! R# R$ R& R' R( R) R* R5 RP ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt test_mainó s t __main__( ( RW R t unittestt testR t BaseHTTPServerR R R R< t ImportErrort NoneR t TestCaseR R R! R# R$ R& R' R( R) R* R/ t
skipUnlessR5 RP Rg R ( ( ( s1 /usr/local/lib/python2.7/test/test_robotparser.pyt <module> s6
&
Zerion Mini Shell 1.0