1#!/usr/bin/env python23importcStringIO#4importformatter#5fromhtmllibimportHTMLParser#We use various classes in these modules for parsing HTML.6importhttplib#We only need an exception from this module7importos#ThisPRovides various file system functions8importsys#We are just using argv for command-line arguments9importurllib#We only need the urlretrieve()function for downloading Web pages10importurlparse#We use the urlparse()and urljoin()functions for URL manipulation1112classRetriever(object):13__slots__= ('url','file')1415def__init__(self,url):16self.url, self.file =self.get_file(url)1718defget_file(self, url, default='index.html'):19'Create usable local filename from URL'20parsed = urlparse.urlparse(url)#ParseResult(scheme='http', netloc='www.baidu.com', path='', params='', query='', fragment='')21host = parsed.netloc.split('@')[-1].split(':')[0]#'www.baidu.com'22filepath ='%s%s'% (host,parsed.path)#'www.baidu.com'23ifnotos.path.splitext(parsed.path)[1]:#''24filepath = os.path.join(filepath, default)#'www.baidu.com\\index.html'25linkdir = os.path.dirname(filepath)#'www.baidu.com'26ifnotos.path.isdir(linkdir):#False27ifos.path.exists(linkdir):#False28os.unlink(linkdir)29os.makedirs(linkdir)#make a directory named by link directory on the hard disc30returnurl, filepath3132defdownload(self):33'Download URL to specific name file'34try:35retval =urllib.urlretrieve(self.url, self.file)36except(IOError, httplib.InvalidURL) as e:37retval = (('*** ERROR:bad URL "%s": %s'%(self.url,e)),)38returnretval3940defparse_links(self):41'Parse out the links found in downloaded HTML file'42f = open(self.file,'r')43data =f.read()44f.close()45parser =HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))46parser.feed(data)47parser.close()48returnparser.anchorlist4950classCrawler(object):51count = 0#the number of objects downloaded from the internet5253def__init__(self, url):54self.q = [url]#a queue of links to download55self.seen = set()#a set containing all the links that we have seen(downloaded) already56parsed =urlparse.urlparse(url)57host = parsed.netloc.split('@')[-1].split(':')[0]58self.dom ='.'.join(host.split('.')[-2:])#'b.a.i.d.u'5960defget_page(self, url, media=False):61'Download page & parse links, add to queue if nec'62r =Retriever(url)63fname = r.download()[0]#'www.baidu.com\\index.html'64iffname[0] =='*':#'w'65printfname,'... skipping parse'66return67Crawler.count += 1#168print'\n(', Crawler.count,')'#(1)69print'URL:', url#URL: http://www.baidu.com70print'FILE:', fname#FILE: www.baidu.com\\index.html71self.seen.add(url)#set(['http://www.baidu.com'])72ftype = os.path.splitext(fname)[1]#'.html'73ifftypenotin('.htm','.html'):#False74return7576forlinkinr.parse_links():77iflink.startswith('mailto:'):#False78print'... discarded, mailto link'79continue80ifnotmedia:#False81ftype = os.path.splitext(link)[1]82ifftypein('.mp3','.mp4','.m4v','.wav'):83print'... discarded, media file'84continue85ifnotlink.startswith('http://'):#False86link =urlparse.urljoin(url, link)87print'*', link,88iflinknotinself.seen:#True89ifself.domnotinlink:#False90print'... discarded, not in domain'91else:92iflinknotinself.q:93self.q.append(link)94print'... new, added to Q'95else:96print'... discarded, already in Q'97else:98print'... discarded, already processed'99100defgo(self, media=False):101'Process next page in queue (if any)'102whileself.q:103url =self.q.pop()104self.get_page(url, media)105106defmain():107iflen(sys.argv) > 1:108url = sys.argv[1]109else:110try:111url = raw_input('Enter starting URL:')112except(KeyboardInterrupt, EOFError):113url =''114ifnoturl:115return116ifnoturl.startswith('http://')andnoturl.startswith('ftp://'):117url ='http://%s/'%url118robot =Crawler(url)119robot.go()120121if__name__=='__main__':122main()