Python爬蟲實現(xiàn)爬取百度百科詞條功能實例
本文實例講述了Python爬蟲實現(xiàn)爬取百度百科詞條功能。分享給大家供大家參考，具體如下：
以下我寫了一個爬取百度百科詞條的實例。
爬蟲主程序入口
from crawler_test.html_downloader import UrlDownLoaderfrom crawler_test.html_outer import HtmlOuterfrom crawler_test.html_parser import HtmlParserfrom crawler_test.url_manager import UrlManager# 爬蟲主程序入口class MainCrawler():def __init__(self):# 初始值，實例化四大處理器：url管理器，下載器，解析器，輸出器self.urls = UrlManager()self.downloader = UrlDownLoader()self.parser = HtmlParser()self.outer = HtmlOuter()# 開始爬蟲方法def start_craw(self, main_url):print('爬蟲開始...')count = 1self.urls.add_new_url(main_url)while self.urls.has_new_url():try:new_url = self.urls.get_new_url()print('爬蟲%d,%s' % (count, new_url))html_cont = self.downloader.down_load(new_url)new_urls, new_data = self.parser.parse(new_url, html_cont)# 將解析出的url放入url管理器，解析出的數(shù)據放入輸出器中self.urls.add_new_urls(new_urls)self.outer.conllect_data(new_data)if count >= 10:# 控制爬取的數(shù)量breakcount += 1except:print('爬蟲失敗一條')self.outer.output()print('爬蟲結束。')if __name__ == '__main__':main_url = 'https://baike.baidu.com/item/Python/407313'mc = MainCrawler()mc.start_craw(main_url)
URL管理器
# URL管理器class UrlManager():def __init__(self):self.new_urls = set() # 待爬取self.old_urls = set() # 已爬取# 添加一個新的urldef add_new_url(self, url):if url is None:returnelif url not in self.new_urls and url not in self.old_urls:self.new_urls.add(url)# 批量添加urldef add_new_urls(self, urls):if urls is None or len(urls) == 0:returnelse:for url in urls:self.add_new_url(url)# 判斷是否有urldef has_new_url(self):return len(self.new_urls) != 0# 從待爬取的集合中獲取一個urldef get_new_url(self):new_url = self.new_urls.pop()self.old_urls.add(new_url)return new_url
網頁下載器
from urllib import request# 網頁下載器class UrlDownLoader():def down_load(self, url):if url is None:return Noneelse:rt = request.Request(url=url, method='GET') # 發(fā)GET請求with request.urlopen(rt) as rp: # 打開網頁if rp.status != 200:return Noneelse:return rp.read() # 讀取網頁內容
網頁解析器
import refrom urllib import parsefrom bs4 import BeautifulSoup# 網頁解析器，使用BeautifulSoupclass HtmlParser():# 每個詞條中，可以有多個超鏈接# main_url指url公共部分，如“https://baike.baidu.com/”def _get_new_url(self, main_url, soup):# baike.baidu.com/# <a target="_blank" href="/item/%E8%AE%A1%E7%AE%97%E6%9C%BA%E7%A8%8B%E5%BA%8F%E8%AE%BE%E8%AE%A1%E8%AF%AD%E8%A8%80" rel="external nofollow" >計算機程序設計語言</a>new_urls = set()# 解析出main_url之后的url部分child_urls = soup.find_all('a', href=re.compile(r'/item/(\%\w{2})+'))for child_url in child_urls:new_url = child_url['href']# 再拼接成完整的urlfull_url = parse.urljoin(main_url, new_url)new_urls.add(full_url)return new_urls# 每個詞條中，只有一個描述內容，解析出數(shù)據（詞條，內容）def _get_new_data(self, main_url, soup):new_datas = {}new_datas['url'] = main_url# <dd class="lemmaWgt-lemmaTitle-title"><h1>計算機程序設計語言</h1>...new_datas['title'] = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1').get_text()# class="lemma-summary" label-module="lemmaSummary"...new_datas['content'] = soup.find('div', attrs={'label-module': 'lemmaSummary'},class_='lemma-summary').get_text()return new_datas# 解析出url和數(shù)據（詞條，內容）def parse(self, main_url, html_cont):if main_url is None or html_cont is None:returnsoup = BeautifulSoup(html_cont, 'lxml', from_encoding='utf-8')new_url = self._get_new_url(main_url, soup)new_data = self._get_new_data(main_url, soup)return new_url, new_data
輸出處理器
# 輸出器class HtmlOuter():def __init__(self):self.datas = []# 先收集數(shù)據def conllect_data(self, data):if data is None:returnself.datas.append(data)return self.datas# 輸出為HTMLdef output(self, file='output_html.html'):with open(file, 'w', encoding='utf-8') as fh:fh.write('<html>')fh.write('<head>')fh.write('<meta charset="utf-8"></meta>')fh.write('<title>爬蟲數(shù)據結果</title>')fh.write('</head>')fh.write('<body>')fh.write('<table style="border-collapse:collapse; border:1px solid gray; width:80%; word-break:break-all; margin:20px auto;">')fh.write('<tr>')fh.write('<th style="border:1px solid black; width:35%;">URL</th>')fh.write('<th style="border:1px solid black; width:15%;">詞條</th>')fh.write('<th style="border:1px solid black; width:50%;">內容</th>')fh.write('</tr>')for data in self.datas:fh.write('<tr>')fh.write('<td style="border:1px solid black">{0}</td>'.format(data['url']))fh.write('<td style="border:1px solid black">{0}</td>'.format(data['title']))fh.write('<td style="border:1px solid black">{0}</td>'.format(data['content']))fh.write('</tr>')fh.write('</table>')fh.write('</body>')fh.write('</html>')
效果（部分）：
原文鏈接：https://www.cnblogs.com/wcwnina/p/8619084.html
文章轉載：Python編程學習圈
（版權歸原作者所有，侵刪）
點擊下方“閱讀原文”查看更多
評論
圖片
表情