Selenium自動(dòng)化｜輕松爬取公眾號(hào)文章

Selenium介紹

需求分析和代碼實(shí)現(xiàn)

from?selenium?import?webdriver
from?selenium.webdriver.common.by?import?By
from?selenium.webdriver.support?import?expected_conditions?as?EC
from?selenium.webdriver.support.wait?import?WebDriverWait
#?導(dǎo)入第2-4行是為了馬上會(huì)提到的?顯式等待
import?time
import?datetime

driver?=?webdriver.Chrome()
driver.get('https://weixin.sogou.com/')

wait?=?WebDriverWait(driver,?10)
input?=?wait.until(EC.presence_of_element_located((By.NAME,?'query')))
input.send_keys('早起Python')
driver.find_element_by_xpath("http://input[@class='swz']").click()

num?=?0

def?get_news():
????global?num?#?放全局變量是為了給符合條件的文章記序
????time.sleep(1)
????news_lst?=?driver.find_elements_by_xpath("http://li[contains(@id,'sogou_vr_11002601_box')]")
????for?news?in?news_lst:
????????#?獲取公眾號(hào)來(lái)源
????????source?=?news.find_elements_by_xpath('div[2]/div/a')[0].text
????????if?'早起'?not?in?source:
????????????continue
????????num?+=?1
????????#?獲取文章標(biāo)題
????????title?=?news.find_elements_by_xpath('div[2]/h3/a')[0].text
????????#?獲取文章發(fā)表日期
????????date?=?news.find_elements_by_xpath('div[2]/div/span')[0].text
????????#?文章發(fā)表的日期如果較近可能會(huì)顯示“1天前”?“12小時(shí)前”?“30分鐘前”
????????#?這里可以用`datetime`模塊根據(jù)時(shí)間差求出具體時(shí)間
????????#?然后解析為`YYYY-MM-DD`格式
????????if?'前'?in?date:
????????????today?=?datetime.datetime.today()
????????????if?'天'?in?date:
????????????????delta?=?datetime.timedelta(days=int(date[0]))
????????????elif?'小時(shí)'?in?date:
????????????????delta?=?datetime.timedelta(hours=int(date.replace('小時(shí)前',?'?')))
????????????else:
????????????????delta?=?datetime.timedelta(minutes=int(date.replace('分鐘前',?'?')))
????????????date?=?str((today?-?delta).strftime('%Y-%m-%d'))
????????date?=?datetime.datetime.strptime(date,?'%Y-%m-%d').strftime('%Y-%m-%d')
????????#?獲取url
????????url?=?news.find_elements_by_xpath('div[2]/h3/a')[0].get_attribute('href')
????????print(num,?title,?date)
????????print(url)
????????print('-'?*?10)

for?i?in?range(10):
????get_news()
????if?i?==?9:
????????#?如果遍歷到第十頁(yè)則跳出循環(huán)不需要點(diǎn)擊“下一頁(yè)”
????????break
????driver.find_element_by_id("sogou_next").click()

driver.find_element_by_name('top_login').click()
while?True:
????try:
????????next_page?=?driver.find_element_by_id("sogou_next")
????????break
????except:
????????time.sleep(3)
next_page.click()

while?True:
????get_news()
????try:
????????driver.find_element_by_id("sogou_next").click()
????except:
????????break

#?最后退出瀏覽器即可
driver.quit()