環境:

ubuntu16.04
python3.5
python庫: selenium, pyquery,pymongo, re

要求:

設置×××面瀏覽器訪問,并將商品列表存入mongoDB數據庫.

分析過程暫時略過

代碼：

#!/usr/bin/env python# -*- coding:utf-8 -*-"""1.爬取淘寶美食的流程- 搜索關鍵字: 用selenium打開瀏覽器,模擬輸入關鍵字,并搜索對應的商品列表.- 分析頁碼并翻頁,模擬翻頁,查看到所有頁面的商品列表.- 分析并提取商品,利用Pyquery分析源碼,解析得到商品列表.- 存儲到MONGODB數據庫,將商品列表信息存儲到mongoDB數據庫."""from selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.common.exceptions import TimeoutExceptionfrom pyquery import PyQuery as pqimport refrom anli.mongoconfig import *import pymongoclient = pymongo.MongoClient(MONGO_URL)db = client[MONGO_DB]#設置×××面訪問opt = webdriver.FirefoxOptions()opt.set_headless()browser = webdriver.Firefox(options=opt)#等待瀏覽器加載頁面成功.wait = WebDriverWait(browser,10)def search():  try:    # 后臺打開瀏覽器    browser.get('https://www.taobao.com')    # 用CSS選擇器復制搜索框    input = wait.until(      EC.presence_of_element_located((By.CSS_SELECTOR, '#q'))    )    # 找到搜索按鈕.    submit = wait.until(      # EC.element_to_be_clickable((By.CSS_SELECTOR,'#J_TSearchForm .search-button')))      EC.element_to_be_clickable((By.CSS_SELECTOR,'#J_TSearchForm > div.search-button > button')))    # 輸入關鍵字    input.send_keys('美食')    # 點擊搜索按鈕.    submit.click()    # 輸出總共的頁數.    total=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'.total')))    # 調取商品列表的函數.    get_products()    return total.text  except TimeoutException: #超時錯誤.    return search()# 翻頁def next_page(page_number):  try:    #注意在firefox和chrome瀏覽器復制出來的元素不太一樣.    #要傳入的頁碼: 到第幾頁    input = wait.until(      EC.presence_of_element_located((By.CSS_SELECTOR,'input.input:nth-child(2)'))    )    #復制確定按鈕的元素:    submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'span.btn:nth-child(4)')))    #清除頁碼    input.clear()    #輸入當前頁碼    input.send_keys(page_number)    #點擊確定按鈕.    submit.click()    #判斷當前頁碼是否是當前數字: 復制高亮頁碼數.    wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,'span.num'),str(page_number)))    get_products()  except TimeoutException:    next_page(page_number)#解析jquery源碼def get_products():  #商品列表:  wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item')))  # 獲取網頁源碼  html = browser.page_source  doc = pq(html)  items = doc('#mainsrp-itemlist .items .item').items()  for item in items:    #定義商品列表詳細信息的字典    product = {      'title': item.find('.title').text(),      'price': item.find('.price').text(),      'image': item.find('.pic .img').attr('src'),      'shop': item.find('.shop').text(),      'deal': item.find('.deal-cnt').text()[:-3],      'location': item.find('.location').text()    }    print(product)    #將商品列表信息保存到mongoDB數據庫.    save_to_mongo(product)def save_to_mongo(result):  try:    if db[MONGO_TABLE].insert(result):      print('存儲到mongodb成功',result)  except Exception:    print('存儲到mongodb失敗',result)def main():  total = search()  # 用正則表達式只匹配出數字,并打印數字.  total = int(re.compile('(/d+)').search(total).group(1))  print(total)  for i in range(2,total + 1):    next_page(i)if __name__=='__main__':  main()  #關閉×××面瀏覽器.  browser.quit()#mongoconfig.py#!/usr/bin/env python# -*- coding:utf-8 -*-MONGO_URL = 'localhost'MONGO_DB = 'taobao'MONGO_TABLE = 'meishi'