selenium 持久化 | 易学教程

1、安装pyexcel

 pip3 install -i https://pypi.douban.com/simple pyexcel

2、四步走战略

import time
import sys
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
import pyexcel     # 1.导入模块
#

if __name__ == '__main__':
    keyword = 'iphone'
    if len(sys.argv) > 1:
        keyword = sys.argv[1]

    option = Options()
    option.add_argument('--headless')
    # 打开浏览器
    browser = webdriver.Chrome(chrome_options=option)
    browser.get('https://www.jd.com')
    # 搜索iphone
    _input = browser.find_element_by_id('key')
    _input.send_keys(keyword)
    _input.send_keys(Keys.ENTER)
    time.sleep(5)

    # 按销量排序
    sales = browser.find_element_by_xpath('//div[@class="f-sort"]/a[2]')
    sales.click()

    # 2.定义一个列表
    rows = []
    has_next = True
    while has_next:
        # 获取当前的页码
        time.sleep(5)
        cur_page = browser.find_element_by_xpath('//div[@id="J_bottomPage"]/span[@class="p-skip"]/input').get_attribute('value')
        print('-------------------------   当前页码 {}  -------------------------'.format(cur_page))

        # 加载全部数据，数据随着滚动条的下来而加载
        # good_list = browser.find_element_by_id('J_goodsList')
        # y = good_list.rect['y'] + good_list.rect['height']
        next_page = browser.find_element_by_class_name('pn-next')
        y = next_page.location['y']
        browser.execute_script('window.scrollTo(0, {})'.format(y))
        time.sleep(3)
        # 获取当前页面所有商品列表
        p_list = browser.find_elements_by_class_name('gl-item')
        for p in p_list:
            production = {}
            sku = p.get_attribute('data-sku')
            production['price'] = p.find_element_by_css_selector('strong.J_{}'.format(sku)).text
            production['name'] = p.find_element_by_css_selector('div.p-name>a>em').text
            production['comment'] = p.find_element_by_id('J_comment_{}'.format(sku)).text
            production['shop'] = p.find_element_by_css_selector('div.p-shop>span>a').get_attribute('title')
            print(production)
            # 3. 把数据添加到自定义的列表中
            rows.append(production)

        # 下一页
        cur_next_page = browser.find_element_by_class_name('pn-next')
        # 判断是否是最后一页
        if 'disabled' in cur_next_page.get_attribute('class'):
            has_next = False
        else:
            cur_next_page.click()
    # 4. 保存数据到表格，并命名
    pyexcel.save_as(records=rows, dest_file_name='{}.xlsx'.format(keyword))
    browser.quit()

来源：https://www.cnblogs.com/wt7018/p/11874944.html

标签

selenium

持久化

webdriver