昨日作业
自动登录抽屉网
1 from selenium import webdriver
2 import time
3
4 driver = webdriver.Chrome()
5 driver.maximize_window()
6
7 try:
8 driver.get('https://dig.chouti.com/')
9 driver.implicitly_wait(10)
10 time.sleep()
11
12 login_bin = driver.find_element_by_id('login_btn')
13 login_bin.click()
14 time.sleep(2)
15
16 phone = driver.find_element_by_class_name('login-phone')
17 phone.send_keys('17718268511')
18
19 pwd = driver.find_element_by_class_name('pwd-password-imput')
20 pwd.send_keys('tututu')
21
22 login_submit = driver.find_element_by_class_name('btn-large')
23 login_submit.click()
24 time.sleep(20)
25
26 #捕获异常并打印
27 except Exception as e:
28 print(e)
29
30 finally:
31 driver.close()
今日内容
1、selenium剩余用法
2、selenium万能登陆破解
3、破解极验滑动验证码
1、selenium剩余用法
selenuim选择器之Xpath
1 from selenium import webdriver
2
3 driver = webdriver.Chrome()
4
5 try:
6 driver.get('https://doc.scrapy.org/en/latest/_static/selectors-sample1.html')
7 driver.implicitly_wait(5)
8
9 根据Xpath语法查找元素
10 /从根节点开始找第一个
11 html = driver.find_element_by_xpath('/html')
12 print(html.tag_name)
13
14 #//从根节点找任意一个节点
15 div = driver.find_element_by_xpath('//div')
16 print(div.tag_name)
17
18
19 #@
20 #查找id为images的div节点
21 div = driver.find_element_by_xpath('//div[@id="images"]')
22 print(div.tag_name)
23 print(div.text)
24
25 #找到一个a节点
26 a = driver.find_element_by_xpath('//a')
27 print(a.tag_name)
28
29 #找到所有a节点
30 a_s = driver.find_elements_by_xpath('//a')
31 print(a_s)
32
33 #找到第一个a节点的href属性
34 a = driver.find_element_by_xpath('//a').get_attribute('href')
35 print(a)
36
37 finally:
38 driver.close()
39
40
41 找到所有a节点
42
43
44 '''
45
46 添加cookie
47 '''
48 from selenium import webdriver
49 driver = webdriver.Chrome()
50 import time
51
52
53
54 try:
55 driver.implicitly_wait(10)
56 driver.get('https://www.zhihu.com/explore')
57 print(driver.get_cookie())
58
59 time.sleep(10)
60
61
62 finally:
63 driver.close()
64
65
66
67 '''
68 #
69 # 选项卡
70 #
71 # '''
72
73 import time
74 from selenium import webdriver
75
76 browser = webdriver.Chrome()
77
78 try:
79 browser.get('https://')
80
81
82
83 finally:
84 browser.close()
85
86
87
88 '''
89 前进、后退
90 '''
91 from selenium import webdriver
92 import time
93
94 driver = webdriver.Chrome()
95
96
97 try:
98 driver.implicitly_wait(10)
99 driver.get('https://www.jd.com/')
100 driver.get('https://www.baidu.com/')
101 driver.get('https://www.cnblogs.com/')
102
103 time.sleep(2)
104
105 # 回退操作
106 driver.back()
107 time.sleep(1)
108 # 前进操作
109 driver.forward()
110 time.sleep(1)
111 driver.back()
112 time.sleep(10)
113
114 finally:
115 driver.close()
selenium剩余操作
1 ''''''
2 '''
3 点击、清除操作
4 '''
5 # from selenium import webdriver
6 # from selenium.webdriver.common.keys import Keys
7 # import time
8 #
9 # driver = webdriver.Chrome(r'D:\BaiduNetdiskDownload\chromedriver_win32\chromedriver.exe')
10 #
11 # try:
12 # driver.implicitly_wait(10)
13 # # 1、往jd发送请求
14 # driver.get('https://www.jd.com/')
15 # # 找到输入框输入围城
16 # input_tag = driver.find_element_by_id('key')
17 # input_tag.send_keys('围城')
18 # # 键盘回车
19 # input_tag.send_keys(Keys.ENTER)
20 # time.sleep(2)
21 # # 找到输入框输入墨菲定律
22 # input_tag = driver.find_element_by_id('key')
23 # input_tag.clear()
24 # input_tag.send_keys('墨菲定律')
25 # # 找到搜索按钮点击搜索
26 # button = driver.find_element_by_class_name('button')
27 # button.click()
28 # time.sleep(10)
29 #
30 # finally:
31 # driver.close()
32
33
34 '''
35 获取cookies （了解）
36 '''
37 # from selenium import webdriver
38 # import time
39 #
40 # driver = webdriver.Chrome(r'D:\BaiduNetdiskDownload\chromedriver_win32\chromedriver.exe')
41 #
42 # try:
43 # driver.implicitly_wait(10)
44 # driver.get('https://www.zhihu.com/explore')
45 # print(driver.get_cookies())
46 #
47 # time.sleep(10)
48 # finally:
49 # driver.close()
50
51 '''
52 选项卡
53 '''
54 #选项卡管理：切换选项卡，有js的方式windows.open,有windows快捷键：
55 # ctrl+t等，最通用的就是js的方式
56 # import time
57 # from selenium import webdriver
58 #
59 # browser = webdriver.Chrome()
60 # try:
61 # browser.get('https://www.baidu.com')
62 #
63 # # execute_script: 执行javascrpit代码
64 # # 弹窗操作
65 # # browser.execute_script('alert("tank")')
66 # # 新建浏览器窗口
67 # browser.execute_script(
68 # '''
69 # window.open();
70 # '''
71 # )
72 # time.sleep(1)
73 # print(browser.window_handles) # 获取所有的选项卡
74 # # 切换到第二个窗口
75 # # 新:
76 # browser.switch_to.window(browser.window_handles[1])
77 # # 旧:
78 # # browser.switch_to_window(browser.window_handles[1])
79 #
80 # # 第二个窗口往淘宝发送请求
81 # browser.get('https://www.taobao.com')
82 # time.sleep(5)
83 #
84 # # 切换到第一个窗口
85 # browser.switch_to_window(browser.window_handles[0])
86 # browser.get('https://www.sina.com.cn')
87 #
88 # time.sleep(10)
89 # finally:
90 # browser.close()
91
92
93 '''
94 ActionChangs动作链
95 '''
96 # from selenium import webdriver
97 # from selenium.webdriver import ActionChains
98 # import time
99 #
100 # driver = webdriver.Chrome()
101 # driver.implicitly_wait(10)
102 # driver.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
103 #
104 # try:
105 #
106 # # driver.switch_to_frame('iframeResult')
107 # # 切换到id为iframeResult的窗口内
108 # driver.switch_to.frame('iframeResult')
109 #
110 # # 源位置
111 # draggable = driver.find_element_by_id('draggable')
112 #
113 # # 目标位置
114 # droppable = driver.find_element_by_id('droppable')
115 #
116 # # 调用ActionChains，必须把驱动对象传进去
117 # # 得到一个动作链对象，复制给一个变量
118 # actions = ActionChains(driver)
119 #
120 # # 方式一: 机器人
121 # # 瞬间把源图片位置秒移到目标图片位置
122 # # actions.drag_and_drop(draggable, droppable) # 编写一个行为
123 # # actions.perform() # 执行编写好的行为
124 #
125 #
126 # # 方式二: 模拟人的行为
127 # source = draggable.location['x']
128 # target = droppable.location['x']
129 # print(source, target)
130 #
131 # distance = target - source
132 # print(distance)
133 #
134 # # perform：每个动作都要调用perform执行
135 #
136 # # 点击并摁住源图片
137 # ActionChains(driver).click_and_hold(draggable).perform()
138 #
139 # s = 0
140 # while s < distance:
141 # # 执行位移操作
142 # ActionChains(driver).move_by_offset(xoffset=2, yoffset=0).perform()
143 # s += 2
144 #
145 # # 释放动作链
146 # ActionChains(driver).release().perform()
147 #
148 # time.sleep(10)
149 #
150 #
151 # finally:
152 # driver.close()
153
154
155 '''
156 前进、后退
157 '''
158 # from selenium import webdriver
159 # import time
160 #
161 # driver = webdriver.Chrome()
162 #
163 # try:
164 # driver.implicitly_wait(10)
165 # driver.get('https://www.jd.com/')
166 # driver.get('https://www.baidu.com/')
167 # driver.get('https://www.cnblogs.com/')
168 #
169 # time.sleep(2)
170 #
171 # # 回退操作
172 # driver.back()
173 # time.sleep(1)
174 # # 前进操作
175 # driver.forward()
176 # time.sleep(1)
177 # driver.back()
178 # time.sleep(10)
179 #
180 # finally:
181 # driver.close()
2、selenium万能登陆破解
1 from selenium import webdriver
2 from selenium.webdriver import ChromeOptions
3 import time
4 r'''
5 步骤:
6 1、打开文件的查看，显示隐藏文件
7 2、找到C:\Users\administortra\AppData\Local\Google\Chrome\User Data
8 删除Default文件
9 3、重新打开浏览器，并登陆百度账号
10 - 此时会创建一个新的Default缓存文件
11 4、添加cookies
12 5、关闭谷歌浏览器后执行程序
13 '''
14 # 获取options对象，参数对象
15 options = ChromeOptions()
16
17 # 获取cookies保存路径
18 # 'C:\Users\administortra\AppData\Local\Google\Chrome\User Data'
19 profile_directory = r'--user-data-dir=C:\Users\administortra\AppData\Local\Google\Chrome\User Data'
20
21 # 添加用户信息目录
22 options.add_argument(profile_directory)
23
24 # 把参数加载到当前驱动中 chrome_options默认参数，用来接收options对象
25 driver = webdriver.Chrome(chrome_options=options)
26
27 try:
28 driver.implicitly_wait(10)
29 driver.get('https://www.baidu.com/')
30 '''
31 BDUSS:*****
32 '''
33 # 添加用户cookies信息
34 # name、value必须小写
35 driver.add_cookie({"name": "BDUSS", "value": "用户session字符串"})
36
37 # 刷新操作
38 driver.refresh()
39
40 time.sleep(10)
41
42 finally:
43 driver.close()
今日作业
1、总结课堂知识点，写博客
2、爬取京东商品信息
3、滑动验证（提高题）
爬取京东商品信息
1 from selenium import webdriver
2 from selenium.webdriver.support.ui import WebDriverWait
3 from selenium.webdriver.support import expected_conditions as EC
4 from selenium.webdriver.common.by import By
5 from selenium.webdriver.common.keys import Keys
6 import time
7 import json
8 import csv
9 import random
10
11
12 # 声明一个谷歌驱动器，并设置不加载图片，间接加快访问速度
13 options = webdriver.ChromeOptions()
14 options.add_experimental_option('prefs', {'profile.managed_default_content_settings.images': 2})
15 browser = webdriver.Chrome(options=options)
16 # url
17 url = 'https://www.jd.com/'
18 # 声明一个list，存储dict
19 data_list = []
20
21
22 def start_spider():
23 # 请求url
24 browser.get(url)
25 # 获取输入框的id，并输入关键字python爬虫
26 browser.find_element_by_id('key').send_keys('python爬虫')
27 # 输入回车进行搜索
28 browser.find_element_by_id('key').send_keys(Keys.ENTER)
29 # 显示等待下一页的元素加载完成
30 WebDriverWait(browser, 1000).until(
31 EC.presence_of_all_elements_located(
32 (By.CLASS_NAME, 'pn-next')
33 )
34 )
35 # 先获取一个有多少页
36 all_page = eval(browser.find_element_by_css_selector('span.p-skip em b').text)
37 # 设置一个计数器
38 count = 0
39 # 无限循环
40 while True:
41 try:
42 count += 1
43 # 显示等待商品信息加载完成
44 WebDriverWait(browser, 1000).until(
45 EC.presence_of_all_elements_located(
46 (By.CLASS_NAME, 'gl-item')
47 )
48 )
49 # 将滚动条拉到最下面的位置，因为往下拉才能将这一页的商品信息全部加载出来
50 browser.execute_script('document.documentElement.scrollTop=10000')
51 # 随机延迟,等下元素全部刷新
52 time.sleep(random.randint(1, 3))
53 browser.execute_script('document.documentElement.scrollTop=0')
54
55 # 开始提取信息,找到ul标签下的全部li标签
56 lis = browser.find_elements_by_class_name('gl-item')
57 # 遍历
58 for li in lis:
59 # 名字
60 name = li.find_element_by_xpath('.//div[@class="p-name p-name-type-2"]//em').text
61 # 价格
62 price = li.find_element_by_xpath('.//div[@class="p-price"]//i').text
63 # 评论数
64 comment = li.find_elements_by_xpath('.//div[@class="p-commit"]//a')
65 if comment:
66 comment = comment[0].text
67 else:
68 comment = None
69 # 商铺名字
70 shop_name = li.find_elements_by_class_name('J_im_icon')
71 if shop_name:
72 shop_name = shop_name[0].text
73 else:
74 shop_name = None
75 # 商家类型
76 shop_type = li.find_elements_by_class_name('goods-icons')
77 if shop_type:
78 shop_type = shop_type[0].text
79 else:
80 shop_type = None
81
82 # 声明一个字典存储数据
83 data_dict = {}
84 data_dict['name'] = name
85 data_dict['price'] = price
86 data_dict['comment'] = comment
87 data_dict['shop_name'] = shop_name
88 data_dict['shop_type'] = shop_type
89
90 data_list.append(data_dict)
91 print(data_dict)
92 except Exception as e:
93 continue
94
95 # 如果count==all_page就退出循环
96 if count == all_page:
97 break
98 # 找到下一页的元素pn-next
99 fp_next = browser.find_element_by_css_selector('a.fp-next')
100 # 点击下一页
101 fp_next.click()
102
103
104 def main():
105
106 start_spider()
107 # 将数据写入jsonwenj
108 with open('data_json.json', 'a+', encoding='utf-8') as f:
109 json.dump(data_list, f, ensure_ascii=False, indent=4)
110 print('json文件写入完成')
111
112 with open('data_csv.csv', 'w', encoding='utf-8', newline='') as f:
113 # 表头
114 title = data_list[0].keys()
115 # 声明writer
116 writer = csv.DictWriter(f, title)
117 # 写入表头
118 writer.writeheader()
119 # 批量写入数据
120 writer.writerows(data_list)
121 print('csv文件写入完成')
122
123
124 if __name__ == '__main__':
125
126 main()
127 # 退出浏览器
128 browser.quit()
滑动验证
1 from selenium import webdriver
2 from selenium.webdriver.support.ui import WebDriverWait # 等待元素加载的
3 from selenium.webdriver.common.action_chains import ActionChains #拖拽
4 from selenium.webdriver.support import expected_conditions as EC
5 from selenium.common.exceptions import TimeoutException, NoSuchElementException
6 from selenium.webdriver.common.by import By
7 from PIL import Image
8 import requests
9 import time
10 import re
11 import random
12 from io import BytesIO
13
14
15 def merge_image(image_file,location_list):
16 """
17 拼接图片
18 :param image_file:
19 :param location_list:
20 :return:
21 """
22 im = Image.open(image_file)
23 im.save('code.jpg')
24 new_im = Image.new('RGB',(260,116))
25 # 把无序的图片 切成52张小图片
26 im_list_upper = []
27 im_list_down = []
28 # print(location_list)
29 for location in location_list:
30 # print(location['y'])
31 if location['y'] == -58: # 上半边
32 im_list_upper.append(im.crop((abs(location['x']),58,abs(location['x'])+10,116)))
33 if location['y'] == 0: # 下半边
34 im_list_down.append(im.crop((abs(location['x']),0,abs(location['x'])+10,58)))
35
36 x_offset = 0
37 for im in im_list_upper:
38 new_im.paste(im,(x_offset,0)) # 把小图片放到 新的空白图片上
39 x_offset += im.size[0]
40
41 x_offset = 0
42 for im in im_list_down:
43 new_im.paste(im,(x_offset,58))
44 x_offset += im.size[0]
45 new_im.show()
46 return new_im
47
48 def get_image(driver,div_path):
49 '''
50 下载无序的图片 然后进行拼接 获得完整的图片
51 :param driver:
52 :param div_path:
53 :return:
54 '''
55 time.sleep(2)
56 background_images = driver.find_elements_by_xpath(div_path)
57 location_list = []
58 for background_image in background_images:
59 location = {}
60 result = re.findall('background-image: url\("(.*?)"\); background-position: (.*?)px (.*?)px;',background_image.get_attribute('style'))
61 # print(result)
62 location['x'] = int(result[0][1])
63 location['y'] = int(result[0][2])
64
65 image_url = result[0][0]
66 location_list.append(location)
67
68 print('==================================')
69 image_url = image_url.replace('webp','jpg')
70 # '替换url http://static.geetest.com/pictures/gt/579066de6/579066de6.webp'
71 image_result = requests.get(image_url).content
72 # with open('1.jpg','wb') as f:
73 # f.write(image_result)
74 image_file = BytesIO(image_result) # 是一张无序的图片
75 image = merge_image(image_file,location_list)
76
77 return image
78
79 def get_track(distance):
80 '''
81 拿到移动轨迹，模仿人的滑动行为，先匀加速后匀减速
82 匀变速运动基本公式：
83 ①v=v0+at
84 ②s=v0t+(1/2)at²
85 ③v²-v0²=2as
86
87 :param distance: 需要移动的距离
88 :return: 存放每0.2秒移动的距离
89 '''
90 # 初速度
91 v=0
92 # 单位时间为0.2s来统计轨迹，轨迹即0.2内的位移
93 t=0.2
94 # 位移/轨迹列表，列表内的一个元素代表0.2s的位移
95 tracks=[]
96 # 当前的位移
97 current=0
98 # 到达mid值开始减速
99 mid=distance * 7/8
100
101 distance += 10 # 先滑过一点，最后再反着滑动回来
102 # a = random.randint(1,3)
103 while current < distance:
104 if current < mid:
105 # 加速度越小，单位时间的位移越小,模拟的轨迹就越多越详细
106 a = random.randint(2,4) # 加速运动
107 else:
108 a = -random.randint(3,5) # 减速运动
109
110 # 初速度
111 v0 = v
112 # 0.2秒时间内的位移
113 s = v0*t+0.5*a*(t**2)
114 # 当前的位置
115 current += s
116 # 添加到轨迹列表
117 tracks.append(round(s))
118
119 # 速度已经达到v,该速度作为下次的初速度
120 v= v0+a*t
121
122 # 反着滑动到大概准确位置
123 for i in range(4):
124 tracks.append(-random.randint(2,3))
125 for i in range(4):
126 tracks.append(-random.randint(1,3))
127 return tracks
128
129
130 def get_distance(image1,image2):
131 '''
132 拿到滑动验证码需要移动的距离
133 :param image1:没有缺口的图片对象
134 :param image2:带缺口的图片对象
135 :return:需要移动的距离
136 '''
137 # print('size', image1.size)
138
139 threshold = 50
140 for i in range(0,image1.size[0]): # 260
141 for j in range(0,image1.size[1]): # 160
142 pixel1 = image1.getpixel((i,j))
143 pixel2 = image2.getpixel((i,j))
144 res_R = abs(pixel1[0]-pixel2[0]) # 计算RGB差
145 res_G = abs(pixel1[1] - pixel2[1]) # 计算RGB差
146 res_B = abs(pixel1[2] - pixel2[2]) # 计算RGB差
147 if res_R > threshold and res_G > threshold and res_B > threshold:
148 return i # 需要移动的距离
149
150
151
152 def main_check_code(driver, element):
153 """
154 拖动识别验证码
155 :param driver:
156 :param element:
157 :return:
158 """
159 image1 = get_image(driver, '//div[@class="gt_cut_bg gt_show"]/div')
160 image2 = get_image(driver, '//div[@class="gt_cut_fullbg gt_show"]/div')
161 # 图片上 缺口的位置的x坐标
162
163 # 2 对比两张图片的所有RBG像素点，得到不一样像素点的x值，即要移动的距离
164 l = get_distance(image1, image2)
165 print('l=',l)
166 # 3 获得移动轨迹
167 track_list = get_track(l)
168 print('第一步,点击滑动按钮')
169 ActionChains(driver).click_and_hold(on_element=element).perform() # 点击鼠标左键，按住不放
170 time.sleep(1)
171 print('第二步,拖动元素')
172 for track in track_list:
173 ActionChains(driver).move_by_offset(xoffset=track, yoffset=0).perform() # 鼠标移动到距离当前位置（x,y） time.sleep(0.002)
174 # if l>100:
175
176 ActionChains(driver).move_by_offset(xoffset=-random.randint(2,5), yoffset=0).perform()
177 time.sleep(1)
178 print('第三步,释放鼠标')
179 ActionChains(driver).release(on_element=element).perform()
180 time.sleep(5)
181
182
183 def main_check_slider(driver):
184 """
185 检查滑动按钮是否加载
186 :param driver:
187 :return:
188 """
189 while True:
190 try :
191 driver.get('http://www.cnbaowen.net/api/geetest/')
192 element = WebDriverWait(driver, 30, 0.5).until(EC.element_to_be_clickable((By.CLASS_NAME, 'gt_slider_knob')))
193 if element:
194 return element
195 except TimeoutException as e:
196 print('超时错误，继续')
197 time.sleep(5)
198
199
200 if __name__ == '__main__':
201 try:
202 count = 6 # 最多识别6次
203 driver = webdriver.Chrome()
204 # 等待滑动按钮加载完成
205 element = main_check_slider(driver)
206 while count > 0:
207 main_check_code(driver,element)
208 time.sleep(2)
209 try:
210 success_element = (By.CSS_SELECTOR, '.gt_holder .gt_ajax_tip.gt_success')
211 # 得到成功标志
212 print('suc=',driver.find_element_by_css_selector('.gt_holder .gt_ajax_tip.gt_success'))
213 success_images = WebDriverWait(driver, 20).until(EC.presence_of_element_located(success_element))
214 if success_images:
215 print('成功识别！！！！！！')
216 count = 0
217 break
218 except NoSuchElementException as e:
219 print('识别错误，继续')
220 count -= 1
221 time.sleep(2)
222 else:
223 print('too many attempt check code ')
224 exit('退出程序')
225 finally:
226 driver.close()
来源：https://www.cnblogs.com/jrc123/p/11048187.html