Creating a page count down in Python Selenium

问题

I have a simple script that navigates to a series of pages.

My question is, how do I create a page counter so I know where the job is at for this?

Ideally it would read:

Page 5
Page 4 #For each loop

The following tends to work, though I am having issues with the following:

elements = [x.get_text() for x in
        wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH,'(//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()]')))]

Related error:

 line 36, in <module>
        wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH,'(//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()]')))]
      File "C:\Users\Django\AppData\Local\Continuum\miniconda3\lib\site-packages\selenium\webdriver\support\wait.py", line 80, in until
        raise TimeoutException(message, screen, stacktrace)
    selenium.common.exceptions.TimeoutException: Message: 

Full code:

import collections
from random import shuffle

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait as wait

driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()


driver.get('https://www.bet365.com.au/#/AS/B1/')
driver.get('https://www.bet365.com.au/#/AS/B1/')



def page_counter():
    for x in range(1000):
        yield x


clickMe = wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH,'(//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()])')))
options = driver.find_elements_by_xpath('//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()]')

indexes = [index for index in range(len(options))]
shuffle(indexes)
for index in indexes:


    count = page_counter()
    driver.get('https://www.bet365.com.au/#/AS/B1/')
    elements = [x.get_text("*") for x in
            wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH,'(//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()]')))]


       #elements = [x.get_attribute("href") for x in
       # driver.find_elements_by_xpath('//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()]')))]

    clickMe.click()
    shuffle(elements)

    links = dict((next(count) + 1, e) for e in elements)

desc_links = collections.OrderedDict(sorted(links.items(), reverse=True))
for key, value in desc_links.items():
    try:
        driver.get(value)
        print('Page ' + str(key))
    except TimeoutException as ex:
        pass

Alternative:

import collections
import time

from selenium import webdriver

driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()

driver.get('https://www.bet365.com.au/#/AS/B1/')

driver.find_element_by_id('TopPromotionBetNow').click()


def page_counter():
    for x in range(1000):
        yield x


count = page_counter()
time.sleep(10)
classifications = driver.find_elements_by_class_name('wn-Classification')
for classification in classifications:
    if classification.text == 'Soccer':
        classification.click()
        break

time.sleep(10)
markets = driver.find_elements_by_class_name('sm-Market')
for market in markets:
    group_name = market.find_element_by_class_name('sm-Market_GroupName')
    if group_name.text == 'Main Lists':
        coupon_lables = [x for x in market.find_elements_by_class_name('sm-CouponLink_Label')]
    break

coupon_lables = [x.text for x in market.find_elements_by_class_name('sm-CouponLink_Label')]

links = dict((next(count) + 1, e) for e in coupon_lables)

desc_links = collections.OrderedDict(sorted(links.items(), reverse=True))
for key, value in desc_links.items():

    for label in coupon_lables:
        print('Page ' + str(key))
        print('executing:' + label)
        time.sleep(5)
        driver.find_element_by_xpath(f'//div[contains(text(), "' + label + '")]').click()
        time.sleep(5)
        driver.find_element_by_class_name('cl-BreadcrumbTrail_BackButton').click()

My output:

Page 12
executing:UK Saturday's Matches
Page 12
executing:Spanish Saturday's Matches
Page 12
executing:Italian Saturday's Matches
Page 12
executing:International Saturday's Matches
Page 12
executing:Europe Saturday's Matches

Desired:

Page 12
Page 11
Page 10

Any idea how I can fix this

回答1:

The code should look like this:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import sys, time
driver = webdriver.Chrome()
driver.get('https://www.bet365.com.au/#/AS/B1/')

try:
    WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.ID, 'TopPromotionInfo'))).click()
except:
    print("Unexpected error:", sys.exc_info()[0])


try:
    elements = WebDriverWait(driver, 25).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'wn-Classification')))
    for element in elements:
        if 'Soccer' in element.text:
            element.click()
            break
except:
    print("Unexpected error:", sys.exc_info()[0])

market = []

try:
    market = WebDriverWait(driver, 25).until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='sm-MarketGroup ']/div[@class='sm-MarketGroup_Open ']/div[@class='sm-Market ']")))
except:
    print("Unexpected error:", sys.exc_info()[0])

for m1 in market:
    if 'Main Lists' in m1.find_element_by_class_name('sm-Market_GroupName ').text:
        elements = m1.find_elements_by_class_name('sm-CouponLink ')
        labels = [x.text for x in elements]
        c=0
        for label in labels:
            c+=1
            print('Executing... ', c, len(labels))
            driver.find_element_by_xpath('//div[contains(text(),"' + label +'")]').click()
            time.sleep(5)
            WebDriverWait(driver, 25).until(EC.presence_of_element_located((By.CLASS_NAME, 'cl-BreadcrumbTrail_BackButton '))).click()
            time.sleep(5)
        break

回答2:

Your xpath is not valid, you need to remove the opening bracket from it

'(//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()])'
 ^ remove this

Should be

'//div[div/div/text()="Main Lists"]//div[starts-with(@class, "sm-CouponLink_Label") and normalize-space()])'

The interesting thing is why you didn't get InvalidSelectorException. Looking at expected_conditions source code you can see element_to_be_clickable is using visibility_of_element_located which in turn use _find_element (defined in the bottom of the source code).

_find_element catch WebDriverException, so the InvalidSelectorException, which could direct you to the problem, is caught and you received TimeoutException instead.

Edit

In addition, element_to_be_clickable returns single WebElement, its not possible to iterate over it. Instead you can use visibility_of_all_elements_located

elements = [x.get_text("*") for x in
    wait(driver, 10).until(EC.visibility_of_all_elements_located(...))

or locate the elements using driver.find_elements and test the elements one by one in loop

elements = driver.find_elements_by_xpath(...)
for element in elements:
    wait(driver, 10).until(EC.element_to_be_clickable(element))

来源：https://stackoverflow.com/questions/47854129/creating-a-page-count-down-in-python-selenium