How to automate login credential with selenium chrome web driver

问题

I am trying to extract data from the [this][1] website:

The manual procedure is to enter String such as 'CCOCCO' in the search box , click on "Predict Properties" and record the 'Glass Transition Temperature (K)' from the table.

The following code will automate the above task if the number of html POST is less than 5:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options 
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

options=Options()
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument('--disable-extensions')
driver=webdriver.Chrome(chrome_options=options)

def get_glass_temperature(smiles):
    driver.get('https://www.polymergenome.org/explore/index.php?m=1')
    x_path_click="//input[@class='large_input_no_round ui-autocomplete-input' and @id='keyword_original']"
    x_path_find="//input[@class='dark_blue_button_no_round' and @value='Predict Properties']"
    x_path_get="//table[@class='record']//tbody/tr[@class='record']//following::td[7]/center/font/font"
    WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_click))).send_keys(smiles)
    driver.find_element_by_xpath(x_path_find).click()
    return WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH,x_path_get))).get_attribute("innerHTML")

I am applying the above function on a pandas dataframe that has up tp 400 value of String similar to 'CCOCCO'. However, after returning 5 "Glass Temperature" there will be WebdriverException error as the website throws the following message:

"Visits of more than 5 times per day to the property prediction capability requires login. "

Before running the code, I login to the website and check the "remember me" box but the error is the same.

I have tried to modify the code as below:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options 
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd 
import os 

options=Options()
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument('--disable-extensions')
driver=webdriver.Chrome(chrome_options=options, executable_path='/Users/ae/Downloads/chromedriver')

def get_glass_temperature(smiles):
    driver.get('https://www.polymergenome.org/explore/index.php?m=1')
    user_name='my_user_name'
    password='my_password'
    x_path_id="//input[@class='large_input_no_round' and @placeholder='User ID']"
    x_path_pass="//input[@class='large_input_no_round' and @placeholder='Password']"
    x_path_sign="//input[@class='orange_button_no_round' and @value='Sign In']"
    WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_id))).send_keys(user_name)
    WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_pass))).send_keys(password)
    driver.find_element_by_xpath(x_path_sign).click()

    x_path_click="//input[@class='large_input_no_round ui-autocomplete-input' and @id='keyword_original']"
    x_path_find="//input[@class='dark_blue_button_no_round' and @value='Predict Properties']"
    x_path_get="//table[@class='record']//tbody/tr[@class='record']//following::td[7]/center/font/font"
    WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_click))).send_keys(smiles)
    driver.find_element_by_xpath(x_path_find).click()
    return WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH,x_path_get))).get_attribute("innerHTML")

test_smiles=['CC(F)(F)CC(F)(F)','CCCCC(=O)OCCOC(=O)','CNS-C6H3-CSN-C6H3','CCOCCO','NH-CS-NH-C6H4','C4H8','C([*])C([*])(COOc1cc(Cl)ccc1)']
test_polymer=pd.DataFrame({'SMILES': test_smiles})
test_polymer['test_tg']=test_polymer['SMILES'].apply(get_glass_temperature)
print (test_polymer)

After this modification, I am getting TimeOut Error:

Traceback (most recent call last):
  File "/Users/alieftekhari/Desktop/extract_TG.py", line 42, in <module>
    test_polymer['test_tg']=test_polymer['SMILES'].apply(get_glass_temperature)
  File "/anaconda/lib/python2.7/site-packages/pandas/core/series.py", line 3194, in apply
    mapped = lib.map_infer(values, f, convert=convert_dtype)
  File "pandas/_libs/src/inference.pyx", line 1472, in pandas._libs.lib.map_infer
  File "/Users/user/Desktop/extract_TG.py", line 22, in get_glass_temperature
    WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_id))).send_keys(user_name)
  File "/anaconda/lib/python2.7/site-packages/selenium/webdriver/support/wait.py", line 80, in until
    raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
  [1]: https://www.polymergenome.org/explore/index.php?m=1

回答1:

See last line of stacktrace File "/anaconda/lib/python2.7/site-packages/selenium/webdriver/support/wait.py", line 80, in until raise TimeoutException(message, screen, stacktrace) selenium.common.exceptions.TimeoutException: Message:

It clearly mentioned there is no such element, that's why it is giving TimeoutException. What I see here, your xpath are wrong..

x_path_id="//input[@class='large_input_no_round ui-autocomplete-input' and @placeholder='User ID']"
x_path_pass="//input[@class='large_input_no_round ui-autocomplete-input' and @placeholder='Password']"

There is no class large_input_no_round ui-autocomplete-input, so modify xpath with correct class as below..

x_path_id="//input[@class='large_input_no_round' and @placeholder='User ID']"
x_path_pass="//input[@class='large_input_no_round' and @placeholder='Password']"

Problem

• driver.get('https://www.polymergenome.org/explore/index.php?m=1') this page doesn't have login window, so TimeoutException in line WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_id))).send_keys(user_name)

  In other words when you run script, it launches a fresh browser instance, means your previous login has been gone, now you need to login to pass this limit Visits of more than 5 times per day to the property prediction capability requires login.; and login window will populate after 5 successful extract iterations, the script failing here is because it is trying to login straight forward without waiting for login dialog, and as there is no login window, it gives TimeoutException.

Solution is you should put extract data part into try block and login into catch, this will execute login part only if there is exception in extracting data. My Java implementation would be like this,

@Test(invocationCount = 7)
    public void getList(){
        wait = new WebDriverWait(driver, 20);
        By locator = By.xpath("//table[@class='record']//tbody/tr[@class='record']//following::td[7]/center/font/font");
        try {
            driver.findElement(By.xpath("//input[@class='large_input_no_round ui-autocomplete-input' and @id='keyword_original']")).clear();
            driver.findElement(By.xpath("//input[@class='large_input_no_round ui-autocomplete-input' and @id='keyword_original']")).sendKeys("CCOCCO");
            driver.findElement(By.xpath("//input[@class='dark_blue_button_no_round' and @value='Predict Properties']")).click();
            String text = wait.until(ExpectedConditions.visibilityOfElementLocated(locator)).getAttribute("innerHTML");
            System.out.println(text);
        }catch(Exception e){
            System.out.println("In Exception Block");
            wait.until(ExpectedConditions.elementToBeClickable(By.xpath("//input[@class='large_input_no_round' and @placeholder='User ID']")));
            driver.findElement(By.xpath("//input[@class='large_input_no_round' and @placeholder='User ID']")).sendKeys("testing");
            driver.findElement(By.xpath("//input[@class='large_input_no_round' and @placeholder='Password']")).sendKeys("testing");
            driver.findElement(By.xpath("//input[@class='orange_button_no_round' and @value='Sign In']")).click();
        }
    }       

Other way around

• Best way is to browse site, navigate to sign in dialog, and do login, on successful login, browse search page and continue extracting.
• Or you can put a limit of 5 (means extract 5 times) before going for login.

回答2:

One way to login automatically every time you request the website is to use specific chrome profile on initialization. If you want to use your existing profile of your google chrome check out this post.

Therefore, you have to add one more option:

options=Options()
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument('--disable-extensions')
options.add_argument('user-data-dir=/path/to/chrome/profile')
driver=webdriver.Chrome(chrome_options=options, executable_path='/Users/ae/Downloads/chromedriver')

So, if you have logged in using this profile and you have checked "remember me", every time you request this website you will be logged in automatically.

回答3:

here is implementation in python:

def time_out_handling (smiles):
        try:
            user_name='my_user_name'
            password='my_password'
            x_path_id="//input[@class='large_input_no_round' and @placeholder='User ID']"
            x_path_pass="//input[@class='large_input_no_round' and @placeholder='Password']"
            x_path_sign="//input[@class='orange_button_no_round' and @value='Sign In']"
            WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_id))).send_keys(user_name)
            WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_pass))).send_keys(password)
            driver.find_element_by_xpath(x_path_sign).click()
            driver.get('https://www.polymergenome.org/explore/index.php?m=1')
            x_path_click="//input[@class='large_input_no_round ui-autocomplete-input' and @id='keyword_original']"
            x_path_find="//input[@class='dark_blue_button_no_round' and @value='Predict Properties']"
            x_path_get="//table[@class='record']//tbody/tr[@class='record']//following::td[7]/center/font/font"
            WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_click))).send_keys(smiles)
            driver.find_element_by_xpath(x_path_find).click()
            return WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH,x_path_get))).get_attribute("innerHTML").encode('ascii', 'ignore').split(' ')[0]
        except TimeoutException:
            return "Nan" 


def get_glass_temperature(smiles):
        try: 
            driver.get('https://www.polymergenome.org/explore/index.php?m=1')
            x_path_click="//input[@class='large_input_no_round ui-autocomplete-input' and @id='keyword_original']"
            x_path_find="//input[@class='dark_blue_button_no_round' and @value='Predict Properties']"
            x_path_get="//table[@class='record']//tbody/tr[@class='record']//following::td[7]/center/font/font"
            WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, x_path_click))).send_keys(smiles)
            driver.find_element_by_xpath(x_path_find).click()
            return WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH,x_path_get))).get_attribute("innerHTML").encode('ascii', 'ignore').split(' ')[0]
        except WebDriverException:
            time_out_handling (smiles)

来源：https://stackoverflow.com/questions/51887473/how-to-automate-login-credential-with-selenium-chrome-web-driver