Beautifulsoup - scrape webpage - dynamically loading page

问题

i want to scrape a webpage : https://www.justdial.com/Mumbai/Dairy-Product-Retailers-in-Thane/nct-10152687

i need a data of all store name, tel- num and their address

But i can only do it upto 10 cause to load other items you need to scroll the webpage

my code :

import requests
import bs4

crawl_url = requests.get('https://www.justdial.com/Mumbai/Dairy-Product-
Retailers-in-Thane/nct-10152687', headers={'User-Agent': 'Mozilla/5.0'})
crawl_url.raise_for_status()


soup = bs4.BeautifulSoup(crawl_url.text, 'lxml')

for elems in soup.find_all('span', class_="jcn"):
    select_a = elems.select('a')
    for links in select_a:
        href = links.get('href')
        res = requests.get(href, headers={'User-Agent': 'Mozilla/5.0'})

        xsoup = bs4.BeautifulSoup(res.text, 'lxml')

        Name = xsoup.select('.fn')
        tel = xsoup.select('.tel')
        add = xsoup.select('.adrstxtr')
        a = Name[0]
        b = tel[0]
        c = add[0]
        print(a.getText())
        print("--"*10)
        print(b.getText())
        print("--"*10)
        print(c.getText())
        print("=="*25)

When We Scroll Down the Page other Items Load up So i Want to Know How can get any numbers of Data/Items I want

I tried this

But didn't quiet understood Well, and also i did not got that POST method :/

If need more Info Tell me

回答1:

Solution given by t.m.adam worked Here is Code

import requests
import bs4

def spider(max_pages):
    page = 1
    while page < max_pages:
        url = "https://www.justdial.com/Mumbai/Dairy-Product-Retailers-in-
Thane/nct-10152687/page-%s" % page
        crawl_url = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'})
        crawl_url.raise_for_status()
        soup = bs4.BeautifulSoup(crawl_url.text, 'lxml')
        for elems in soup.find_all('span', class_="jcn"):
            select_a = elems.select('a')
            for links in select_a:
               href = links.get('href')
               res = requests.get(href, headers={'User-Agent': 
'Mozilla/5.0'})
                xsoup = bs4.BeautifulSoup(res.text, 'lxml')
                Name = xsoup.select('.fn')
                tel = xsoup.select('.tel')
                add = xsoup.select('.adrstxtr')
                a = Name[0]
                b = tel[0]
                c = add[0]
                print(a.getText())
                print("--"*10)
                print(b.getText())
                print("--"*10)
                print(c.getText())
                print("=="*25)
        page += 1


spider(3)

来源：https://stackoverflow.com/questions/45616908/beautifulsoup-scrape-webpage-dynamically-loading-page