本文以“Mate30”为关键字进行分析
import requests as re import time #搜索关键字Mate30并获取url target_url = "https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D1%26q%3DMate30&page_type=searchall&page=" # 爬取的微博总页数 total_page=400 #存放微博与用户数据 mblog = list() user = list() # 根据是否为长文本采集不同的微博文本数据 long_text_count = 0 for page in range(total_page): print('Crawling page:%d/%d'%(page+1,total_page)) cur_url=target_url + str(page+1) source_json = re.get(cur_url).json() time.sleep(1) source_data=source_json['data'] cards = source_data['cards'] if 'cards' in source_data else [] for card in cards: if 'mblog' in card: cur_blog = card['mblog'] cur_text = '' is_long_text = 0 if cur_blog['isLongText'] == True: cur_text = cur_blog['longText']['longTextContent'] is_long_text = 1 else: cur_text = cur_blog['text'] verified_reason = cur_blog['user']['verified_reason'] if 'verified_reason' in cur_blog['user'] else None # 微博数据记录 mblog.append({'mid':cur_blog['mid'] ,'raw_text':cur_text ,'isLongText':is_long_text ,'like_count':cur_blog['attitudes_count'] ,'comments_count':cur_blog['comments_count'] ,'reposts_count':cur_blog['reposts_count'] ,'created_at':cur_blog['created_at'] ,'source':cur_blog['source'] ,'user_id':cur_blog['user']['id'] }) # 用户数据记录 user.append({'user_id':cur_blog['user']['id'] ,'screen_name':cur_blog['user']['screen_name'] ,'description':cur_blog['user']['description'] ,'follow_count':cur_blog['user']['follow_count'] ,'followers_count':cur_blog['user']['followers_count'] ,'gender':cur_blog['user']['gender'] ,'verified':cur_blog['user']['verified'] ,'verified_reason':verified_reason ,'profile_url':cur_blog['user']['profile_url'] }) print('Crawl finished') print('共爬取数据条数：',len(mblog)) 将微博数据转换为pandas数据，便于分析与保存
import pandas as pd mblog_frame = pd.DataFrame(mblog) user_frame = pd.DataFrame(user) # 去除重复数据 mblog_frame.drop_duplicates(subset=['mid'],inplace=True) user_frame.drop_duplicates(subset=['user_id'],inplace=True) #数据保存到文件 mblog_frame.to_csv('mblog.csv',index=None,encoding='utf_8_sig') user_frame.to_csv('users.csv', encoding='utf_8_sig',index=False) 获取用户地理位置信息（由于在微博网站在用户信息部分没有地理信息，这个部分花费了很多时间，最终选择跳转到用户主页来爬取用户的地理信息）
def get_containerid(id): """ 获取用户的containerid """ containerid_url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value='+id containerid_json = re.get(containerid_url).json() containerid_data = containerid_json['data'] if 'data' in containerid_json else None if not containerid_data: return None tabsInfo = containerid_data['tabsInfo'] if 'tabsInfo'in containerid_data else None if not tabsInfo: return None tabs = tabsInfo['tabs'] if 'tabs' in tabsInfo else None if not tabs: return None containerid = tabs[0]['containerid'] if 'containerid'in tabs[0] else None return containerid def get_user_location(raw): profile_url = raw['profile_url'] left,right = profile_url.split('?') uid = left.split('/')[-1] pre = 'https://m.weibo.cn/api/container/getIndex?' mid = '&type=uid&value=' containerid = get_containerid(uid) if not containerid: return None final_url = pre+right+mid+uid+'&containerid='+containerid local_json = re.get(final_url).json() data = local_json['data'] if 'data' in local_json else None if not data: return None cards = data['cards'] if 'cards' in data else None if not cards: return None card_group = cards[0]['card_group'] if 'card_group' in cards[0] else None if not card_group: return None location = card_group[0]['item_content'] if 'item_content' in card_group[0] else None return location #批量处理获取用户所在区域并将结果更新保存到文件 user_frame['location'] = user_frame.apply(get_user_location,axis=1) user_frame.to_csv('users.csv',index=None,encoding='utf_8_sig') #若要保存到数据库，运行下面的代码 import pymysql from sqlalchemy import create_engine engine = create_engine('mysql+pymysql://root:password@localhost:3306/weibo_keyword?charset=utf8mb4',echo=False) mblog_frame.to_sql(name='ori_mblog',con=engine,if_exists='append',index=False) user_frame.to_sql(name='ori_users',con=engine,if_exists='append',index=False) 来源：51CTO
作者：Spratumn
链接：https://blog.csdn.net/Spratumn/article/details/100853846