web crawler案例篇（一）

#科学科普——知识——UP主（'历史累计弹幕数','评论','收藏人数','投硬币枚数','分享数','点赞数'）
import requests
from bs4 import  BeautifulSoup
import json
import urllib.parse
import pandas as pd
import re
headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'
}
all_content=pd.DataFrame()
for i in range(1,10):
    print('正在爬取第{}页'.format(i))
    url="https://s.search.bilibili.com/cate/search?callback=jqueryCallback_bili_&main_ver=v3&search_type=video&view_type=hot_rank&order=click&copy_right=-1&cate_id=201&pagesize=20&jsonp=jsonp&time_from=20201031&time_to=20201107&_="+str(i)
    req=requests.get(url,headers=headers,timeout=30)
    res=req.text
    res=res.replace("jqueryCallback_bili_","")
    res=res[1:(len(res)-1)]
    res=json.loads(res)
    result=res['result']
    for val in result:
        num=re.findall("\d+", val['arcurl'])
        url = r'https://api.bilibili.com/x/web-interface/archive/stat?aid='+ num[0]   
        response = requests.get(url , timeout=30,headers = headers)
        text = response.text
        jsonobj = json.loads(text)

        #从Json对象获取视频基本信息并转入词典中
        video_dict = {'author':val['author'],
                     'aid':jsonobj['data']['aid'],
                     'bvid':jsonobj['data']['bvid'],
                     'view':jsonobj['data']['view'],
                     'danmuku_num':jsonobj['data']['danmaku'],
                     'reply_num':jsonobj['data']['reply'],
                     'favorite_num':jsonobj['data']['favorite'],
                     'coin_num':jsonobj['data']['coin'],
                     'share_num':jsonobj['data']['share'],
                     'like_num':jsonobj['data']['like']
                     }
        #all_content=all_content.append(pd.DataFrame(video_dict,columns=['author','aid', 'bvid','view','danmuku_num','reply_num','favorite_num','coin_num','share_num','like_num'],index=[0])) #获取的内容存入数据框中
        all_content=all_content.append(pd.DataFrame(video_dict,columns=['author','aid', 'bvid','总播放数','历史累计弹幕数','评论','收藏人数','投硬币枚数','分享数','点赞数'],index=[0])) #获取的内容存入数据框中
#all_content
all_content.to_csv('E:\\content.csv',index=False,encoding='GBK')