用Python爬了下知乎的“沙雕”问题,我们得到了一个15强排行榜
import re
import selenium
from selenium import webdriver
import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
driver = webdriver.Chrome()
driver.maximize_window()
url = 'https://www.zhihu.com/question/37453271'
js='window.open("'+url+'")'
driver.execute_script(js)
driver.close()
driver.switch_to_window(driver.window_handles[0])
for i in range(100):
js="var q=document.documentElement.scrollTop=10000000"
driver.execute_script(js)
all_html = [k.get_property('innerHTML') for k in driver.find_elements_by_class_name('AnswerItem')]
all_text = ''.join(all_html)
#all_text = all_text.replace('u002F','/')
all_text = all_text.replace('questions','question')
pat = 'question/d+'
questions = list(set([k for k in re.findall(pat,all_text)]))
header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win32; x32; rv:54.0) Gecko/20100101 Firefox/54.0',
'Connection': 'keep-alive'}
cookies ='v=3; iuuid=1A6E888B4A4B29B16FBA1299108DBE9CDCB327A9713C232B36E4DB4FF222CF03; webp=true; ci=1%2C%E5%8C%97%E4%BA%AC; __guid=26581345.3954606544145667000.1530879049181.8303; _lxsdk_cuid=1646f808301c8-0a4e19f5421593-5d4e211f-100200-1646f808302c8; _lxsdk=1A6E888B4A4B29B16FBA1299108DBE9CDCB327A9713C232B36E4DB4FF222CF03; monitor_count=1; _lxsdk_s=16472ee89ec-de2-f91-ed0%7C%7C5; __mta=189118996.1530879050545.1530936763555.1530937843742.18'
cookie = {}
for line in cookies.split(';'):
name, value = cookies.strip().split('=', 1)
cookie[name] = value
questions_df = pd.DataFrame(columns = ['title','visit','follower','answer','is_open'])
for i in range(len(questions)):
try:
url = 'https://www.zhihu.com/'+questions[i]
html = requests.get(url,cookies=cookie, headers=header).content
bsObj = BeautifulSoup(html.decode('utf-8'),"html.parser")
text = str(bsObj)
title = bsObj.find('h1',attrs={'class':'QuestionHeader-title'}).text
visit = int(re.findall('"visitCount":d+',text)[0].replace('"visitCount":',''))
follower = int(re.findall('"followerCount":d+',text)[0].replace('"followerCount":',''))
answer = int(re.findall('"answerCount":d+',text)[0].replace('"answerCount":',''))
is_open = int(len(re.findall('问题已关闭',text))==0)
questions_df = questions_df.append({'title':title,'visit':visit,
'follower':follower,'answer':answer,
'is_open':is_open},ignore_index=True)
time.sleep(2)
print(i)
except:
print('错误'+str(i))
https://www.zhihu.com/question/276876453
https://www.zhihu.com/question/37160000
https://www.zhihu.com/question/36805199
https://www.zhihu.com/question/52083864
https://www.zhihu.com/question/24393996
https://www.zhihu.com/question/21370688/
https://www.zhihu.com/question/20831390/
实习/全职编辑记者招聘ing
加入我们,亲身体验一家专业科技媒体采写的每个细节,在最有前景的行业,和一群遍布全球最优秀的人一起成长。坐标北京·清华东门,在大数据文摘主页对话页回复“招聘”了解详情。简历请直接发送至zz@bigdatadigest.cn
关注公众号:拾黑(shiheibook)了解更多
[广告]赞助链接:
四季很好,只要有你,文娱排行榜:https://www.yaopaiming.com/
让资讯触达的更精准有趣:https://www.0xu.cn/
关注网络尖刀微信公众号
随时掌握互联网精彩
随时掌握互联网精彩
赞助链接
排名
热点
搜索指数
- 1 习近平主席拉美之行高光时刻 7944281
- 2 北方这些地方有暴雪 7952552
- 3 菲律宾副总统称已安排杀手 7881074
- 4 特色文旅释放消费“新”热潮 7700644
- 5 2.8亿!南京富家千金买下父亲公司股份 7656321
- 6 专家:“卷学历”性价比急剧下降 7542296
- 7 冰雪大世界糖葫芦摊位100万起拍 7481152
- 8 长沙公交站台奇怪座椅引吐槽 7377206
- 9 黄奕 只要不是外公姐就行 7285548
- 10 上海网红小区疑有人高空抛粪 7172489