# 抓取票房数据
def crawl(path):
    duration = (datetime.datetime.now()-datetime.datetime.strptime('20200720', '%Y%m%d')).days
    days = '20200720'
    df =pd.DataFrame()
    for i in range(duration):
        try:
            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36',
                       'X-FOR-WITH':'Rz7i6ufGHjmGodGgu8BbUrav6q0DVrkeC5I7iR1aieo0JKLMIWrKB23CUCa30sB5g2mut8nHm5zagQwr8n9WzUV1B2MdplRa38rPEdN5FR5Ei9Op0hWjBndw7WgEr+4N2K5oMAp85ve1WArjpDa2XhOtge64qDRoEy53qsLl13UrBMVcg7Z2fQU48wHxEjkHsSGWySnSWHBs/flWxyj3OX+OSXf2fzg8fIVPTLI/k+w='}
            url = 'http://piaofang.maoyan.com/dashboard-ajax/movie?showDate={}'.format(days)
            r = requests.get(url, headers=headers)
            films_message = r.json()['movieList']['list']
            # 结果存入dataframe
            df2 = pd.DataFrame(films_message)
            # 插入时间列
            df2.insert(0, 'date', days)
            # 插入每日排名
            df2.insert(1, 'rank', df2.index)
            # 合并多个dataframe
            df = pd.concat([df, df2], axis=0, ignore_index=True)
            # 日期+1天
            days= (datetime.datetime.strptime(days, '%Y%m%d')+ datetime.timedelta(days=1)).strftime('%Y%m%d')
            print(df.shape,days)
        except:
            print(days)
            days= (datetime.datetime.strptime(days, '%Y%m%d')+ datetime.timedelta(days=1)).strftime('%Y%m%d')
            pass
    # 保存
    writer = pd.ExcelWriter(path)
    df.to_excel(writer,index=False, encoding="utf_8_sig")
    writer.save()

# 获取豆瓣电影ID
def crawl_ID():
    data = pd.read_excel('票房数据.xlsx',sheet_name='上映')
    movies = list(data['电影名称'].unique())
    for name in movies:
        url = 'https://movie.douban.com/subject_search?search_text={}'.format(name)
        # 用chrome浏览器打开
        driver = webdriver.Chrome()
        # 打开网址
        driver.get(url)
        time.sleep(2)
        html = driver.page_source
        soup1 = BeautifulSoup(html, 'lxml')
        if len(soup1.select('a.title-text')) > 0:
            targetUrl = soup1.select('a.title-text')[0].get('href')
            print(targetUrl)
            redis_db.sadd('douban_movieId', targetUrl)
        else:
            print("无结果")
        driver.close()
# 构建豆瓣影评链接
def get_url():
    douban_movieId_list = redis_db.smembers('douban_movieId')
    for douban_movieId in douban_movieId_list:
        start = 20
        for i in range(25):
            # 构建URL
            page = start * i
            url = douban_movieId + 'comments?start={}&limit=20&sort=new_score&status=P'.format(page)
            print(url)
            redis_db.sadd('douban_movieURL', url)