123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107 |
- # Author renyupeng
- """
- coding=utf-8
- @Time : 2023/2/8 10:27 上午
- @Site :
- @File : PromoterLiveInfoSpider.py
- @Software: PyCharm
- @contact: renyupeng@c-top.com.cn
- @Tel 1501435553
- """
- import json
- import requests
- from utils.mysql_helper import insert, batch_insert
- from utils.mysql_utils import MysqlUtils
- from utils.send_feishu_msg import SendFeiShuMsg
- class PromoterLiveInfoSpider:
- def __init__(self):
- self.conn = MysqlUtils()
- self.list = [1, 2, 3]
- def PromoterLiveInfoSpiderHander(self, promoterId, cookie):
- headers = {'User-Agent': 'Mozilla/5.0',
- 'Cookie': cookie}
- try:
- for i in self.list:
- url = "https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/live/info?" \
- "timeRangeType={timeRangeType}&promoterId={promoterId}&type=1".format(timeRangeType=i,
- promoterId=promoterId)
- rep = requests.get(url=url, headers=headers)
- table_name = 'kwai_promoter_live_info'
- data = json.loads(rep.text)["data"]
- if data["likesCountInfo"] is None:
- SendFeiShuMsg.send_robot_msg("达人{promoterId}直播数据 is None".format(promoterId=promoterId))
- else:
- promoter_info_item = {"commentsCountInfo": json.dumps(data["commentsCountInfo"]),
- "likesCountInfo": json.dumps(data["likesCountInfo"]),
- "liveMinuteDurationInfo": json.dumps(data["liveMinuteDurationInfo"]),
- "liveVisitorCountInfo": json.dumps(data["liveVisitorCountInfo"]),
- "maxVisitorCountInfo": json.dumps(data["maxVisitorCountInfo"]),
- "sharesCountInfo": json.dumps(data["sharesCountInfo"]),
- "promoterId": data["promoterId"],
- "staticDataInfoView": json.dumps(data["staticDataInfoView"]),
- "timeRangeType": i}
- insert(table_name=table_name, item=promoter_info_item)
- base_table_name = 'kwai_promoter_live_base_info'
- promoter_base_info_item = {
- "promoterId": data["promoterId"],
- "timeRangeType": i,
- "avgLiveStreamDuration": data["staticDataInfoView"]["avgLiveStreamDuration"],
- "avgLiveVisitorCount": data["staticDataInfoView"]["avgLiveVisitorCount"],
- "avgMinuteDuration": data["staticDataInfoView"]["avgMinuteDuration"],
- "avgPromoteSaleVolume": data["staticDataInfoView"]["avgPromoteSaleVolume"],
- "avgSameTimeVisitorCount": data["staticDataInfoView"]["avgSameTimeVisitorCount"],
- "avgViewTime": data["staticDataInfoView"]["avgViewTime"],
- "avgVisitorOnlineCount": data["staticDataInfoView"]["avgVisitorOnlineCount"],
- "interactionRate": data["staticDataInfoView"]["interactionRate"],
- "liveStreamVisitorCount": data["staticDataInfoView"]["liveStreamVisitorCount"],
- "maxLiveVisitorCount": data["staticDataInfoView"]["maxLiveVisitorCount"],
- "maxMinuteDuration": data["staticDataInfoView"]["maxMinuteDuration"],
- "maxPromoteSaleVolume": data["staticDataInfoView"]["maxPromoteSaleVolume"],
- "maxSameTimeVisitorCount": data["staticDataInfoView"]["maxSameTimeVisitorCount"],
- "minLiveVisitorCount": data["staticDataInfoView"]["minLiveVisitorCount"],
- "minMinuteDuration": data["staticDataInfoView"]["minMinuteDuration"],
- "minPromoteSaleVolume": data["staticDataInfoView"]["minPromoteSaleVolume"],
- "promoteItemCount": data["staticDataInfoView"]["promoteItemCount"],
- "promoteLiveCount": data["staticDataInfoView"]["promoteLiveCount"],
- "promoteLiveDays": data["staticDataInfoView"]["promoteLiveDays"],
- "promoteSaleVolume": data["staticDataInfoView"]["promoteSaleVolume"],
- }
- insert(table_name=base_table_name, item=promoter_base_info_item)
- if i == 3:
- live_analyse_table_name = 'kwai_promoter_live_analyse_info'
- commentsCountInfo = tuple(data["commentsCountInfo"])
- likesCountInfo = tuple(data["likesCountInfo"])
- liveMinuteDurationInfo = tuple(data["liveMinuteDurationInfo"])
- liveVisitorCountInfo = tuple(data["liveVisitorCountInfo"])
- maxVisitorCountInfo = tuple(data["maxVisitorCountInfo"])
- sharesCountInfo = tuple(data["sharesCountInfo"])
- live_analyse_item_list = []
- for i in range(len(commentsCountInfo)):
- live_analyse_item = {}
- commentsCount = commentsCountInfo[i]
- likesCount = likesCountInfo[i]
- liveMinuteDuration = liveMinuteDurationInfo[i]
- liveVisitorCount = liveVisitorCountInfo[i]
- maxVisitorCount = maxVisitorCountInfo[i]
- sharesCount = sharesCountInfo[i]
- live_analyse_item["date"] = commentsCount["date"]
- live_analyse_item["promoterId"] = commentsCount["promoterId"]
- live_analyse_item["commentsCount"] = commentsCount["commentsCount"]
- live_analyse_item["likesCount"] = likesCount["likesCount"]
- live_analyse_item["liveMinuteDuration"] = liveMinuteDuration["minuteDuration"]
- live_analyse_item["liveVisitorCount"] = liveVisitorCount["visitorCount"]
- live_analyse_item["maxVisitorCount"] = maxVisitorCount["maxVisitorCount"]
- live_analyse_item["sharesCount"] = sharesCount["sharesCount"]
- live_analyse_item_list.append(live_analyse_item)
- batch_insert(table_name=live_analyse_table_name, item_list=live_analyse_item_list)
- except Exception as e:
- SendFeiShuMsg.send_robot_msg(
- 'PromoterLiveInfoSpider {promoterId}请求错误请检查cookie{e}'.format(promoterId=promoterId, e=e))
|