| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107 | # Author renyupeng"""coding=utf-8@Time    : 2023/2/8 10:27 上午@Site    :@File    : PromoterLiveInfoSpider.py@Software: PyCharm@contact: renyupeng@c-top.com.cn@Tel 1501435553"""import jsonimport requestsfrom utils.mysql_helper import insert, batch_insertfrom utils.mysql_utils import MysqlUtilsfrom utils.send_feishu_msg import SendFeiShuMsgclass PromoterLiveInfoSpider:    def __init__(self):        self.conn = MysqlUtils()        self.list = [1, 2, 3]    def PromoterLiveInfoSpiderHander(self, promoterId, cookie):        headers = {'User-Agent': 'Mozilla/5.0',                   'Cookie': cookie}        try:            for i in self.list:                url = "https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/live/info?" \                      "timeRangeType={timeRangeType}&promoterId={promoterId}&type=1".format(timeRangeType=i,                                                                                            promoterId=promoterId)                rep = requests.get(url=url, headers=headers)                table_name = 'kwai_promoter_live_info'                data = json.loads(rep.text)["data"]                if data["likesCountInfo"] is None:                    SendFeiShuMsg.send_robot_msg("达人{promoterId}直播数据 is None".format(promoterId=promoterId))                else:                    promoter_info_item = {"commentsCountInfo": json.dumps(data["commentsCountInfo"]),                                          "likesCountInfo": json.dumps(data["likesCountInfo"]),                                          "liveMinuteDurationInfo": json.dumps(data["liveMinuteDurationInfo"]),                                          "liveVisitorCountInfo": json.dumps(data["liveVisitorCountInfo"]),                                          "maxVisitorCountInfo": json.dumps(data["maxVisitorCountInfo"]),                                          "sharesCountInfo": json.dumps(data["sharesCountInfo"]),                                          "promoterId": data["promoterId"],                                          "staticDataInfoView": json.dumps(data["staticDataInfoView"]),                                          "timeRangeType": i}                    insert(table_name=table_name, item=promoter_info_item)                    base_table_name = 'kwai_promoter_live_base_info'                    promoter_base_info_item = {                        "promoterId": data["promoterId"],                        "timeRangeType": i,                        "avgLiveStreamDuration": data["staticDataInfoView"]["avgLiveStreamDuration"],                        "avgLiveVisitorCount": data["staticDataInfoView"]["avgLiveVisitorCount"],                        "avgMinuteDuration": data["staticDataInfoView"]["avgMinuteDuration"],                        "avgPromoteSaleVolume": data["staticDataInfoView"]["avgPromoteSaleVolume"],                        "avgSameTimeVisitorCount": data["staticDataInfoView"]["avgSameTimeVisitorCount"],                        "avgViewTime": data["staticDataInfoView"]["avgViewTime"],                        "avgVisitorOnlineCount": data["staticDataInfoView"]["avgVisitorOnlineCount"],                        "interactionRate": data["staticDataInfoView"]["interactionRate"],                        "liveStreamVisitorCount": data["staticDataInfoView"]["liveStreamVisitorCount"],                        "maxLiveVisitorCount": data["staticDataInfoView"]["maxLiveVisitorCount"],                        "maxMinuteDuration": data["staticDataInfoView"]["maxMinuteDuration"],                        "maxPromoteSaleVolume": data["staticDataInfoView"]["maxPromoteSaleVolume"],                        "maxSameTimeVisitorCount": data["staticDataInfoView"]["maxSameTimeVisitorCount"],                        "minLiveVisitorCount": data["staticDataInfoView"]["minLiveVisitorCount"],                        "minMinuteDuration": data["staticDataInfoView"]["minMinuteDuration"],                        "minPromoteSaleVolume": data["staticDataInfoView"]["minPromoteSaleVolume"],                        "promoteItemCount": data["staticDataInfoView"]["promoteItemCount"],                        "promoteLiveCount": data["staticDataInfoView"]["promoteLiveCount"],                        "promoteLiveDays": data["staticDataInfoView"]["promoteLiveDays"],                        "promoteSaleVolume": data["staticDataInfoView"]["promoteSaleVolume"],                    }                    insert(table_name=base_table_name, item=promoter_base_info_item)                    if i == 3:                        live_analyse_table_name = 'kwai_promoter_live_analyse_info'                        commentsCountInfo = tuple(data["commentsCountInfo"])                        likesCountInfo = tuple(data["likesCountInfo"])                        liveMinuteDurationInfo = tuple(data["liveMinuteDurationInfo"])                        liveVisitorCountInfo = tuple(data["liveVisitorCountInfo"])                        maxVisitorCountInfo = tuple(data["maxVisitorCountInfo"])                        sharesCountInfo = tuple(data["sharesCountInfo"])                        live_analyse_item_list = []                        for i in range(len(commentsCountInfo)):                            live_analyse_item = {}                            commentsCount = commentsCountInfo[i]                            likesCount = likesCountInfo[i]                            liveMinuteDuration = liveMinuteDurationInfo[i]                            liveVisitorCount = liveVisitorCountInfo[i]                            maxVisitorCount = maxVisitorCountInfo[i]                            sharesCount = sharesCountInfo[i]                            live_analyse_item["date"] = commentsCount["date"]                            live_analyse_item["promoterId"] = commentsCount["promoterId"]                            live_analyse_item["commentsCount"] = commentsCount["commentsCount"]                            live_analyse_item["likesCount"] = likesCount["likesCount"]                            live_analyse_item["liveMinuteDuration"] = liveMinuteDuration["minuteDuration"]                            live_analyse_item["liveVisitorCount"] = liveVisitorCount["visitorCount"]                            live_analyse_item["maxVisitorCount"] = maxVisitorCount["maxVisitorCount"]                            live_analyse_item["sharesCount"] = sharesCount["sharesCount"]                            live_analyse_item_list.append(live_analyse_item)                        batch_insert(table_name=live_analyse_table_name, item_list=live_analyse_item_list)        except Exception as e:            SendFeiShuMsg.send_robot_msg(                'PromoterLiveInfoSpider   {promoterId}请求错误请检查cookie{e}'.format(promoterId=promoterId, e=e))
 |