PromoterLiveInfoSpider.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107
  1. # Author renyupeng
  2. """
  3. coding=utf-8
  4. @Time : 2023/2/8 10:27 上午
  5. @Site :
  6. @File : PromoterLiveInfoSpider.py
  7. @Software: PyCharm
  8. @contact: renyupeng@c-top.com.cn
  9. @Tel 1501435553
  10. """
  11. import json
  12. import requests
  13. from utils.mysql_helper import insert, batch_insert
  14. from utils.mysql_utils import MysqlUtils
  15. from utils.send_feishu_msg import SendFeiShuMsg
  16. class PromoterLiveInfoSpider:
  17. def __init__(self):
  18. self.conn = MysqlUtils()
  19. self.list = [1, 2, 3]
  20. def PromoterLiveInfoSpiderHander(self, promoterId, cookie):
  21. headers = {'User-Agent': 'Mozilla/5.0',
  22. 'Cookie': cookie}
  23. try:
  24. for i in self.list:
  25. url = "https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/live/info?" \
  26. "timeRangeType={timeRangeType}&promoterId={promoterId}&type=1".format(timeRangeType=i,
  27. promoterId=promoterId)
  28. rep = requests.get(url=url, headers=headers)
  29. table_name = 'kwai_promoter_live_info'
  30. data = json.loads(rep.text)["data"]
  31. if data["likesCountInfo"] is None:
  32. SendFeiShuMsg.send_robot_msg("达人{promoterId}直播数据 is None".format(promoterId=promoterId))
  33. else:
  34. promoter_info_item = {"commentsCountInfo": json.dumps(data["commentsCountInfo"]),
  35. "likesCountInfo": json.dumps(data["likesCountInfo"]),
  36. "liveMinuteDurationInfo": json.dumps(data["liveMinuteDurationInfo"]),
  37. "liveVisitorCountInfo": json.dumps(data["liveVisitorCountInfo"]),
  38. "maxVisitorCountInfo": json.dumps(data["maxVisitorCountInfo"]),
  39. "sharesCountInfo": json.dumps(data["sharesCountInfo"]),
  40. "promoterId": data["promoterId"],
  41. "staticDataInfoView": json.dumps(data["staticDataInfoView"]),
  42. "timeRangeType": i}
  43. insert(table_name=table_name, item=promoter_info_item)
  44. base_table_name = 'kwai_promoter_live_base_info'
  45. promoter_base_info_item = {
  46. "promoterId": data["promoterId"],
  47. "timeRangeType": i,
  48. "avgLiveStreamDuration": data["staticDataInfoView"]["avgLiveStreamDuration"],
  49. "avgLiveVisitorCount": data["staticDataInfoView"]["avgLiveVisitorCount"],
  50. "avgMinuteDuration": data["staticDataInfoView"]["avgMinuteDuration"],
  51. "avgPromoteSaleVolume": data["staticDataInfoView"]["avgPromoteSaleVolume"],
  52. "avgSameTimeVisitorCount": data["staticDataInfoView"]["avgSameTimeVisitorCount"],
  53. "avgViewTime": data["staticDataInfoView"]["avgViewTime"],
  54. "avgVisitorOnlineCount": data["staticDataInfoView"]["avgVisitorOnlineCount"],
  55. "interactionRate": data["staticDataInfoView"]["interactionRate"],
  56. "liveStreamVisitorCount": data["staticDataInfoView"]["liveStreamVisitorCount"],
  57. "maxLiveVisitorCount": data["staticDataInfoView"]["maxLiveVisitorCount"],
  58. "maxMinuteDuration": data["staticDataInfoView"]["maxMinuteDuration"],
  59. "maxPromoteSaleVolume": data["staticDataInfoView"]["maxPromoteSaleVolume"],
  60. "maxSameTimeVisitorCount": data["staticDataInfoView"]["maxSameTimeVisitorCount"],
  61. "minLiveVisitorCount": data["staticDataInfoView"]["minLiveVisitorCount"],
  62. "minMinuteDuration": data["staticDataInfoView"]["minMinuteDuration"],
  63. "minPromoteSaleVolume": data["staticDataInfoView"]["minPromoteSaleVolume"],
  64. "promoteItemCount": data["staticDataInfoView"]["promoteItemCount"],
  65. "promoteLiveCount": data["staticDataInfoView"]["promoteLiveCount"],
  66. "promoteLiveDays": data["staticDataInfoView"]["promoteLiveDays"],
  67. "promoteSaleVolume": data["staticDataInfoView"]["promoteSaleVolume"],
  68. }
  69. insert(table_name=base_table_name, item=promoter_base_info_item)
  70. if i == 3:
  71. live_analyse_table_name = 'kwai_promoter_live_analyse_info'
  72. commentsCountInfo = tuple(data["commentsCountInfo"])
  73. likesCountInfo = tuple(data["likesCountInfo"])
  74. liveMinuteDurationInfo = tuple(data["liveMinuteDurationInfo"])
  75. liveVisitorCountInfo = tuple(data["liveVisitorCountInfo"])
  76. maxVisitorCountInfo = tuple(data["maxVisitorCountInfo"])
  77. sharesCountInfo = tuple(data["sharesCountInfo"])
  78. live_analyse_item_list = []
  79. for i in range(len(commentsCountInfo)):
  80. live_analyse_item = {}
  81. commentsCount = commentsCountInfo[i]
  82. likesCount = likesCountInfo[i]
  83. liveMinuteDuration = liveMinuteDurationInfo[i]
  84. liveVisitorCount = liveVisitorCountInfo[i]
  85. maxVisitorCount = maxVisitorCountInfo[i]
  86. sharesCount = sharesCountInfo[i]
  87. live_analyse_item["date"] = commentsCount["date"]
  88. live_analyse_item["promoterId"] = commentsCount["promoterId"]
  89. live_analyse_item["commentsCount"] = commentsCount["commentsCount"]
  90. live_analyse_item["likesCount"] = likesCount["likesCount"]
  91. live_analyse_item["liveMinuteDuration"] = liveMinuteDuration["minuteDuration"]
  92. live_analyse_item["liveVisitorCount"] = liveVisitorCount["visitorCount"]
  93. live_analyse_item["maxVisitorCount"] = maxVisitorCount["maxVisitorCount"]
  94. live_analyse_item["sharesCount"] = sharesCount["sharesCount"]
  95. live_analyse_item_list.append(live_analyse_item)
  96. batch_insert(table_name=live_analyse_table_name, item_list=live_analyse_item_list)
  97. except Exception as e:
  98. SendFeiShuMsg.send_robot_msg(
  99. 'PromoterLiveInfoSpider {promoterId}请求错误请检查cookie{e}'.format(promoterId=promoterId, e=e))