SpiderWeekPromoterInfo.py 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. # Author renyupeng
  2. # coding=utf-8
  3. # @Time : 2023/9/11 下午4:19
  4. # @Site :
  5. # @File : SpiderWeekPromoterInfo.py
  6. # @Software: PyCharm
  7. # @contact: renyupeng@c-top.com.cn
  8. # @Tel 1501435553
  9. import json
  10. import threading
  11. import time
  12. import random
  13. from multiprocessing import Process
  14. import redis
  15. from spider.PromoterFansInfo import PromoterFansInfo
  16. from spider.PromoterInfoSpider import PromoterInfoSpider
  17. from spider.PromoterLiveInfoSpider import PromoterLiveInfoSpider
  18. from spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfo
  19. from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend
  20. from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
  21. from utils.cookie_update import cookie_update
  22. from utils.mysql_utils_pro import MysqlProUtils
  23. from utils.send_feishu_msg import SendFeiShuMsg
  24. class SpiderWeekPromoterInfo:
  25. def __init__(self):
  26. self.r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
  27. self.conn = MysqlProUtils()
  28. def get_promoter(self, promoterId):
  29. if self.r.zcard('kuaishou_shop_token') == 0:
  30. cookie_update.get_coookie_to_redis()
  31. else:
  32. random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
  33. random_item = random.choice(random_member)
  34. # cookie = random_item[0].decode('utf-8')
  35. cookie=cookie_update.get_cookie_handler()[0].encode('utf-8')
  36. phone_num = int(random_item[1])
  37. roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
  38. if json.loads(roll).__contains__("result"):
  39. SendFeiShuMsg.send_cookie_robot_msg(
  40. "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
  41. code=json.loads(roll)["result"]))
  42. cookie_update.update_temporary_cookie_handler(phone_num)
  43. cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
  44. retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
  45. retry_cookie = retry_item[0].decode('utf-8')
  46. roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
  47. try:
  48. roll = json.loads(roll)
  49. fanNum = roll["fanNum"]
  50. totalSale = roll["totalSale"]
  51. avgVideoSales = roll["avgVideoSales"]
  52. videoSales = roll["videoSales"]
  53. sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
  54. {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
  55. ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
  56. videoSales=videoSales,
  57. promoterId=promoterId)
  58. MysqlProUtils().Operate(sql=sql)
  59. cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
  60. except Exception as e:
  61. print(e)
  62. self.r.sadd('promoter_week_err_info', str(promoterId))
  63. else:
  64. roll = json.loads(roll)
  65. print(roll, '--roll--', promoterId)
  66. fanNum = roll["fanNum"]
  67. totalSale = roll["totalSale"]
  68. avgVideoSales = roll["avgVideoSales"]
  69. videoSales = roll["videoSales"]
  70. sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
  71. {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
  72. ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
  73. promoterId=promoterId)
  74. MysqlProUtils().Operate(sql=sql)
  75. cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
  76. def get_promoter_info(self):
  77. sql = """
  78. select promoter_id from ruixuan.promoter_info where promoter_id!=0
  79. """
  80. promoter_result = self.conn.QueryAll(sql)
  81. self.r.delete('promoter_week_id_list')
  82. for promoter in set(promoter_result):
  83. promoter_id = str(int(promoter[0]))
  84. self.r.sadd('promoter_week_id_list', promoter_id)
  85. set_data = self.r.smembers('promoter_week_id_list')
  86. for promoter_id in set_data:
  87. promoter_id = promoter_id.decode('utf-8')
  88. self.get_promoter(promoter_id)
  89. self.r.srem('promoter_week_id_list', promoter_id)
  90. time.sleep(1)
  91. self.r.close()
  92. if __name__ == '__main__':
  93. SpiderWeekPromoterInfo().get_promoter_info()