SpiderPromoterInfo.py 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899
  1. # Author renyupeng
  2. # coding=utf-8
  3. # @Time : 2023/9/11 下午4:19
  4. # @Site :
  5. # @File : SpiderPromoterInfo.py
  6. # @Software: PyCharm
  7. # @contact: renyupeng@c-top.com.cn
  8. # @Tel 1501435553
  9. import json
  10. import threading
  11. import time
  12. import random
  13. import redis
  14. from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
  15. from utils.cookie_update import cookie_update
  16. from utils.mysql_utils_pro import MysqlProUtils
  17. from utils.send_feishu_msg import SendFeiShuMsg
  18. class SpiderPromoterInfo:
  19. r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
  20. conn = MysqlProUtils()
  21. @staticmethod
  22. def get_promoter(promoterId):
  23. print(promoterId, '22222')
  24. if SpiderPromoterInfo.r.zcard('kuaishou_shop_token') == 0:
  25. cookie_update.get_coookie_to_redis()
  26. else:
  27. random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
  28. random_item = random.choice(random_member)
  29. # cookie = random_item[0].decode('utf-8')
  30. cookie = cookie_update.get_cookie_handler()
  31. phone_num = int(random_item[1])
  32. roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
  33. if json.loads(roll).__contains__("result"):
  34. SendFeiShuMsg.send_cookie_robot_msg(
  35. "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
  36. code=json.loads(roll)["result"]))
  37. cookie_update.update_temporary_cookie_handler(phone_num)
  38. cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
  39. retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
  40. retry_cookie = retry_item[0].decode('utf-8')
  41. roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
  42. try:
  43. roll = json.loads(roll)
  44. fanNum = roll["fanNum"]
  45. totalSale = roll["totalSale"]
  46. avgVideoSales = roll["avgVideoSales"]
  47. videoSales = roll["videoSales"]
  48. sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
  49. {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
  50. ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
  51. videoSales=videoSales,
  52. promoterId=promoterId)
  53. MysqlProUtils().Operate(sql=sql)
  54. cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
  55. except Exception as e:
  56. print(e)
  57. SpiderPromoterInfo.r.sadd('promoter_err_info', str(promoterId))
  58. else:
  59. roll = json.loads(roll)
  60. print(roll, '--roll--',promoterId)
  61. fanNum = roll["fanNum"]
  62. totalSale = roll["totalSale"]
  63. avgVideoSales = roll["avgVideoSales"]
  64. videoSales = roll["videoSales"]
  65. sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
  66. {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
  67. ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
  68. promoterId=promoterId)
  69. MysqlProUtils().Operate(sql=sql)
  70. cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
  71. @staticmethod
  72. def get_promoter_info():
  73. sql = """
  74. select promoter_id from ruixuan.kuaishou_promoter where media_id =2
  75. group by promoter_id
  76. """
  77. promoter_result = SpiderPromoterInfo.conn.QueryAll(sql)
  78. SpiderPromoterInfo.r.delete('promoter_id_list')
  79. for promoter in promoter_result:
  80. promoter_id = promoter[0]
  81. SpiderPromoterInfo.r.sadd('promoter_id_list', promoter_id)
  82. set_data = SpiderPromoterInfo.r.smembers('promoter_id_list')
  83. for promoter_id in set_data:
  84. promoter_id = promoter_id.decode('utf-8')
  85. SpiderPromoterInfo.get_promoter(promoter_id)
  86. SpiderPromoterInfo.r.srem('promoter_id_list', promoter_id)
  87. time.sleep(3)
  88. SpiderPromoterInfo.r.close()
  89. if __name__ == '__main__':
  90. SpiderPromoterInfo().get_promoter_info()