123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899 |
- # Author renyupeng
- # coding=utf-8
- # @Time : 2023/9/11 下午4:19
- # @Site :
- # @File : SpiderPromoterInfo.py
- # @Software: PyCharm
- # @contact: renyupeng@c-top.com.cn
- # @Tel 1501435553
- import json
- import threading
- import time
- import random
- import redis
- from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
- from utils.cookie_update import cookie_update
- from utils.mysql_utils_pro import MysqlProUtils
- from utils.send_feishu_msg import SendFeiShuMsg
- class SpiderPromoterInfo:
- r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
- conn = MysqlProUtils()
- @staticmethod
- def get_promoter(promoterId):
- print(promoterId, '22222')
- if SpiderPromoterInfo.r.zcard('kuaishou_shop_token') == 0:
- cookie_update.get_coookie_to_redis()
- else:
- random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
- random_item = random.choice(random_member)
- # cookie = random_item[0].decode('utf-8')
- cookie = cookie_update.get_cookie_handler()
- phone_num = int(random_item[1])
- roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
- if json.loads(roll).__contains__("result"):
- SendFeiShuMsg.send_cookie_robot_msg(
- "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
- code=json.loads(roll)["result"]))
- cookie_update.update_temporary_cookie_handler(phone_num)
- cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
- retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
- retry_cookie = retry_item[0].decode('utf-8')
- roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
- try:
- roll = json.loads(roll)
- fanNum = roll["fanNum"]
- totalSale = roll["totalSale"]
- avgVideoSales = roll["avgVideoSales"]
- videoSales = roll["videoSales"]
- sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
- {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
- ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
- videoSales=videoSales,
- promoterId=promoterId)
- MysqlProUtils().Operate(sql=sql)
- cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
- except Exception as e:
- print(e)
- SpiderPromoterInfo.r.sadd('promoter_err_info', str(promoterId))
- else:
- roll = json.loads(roll)
- print(roll, '--roll--',promoterId)
- fanNum = roll["fanNum"]
- totalSale = roll["totalSale"]
- avgVideoSales = roll["avgVideoSales"]
- videoSales = roll["videoSales"]
- sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
- {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
- ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
- promoterId=promoterId)
- MysqlProUtils().Operate(sql=sql)
- cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
- @staticmethod
- def get_promoter_info():
- sql = """
- select promoter_id from ruixuan.kuaishou_promoter where media_id =2
- group by promoter_id
- """
- promoter_result = SpiderPromoterInfo.conn.QueryAll(sql)
- SpiderPromoterInfo.r.delete('promoter_id_list')
- for promoter in promoter_result:
- promoter_id = promoter[0]
- SpiderPromoterInfo.r.sadd('promoter_id_list', promoter_id)
- set_data = SpiderPromoterInfo.r.smembers('promoter_id_list')
- for promoter_id in set_data:
- promoter_id = promoter_id.decode('utf-8')
- SpiderPromoterInfo.get_promoter(promoter_id)
- SpiderPromoterInfo.r.srem('promoter_id_list', promoter_id)
- time.sleep(3)
- SpiderPromoterInfo.r.close()
- if __name__ == '__main__':
- SpiderPromoterInfo().get_promoter_info()
|