# Author renyupeng # coding=utf-8 # @Time : 2023/9/11 下午4:19 # @Site : # @File : SpiderPromoterInfo.py # @Software: PyCharm # @contact: renyupeng@c-top.com.cn # @Tel 1501435553 import json import threading import time import random import redis from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider from utils.cookie_update import cookie_update from utils.mysql_utils_pro import MysqlProUtils from utils.send_feishu_msg import SendFeiShuMsg class SpiderPromoterInfo: r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0) conn = MysqlProUtils() @staticmethod def get_promoter(promoterId): print(promoterId, '22222') if SpiderPromoterInfo.r.zcard('kuaishou_shop_token') == 0: cookie_update.get_coookie_to_redis() else: random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True) random_item = random.choice(random_member) # cookie = random_item[0].decode('utf-8') cookie = cookie_update.get_cookie_handler() phone_num = int(random_item[1]) roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie) if json.loads(roll).__contains__("result"): SendFeiShuMsg.send_cookie_robot_msg( "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num, code=json.loads(roll)["result"])) cookie_update.update_temporary_cookie_handler(phone_num) cookie_update.r.zrem('kuaishou_shop_token', random_item[0]) retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True) retry_cookie = retry_item[0].decode('utf-8') roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie) try: roll = json.loads(roll) fanNum = roll["fanNum"] totalSale = roll["totalSale"] avgVideoSales = roll["avgVideoSales"] videoSales = roll["videoSales"] sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values ( {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}' ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales, promoterId=promoterId) MysqlProUtils().Operate(sql=sql) cookie_update.r.zrem('kuaishou_shop_token', random_item[0]) except Exception as e: print(e) SpiderPromoterInfo.r.sadd('promoter_err_info', str(promoterId)) else: roll = json.loads(roll) print(roll, '--roll--',promoterId) fanNum = roll["fanNum"] totalSale = roll["totalSale"] avgVideoSales = roll["avgVideoSales"] videoSales = roll["videoSales"] sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values ( {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}' ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales, promoterId=promoterId) MysqlProUtils().Operate(sql=sql) cookie_update.r.zrem('kuaishou_shop_token', random_item[0]) @staticmethod def get_promoter_info(): sql = """ select promoter_id from ruixuan.kuaishou_promoter where media_id =2 group by promoter_id """ promoter_result = SpiderPromoterInfo.conn.QueryAll(sql) SpiderPromoterInfo.r.delete('promoter_id_list') for promoter in promoter_result: promoter_id = promoter[0] SpiderPromoterInfo.r.sadd('promoter_id_list', promoter_id) set_data = SpiderPromoterInfo.r.smembers('promoter_id_list') for promoter_id in set_data: promoter_id = promoter_id.decode('utf-8') SpiderPromoterInfo.get_promoter(promoter_id) SpiderPromoterInfo.r.srem('promoter_id_list', promoter_id) time.sleep(3) SpiderPromoterInfo.r.close() if __name__ == '__main__': SpiderPromoterInfo().get_promoter_info()