# Author renyupeng # coding=utf-8 # @Time : 2023/9/11 下午4:19 # @Site : # @File : SpiderWeekPromoterInfo.py # @Software: PyCharm # @contact: renyupeng@c-top.com.cn # @Tel 1501435553 import json import threading import time import random from multiprocessing import Process import redis from spider.PromoterFansInfo import PromoterFansInfo from spider.PromoterInfoSpider import PromoterInfoSpider from spider.PromoterLiveInfoSpider import PromoterLiveInfoSpider from spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfo from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider from utils.cookie_update import cookie_update from utils.mysql_utils_pro import MysqlProUtils from utils.send_feishu_msg import SendFeiShuMsg class SpiderWeekPromoterInfo: def __init__(self): self.r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0) self.conn = MysqlProUtils() def get_promoter(self, promoterId): if self.r.zcard('kuaishou_shop_token') == 0: cookie_update.get_coookie_to_redis() else: random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True) random_item = random.choice(random_member) # cookie = random_item[0].decode('utf-8') cookie=cookie_update.get_cookie_handler()[0] phone_num = int(random_item[1]) roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie) if json.loads(roll).__contains__("result"): SendFeiShuMsg.send_cookie_robot_msg( "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num, code=json.loads(roll)["result"])) cookie_update.update_temporary_cookie_handler(phone_num) cookie_update.r.zrem('kuaishou_shop_token', random_item[0]) retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True) retry_cookie = retry_item[0].decode('utf-8') roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie) try: roll = json.loads(roll) fanNum = roll["fanNum"] totalSale = roll["totalSale"] avgVideoSales = roll["avgVideoSales"] videoSales = roll["videoSales"] sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values ( {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}' ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales, promoterId=promoterId) MysqlProUtils().Operate(sql=sql) cookie_update.r.zrem('kuaishou_shop_token', random_item[0]) except Exception as e: print(e) self.r.sadd('promoter_week_err_info', str(promoterId)) else: roll = json.loads(roll) print(roll, '--roll--', promoterId) fanNum = roll["fanNum"] totalSale = roll["totalSale"] avgVideoSales = roll["avgVideoSales"] videoSales = roll["videoSales"] sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values ( {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}' ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales, promoterId=promoterId) MysqlProUtils().Operate(sql=sql) cookie_update.r.zrem('kuaishou_shop_token', random_item[0]) def get_promoter_info(self): sql = """ select promoter_id from ruixuan.promoter_info where promoter_id!=0 """ promoter_result = self.conn.QueryAll(sql) self.r.delete('promoter_week_id_list') for promoter in set(promoter_result): promoter_id = str(int(promoter[0])) self.r.sadd('promoter_week_id_list', promoter_id) set_data = self.r.smembers('promoter_week_id_list') for promoter_id in set_data: promoter_id = promoter_id.decode('utf-8') self.get_promoter(promoter_id) self.r.srem('promoter_week_id_list', promoter_id) time.sleep(1) self.r.close() if __name__ == '__main__': SpiderWeekPromoterInfo().get_promoter_info()