| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106 | # Author renyupeng# coding=utf-8# @Time    : 2023/9/11 下午4:19# @Site    : # @File    : SpiderWeekPromoterInfo.py# @Software: PyCharm# @contact: renyupeng@c-top.com.cn# @Tel 1501435553import jsonimport threadingimport timeimport randomfrom multiprocessing import Processimport redisfrom spider.PromoterFansInfo import PromoterFansInfofrom spider.PromoterInfoSpider import PromoterInfoSpiderfrom spider.PromoterLiveInfoSpider import PromoterLiveInfoSpiderfrom spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfofrom spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrendfrom spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpiderfrom utils.cookie_update import cookie_updatefrom utils.mysql_utils_pro import MysqlProUtilsfrom utils.send_feishu_msg import SendFeiShuMsgclass SpiderWeekPromoterInfo:    def __init__(self):        self.r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)        self.conn = MysqlProUtils()    def get_promoter(self, promoterId):        if self.r.zcard('kuaishou_shop_token') == 0:            cookie_update.get_coookie_to_redis()        else:            random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)            random_item = random.choice(random_member)            # cookie = random_item[0].decode('utf-8')            cookie=cookie_update.get_cookie_handler()[0]            phone_num = int(random_item[1])            roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)            if json.loads(roll).__contains__("result"):                SendFeiShuMsg.send_cookie_robot_msg(                    "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,                                                                         code=json.loads(roll)["result"]))                cookie_update.update_temporary_cookie_handler(phone_num)                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])                retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)                retry_cookie = retry_item[0].decode('utf-8')                roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)                try:                    roll = json.loads(roll)                    fanNum = roll["fanNum"]                    totalSale = roll["totalSale"]                    avgVideoSales = roll["avgVideoSales"]                    videoSales = roll["videoSales"]                    sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (                        {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'                        ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,                                     videoSales=videoSales,                                     promoterId=promoterId)                    MysqlProUtils().Operate(sql=sql)                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])                except Exception as e:                    print(e)                    self.r.sadd('promoter_week_err_info', str(promoterId))            else:                roll = json.loads(roll)                print(roll, '--roll--', promoterId)                fanNum = roll["fanNum"]                totalSale = roll["totalSale"]                avgVideoSales = roll["avgVideoSales"]                videoSales = roll["videoSales"]                sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (                {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'                ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,                             promoterId=promoterId)                MysqlProUtils().Operate(sql=sql)                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])    def get_promoter_info(self):        sql = """        select promoter_id from ruixuan.promoter_info  where promoter_id!=0        """        promoter_result = self.conn.QueryAll(sql)        self.r.delete('promoter_week_id_list')        for promoter in set(promoter_result):            promoter_id = str(int(promoter[0]))            self.r.sadd('promoter_week_id_list', promoter_id)        set_data = self.r.smembers('promoter_week_id_list')        for promoter_id in set_data:            promoter_id = promoter_id.decode('utf-8')            self.get_promoter(promoter_id)            self.r.srem('promoter_week_id_list', promoter_id)            time.sleep(1)        self.r.close()if __name__ == '__main__':    SpiderWeekPromoterInfo().get_promoter_info()
 |