renyupeng 1 gadu atpakaļ
vecāks
revīzija
6651df66bd
3 mainītis faili ar 232 papildinājumiem un 1 dzēšanām
  1. 106 0
      spider/SpiderPromoterInfo.py
  2. 107 0
      spider/SpiderWeekPromoterInfo.py
  3. 19 1
      utils/cookie_update.py

+ 106 - 0
spider/SpiderPromoterInfo.py

@@ -0,0 +1,106 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/9/11 下午4:19
+# @Site    : 
+# @File    : SpiderPromoterInfo.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import json
+import time
+import random
+from multiprocessing import Process
+import redis
+
+from spider.PromoterFansInfo import PromoterFansInfo
+from spider.PromoterInfoSpider import PromoterInfoSpider
+from spider.PromoterLiveInfoSpider import PromoterLiveInfoSpider
+from spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfo
+from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend
+from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
+from utils.cookie_update import cookie_update
+from utils.mysql_utils_pro import MysqlProUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class SpiderPromoterInfo:
+    def __init__(self):
+        self.r = redis.Redis(host='192.168.0.193', password='hcst@2022', port=6379, db=0)
+        self.conn = MysqlProUtils()
+
+    def get_promoter(self, promoterId):
+        if self.r.zcard('kuaishou_shop_token') == 0:
+            cookie_update.get_coookie_to_redis()
+        else:
+            random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+            random_item = random.choice(random_member)
+            cookie = random_item[0].decode('utf-8')
+            phone_num = int(random_item[1])
+            roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
+
+            if json.loads(roll).__contains__("result"):
+                print(11111)
+                if json.loads(roll)["result"] == 100110000:
+                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+                    SendFeiShuMsg.send_cookie_robot_msg(
+                        "cookie 永久失效 轻更新cookie 唯一电话为{phone_num}".format(phone_num=phone_num))
+                    retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+                    retry_cookie = retry_item[0].decode('utf-8')
+                    cookie_update.update_cookie_handler(phone_num)
+                    PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+                    PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
+                    PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
+                    PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
+                    PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
+
+                else:
+
+                    SendFeiShuMsg.send_cookie_robot_msg(
+                        "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
+                                                                             code=json.loads(roll)["result"]))
+                    cookie_update.update_temporary_cookie_handler(phone_num)
+                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+
+                    retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+                    retry_cookie = retry_item[0].decode('utf-8')
+                    PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+                    PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
+                    PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
+                    PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
+                    PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
+
+            else:
+                self.update_promoter_info(promoterId)
+                PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
+                PromoterFansInfo().PromoterFansInfoHandler(promoterId, cookie)
+                PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, cookie)
+                PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, cookie)
+                PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, cookie)
+
+                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+                return roll
+
+    def get_promoter_info(self):
+        sql = """
+        select promoter_id from ruixuan.kuaishou_promoter where media_id =2 
+        """
+        promoter_result = self.conn.QueryAll(sql)
+        for promoter in promoter_result:
+            promoter_id = int(promoter[0].decode('utf-8'))
+            self.get_promoter(promoter_id)
+            time.sleep(2)
+
+    def update_promoter_info(self, promoter_id):
+        roll = self.get_promoter(promoter_id)
+        fanNum = roll['fanNum']
+        totalSale = roll["promoteBaseInfo"]["totalSale"]
+        avgVideoSales = roll["promoteBaseInfo"]["avgVideoSales"]
+        videoSales = roll["promoteBaseInfo"]["videoSales"]
+        sql = """ update ruixuan.kuaishou_promoter set   total_sale={totalSale} ,fans_number={fanNum},avg_video_sales={avgVideoSales}
+         video_sales={videoSales} where promoter_id={promoterId}
+         """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,promoterId=promoter_id)
+        MysqlProUtils().Operate(sql=sql)
+
+
+if __name__ == '__main__':
+    SpiderPromoterInfo().get_promoter_info()

+ 107 - 0
spider/SpiderWeekPromoterInfo.py

@@ -0,0 +1,107 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/9/11 下午4:19
+# @Site    : 
+# @File    : SpiderWeekPromoterInfo.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import json
+import time
+import random
+from multiprocessing import Process
+import redis
+
+from spider.PromoterFansInfo import PromoterFansInfo
+from spider.PromoterInfoSpider import PromoterInfoSpider
+from spider.PromoterLiveInfoSpider import PromoterLiveInfoSpider
+from spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfo
+from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend
+from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
+from utils.cookie_update import cookie_update
+from utils.mysql_utils_pro import MysqlProUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class SpiderWeekPromoterInfo:
+    def __init__(self):
+        self.r = redis.Redis(host='192.168.0.193', password='hcst@2022', port=6379, db=0)
+        self.conn = MysqlProUtils()
+
+    def get_promoter(self, promoterId):
+        if self.r.zcard('kuaishou_shop_token') == 0:
+            cookie_update.get_coookie_to_redis()
+        else:
+            random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+            random_item = random.choice(random_member)
+            cookie = random_item[0].decode('utf-8')
+            phone_num = int(random_item[1])
+            roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
+
+            if json.loads(roll).__contains__("result"):
+                print(11111)
+                if json.loads(roll)["result"] == 100110000:
+                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+                    SendFeiShuMsg.send_cookie_robot_msg(
+                        "cookie 永久失效 轻更新cookie 唯一电话为{phone_num}".format(phone_num=phone_num))
+                    retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+                    retry_cookie = retry_item[0].decode('utf-8')
+                    cookie_update.update_cookie_handler(phone_num)
+                    PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+                    PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
+                    PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
+                    PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
+                    PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
+
+                else:
+
+                    SendFeiShuMsg.send_cookie_robot_msg(
+                        "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
+                                                                             code=json.loads(roll)["result"]))
+                    cookie_update.update_temporary_cookie_handler(phone_num)
+                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+
+                    retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+                    retry_cookie = retry_item[0].decode('utf-8')
+                    PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+                    PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
+                    PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
+                    PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
+                    PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
+
+            else:
+                self.update_promoter_info(promoterId)
+                PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
+                PromoterFansInfo().PromoterFansInfoHandler(promoterId, cookie)
+                PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, cookie)
+                PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, cookie)
+                PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, cookie)
+
+                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+                return roll
+
+    def get_promoter_info(self):
+        sql = """
+        select promoter_id from ruixuan.promoter_info 
+        """
+        promoter_result = self.conn.QueryAll(sql)
+        for promoter in promoter_result:
+            promoter_id = int(promoter[0].decode('utf-8'))
+            self.get_promoter(promoter_id)
+            time.sleep(2)
+
+    def update_promoter_info(self, promoter_id):
+        roll = self.get_promoter(promoter_id)
+        fanNum = roll['fanNum']
+        totalSale = roll["promoteBaseInfo"]["totalSale"]
+        avgVideoSales = roll["promoteBaseInfo"]["avgVideoSales"]
+        videoSales = roll["promoteBaseInfo"]["videoSales"]
+        sql = """ update ruixuan.promoter_sales_info set   total_sale={totalSale} ,fans_number={fanNum},avg_video_sales={avgVideoSales}
+         video_sales={videoSales} where promoter_id={promoterId}
+         """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
+                    promoterId=promoter_id)
+        MysqlProUtils().Operate(sql=sql)
+
+
+if __name__ == '__main__':
+    SpiderWeekPromoterInfo().get_promoter_info()

+ 19 - 1
utils/cookie_update.py

@@ -7,9 +7,13 @@
 # @contact: renyupeng@c-top.com.cn
 # @Tel 1501435553
 from utils.mysql_utils import MysqlUtils
+import redis
+import random
+
 
 
 class cookie_update:
+    r = redis.Redis(host='192.168.0.193', password='hcst@2022', port=6379, db=0)
 
     @staticmethod
     def get_cookie_handler():
@@ -30,6 +34,20 @@ class cookie_update:
             phone_num=phone_num)
         MysqlUtils().Operate(sql=sql, params=None, DML=True)
 
+    @staticmethod
+    def get_coookie_to_redis():
+        sql = "select cookie ,phone_num from kwai_promoter.kuaishou_supply_chain_cookie where status =0"
+        results = MysqlUtils().QueryAll(sql)
+        for result in results:
+            cookie_update.r.zadd('kuaishou_shop_token', {result[0]: result[1]})
 
 if __name__ == '__main__':
-    cookie_update().get_cookie_handler()
+    # cookie_update.get_coookie_to_redis()
+
+    random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1,withscores=True)
+    random_item = random.choice(random_member)
+
+    cookie_update.r.zrem('kuaishou_shop_token',random_item[0])
+
+
+