renyupeng пре 1 година
родитељ
комит
e184cad3e6
3 измењених фајлова са 76 додато и 118 уклоњено
  1. 0 1
      spider/ReturnPromoterInfoSpider.py
  2. 32 70
      spider/SpiderPromoterInfo.py
  3. 44 47
      spider/SpiderWeekPromoterInfo.py

+ 0 - 1
spider/ReturnPromoterInfoSpider.py

@@ -33,7 +33,6 @@ class ReturnPromoterInfoSpider:
 
         try:
             rep = requests.get(url=url, headers=headers)
-            print(rep.text, '-----rep.txt---------')
             if json.loads(rep.text)["result"] != 1:
                 errItem = {"result": json.loads(rep.text)["result"]}
                 return json.dumps(errItem)

+ 32 - 70
spider/SpiderPromoterInfo.py

@@ -7,16 +7,10 @@
 # @contact: renyupeng@c-top.com.cn
 # @Tel 1501435553
 import json
+import threading
 import time
 import random
-from multiprocessing import Process
 import redis
-
-from spider.PromoterFansInfo import PromoterFansInfo
-from spider.PromoterInfoSpider import PromoterInfoSpider
-from spider.PromoterLiveInfoSpider import PromoterLiveInfoSpider
-from spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfo
-from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend
 from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
 from utils.cookie_update import cookie_update
 from utils.mysql_utils_pro import MysqlProUtils
@@ -29,6 +23,7 @@ class SpiderPromoterInfo:
 
     @staticmethod
     def get_promoter(promoterId):
+        print(promoterId, '22222')
         if SpiderPromoterInfo.r.zcard('kuaishou_shop_token') == 0:
             cookie_update.get_coookie_to_redis()
         else:
@@ -39,69 +34,36 @@ class SpiderPromoterInfo:
             roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
 
             if json.loads(roll).__contains__("result"):
-                print(11111)
-                if json.loads(roll)["result"] == 100110000:
-                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-                    SendFeiShuMsg.send_cookie_robot_msg(
-                        "cookie 永久失效 轻更新cookie 唯一电话为{phone_num}".format(phone_num=phone_num))
-                    retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
-                    retry_cookie = retry_item[0].decode('utf-8')
-                    roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
-
-                    cookie_update.update_cookie_handler(phone_num)
-                    PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
-                    PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
-                    PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
-                    PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
-                    PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
-                    roll = json.loads(roll)
-                    fanNum = roll["fanNum"]
-                    totalSale = roll["totalSale"]
-                    avgVideoSales = roll["avgVideoSales"]
-                    videoSales = roll["videoSales"]
-                    sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
-                    {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
-                    ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
-                                 promoterId=promoterId)
-                    MysqlProUtils().Operate(sql=sql)
-                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-
-                else:
-
-                    SendFeiShuMsg.send_cookie_robot_msg(
-                        "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
-                                                                             code=json.loads(roll)["result"]))
-                    cookie_update.update_temporary_cookie_handler(phone_num)
-                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+                SendFeiShuMsg.send_cookie_robot_msg(
+                    "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
+                                                                         code=json.loads(roll)["result"]))
+                cookie_update.update_temporary_cookie_handler(phone_num)
+                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
 
-                    retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
-                    retry_cookie = retry_item[0].decode('utf-8')
-                    roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
-                    PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
-                    PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
-                    PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
-                    PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
-                    PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
+                retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+                retry_cookie = retry_item[0].decode('utf-8')
+                roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+                try:
                     roll = json.loads(roll)
                     fanNum = roll["fanNum"]
                     totalSale = roll["totalSale"]
                     avgVideoSales = roll["avgVideoSales"]
                     videoSales = roll["videoSales"]
                     sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
-                    {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
-                    ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
-                                 promoterId=promoterId)
+                        {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
+                        ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
+                                     videoSales=videoSales,
+                                     promoterId=promoterId)
                     MysqlProUtils().Operate(sql=sql)
                     cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+                except Exception as e:
+                    print(e)
+                    SpiderPromoterInfo.r.sadd('promoter_err_info', str(promoterId))
 
             else:
-                PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
-                PromoterFansInfo().PromoterFansInfoHandler(promoterId, cookie)
-                PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, cookie)
-                PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, cookie)
-                PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, cookie)
 
                 roll = json.loads(roll)
+                print(roll, '--roll--',promoterId)
                 fanNum = roll["fanNum"]
                 totalSale = roll["totalSale"]
                 avgVideoSales = roll["avgVideoSales"]
@@ -112,24 +74,24 @@ class SpiderPromoterInfo:
                              promoterId=promoterId)
                 MysqlProUtils().Operate(sql=sql)
                 cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-                return roll
 
     @staticmethod
     def get_promoter_info():
         sql = """
-        select promoter_id from ruixuan.kuaishou_promoter where media_id =2 
-        """
+      select promoter_id from ruixuan.kuaishou_promoter where media_id =2 
+      group by promoter_id
+ """
         promoter_result = SpiderPromoterInfo.conn.QueryAll(sql)
-        # SpiderPromoterInfo.r.rpush('promoter_list', *promoter_result)
-
-        try:
-            for promoter in promoter_result:
-                promoter_id = int(promoter[0].decode('utf-8'))
-
-                SpiderPromoterInfo.get_promoter(promoter_id)
-                time.sleep(2)
-        except Exception as e:
-            return
+        SpiderPromoterInfo.r.delete('promoter_id_list')
+        for promoter in promoter_result:
+            promoter_id = promoter[0]
+            SpiderPromoterInfo.r.sadd('promoter_id_list', promoter_id)
+        set_data = SpiderPromoterInfo.r.smembers('promoter_id_list')
+        for promoter_id in set_data:
+            promoter_id = promoter_id.decode('utf-8')
+            SpiderPromoterInfo.get_promoter(promoter_id)
+            SpiderPromoterInfo.r.srem('promoter_id_list', promoter_id)
+            time.sleep(1)
 
 
 if __name__ == '__main__':

+ 44 - 47
spider/SpiderWeekPromoterInfo.py

@@ -7,6 +7,7 @@
 # @contact: renyupeng@c-top.com.cn
 # @Tel 1501435553
 import json
+import threading
 import time
 import random
 from multiprocessing import Process
@@ -32,71 +33,67 @@ class SpiderWeekPromoterInfo:
         if self.r.zcard('kuaishou_shop_token') == 0:
             cookie_update.get_coookie_to_redis()
         else:
-            try:
-                random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
-                random_item = random.choice(random_member)
-                cookie = random_item[0].decode('utf-8')
-                phone_num = int(random_item[1])
-                roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
+            random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+            random_item = random.choice(random_member)
+            cookie = random_item[0].decode('utf-8')
+            phone_num = int(random_item[1])
+            roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
 
-                if json.loads(roll).__contains__("result"):
-                    if json.loads(roll)["result"] == 100110000:
-                        cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-                        SendFeiShuMsg.send_cookie_robot_msg(
-                            "cookie 永久失效 轻更新cookie 唯一电话为{phone_num}".format(phone_num=phone_num))
-                        retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
-                        retry_cookie = retry_item[0].decode('utf-8')
-                        cookie_update.update_cookie_handler(phone_num)
-                        # PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
-                        # PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
-                        # PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
-                        # PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
-                        # PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
+            if json.loads(roll).__contains__("result"):
+                SendFeiShuMsg.send_cookie_robot_msg(
+                    "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
+                                                                         code=json.loads(roll)["result"]))
+                cookie_update.update_temporary_cookie_handler(phone_num)
+                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
 
-                    else:
-
-                        SendFeiShuMsg.send_cookie_robot_msg(
-                            "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
-                                                                                 code=json.loads(roll)["result"]))
-                        cookie_update.update_temporary_cookie_handler(phone_num)
-                        cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-
-                        retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
-                        retry_cookie = retry_item[0].decode('utf-8')
-                        # PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
-                        # PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
-                        # PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
-                        # PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
-                        # PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
-
-                else:
-                    # PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
-                    # PromoterFansInfo().PromoterFansInfoHandler(promoterId, cookie)
-                    # PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, cookie)
-                    # PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, cookie)
-                    # PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, cookie)
+                retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
+                retry_cookie = retry_item[0].decode('utf-8')
+                roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+                try:
                     roll = json.loads(roll)
                     fanNum = roll["fanNum"]
                     totalSale = roll["totalSale"]
                     avgVideoSales = roll["avgVideoSales"]
                     videoSales = roll["videoSales"]
                     sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
-                    {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
-                    ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
-                                 promoterId=promoterId)
+                        {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
+                        ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
+                                     videoSales=videoSales,
+                                     promoterId=promoterId)
                     MysqlProUtils().Operate(sql=sql)
                     cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-            except Exception as e:
-                return
+                except Exception as e:
+                    print(e)
+                    self.r.sadd('promoter_week_err_info', str(promoterId))
+
+            else:
+
+                roll = json.loads(roll)
+                fanNum = roll["fanNum"]
+                totalSale = roll["totalSale"]
+                avgVideoSales = roll["avgVideoSales"]
+                videoSales = roll["videoSales"]
+                sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
+                {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
+                ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
+                             promoterId=promoterId)
+                MysqlProUtils().Operate(sql=sql)
+                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
 
     def get_promoter_info(self):
         sql = """
         select promoter_id from ruixuan.promoter_info  where promoter_id!=0
         """
         promoter_result = self.conn.QueryAll(sql)
+        self.r.delete('promoter_week_id_list')
         for promoter in promoter_result:
-            promoter_id = int(promoter[0].decode('utf-8'))
+            promoter_id = promoter[0]
+            self.r.sadd('promoter_week_id_list', promoter_id)
+        set_data = self.r.smembers('promoter_week_id_list')
+        for promoter_id in set_data:
+            promoter_id = promoter_id.decode('utf-8')
             self.get_promoter(promoter_id)
+            self.r.srem('promoter_week_id_list', promoter_id)
             time.sleep(1)