renyupeng 11 月之前
父節點
當前提交
9cddbb4048
共有 1 個文件被更改,包括 37 次插入57 次删除
  1. 37 57
      spider/SpiderPromoterInfo.py

+ 37 - 57
spider/SpiderPromoterInfo.py

@@ -1,7 +1,7 @@
 # Author renyupeng
 # coding=utf-8
 # @Time    : 2023/9/11 下午4:19
-# @Site    : 
+# @Site    :
 # @File    : SpiderPromoterInfo.py
 # @Software: PyCharm
 # @contact: renyupeng@c-top.com.cn
@@ -14,86 +14,66 @@ import redis
 from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
 from utils.cookie_update import cookie_update
 from utils.mysql_utils_pro import MysqlProUtils
-from utils.send_feishu_msg import SendFeiShuMsg
 
 
 class SpiderPromoterInfo:
-    r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
     conn = MysqlProUtils()
 
     @staticmethod
     def get_promoter(promoterId):
         print(promoterId, '22222')
-        if SpiderPromoterInfo.r.zcard('kuaishou_shop_token') == 0:
-            cookie_update.get_coookie_to_redis()
-        else:
-            random_member = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
-            random_item = random.choice(random_member)
-            # cookie = random_item[0].decode('utf-8')
-            cookie = cookie_update.get_cookie_handler()
-            phone_num = int(random_item[1])
-            roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
-
-            if json.loads(roll).__contains__("result"):
-                SendFeiShuMsg.send_cookie_robot_msg(
-                    "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
-                                                                         code=json.loads(roll)["result"]))
-                cookie_update.update_temporary_cookie_handler(phone_num)
-                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-
-                retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
-                retry_cookie = retry_item[0].decode('utf-8')
-                roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
-                try:
-                    roll = json.loads(roll)
-                    fanNum = roll["fanNum"]
-                    totalSale = roll["totalSale"]
-                    avgVideoSales = roll["avgVideoSales"]
-                    videoSales = roll["videoSales"]
-                    sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
-                        {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
-                        ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
-                                     videoSales=videoSales,
-                                     promoterId=promoterId)
-                    MysqlProUtils().Operate(sql=sql)
-                    cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
-                except Exception as e:
-                    print(e)
-                    SpiderPromoterInfo.r.sadd('promoter_err_info', str(promoterId))
-
-            else:
+        cookie = cookie_update.get_cookie_handler()
+        roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
+        print(roll, '------')
 
+        if json.loads(roll).__contains__("result"):
+            retry_cookie = cookie_update.get_cookie_handler()
+            roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+            try:
                 roll = json.loads(roll)
-                print(roll, '--roll--',promoterId)
                 fanNum = roll["fanNum"]
                 totalSale = roll["totalSale"]
                 avgVideoSales = roll["avgVideoSales"]
                 videoSales = roll["videoSales"]
                 sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
+                        {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
+                        ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
+                                     videoSales=videoSales,
+                                     promoterId=promoterId)
+                MysqlProUtils().Operate(sql=sql)
+            except Exception as e:
+                print(e)
+
+        else:
+
+            roll = json.loads(roll)
+            print(roll, '--roll--', promoterId)
+            fanNum = roll["fanNum"]
+            totalSale = roll["totalSale"]
+            avgVideoSales = roll["avgVideoSales"]
+            videoSales = roll["videoSales"]
+            sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
                 {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
                 ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
                              promoterId=promoterId)
-                MysqlProUtils().Operate(sql=sql)
-                cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
+            MysqlProUtils().Operate(sql=sql)
 
     @staticmethod
     def get_promoter_info():
         sql = """
-      select promoter_id from ruixuan.kuaishou_promoter where media_id =2 
-      group by promoter_id
- """
+   select a.promoter_id from ruixuan.kuaishou_promoter  a
+   left join ruixuan.promoter_sales_info b
+   on a.promoter_id=b.promoter_id
+   where b.promoter_id is null
+         and a.media_id=2
+     and a.create_time>=date_format(date_sub(now(),interval 10 day),'%Y-%m-%d')
+     """
         promoter_result = SpiderPromoterInfo.conn.QueryAll(sql)
-        SpiderPromoterInfo.r.delete('promoter_id_list')
         for promoter in promoter_result:
-            promoter_id = promoter[0]
-            SpiderPromoterInfo.r.sadd('promoter_id_list', promoter_id)
-        set_data = SpiderPromoterInfo.r.smembers('promoter_id_list')
-        for promoter_id in set_data:
-            promoter_id = promoter_id.decode('utf-8')
+            promoter_id = promoter[0].decode('utf-8')
             SpiderPromoterInfo.get_promoter(promoter_id)
-            SpiderPromoterInfo.r.srem('promoter_id_list', promoter_id)
-            time.sleep(3)
-        SpiderPromoterInfo.r.close()
+            time.sleep(30)
+
 
 if __name__ == '__main__':
-    SpiderPromoterInfo().get_promoter_info()
+    SpiderPromoterInfo.get_promoter_info()