renyupeng před 9 měsíci
rodič
revize
d840e952d4

+ 1 - 1
ETL/CanalManagement.py

@@ -123,6 +123,6 @@ and b.stat_date is not null
 
 
 if __name__ == '__main__':
-    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan'}
+    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan','yuxiu'}
     for database in databases:
         CanalManagementThread().KuaishouRuleOrderDataHandler(database=database)

+ 1 - 1
ETL/ItermOrderAmountThread.py

@@ -66,6 +66,6 @@ on a.item_id=b.item_id
 
 
 if __name__ == '__main__':
-    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan'}
+    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan','yuxiu'}
     for database in databases:
         ItermOrderAmountThread().KuaishouRuleOrderDataHandler(database=database)

+ 1 - 1
ETL/KuaishouRuleOrderDataTherad.py

@@ -109,7 +109,7 @@ group by promoter_id
 
 
 if __name__ == '__main__':
-    databases = {'ruixuan', 'yufu', 'miaogousi', 'rocket'}
+    databases = {'ruixuan', 'yufu', 'miaogousi', 'rocket','yuxiu'}
     for database in databases:
         KuaishouRuleOrderData().KuaishouRuleOrderDataHandler(database=database)
 

+ 1 - 1
ETL/PromoterItemOrderCountThread.py

@@ -119,6 +119,6 @@ where
 
 
 if __name__ == '__main__':
-    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan'}
+    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan','yuxiu'}
     for database in databases:
         CanalManagementThread().KuaishouRuleOrderDataHandler(database=database)

+ 1 - 1
ETL/PromoterItermOrderAmountThread.py

@@ -66,6 +66,6 @@ on a.item_id=b.item_id and a.promoter_id=b.promoter_id
 
 
 if __name__ == '__main__':
-    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan'}
+    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan','yuxiu'}
     for database in databases:
         PromoterItermOrderAmountThread().KuaishouRuleOrderDataHandler(database=database)

+ 1 - 1
ETL/PromoterOrderAmountThread.py

@@ -66,6 +66,6 @@ on a.promoter_id=b.promoter_id;
 
 
 if __name__ == '__main__':
-    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan'}
+    databases = {'rocket', 'miaogousi', 'yufu', 'ruixuan','yuxiu'}
     for database in databases:
         PromoterOrderAmountThread().KuaishouRuleOrderDataHandler(database=database)

+ 111 - 0
spider/SpiderPromoterHeaderYuXiu.py

@@ -0,0 +1,111 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/3/23 2:49 下午
+# @Site    : 
+# @File    : SpiderPromoterHeaderYuXiu.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import json
+import random
+import time
+
+import redis
+import requests
+
+from utils.cookie_update import cookie_update
+from utils.mysql_utils_pro import MysqlProUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class SpiderPromoterHeaderYuXiu:
+    def __init__(self):
+        self.conn = MysqlProUtils()
+
+    @staticmethod
+    def GetPromoterInfo(promoter_id):
+        r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
+        listItem = []
+        url = 'https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/info?promoterId={promoterId}&type=1'.format(
+            promoterId=promoter_id)
+
+        if r.zcard('kuaishou_shop_click_token') == 0:
+            cookie_update.get_click_coookie_to_redis()
+        else:
+            random_member = r.zrange('kuaishou_shop_click_token', 0, -1, withscores=True)
+            random_item = random.choice(random_member)
+            # cookie = random_item[0].decode('utf-8')
+            cookie = cookie_update.get_cookie_handler()[0]
+            phone_num = int(random_item[1])
+
+            headers = {'User-Agent': 'Mozilla/5.0',
+                       'Cookie': cookie}
+
+            try:
+                rep = requests.get(url=url, headers=headers)
+                print(rep.text, '-----rep.txt---------')
+                # if json.loads(rep.text)["result"] != 1 and json.loads(rep.text)["result"] != 100110000:
+                #     cookie_update.update_temporary_cookie_handler(phone_num)
+                #     retry_cookie = cookie_update.get_cookie_handler()[0]
+                #     headers = {'User-Agent': 'Mozilla/5.0',
+                #                'Cookie': retry_cookie}
+                #     rep = requests.get(url=url, headers=headers)
+                #     data = json.loads(rep.text)["data"]
+                #     item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                #             "promoter_name": data["promoterNickName"],
+                #             "promoter_id": promoter_id}
+                #     listItem.append(item)
+                #     return listItem
+                # elif json.loads(rep.text)["result"] == 100110000:
+                #     cookie_update.update_cookie_handler(phone_num)
+                #     return
+                if json.loads(rep.text)["result"] == 1:
+                    data = json.loads(rep.text)["data"]
+                    print(data)
+                    item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                            "promoter_name": data["promoterNickName"],
+                            "promoter_id": promoter_id}
+                    listItem.append(item)
+                    return listItem
+                else:
+                    return
+            except Exception as e:
+                SendFeiShuMsg.send_robot_msg(
+                    'GetPromoterInfo {promoterId}请求错误请检查cookie{e}'.format(promoterId=promoter_id, e=e))
+
+    def GetPromoterID(self):
+        try:
+            sql = """select a.promoter_id from yuxiu.promoter_oerder_amount a
+    left join yuxiu.promoter_info b
+    on a.promoter_id=b.promoter_id
+    where b.promoter_id is null and a.`30day_gmv` is not null
+    group by a.promoter_id"""
+            promoterIdList = self.conn.QueryAll(sql)
+            return promoterIdList
+        except Exception as e:
+            print(e)
+
+    def PromoterInsert(self):
+        promoterIdList = self.GetPromoterID()
+        try:
+            for promoterId in promoterIdList:
+                print(promoterId)
+                try:
+                    listItem = SpiderPromoterHeaderYuXiu.GetPromoterInfo(int(promoterId[0]))
+                    time.sleep(60)
+                    if listItem is not None:
+                        if listItem.__len__() > 0:
+                            for item in listItem:
+                                try:
+                                    self.conn.yuxiuInsert(item)
+                                except Exception as e:
+                                    print('插入错误', item)
+                except Exception as e:
+                    print(e)
+
+        except Exception as e:
+            print('获取达人详情错误', e)
+
+
+if __name__ == '__main__':
+    SpiderPromoterHeaderYuXiu().PromoterInsert()

+ 111 - 0
spider/SpiderPromoterHeaderYufu.py

@@ -0,0 +1,111 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/3/23 2:49 下午
+# @Site    : 
+# @File    : SpiderPromoterHeaderYufu.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import json
+import random
+import time
+
+import redis
+import requests
+
+from utils.cookie_update import cookie_update
+from utils.mysql_utils_pro import MysqlProUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class SpiderPromoterHeaderYufu:
+    def __init__(self):
+        self.conn = MysqlProUtils()
+
+    @staticmethod
+    def GetPromoterInfo(promoter_id):
+        r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
+        listItem = []
+        url = 'https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/info?promoterId={promoterId}&type=1'.format(
+            promoterId=promoter_id)
+
+        if r.zcard('kuaishou_shop_click_token') == 0:
+            cookie_update.get_click_coookie_to_redis()
+        else:
+            random_member = r.zrange('kuaishou_shop_click_token', 0, -1, withscores=True)
+            random_item = random.choice(random_member)
+            # cookie = random_item[0].decode('utf-8')
+            cookie = cookie_update.get_cookie_handler()[0]
+            phone_num = int(random_item[1])
+
+            headers = {'User-Agent': 'Mozilla/5.0',
+                       'Cookie': cookie}
+
+            try:
+                rep = requests.get(url=url, headers=headers)
+                print(rep.text, '-----rep.txt---------')
+                # if json.loads(rep.text)["result"] != 1 and json.loads(rep.text)["result"] != 100110000:
+                #     cookie_update.update_temporary_cookie_handler(phone_num)
+                #     retry_cookie = cookie_update.get_cookie_handler()[0]
+                #     headers = {'User-Agent': 'Mozilla/5.0',
+                #                'Cookie': retry_cookie}
+                #     rep = requests.get(url=url, headers=headers)
+                #     data = json.loads(rep.text)["data"]
+                #     item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                #             "promoter_name": data["promoterNickName"],
+                #             "promoter_id": promoter_id}
+                #     listItem.append(item)
+                #     return listItem
+                # elif json.loads(rep.text)["result"] == 100110000:
+                #     cookie_update.update_cookie_handler(phone_num)
+                #     return
+                if json.loads(rep.text)["result"] == 1:
+                    data = json.loads(rep.text)["data"]
+                    print(data)
+                    item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                            "promoter_name": data["promoterNickName"],
+                            "promoter_id": promoter_id}
+                    listItem.append(item)
+                    return listItem
+                else:
+                    return
+            except Exception as e:
+                SendFeiShuMsg.send_robot_msg(
+                    'GetPromoterInfo {promoterId}请求错误请检查cookie{e}'.format(promoterId=promoter_id, e=e))
+
+    def GetPromoterID(self):
+        try:
+            sql = """select a.promoter_id from yufu.promoter_oerder_amount a
+    left join yufu.promoter_info b
+    on a.promoter_id=b.promoter_id
+    where b.promoter_id is null and a.`30day_gmv` is not null
+    group by a.promoter_id"""
+            promoterIdList = self.conn.QueryAll(sql)
+            return promoterIdList
+        except Exception as e:
+            print(e)
+
+    def PromoterInsert(self):
+        promoterIdList = self.GetPromoterID()
+        try:
+            for promoterId in promoterIdList:
+                print(promoterId)
+                try:
+                    listItem = SpiderPromoterHeaderYufu.GetPromoterInfo(int(promoterId[0]))
+                    time.sleep(60)
+                    if listItem is not None:
+                        if listItem.__len__() > 0:
+                            for item in listItem:
+                                try:
+                                    self.conn.yufuInsert(item)
+                                except Exception as e:
+                                    print('插入错误', item)
+                except Exception as e:
+                    print(e)
+
+        except Exception as e:
+            print('获取达人详情错误', e)
+
+
+if __name__ == '__main__':
+    SpiderPromoterHeaderYufu().PromoterInsert()

+ 78 - 0
spider/SpiderPromoterInfoYuXiu.py

@@ -0,0 +1,78 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/9/11 下午4:19
+# @Site    :
+# @File    : SpiderPromoterInfoYuXiu.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import json
+import threading
+import time
+import random
+import redis
+from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
+from utils.cookie_update import cookie_update
+from utils.mysql_utils_pro import MysqlProUtils
+
+
+class SpiderPromoterInfoYuXiu:
+    conn = MysqlProUtils()
+
+    @staticmethod
+    def get_promoter(promoterId):
+        print(promoterId, '22222')
+        cookie = cookie_update.get_cookie_handler()[0]
+        roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
+        print(roll, '------')
+
+        if json.loads(roll).__contains__("result"):
+            retry_cookie = cookie_update.get_cookie_handler()[0]
+            roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
+            try:
+                roll = json.loads(roll)
+                fanNum = roll["fanNum"]
+                totalSale = roll["totalSale"]
+                avgVideoSales = roll["avgVideoSales"]
+                videoSales = roll["videoSales"]
+                sql = """ replace into yuxiu.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
+                        {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
+                        ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
+                                     videoSales=videoSales,
+                                     promoterId=promoterId)
+                MysqlProUtils().Operate(sql=sql)
+            except Exception as e:
+                print(e)
+
+        else:
+
+            roll = json.loads(roll)
+            print(roll, '--roll--', promoterId)
+            fanNum = roll["fanNum"]
+            totalSale = roll["totalSale"]
+            avgVideoSales = roll["avgVideoSales"]
+            videoSales = roll["videoSales"]
+            sql = """ replace into yufu.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
+                {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
+                ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
+                             promoterId=promoterId)
+            MysqlProUtils().Operate(sql=sql)
+
+    @staticmethod
+    def get_promoter_info():
+        sql = """
+   select a.promoter_id from yuxiu.kuaishou_promoter  a
+   left join yuxiu.promoter_sales_info b
+   on a.promoter_id=b.promoter_id
+   where  a.media_id=2
+     and a.create_time>=date_format(date_sub(now(),interval 10 day),'%Y-%m-%d')
+     """
+        promoter_result = SpiderPromoterInfoYuXiu.conn.QueryAll(sql)
+        for promoter in promoter_result:
+            promoter_id = promoter[0].decode('utf-8')
+            SpiderPromoterInfoYuXiu.get_promoter(promoter_id)
+            time.sleep(30)
+
+
+if __name__ == '__main__':
+    SpiderPromoterInfoYuXiu.get_promoter_info()