renyupeng 1 anno fa
parent
commit
7b72d900e5

+ 110 - 0
spider/SpiderPromoterHeaderMgs.py

@@ -0,0 +1,110 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/3/23 2:49 下午
+# @Site    : 
+# @File    : SpiderPromoterHeader.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import json
+import random
+import time
+
+import redis
+import requests
+
+from utils.cookie_update import cookie_update
+from utils.mysql_utils_pro import MysqlProUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class SpiderPromoterHeader:
+    def __init__(self):
+        self.conn = MysqlProUtils()
+
+    @staticmethod
+    def GetPromoterInfo(promoter_id):
+        r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
+        listItem = []
+        url = 'https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/info?promoterId={promoterId}&type=1'.format(
+            promoterId=promoter_id)
+
+        if r.zcard('kuaishou_shop_click_token') == 0:
+            cookie_update.get_click_coookie_to_redis()
+        else:
+            random_member = r.zrange('kuaishou_shop_click_token', 0, -1, withscores=True)
+            random_item = random.choice(random_member)
+            cookie = random_item[0].decode('utf-8')
+            phone_num = int(random_item[1])
+
+            headers = {'User-Agent': 'Mozilla/5.0',
+                       'Cookie': cookie}
+
+            try:
+                rep = requests.get(url=url, headers=headers)
+                print(rep.text, '-----rep.txt---------')
+                # if json.loads(rep.text)["result"] != 1 and json.loads(rep.text)["result"] != 100110000:
+                #     cookie_update.update_temporary_cookie_handler(phone_num)
+                #     retry_cookie = cookie_update.get_cookie_handler()[0]
+                #     headers = {'User-Agent': 'Mozilla/5.0',
+                #                'Cookie': retry_cookie}
+                #     rep = requests.get(url=url, headers=headers)
+                #     data = json.loads(rep.text)["data"]
+                #     item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                #             "promoter_name": data["promoterNickName"],
+                #             "promoter_id": promoter_id}
+                #     listItem.append(item)
+                #     return listItem
+                # elif json.loads(rep.text)["result"] == 100110000:
+                #     cookie_update.update_cookie_handler(phone_num)
+                #     return
+                if json.loads(rep.text)["result"] == 1:
+                    data = json.loads(rep.text)["data"]
+                    print(data)
+                    item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                            "promoter_name": data["promoterNickName"],
+                            "promoter_id": promoter_id}
+                    listItem.append(item)
+                    return listItem
+                else:
+                    return
+            except Exception as e:
+                SendFeiShuMsg.send_robot_msg(
+                    'GetPromoterInfo {promoterId}请求错误请检查cookie{e}'.format(promoterId=promoter_id, e=e))
+
+    def GetPromoterID(self):
+        try:
+            sql = """select a.promoter_id from miaogousi.promoter_oerder_amount a
+    left join miaogousi.promoter_info b
+    on a.promoter_id=b.promoter_id
+    where b.promoter_id is null and a.`30day_gmv` is not null
+    group by a.promoter_id"""
+            promoterIdList = self.conn.QueryAll(sql)
+            return promoterIdList
+        except Exception as e:
+            print(e)
+
+    def PromoterInsert(self):
+        promoterIdList = self.GetPromoterID()
+        try:
+            for promoterId in promoterIdList:
+                print(promoterId)
+                try:
+                    listItem = SpiderPromoterHeader.GetPromoterInfo(int(promoterId[0]))
+                    time.sleep(60)
+                    if listItem is not None:
+                        if listItem.__len__() > 0:
+                            for item in listItem:
+                                try:
+                                    self.conn.insert(item)
+                                except Exception as e:
+                                    print('插入错误', item)
+                except Exception as e:
+                    print(e)
+
+        except Exception as e:
+            print('获取达人详情错误', e)
+
+
+if __name__ == '__main__':
+    SpiderPromoterHeader().PromoterInsert()

+ 110 - 0
spider/SpiderPromoterHeaderRocket.py

@@ -0,0 +1,110 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/3/23 2:49 下午
+# @Site    : 
+# @File    : SpiderPromoterHeader.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import json
+import random
+import time
+
+import redis
+import requests
+
+from utils.cookie_update import cookie_update
+from utils.mysql_utils_pro import MysqlProUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class SpiderPromoterHeader:
+    def __init__(self):
+        self.conn = MysqlProUtils()
+
+    @staticmethod
+    def GetPromoterInfo(promoter_id):
+        r = redis.Redis(host='192.168.0.99', password='', port=6379, db=0)
+        listItem = []
+        url = 'https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/info?promoterId={promoterId}&type=1'.format(
+            promoterId=promoter_id)
+
+        if r.zcard('kuaishou_shop_click_token') == 0:
+            cookie_update.get_click_coookie_to_redis()
+        else:
+            random_member = r.zrange('kuaishou_shop_click_token', 0, -1, withscores=True)
+            random_item = random.choice(random_member)
+            cookie = random_item[0].decode('utf-8')
+            phone_num = int(random_item[1])
+
+            headers = {'User-Agent': 'Mozilla/5.0',
+                       'Cookie': cookie}
+
+            try:
+                rep = requests.get(url=url, headers=headers)
+                print(rep.text, '-----rep.txt---------')
+                # if json.loads(rep.text)["result"] != 1 and json.loads(rep.text)["result"] != 100110000:
+                #     cookie_update.update_temporary_cookie_handler(phone_num)
+                #     retry_cookie = cookie_update.get_cookie_handler()[0]
+                #     headers = {'User-Agent': 'Mozilla/5.0',
+                #                'Cookie': retry_cookie}
+                #     rep = requests.get(url=url, headers=headers)
+                #     data = json.loads(rep.text)["data"]
+                #     item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                #             "promoter_name": data["promoterNickName"],
+                #             "promoter_id": promoter_id}
+                #     listItem.append(item)
+                #     return listItem
+                # elif json.loads(rep.text)["result"] == 100110000:
+                #     cookie_update.update_cookie_handler(phone_num)
+                #     return
+                if json.loads(rep.text)["result"] == 1:
+                    data = json.loads(rep.text)["data"]
+                    print(data)
+                    item = {"fans": data["fanNum"], "handurl": data["promoterHeadImgUrl"],
+                            "promoter_name": data["promoterNickName"],
+                            "promoter_id": promoter_id}
+                    listItem.append(item)
+                    return listItem
+                else:
+                    return
+            except Exception as e:
+                SendFeiShuMsg.send_robot_msg(
+                    'GetPromoterInfo {promoterId}请求错误请检查cookie{e}'.format(promoterId=promoter_id, e=e))
+
+    def GetPromoterID(self):
+        try:
+            sql = """select a.promoter_id from rocket.promoter_oerder_amount a
+    left join rocket.promoter_info b
+    on a.promoter_id=b.promoter_id
+    where b.promoter_id is null and a.`30day_gmv` is not null
+    group by a.promoter_id"""
+            promoterIdList = self.conn.QueryAll(sql)
+            return promoterIdList
+        except Exception as e:
+            print(e)
+
+    def PromoterInsert(self):
+        promoterIdList = self.GetPromoterID()
+        try:
+            for promoterId in promoterIdList:
+                print(promoterId)
+                try:
+                    listItem = SpiderPromoterHeader.GetPromoterInfo(int(promoterId[0]))
+                    time.sleep(60)
+                    if listItem is not None:
+                        if listItem.__len__() > 0:
+                            for item in listItem:
+                                try:
+                                    self.conn.insert(item)
+                                except Exception as e:
+                                    print('插入错误', item)
+                except Exception as e:
+                    print(e)
+
+        except Exception as e:
+            print('获取达人详情错误', e)
+
+
+if __name__ == '__main__':
+    SpiderPromoterHeader().PromoterInsert()

+ 47 - 3
utils/mysql_utils_pro.py

@@ -140,7 +140,7 @@ class MysqlProUtils(object):
 
     @staticmethod
     def insert(item):
-        print(item,'---------')
+        print(item, '---------')
         con = pymysql.connect(host=ConfConstant.TIDB_PRO_HOST, user=ConfConstant.TIDB_PRO_USER,
                               passwd=ConfConstant.TIDB_PRO_PASSWORD, charset='utf8', port=3390)
         async_item = tuple(item.values())
@@ -148,7 +148,29 @@ class MysqlProUtils(object):
         placeholders = ', '.join(['%s'] * len(item))
         columns = ', '.join(item.keys())
         insert_sql = "REPLACE INTO ruixuan.promoter_info ( %s ) VALUES ( %s )" % (
-             columns, placeholders)
+            columns, placeholders)
+        print(insert_sql)
+
+        cur = con.cursor()
+        try:
+            cur.execute(insert_sql, async_item)
+            con.commit()
+        except Exception as e:
+            SendFeiShuMsg.send_robot_msg('数据插入错误{e}'.format(e=e))
+            con.rollback()
+        con.close()
+
+    @staticmethod
+    def MgsInsert(item):
+        print(item, '---------')
+        con = pymysql.connect(host=ConfConstant.TIDB_PRO_HOST, user=ConfConstant.TIDB_PRO_USER,
+                              passwd=ConfConstant.TIDB_PRO_PASSWORD, charset='utf8', port=3390)
+        async_item = tuple(item.values())
+        # 对数据库进行插入操作,并不需要commit,twisted会自动commit
+        placeholders = ', '.join(['%s'] * len(item))
+        columns = ', '.join(item.keys())
+        insert_sql = "REPLACE INTO miaogousi.promoter_info ( %s ) VALUES ( %s )" % (
+            columns, placeholders)
         print(insert_sql)
 
         cur = con.cursor()
@@ -156,6 +178,28 @@ class MysqlProUtils(object):
             cur.execute(insert_sql, async_item)
             con.commit()
         except Exception as e:
-            SendFeiShuMsg.send_robot_msg('{table_name}数据插入错误{e}'.format(table_name=table_name, e=e))
+            SendFeiShuMsg.send_robot_msg('数据插入错误{e}'.format(e=e))
             con.rollback()
         con.close()
+
+    @staticmethod
+    def RocketInsert(item):
+        print(item, '---------')
+        con = pymysql.connect(host=ConfConstant.TIDB_PRO_HOST, user=ConfConstant.TIDB_PRO_USER,
+                              passwd=ConfConstant.TIDB_PRO_PASSWORD, charset='utf8', port=3390)
+        async_item = tuple(item.values())
+        # 对数据库进行插入操作,并不需要commit,twisted会自动commit
+        placeholders = ', '.join(['%s'] * len(item))
+        columns = ', '.join(item.keys())
+        insert_sql = "REPLACE INTO rocket.promoter_info ( %s ) VALUES ( %s )" % (
+            columns, placeholders)
+        print(insert_sql)
+
+        cur = con.cursor()
+        try:
+            cur.execute(insert_sql, async_item)
+            con.commit()
+        except Exception as e:
+            SendFeiShuMsg.send_robot_msg('数据插入错误{e}'.format(e=e))
+            con.rollback()
+        con.close()