renyupeng 2 vuotta sitten
commit
c43503f9b0

+ 30 - 0
config.ini

@@ -0,0 +1,30 @@
+[xxl_job]
+xxl_url = 192.168.1.187
+
+[tidb]
+tidb_host = 192.168.0.184
+tidb_port = 3390
+tidb_user = hcst
+tidb_password = hcst@2021
+tidb_db = kwai_promoter
+
+
+
+[tidb_pro]
+tidb_host = 139.186.27.96
+tidb_port = 3390
+tidb_user = data
+tidb_password = hcst@2021
+tidb_db = ruixuan
+
+
+[webhook]
+url = 192.168.0.203
+
+[log]
+Log_home=/data/PromoterInfo
+LOG_LEVEL = INFO
+
+
+
+

+ 48 - 0
constant/ConfConstant.py

@@ -0,0 +1,48 @@
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2021/9/15 11:14 上午
+@Site    :
+@File    : ConfConstant.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+
+import os
+from configparser import ConfigParser
+dir_name = os.path.dirname(os.path.dirname(__file__))
+config_file = dir_name + '/config.ini'
+_cfp = ConfigParser()
+_cfp.read(config_file, encoding='utf-8')
+
+
+class ConfConstant:
+    """
+    配置文件常量类
+    """
+    # tidb 配置信息
+    TIDB_HOST = _cfp.get('tidb', 'tidb_host')
+    TIDB_PORT = _cfp.getint('tidb', 'tidb_port')
+    TIDB_USER = _cfp.get('tidb', 'tidb_user')
+    TIDB_PASSWORD = _cfp.get('tidb', 'tidb_password')
+    TIDB_DB = _cfp.get('tidb', 'tidb_db')
+
+
+    # tidb 配置信息
+    TIDB_PRO_HOST = _cfp.get('tidb_pro', 'tidb_host')
+    TIDB_PRO_PORT = _cfp.getint('tidb_pro', 'tidb_port')
+    TIDB_PRO_USER = _cfp.get('tidb_pro', 'tidb_user')
+    TIDB_PRO_PASSWORD = _cfp.get('tidb_pro', 'tidb_password')
+    TIDB_PRO_DB = _cfp.get('tidb_pro', 'tidb_db')
+
+
+    # ### log
+    # # 日志存放路径
+    Log_home = _cfp.get('log', 'Log_home')
+    # # 日志等级
+    LOG_LEVEL = _cfp.get('log', 'log_level')
+
+    # webhook配置
+    URL = _cfp.get('webhook', 'url')
+

+ 122 - 0
spider/PromoterFansInfo.py

@@ -0,0 +1,122 @@
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2023/2/9 1:21 下午
+@Site    :
+@File    : PromoterFansInfo.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+import json
+
+import requests
+
+from utils.mysql_helper import insert, batch_insert
+from utils.mysql_utils import MysqlUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class PromoterFansInfo:
+    def __init__(self):
+        self.conn = MysqlUtils()
+        self.list = [1, 2, 3]
+
+    def PromoterFansInfoHandler(self, promoterId):
+        sql = "select cookie from ruixuan.kuaishou_supply_chain_cookie"
+        cookie = self.conn.QueryOne(sql)[0]
+        headers = {'User-Agent': 'Mozilla/5.0',
+                   'Cookie': cookie}
+        try:
+            for i in self.list:
+                url = 'https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/fans/info?' \
+                      'promoterId={promoterId}&timeRangeType={timeRangeType}'.format(promoterId=promoterId,
+                                                                                     timeRangeType=i)
+                rep = requests.get(url=url, headers=headers)
+                table_name = 'kwai_promoter_fans_info'
+                data = json.loads(rep.text)["data"]
+
+                promoter_fans_item = {"fansAgeFeature": json.dumps(data["fansAgeFeature"]),
+                                      "fansCityFeature": json.dumps(data["fansCityFeature"]),
+                                      "fansCityLevelFeature": json.dumps(data["fansCityLevelFeature"]),
+                                      "fansEquipBrandFeature": json.dumps(data["fansEquipBrandFeature"]),
+                                      "fansEquipPriceFeature": json.dumps(data["fansEquipPriceFeature"]),
+                                      "fansGenderFeature": json.dumps(data["fansGenderFeature"]),
+                                      "fansNumTrend": json.dumps(data["fansNumTrend"]),
+                                      "fansProvinceFeature": json.dumps(data["fansProvinceFeature"]),
+                                      "promoteAvgPrice": json.dumps(data["promoteAvgPrice"]),
+                                      "promoterId": promoterId,
+                                      "timeRangeType": i}
+                insert(table_name=table_name, item=promoter_fans_item)
+
+                fans_type_date_list = []
+                fans_base_table = 'kwai_promoter_fans_base_info'
+
+                fansProvinceFeatureList = tuple(data["fansProvinceFeature"])
+                for fansProvinceFeature in fansProvinceFeatureList:
+                    fansProvinceFeatureItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'Province',
+                                               "type_date": fansProvinceFeature["provinceFeature"],
+                                               "FeatureRate": fansProvinceFeature["provinceFeatureRate"]}
+                    fans_type_date_list.append(fansProvinceFeatureItem)
+                fansAgeFeatureList = tuple(data["fansAgeFeature"])
+                for fansAgeFeature in fansAgeFeatureList:
+                    fansAgeFeatureItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'Age',
+                                          "type_date": fansAgeFeature["ageFeature"],
+                                          "FeatureRate": fansAgeFeature["ageFeatureRate"]}
+                    fans_type_date_list.append(fansAgeFeatureItem)
+
+                fansCityFeatureList = tuple(data["fansCityFeature"])
+                for fansCityFeature in fansCityFeatureList:
+                    fansCityFeatureItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'City',
+                                           "type_date": fansCityFeature["cityFeature"],
+                                           "FeatureRate": fansCityFeature["cityFeatureRate"]}
+                    fans_type_date_list.append(fansCityFeatureItem)
+                fansCityLevelFeatureList = tuple(data["fansCityLevelFeature"])
+                for fansCityLevelFeature in fansCityLevelFeatureList:
+                    fansCityLevelFeatureItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'CityLevel',
+                                                "type_date": fansCityLevelFeature["cityLevelFeature"],
+                                                "FeatureRate": fansCityLevelFeature["cityLevelRate"]}
+                    fans_type_date_list.append(fansCityLevelFeatureItem)
+                fansEquipBrandFeatureList = tuple(data["fansEquipBrandFeature"])
+                for fansEquipBrandFeature in fansEquipBrandFeatureList:
+                    fansEquipBrandFeatureItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'EquipBrand',
+                                                 "type_date": fansEquipBrandFeature["equipBrandTitleFeature"],
+                                                 "FeatureRate": fansEquipBrandFeature["equipBrandFeatureRate"]}
+                    fans_type_date_list.append(fansEquipBrandFeatureItem)
+                fansEquipPriceFeatureList = tuple(data["fansEquipPriceFeature"])
+                for fansEquipPriceFeature in fansEquipPriceFeatureList:
+                    fansEquipPriceFeatureItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'EquipPrice',
+                                                 "type_date": fansEquipPriceFeature["equipPriceFeature"],
+                                                 "FeatureRate": fansEquipPriceFeature["equipPriceFeatureRate"]}
+                    fans_type_date_list.append(fansEquipPriceFeatureItem)
+                promoteAvgPriceList = tuple(data["promoteAvgPrice"])
+                for promoteAvgPrice in promoteAvgPriceList:
+                    promoteAvgPriceItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'promoteAvgPrice',
+                                           "type_date": promoteAvgPrice["avgPriceFeature"],
+                                           "FeatureRate": promoteAvgPrice["avgPriceFeatureRate"]}
+                    fans_type_date_list.append(promoteAvgPriceItem)
+                fansGenderFeatureList = tuple(data["fansGenderFeature"])
+                for fansGenderFeature in fansGenderFeatureList:
+                    fansGenderFeatureItem = {"promoterId": promoterId, "timeRangeType": i, "type": 'Gender',
+                                             "type_date": fansGenderFeature["genderFeature"],
+                                             "FeatureRate": fansGenderFeature["genderFeatureRate"]}
+                    fans_type_date_list.append(fansGenderFeatureItem)
+                batch_insert(table_name=fans_base_table, item_list=fans_type_date_list)
+                fans_trend_list = []
+                if i == 3:
+                    fans_trend_table = 'kwai_promoter_fans_trend_info'
+                    fansNumTrendList = tuple(data["fansNumTrend"])
+                    for fansNumTrend in fansNumTrendList:
+                        fansNumTrendItem = {"promoterId": promoterId,
+                                            "date": fansNumTrend["date"],
+                                            "fansTotalAmount": fansNumTrend["fansTotalAmount"],
+                                            "fansIncreaseAmount": fansNumTrend["fansIncreaseAmount"]}
+                        fans_trend_list.append(fansNumTrendItem)
+                    batch_insert(table_name=fans_trend_table, item_list=fans_trend_list)
+
+        except Exception as e:
+            SendFeiShuMsg.send_robot_msg('请求错误请检查cookie{e}'.format(e=e))
+
+
+if __name__ == '__main__':
+    PromoterFansInfo().PromoterFansInfoHandler(1424128656)

+ 96 - 0
spider/PromoterInfoSpider.py

@@ -0,0 +1,96 @@
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2023/2/7 2:34 下午
+@Site    :
+@File    : PromoterInfoSpider.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+import json
+
+import requests
+
+from utils.mysql_helper import insert
+from utils.mysql_utils import MysqlUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class PromoterInfoSpider:
+    def __init__(self):
+        self.conn = MysqlUtils()
+
+    def PromoterInfoSpiderHandler(self, promoterId):
+        url = 'https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/info?promoterId={promoterId}&type=1'.format(
+            promoterId=promoterId)
+        sql = "select cookie from ruixuan.kuaishou_supply_chain_cookie"
+        cookie = self.conn.QueryOne(sql)[0]
+        headers = {'User-Agent': 'Mozilla/5.0',
+                   'Cookie': cookie}
+
+        try:
+            rep = requests.get(url=url, headers=headers)
+            table_name = 'kwai_promoter_info'
+            data = json.loads(rep.text)["data"]
+            promoter_info_item = {"addressinfo": json.dumps(data["addressInfo"]),
+                                  "alreadylookcontact": data["alreadyLookContact"],
+                                  "existEffectInvestment": data["existEffectInvestment"], "fanNum": data["fanNum"],
+                                  "firstLookContact": data["firstLookContact"],
+                                  "hotSaleBrandInfo": json.dumps(data["hotSaleBrandInfo"]),
+                                  "hotSaleChannelInfo": json.dumps(data["hotSaleChannelInfo"]),
+                                  "inviteChannelInfo": json.dumps(data["inviteChannelInfo"]),
+                                  "inviteCommissionRate": data["inviteCommissionRate"],
+                                  "isActivePromoter": data["isActivePromoter"],
+                                  "isAllowedInvite": data["isAllowedInvite"], "lookNumber": data["lookNumber"],
+                                  "phone": data["phone"], "promoteBaseInfo": json.dumps(data["promoteBaseInfo"]),
+                                  "promoterHeadImgUrl": data["promoterHeadImgUrl"], "promoterId": data["promoterId"],
+                                  "promoterInviteFee": data["promoterInviteFee"],
+                                  "promoterNickName": data["promoterNickName"],
+                                  "showContact": data["showContact"], "showContactReason": data["showContactReason"],
+                                  "updateTime": data["updateTime"], "userSex": data["userSex"],
+                                  "weChat": data["weChat"]}
+
+            insert(table_name=table_name, item=promoter_info_item)
+            base_table_name = 'kwai_promoter_base_info'
+            promoter_base_info_item = {"addressinfo": (json.dumps(data["addressInfo"]),),
+                                       "hotSaleBrandInfo": (json.dumps(data["hotSaleBrandInfo"]),),
+                                       "hotSaleChannelInfo": (json.dumps(data["hotSaleChannelInfo"]),),
+                                       "inviteChannelInfo": (json.dumps(data["hotSaleChannelInfo"]),),
+                                       "inviteCommissionRate": data["inviteCommissionRate"],
+                                       "avgLiveVisitorCount": data["promoteBaseInfo"]["avgLiveVisitorCount"],
+                                       "avgLiveVisitorGmv": data["promoteBaseInfo"]["avgLiveVisitorGmv"],
+                                       "avgVideoSales": data["promoteBaseInfo"]["avgVideoSales"],
+                                       "avgVideoViewers": data["promoteBaseInfo"]["avgVideoViewers"],
+                                       "coopStoresNum": data["promoteBaseInfo"]["coopStoresNum"],
+                                       "fansNum": data["promoteBaseInfo"]["fansNum"],
+                                       "liveExperienceMSGap": data["promoteBaseInfo"]["liveExperienceMSGap"],
+                                       "liveStreamCount": data["promoteBaseInfo"]["liveStreamCount"],
+                                       "liveStreamGMV": data["promoteBaseInfo"]["liveStreamGMV"],
+                                       "liveStreamGPM": data["promoteBaseInfo"]["liveStreamGPM"],
+                                       "liveStreamVisitorCount": data["promoteBaseInfo"]["liveStreamVisitorCount"],
+                                       "promoteAvgCustomerPrice": data["promoteBaseInfo"]["promoteAvgCustomerPrice"],
+                                       "promoteAvgPrice": data["promoteBaseInfo"]["promoteAvgPrice"],
+                                       "promoteLiveCount": data["promoteBaseInfo"]["promoteLiveCount"],
+                                       "promoteSaleVolume": data["promoteBaseInfo"]["promoteSaleVolume"],
+                                       "promoteStartTime": data["promoteBaseInfo"]["promoteStartTime"],
+                                       "promotedProductsNum": data["promoteBaseInfo"]["promotedProductsNum"],
+                                       "promoterId": data["promoterId"],
+                                       "totalSale": data["promoteBaseInfo"]["totalSale"],
+                                       "videoGPM": data["promoteBaseInfo"]["videoGPM"],
+                                       "videoNum": data["promoteBaseInfo"]["videoNum"],
+                                       "videoSales": data["promoteBaseInfo"]["videoSales"],
+                                       "videoViews": data["promoteBaseInfo"]["videoViews"],
+                                       "promoterInviteFee": data["promoterInviteFee"],
+                                       "promoterNickName": data["promoterNickName"],
+                                       "showContact": data["showContact"],
+                                       "showContactReason": data["showContactReason"],
+                                       "updateTime": data["updateTime"],
+                                       "userSex": data["userSex"], "weChat": data["userSex"]}
+            insert(table_name=base_table_name, item=promoter_base_info_item)
+            jsons = json.dumps(promoter_info_item)
+            return jsons
+        except Exception as e:
+            SendFeiShuMsg.send_robot_msg('请求错误请检查cookie'.format(e=e))
+
+

+ 107 - 0
spider/PromoterLiveInfoSpider.py

@@ -0,0 +1,107 @@
+# Author renyupeng
+"""
+coding=utf-8
+@Time    : 2023/2/8 10:27 上午
+@Site    :
+@File    : PromoterLiveInfoSpider.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+
+import json
+
+import requests
+from utils.mysql_helper import  insert, batch_insert
+from utils.mysql_utils import MysqlUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class PromoterLiveInfoSpider:
+    def __init__(self):
+        self.conn = MysqlUtils()
+        self.list = [1, 2, 3]
+
+    def PromoterLiveInfoSpiderHander(self, promoterId):
+        sql = "select cookie from ruixuan.kuaishou_supply_chain_cookie"
+        cookie = self.conn.QueryOne(sql)[0]
+        headers = {'User-Agent': 'Mozilla/5.0',
+                   'Cookie': cookie}
+
+        try:
+            for i in self.list:
+                url = "https://cps.kwaixiaodian.com/distribute/pc/seller/promoter/live/info?" \
+                      "timeRangeType={timeRangeType}&promoterId={promoterId}&type=1".format(timeRangeType=i,
+                                                                                            promoterId=promoterId)
+
+                rep = requests.get(url=url, headers=headers)
+
+                table_name = 'kwai_promoter_live_info'
+                data = json.loads(rep.text)["data"]
+                promoter_info_item = {"commentsCountInfo": json.dumps(data["commentsCountInfo"]),
+                                      "likesCountInfo": json.dumps(data["likesCountInfo"]),
+                                      "liveMinuteDurationInfo": json.dumps(data["liveMinuteDurationInfo"]),
+                                      "liveVisitorCountInfo": json.dumps(data["liveVisitorCountInfo"]),
+                                      "maxVisitorCountInfo": json.dumps(data["maxVisitorCountInfo"]),
+                                      "sharesCountInfo": json.dumps(data["sharesCountInfo"]),
+                                      "promoterId": data["promoterId"],
+                                      "staticDataInfoView": json.dumps(data["staticDataInfoView"]),
+                                      "timeRangeType": i}
+                insert(table_name=table_name, item=promoter_info_item)
+                base_table_name = 'kwai_promoter_live_base_info'
+                promoter_base_info_item = {
+                    "promoterId": data["promoterId"],
+                    "timeRangeType": i,
+                    "avgLiveStreamDuration": data["staticDataInfoView"]["avgLiveStreamDuration"],
+                    "avgLiveVisitorCount": data["staticDataInfoView"]["avgLiveVisitorCount"],
+                    "avgMinuteDuration": data["staticDataInfoView"]["avgMinuteDuration"],
+                    "avgPromoteSaleVolume": data["staticDataInfoView"]["avgPromoteSaleVolume"],
+                    "avgSameTimeVisitorCount": data["staticDataInfoView"]["avgSameTimeVisitorCount"],
+                    "avgViewTime": data["staticDataInfoView"]["avgViewTime"],
+                    "avgVisitorOnlineCount": data["staticDataInfoView"]["avgVisitorOnlineCount"],
+                    "interactionRate": data["staticDataInfoView"]["interactionRate"],
+                    "liveStreamVisitorCount": data["staticDataInfoView"]["liveStreamVisitorCount"],
+                    "maxLiveVisitorCount": data["staticDataInfoView"]["maxLiveVisitorCount"],
+                    "maxMinuteDuration": data["staticDataInfoView"]["maxMinuteDuration"],
+                    "maxPromoteSaleVolume": data["staticDataInfoView"]["maxPromoteSaleVolume"],
+                    "maxSameTimeVisitorCount": data["staticDataInfoView"]["maxSameTimeVisitorCount"],
+                    "minLiveVisitorCount": data["staticDataInfoView"]["minLiveVisitorCount"],
+                    "minMinuteDuration": data["staticDataInfoView"]["minMinuteDuration"],
+                    "minPromoteSaleVolume": data["staticDataInfoView"]["minPromoteSaleVolume"],
+                    "promoteItemCount": data["staticDataInfoView"]["promoteItemCount"],
+                    "promoteLiveCount": data["staticDataInfoView"]["promoteLiveCount"],
+                    "promoteLiveDays": data["staticDataInfoView"]["promoteLiveDays"],
+                    "promoteSaleVolume": data["staticDataInfoView"]["promoteSaleVolume"],
+                }
+                insert(table_name=base_table_name, item=promoter_base_info_item)
+
+                if i == 3:
+                    live_analyse_table_name = 'kwai_promoter_live_analyse_info'
+                    commentsCountInfo = tuple(data["commentsCountInfo"])
+                    likesCountInfo = tuple(data["likesCountInfo"])
+                    liveMinuteDurationInfo = tuple(data["liveMinuteDurationInfo"])
+                    liveVisitorCountInfo = tuple(data["liveVisitorCountInfo"])
+                    maxVisitorCountInfo = tuple(data["maxVisitorCountInfo"])
+                    sharesCountInfo = tuple(data["sharesCountInfo"])
+                    live_analyse_item_list = []
+                    for i in range(len(commentsCountInfo)):
+                        live_analyse_item = {}
+                        commentsCount = commentsCountInfo[i]
+                        likesCount = likesCountInfo[i]
+                        liveMinuteDuration = liveMinuteDurationInfo[i]
+                        liveVisitorCount = liveVisitorCountInfo[i]
+                        maxVisitorCount = maxVisitorCountInfo[i]
+                        sharesCount = sharesCountInfo[i]
+                        live_analyse_item["date"] = commentsCount["date"]
+                        live_analyse_item["promoterId"] = commentsCount["promoterId"]
+                        live_analyse_item["commentsCount"] = commentsCount["commentsCount"]
+                        live_analyse_item["likesCount"] = likesCount["likesCount"]
+                        live_analyse_item["liveMinuteDuration"] = liveMinuteDuration["minuteDuration"]
+                        live_analyse_item["liveVisitorCount"] = liveVisitorCount["visitorCount"]
+                        live_analyse_item["maxVisitorCount"] = maxVisitorCount["maxVisitorCount"]
+                        live_analyse_item["sharesCount"] = sharesCount["sharesCount"]
+                        live_analyse_item_list.append(live_analyse_item)
+                    batch_insert(table_name=live_analyse_table_name, item_list=live_analyse_item_list)
+        except Exception as e:
+            SendFeiShuMsg.send_robot_msg('请求错误请检查cookie{e}'.format(e=e))
+

+ 49 - 0
spider/PromoterVideoAnalysisInfo.py

@@ -0,0 +1,49 @@
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2023/2/9 11:04 上午
+@Site    :
+@File    : PromoterVideoAnalysisInfo.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+import json
+
+import requests
+
+from utils.mysql_helper import insert
+from utils.mysql_utils import MysqlUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class PromoterVideoAnalysisInfo:
+    def __init__(self):
+        self.conn = MysqlUtils()
+        self.list = [1, 2, 3]
+
+    def PromoterVideoAnalysisInfoHandler(self, promoterId):
+        sql = "select cookie from ruixuan.kuaishou_supply_chain_cookie"
+        cookie = self.conn.QueryOne(sql)[0]
+        headers = {'User-Agent': 'Mozilla/5.0',
+                   'Cookie': cookie}
+
+        try:
+            for i in self.list:
+                url = "https://cps.kwaixiaodian.com/gateway/distribute/platform/seller" \
+                      "/promoter/video/analysis/key/indicator?promoterId={promoterId}&timeRangeType={timeRangeType}" \
+                    .format(timeRangeType=i, promoterId=promoterId)
+                rep = requests.get(url=url, headers=headers)
+                table_name = 'kwai_promoter_video_info'
+                data = json.loads(rep.text)["data"]
+                promoter_video_item = {"commentCount": data["commentCount"], "likeCount": data["likeCount"],
+                                       "shareCount": data["shareCount"], "totalSale": data["totalSale"],
+                                       "videoCount": data["videoCount"],
+                                       "videoWatchCount": data["videoWatchCount"],
+                                       "timeRangeType": i, "promoterId": data["promoterId"]}
+
+                insert(table_name=table_name, item=promoter_video_item)
+
+        except Exception as e:
+            SendFeiShuMsg.send_robot_msg('请求错误请检查cookie{e}'.format(e=e))
+

+ 72 - 0
spider/PromoterVideoAnalysisTrend.py

@@ -0,0 +1,72 @@
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2023/2/9 11:23 上午
+@Site    :
+@File    : PromoterVideoAnalysisTrend.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+import json
+
+import requests
+
+from utils.mysql_helper import insert, batch_insert
+from utils.mysql_utils import MysqlUtils
+from utils.send_feishu_msg import SendFeiShuMsg
+
+
+class PromoterVideoAnalysisTrend:
+    def __init__(self):
+        self.conn = MysqlUtils()
+        self.list = [1, 2, 3]
+
+    def PromoterVideoAnalysisTrendHandler(self, promoterId):
+        sql = "select cookie from ruixuan.kuaishou_supply_chain_cookie"
+        cookie = self.conn.QueryOne(sql)[0]
+        headers = {'User-Agent': 'Mozilla/5.0',
+                   'Cookie': cookie}
+
+        try:
+            for i in self.list:
+                url = "https://cps.kwaixiaodian.com/gateway/distribute/platform/seller/promoter/video/analysis/trend?" \
+                      "promoterId={promoterId}&timeRangeType={timeRangeType}" \
+                    .format(timeRangeType=i, promoterId=promoterId)
+                rep = requests.get(url=url, headers=headers)
+                table_name = 'kwai_promoter_video_analyse_info'
+                data = json.loads(rep.text)["data"]
+                promoter_video_analyse_item = {"avgCommentCount": data["avgCommentCount"],
+                                               "avgLikeCount": data["avgLikeCount"],
+                                               "avgShareCount": data["avgShareCount"],
+                                               "avgVideoWatchCount": data["avgVideoWatchCount"],
+                                               "maxCommentCount": data["maxCommentCount"],
+                                               "maxLikeCount": data["maxLikeCount"],
+                                               "maxShareCount": data["maxShareCount"],
+                                               "maxVideoWatchCount": data["maxVideoWatchCount"],
+                                               "minCommentCount": data["minCommentCount"],
+                                               "minLikeCount": data["minLikeCount"],
+                                               "minShareCount": data["minShareCount"],
+                                               "minVideoWatchCount": data["minVideoWatchCount"],
+                                               "promoterId": data["promoterId"],
+                                               "timeRangeType": i,
+                                               "trend": json.dumps(data["trend"]),
+                                               }
+                insert(table_name=table_name, item=promoter_video_analyse_item)
+                if i == 3:
+                    trend_name = 'kwai_promoter_video_report_analyse_info'
+                    trends_list = tuple(data["trend"])
+                    trend_list = []
+
+                    for num in range(len(trends_list)):
+                        trend = trends_list[num]
+                        analyse_trend = {"date": trend["date"], "promoterId": promoterId,
+                                         "commentCount": trend["commentCount"], "likeCount": trend["likeCount"],
+                                         "shareCount": trend["shareCount"], "videoWatchCount": trend["videoWatchCount"]}
+                        trend_list.append(analyse_trend)
+                    batch_insert(table_name=trend_name, item_list=trend_list)
+
+        except Exception as e:
+            SendFeiShuMsg.send_robot_msg('请求错误请检查cookie{e}'.format(e=e))
+
+

+ 8 - 0
spider/__init__.py

@@ -0,0 +1,8 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/2/7 2:34 下午
+# @Site    : 
+# @File    : __init__.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553

+ 46 - 0
utils/PromoterInfoWebHook.py

@@ -0,0 +1,46 @@
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2023/2/9 5:07 下午
+@Site    :
+@File    : PromoterInfoWebHook.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+import time
+from concurrent.futures.thread import ThreadPoolExecutor
+
+from flask import Flask, request, json
+
+from spider.PromoterFansInfo import PromoterFansInfo
+from spider.PromoterInfoSpider import PromoterInfoSpider
+from spider.PromoterLiveInfoSpider import PromoterLiveInfoSpider
+from spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfo
+from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend
+
+app = Flask(__name__)
+
+
+def api_root():
+    return 'Welcome guys'
+
+
+@app.route('/promoterInfo/getPromoterId', methods=['POST'])
+def webhook_get_promoter():
+    rep = json.loads(request.data)
+    promoterId = rep["promoterId"]
+    print(promoterId, '-----')
+    pool = ThreadPoolExecutor(max_workers=10)
+    print(time.time(),'----00-----')
+    result = PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId=promoterId)
+    pool.submit(PromoterFansInfo().PromoterFansInfoHandler(promoterId=promoterId))
+    pool.submit(PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId=promoterId))
+    pool.submit(PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId=promoterId))
+    pool.submit(PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId=promoterId))
+    pool.shutdown()
+    return result
+
+
+if __name__ == '__main__':
+    app.run(port=9999, host='127.0.0.1', debug=True)

+ 8 - 0
utils/__init__.py

@@ -0,0 +1,8 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/2/7 2:29 下午
+# @Site    : 
+# @File    : __init__.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553

+ 59 - 0
utils/mysql_helper.py

@@ -0,0 +1,59 @@
+# Author renyupeng
+"""
+coding=utf-8
+@Time    : 2023/2/7
+@Site    :
+@File    : mysql_helper.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+"""
+import MySQLdb
+import logging
+from utils.send_feishu_msg import SendFeiShuMsg
+
+logger = logging.getLogger(__name__)
+
+
+def insert(item, table_name):
+    con = MySQLdb.connect(host='192.168.0.184', user='hcst', passwd='hcst@2021', charset='utf8', port=3390)
+    async_item = tuple(item.values())
+    # 对数据库进行插入操作,并不需要commit,twisted会自动commit
+    placeholders = ', '.join(['%s'] * len(item))
+    columns = ', '.join(item.keys())
+    insert_sql = "REPLACE INTO kwai_promoter.%s ( %s ) VALUES ( %s )" % (
+        table_name, columns, placeholders)
+
+    cur = con.cursor()
+    try:
+        cur.execute(insert_sql, async_item)
+        con.commit()
+    except Exception as e:
+        SendFeiShuMsg.send_robot_msg('{table_name}数据插入错误{e}'.format(table_name=table_name, e=e))
+        con.rollback()
+    con.close()
+
+
+def batch_insert(item_list, table_name):
+    con = MySQLdb.connect(host='192.168.0.184', user='hcst', passwd='hcst@2021', charset='utf8', port=3390)
+    # 对数据库进行插入操作,并不需要commit,twisted会自动commit
+    placeholders = ', '.join(['%s'] * len(tuple(item_list[0])))
+    columns = ', '.join(item_list[0].keys())
+    insert_sql = "REPLACE INTO kwai_promoter.%s ( %s ) VALUES ( %s )" % (
+        table_name, columns, placeholders)
+    cur = con.cursor()
+    data = []
+    try:
+        for item in item_list:
+            value = tuple(item.values())
+            data.append(value)
+        cur.executemany(insert_sql, data)
+        con.commit()
+    except Exception as e:
+        SendFeiShuMsg.send_robot_msg('{table_name}数据批量插入错误{e}'.format(table_name=table_name, e=e))
+        con.rollback()
+    con.close()
+
+
+class MysqlDBHelper:
+    """这个类也是读取settings中的配置,自行修改代码进行操作"""

+ 117 - 0
utils/mysql_utils.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2023/2/7 下午
+@Site    :
+@File    : mysql_utils.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+encoding=utf-8
+"""
+import pymysql
+# 导入所有Mysql配置常量,请自行指定文件
+from constant.ConfConstant import ConfConstant
+
+
+class MysqlUtils(object):
+    """
+    mysql操作类,对mysql数据库进行增删改查
+    """
+
+    def __init__(self):
+
+        config = dict(
+            host=ConfConstant.TIDB_PRO_HOST,
+            db=ConfConstant.TIDB_PRO_DB,
+            user=ConfConstant.TIDB_PRO_USER,
+            password=ConfConstant.TIDB_PRO_PASSWORD,
+            port=ConfConstant.TIDB_PRO_PORT,
+            charset='utf8mb4',  # 编码要加上,否则可能出现中文乱码问题
+            use_unicode=False,
+        )
+        # Connect to the database
+        self.connection = pymysql.connect(**config)
+        self.connection.autocommit(True)
+        self.cursor = self.connection.cursor()
+
+    def QueryAll(self, sql):
+        """
+        查询所有数据
+        :param sql:
+        :return:
+        """
+        # 数据库若断开即重连
+        self.reConnect()
+
+        self.cursor.execute(sql)
+        return self.cursor.fetchall()
+
+    def QueryMany(self, sql, n):
+        """
+        查询某几条数据数据
+        :param sql:
+        :return:
+        """
+        # 数据库若断开即重连
+        self.reConnect()
+
+        self.cursor.execute(sql)
+        return self.cursor.fetchmany(n)
+
+    def QueryOne(self, sql):
+        """
+        查询某几条数据数据
+        :param sql:
+        :return:
+        """
+        # 数据库若断开即重连
+        self.reConnect()
+
+        self.cursor.execute(sql)
+        return self.cursor.fetchone()
+
+    # return self.cursor.fetchone()
+
+    def reConnect(self):
+        """
+        重连机制
+        :return:
+        """
+        try:
+            self.connection.ping()
+        except:
+            self.connection()
+
+    def Operate(self, sql, params=None, DML=True):
+        """
+        数据库操作:增删改查
+        DML: insert / update / delete
+        DDL: CREATE TABLE/VIEW/INDEX/SYN/CLUSTER
+        """
+        try:
+            # 数据库若断开即重连
+            self.reConnect()
+
+            with self.connection.cursor() as cursor:
+                cursor.execute(sql, params)
+
+                self.connection.commit()
+
+        except Exception as e:
+            if DML:
+                # 涉及DML操作时,若抛异常需要回滚
+                self.connection.rollback()
+            print(e)
+
+    def __del__(self):
+        """
+        MysqlConnection实例对象被释放时调用此方法,用于关闭cursor和connection连接
+        """
+        self.cursor.close()
+        self.connection.close()
+
+
+

+ 44 - 0
utils/send_feishu_msg.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+Author renyupeng
+coding=utf-8
+@Time    : 2023/2/7 下午
+@Site    :
+@File    : send_feishu_msg.py
+@Software: PyCharm
+@contact: renyupeng@c-top.com.cn
+@Tel 1501435553
+encoding=utf-8
+"""
+import logging
+import json
+import requests
+import traceback
+
+
+class SendFeiShuMsg:
+    @staticmethod
+    def send_robot_msg(msg_content):
+        try:
+            url = 'https://open.feishu.cn/open-apis/bot/v2/hook/7abdd37f-4a8e-4d6c-840e-ed78e892f019'
+            req_head = {"Content-Type": "application/json"}
+            req_data = {
+                "msg_type": "text",
+                "content": {
+                    "text": msg_content
+                }
+            }
+            data = json.dumps(req_data)
+            logging.info("send_robot_msg req_url:{}, req_data:{}".format(url, data))
+            response = requests.post(url=url, data=data, headers=req_head)
+            result = response.json()
+            logging.info("send_robot_msg response:{}".format(response.json()))
+            if result.get("StatusCode") == 0:
+                logging.info("飞书通知发送成功")
+                return True
+            else:
+                logging.error("飞书通知发送失败,请检查!")
+                return False
+        except Exception as e:
+            logging.error("send_robot_msg Error:{}".format(traceback.format_exc()))

+ 41 - 0
utils/test.py

@@ -0,0 +1,41 @@
+import time
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+
+# method对应于:自行编写的方法,可传参(可传多个参数,形如pool.submit(self.method, a,b))
+def method1():
+    print("我是方法1")
+    return 1
+
+
+def method2():
+    print("我是方法2")
+    return 2
+
+
+def method3():
+    print("我是方法3")
+    return 3
+
+
+def method4():
+    print("我是方法4")
+    return 4
+
+
+def method5():
+    print("我是方法5")
+    return 5
+
+
+def fun():
+    start = time.time()
+    # 创建包含5个线程的线程池
+    pool = ThreadPoolExecutor(max_workers=5)
+    future1 = pool.submit(method1)
+    future2 = pool.submit(method2)
+    future3 = pool.submit(method3)
+    future4 = pool.submit(method4)
+    future5 = pool.submit(method5)
+