|
@@ -7,16 +7,10 @@
|
|
|
# @contact: renyupeng@c-top.com.cn
|
|
|
# @Tel 1501435553
|
|
|
import json
|
|
|
+import threading
|
|
|
import time
|
|
|
import random
|
|
|
-from multiprocessing import Process
|
|
|
import redis
|
|
|
-
|
|
|
-from spider.PromoterFansInfo import PromoterFansInfo
|
|
|
-from spider.PromoterInfoSpider import PromoterInfoSpider
|
|
|
-from spider.PromoterLiveInfoSpider import PromoterLiveInfoSpider
|
|
|
-from spider.PromoterVideoAnalysisInfo import PromoterVideoAnalysisInfo
|
|
|
-from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend
|
|
|
from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
|
|
|
from utils.cookie_update import cookie_update
|
|
|
from utils.mysql_utils_pro import MysqlProUtils
|
|
@@ -29,6 +23,7 @@ class SpiderPromoterInfo:
|
|
|
|
|
|
@staticmethod
|
|
|
def get_promoter(promoterId):
|
|
|
+ print(promoterId, '22222')
|
|
|
if SpiderPromoterInfo.r.zcard('kuaishou_shop_token') == 0:
|
|
|
cookie_update.get_coookie_to_redis()
|
|
|
else:
|
|
@@ -39,69 +34,36 @@ class SpiderPromoterInfo:
|
|
|
roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
|
|
|
|
|
|
if json.loads(roll).__contains__("result"):
|
|
|
- print(11111)
|
|
|
- if json.loads(roll)["result"] == 100110000:
|
|
|
- cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
|
|
|
- SendFeiShuMsg.send_cookie_robot_msg(
|
|
|
- "cookie 永久失效 轻更新cookie 唯一电话为{phone_num}".format(phone_num=phone_num))
|
|
|
- retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
|
|
|
- retry_cookie = retry_item[0].decode('utf-8')
|
|
|
- roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
|
|
|
-
|
|
|
- cookie_update.update_cookie_handler(phone_num)
|
|
|
- PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
|
|
|
- PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
|
|
|
- PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
|
|
|
- PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
|
|
|
- PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
|
|
|
- roll = json.loads(roll)
|
|
|
- fanNum = roll["fanNum"]
|
|
|
- totalSale = roll["totalSale"]
|
|
|
- avgVideoSales = roll["avgVideoSales"]
|
|
|
- videoSales = roll["videoSales"]
|
|
|
- sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
|
|
|
- {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
|
|
|
- ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
|
|
|
- promoterId=promoterId)
|
|
|
- MysqlProUtils().Operate(sql=sql)
|
|
|
- cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
|
|
|
-
|
|
|
- else:
|
|
|
-
|
|
|
- SendFeiShuMsg.send_cookie_robot_msg(
|
|
|
- "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
|
|
|
- code=json.loads(roll)["result"]))
|
|
|
- cookie_update.update_temporary_cookie_handler(phone_num)
|
|
|
- cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
|
|
|
+ SendFeiShuMsg.send_cookie_robot_msg(
|
|
|
+ "cookie 临时失效 唯一电话为{phone_num},临时失效code{code}".format(phone_num=phone_num,
|
|
|
+ code=json.loads(roll)["result"]))
|
|
|
+ cookie_update.update_temporary_cookie_handler(phone_num)
|
|
|
+ cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
|
|
|
|
|
|
- retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
|
|
|
- retry_cookie = retry_item[0].decode('utf-8')
|
|
|
- roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
|
|
|
- PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
|
|
|
- PromoterFansInfo().PromoterFansInfoHandler(promoterId, retry_cookie)
|
|
|
- PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, retry_cookie)
|
|
|
- PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, retry_cookie)
|
|
|
- PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, retry_cookie)
|
|
|
+ retry_item = cookie_update.r.zrange('kuaishou_shop_token', 0, -1, withscores=True)
|
|
|
+ retry_cookie = retry_item[0].decode('utf-8')
|
|
|
+ roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
|
|
|
+ try:
|
|
|
roll = json.loads(roll)
|
|
|
fanNum = roll["fanNum"]
|
|
|
totalSale = roll["totalSale"]
|
|
|
avgVideoSales = roll["avgVideoSales"]
|
|
|
videoSales = roll["videoSales"]
|
|
|
sql = """ replace into ruixuan.promoter_sales_info(promoter_id, total_sale, fans_number, avg_video_sales, video_sales) values (
|
|
|
- {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
|
|
|
- ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales, videoSales=videoSales,
|
|
|
- promoterId=promoterId)
|
|
|
+ {promoterId},'{totalSale}',{fanNum},'{avgVideoSales}','{videoSales}'
|
|
|
+ ) """.format(totalSale=totalSale, fanNum=fanNum, avgVideoSales=avgVideoSales,
|
|
|
+ videoSales=videoSales,
|
|
|
+ promoterId=promoterId)
|
|
|
MysqlProUtils().Operate(sql=sql)
|
|
|
cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
|
|
|
+ except Exception as e:
|
|
|
+ print(e)
|
|
|
+ SpiderPromoterInfo.r.sadd('promoter_err_info', str(promoterId))
|
|
|
|
|
|
else:
|
|
|
- PromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
|
|
|
- PromoterFansInfo().PromoterFansInfoHandler(promoterId, cookie)
|
|
|
- PromoterLiveInfoSpider().PromoterLiveInfoSpiderHander(promoterId, cookie)
|
|
|
- PromoterVideoAnalysisInfo().PromoterVideoAnalysisInfoHandler(promoterId, cookie)
|
|
|
- PromoterVideoAnalysisTrend().PromoterVideoAnalysisTrendHandler(promoterId, cookie)
|
|
|
|
|
|
roll = json.loads(roll)
|
|
|
+ print(roll, '--roll--',promoterId)
|
|
|
fanNum = roll["fanNum"]
|
|
|
totalSale = roll["totalSale"]
|
|
|
avgVideoSales = roll["avgVideoSales"]
|
|
@@ -112,24 +74,24 @@ class SpiderPromoterInfo:
|
|
|
promoterId=promoterId)
|
|
|
MysqlProUtils().Operate(sql=sql)
|
|
|
cookie_update.r.zrem('kuaishou_shop_token', random_item[0])
|
|
|
- return roll
|
|
|
|
|
|
@staticmethod
|
|
|
def get_promoter_info():
|
|
|
sql = """
|
|
|
- select promoter_id from ruixuan.kuaishou_promoter where media_id =2
|
|
|
- """
|
|
|
+ select promoter_id from ruixuan.kuaishou_promoter where media_id =2
|
|
|
+ group by promoter_id
|
|
|
+ """
|
|
|
promoter_result = SpiderPromoterInfo.conn.QueryAll(sql)
|
|
|
- # SpiderPromoterInfo.r.rpush('promoter_list', *promoter_result)
|
|
|
-
|
|
|
- try:
|
|
|
- for promoter in promoter_result:
|
|
|
- promoter_id = int(promoter[0].decode('utf-8'))
|
|
|
-
|
|
|
- SpiderPromoterInfo.get_promoter(promoter_id)
|
|
|
- time.sleep(2)
|
|
|
- except Exception as e:
|
|
|
- return
|
|
|
+ SpiderPromoterInfo.r.delete('promoter_id_list')
|
|
|
+ for promoter in promoter_result:
|
|
|
+ promoter_id = promoter[0]
|
|
|
+ SpiderPromoterInfo.r.sadd('promoter_id_list', promoter_id)
|
|
|
+ set_data = SpiderPromoterInfo.r.smembers('promoter_id_list')
|
|
|
+ for promoter_id in set_data:
|
|
|
+ promoter_id = promoter_id.decode('utf-8')
|
|
|
+ SpiderPromoterInfo.get_promoter(promoter_id)
|
|
|
+ SpiderPromoterInfo.r.srem('promoter_id_list', promoter_id)
|
|
|
+ time.sleep(1)
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|