renyupeng 11 miesięcy temu
rodzic
commit
83920c0dc1

+ 81 - 0
utils/EtlFirstDeliveryValidMaterials.py

@@ -0,0 +1,81 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2022/3/24 9:53 上午
+# @Site    : 
+# @File    : EtlFirstDeliveryValidMaterials.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+import logging
+
+from utils.mysql_utils import MysqlUtils
+
+
+class EtlFirstDeliveryValidMaterials:
+    def __init__(self):
+        self.stat_date = (datetime.date.today() + datetime.timedelta(0)).strftime("%Y%m%d")
+
+    def handler(self):
+        try:
+            sql = """
+      replace  into `jeecg-boot`.`etl_first_delivery_valid_materials`
+        select `delivery_month`,
+               `operation_label`,
+               a.`material_id`,
+               e.signature,
+               `shot_leader_id`,
+               `shot_leader_name`,
+               f.`shot_id`,
+               f.`plan_id`,
+               f.`leader_id`,
+               f.`clip_name`,
+               f.`shot_name`,
+               f.`plan_name`,
+               f.`leader_name`,
+               c.id                    as `project_id`,
+               c.project_name,
+               c.responsible_id           project_leader_id,
+               d.realname              as project_leader_name,
+               `stat_date`,
+               `customer_name`,
+               `rebate_type`,
+               `settlement_1l`,
+               `settlement_2l`,
+               `first_valid`,
+               `cost_7day`,
+               `cost`,
+            date_format(stat_date,'%Y')  as   `delivery_year`,
+               c.design_responsible_id as project_design_leader_id,
+               h.realname              as project_design_leader_name,
+               f.clip_id               as clip_id
+        from `jeecg-boot`.first_delivery_valid_materials a
+                 left join `jeecg-boot`.ctop_user_allocation b
+                           on a.account_id = b.account_id
+                 left join `jeecg-boot`.ctop_project c
+                           on b.project_id = c.id
+                 left join `jeecg-boot`.sys_user d
+                           on c.responsible_id = d.id
+                 left join application.bytedance_video_get e
+                           on a.material_id = e.material_id
+                 left join application.app_bytedance_material_info f
+                           on e.signature = f.material_id
+                 left join `jeecg-boot`.ctop_material_ascription g
+                           on e.signature = g.material_id
+                 left join `jeecg-boot`.sys_user h
+                           on c.design_responsible_id = h.id
+        where c.id is not null
+        group by a.delivery_month, a.material_id, signature, b.project_id, settlement_2l, delivery_year
+            """.format(stat_date=self.stat_date)
+            print(sql)
+            MysqlUtils().Operate(sql=sql)
+            truncate_sql = "truncate table `jeecg-boot`.first_delivery_valid_materials"
+            MysqlUtils().Operate(sql=truncate_sql)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    EtlFirstDeliveryValidMaterials().handler()

+ 54 - 4
utils/PromoterInfoWebHook.py

@@ -8,9 +8,11 @@ coding=utf-8
 @contact: renyupeng@c-top.com.cn
 @Tel 1501435553
 """
+import os
 import random
 
 import redis
+import requests
 from flask import Flask, request, json
 
 from DouyinSpider.SpiderDouyinHeader import SpiderDouyinHeader
@@ -23,6 +25,8 @@ from spider.PromoterVideoAnalysisTrend import PromoterVideoAnalysisTrend
 from multiprocessing import Process
 
 from spider.ReturnPromoterInfoSpider import ReturnPromoterInfoSpider
+from utils.EtlFirstDeliveryValidMaterials import EtlFirstDeliveryValidMaterials
+from utils.UpLoadFile import UpLoadFile
 from utils.cookie_update import cookie_update
 from utils.douyin_cookie_update import douyin_cookie_update
 
@@ -49,7 +53,7 @@ def webhook_get_promoter():
             cookie = random_item[0].decode('utf-8')
             phone_num = int(random_item[1])
             roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
-            print(roll,'-----000-----{promoterId}'.format(promoterId=promoterId))
+            print(roll, '-----000-----{promoterId}'.format(promoterId=promoterId))
             return roll
 
 
@@ -59,7 +63,7 @@ def webhook_get_promoter():
             cookie = random_item[0].decode('utf-8')
             phone_num = int(random_item[1])
             roll = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, cookie)
-            print(roll,'-----111-----{promoterId}'.format(promoterId=promoterId))
+            print(roll, '-----111-----{promoterId}'.format(promoterId=promoterId))
 
             if json.loads(roll).__contains__("result"):
                 if json.loads(roll)["result"] == 109:
@@ -67,7 +71,7 @@ def webhook_get_promoter():
                     SendFeiShuMsg.send_cookie_robot_msg(
                         "cookie 永久失效 轻更新cookie 唯一电话为{phone_num}".format(phone_num=phone_num))
                     retry_item = cookie_update.r.zrange('kuaishou_shop_click_token', 0, -1, withscores=True)
-                    print(retry_item[0][0],'-----cookie-----')
+                    print(retry_item[0][0], '-----cookie-----')
                     retry_cookie = retry_item[0][0].decode('utf-8')
                     cookie_update.update_cookie_handler(phone_num)
                     rollback = ReturnPromoterInfoSpider().PromoterInfoSpiderHandler(promoterId, retry_cookie)
@@ -84,7 +88,7 @@ def webhook_get_promoter():
                     p3.start()
                     p4.start()
                     p5.start()
-                    print(rollback,'--222---{promoterId}'.format(promoterId=promoterId))
+                    print(rollback, '--222---{promoterId}'.format(promoterId=promoterId))
 
                     return rollback
 
@@ -159,5 +163,51 @@ def webhook_update_promoter():
     return '0'
 
 
+@app.route('/webhook/get_file', methods=['POST'])
+def api_webhook_get_file():
+    payload = {}
+    headers = {
+        'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
+        'Accept': '*/*',
+        'Host': 'api.tjyourong.com.cn',
+        'Connection': 'keep-alive'
+    }
+
+    if 'file' not in request.files:
+        url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+        response = requests.request("GET", url, headers=headers, data=payload)
+        print(response.text, 'error1')
+        return '200'
+    file = request.files['file']
+    if file.filename == '':
+        url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+        response = requests.request("GET", url, headers=headers, data=payload)
+        print(response.text, 'error2')
+        return '200'
+    try:
+        file.save('/home/excel/' + 'upload.csv')
+        status = UpLoadFile.uploadFile('upload.csv')
+        if status == 0:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='success')
+            requests.request("GET", url, headers=headers, data=payload)
+        else:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+            requests.request("GET", url, headers=headers, data=payload)
+        material_status = EtlFirstDeliveryValidMaterials().handler()
+        if material_status == 0:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='success')
+            requests.request("GET", url, headers=headers, data=payload)
+        else:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+            requests.request("GET", url, headers=headers, data=payload)
+        os.remove('/data/excel/' + 'upload.csv')
+        return '200'
+    except Exception as e:
+        url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+        response = requests.request("GET", url, headers=headers, data=payload)
+        print(response.text, 'error3')
+        return '200'
+
+
 if __name__ == '__main__':
     app.run(port=9999, host=ConfConstant.URL, debug=True)

+ 56 - 0
utils/UpLoadFile.py

@@ -0,0 +1,56 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2024/6/27 20:28
+# @Site    : 
+# @File    : UpLoadFile.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+from urllib import parse
+
+import pandas as pd
+from sqlalchemy import create_engine
+
+
+class UpLoadFile:
+    @staticmethod
+    def uploadFile(file_name):
+        OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.101", 3390, "jeecg-boot")
+        OuterConn = create_engine(OuterStr,
+                                  pool_size=10,
+                                  max_overflow=5,
+                                  pool_timeout=10,
+                                  pool_recycle=3600)
+
+        chunksize = 100000
+
+        for chunk in pd.read_csv(f'/data/excel/{file_name}'.format(file_name=file_name),
+                                 chunksize=chunksize, encoding='GBK', header=None, low_memory=False, skiprows=1):
+            # 将每个块存入MySQL,'表名'需要替换为目标表名
+            chunk.columns = ['delivery_month',
+                             'operation_label',
+                             'material_id',
+                             'str_material_id',
+                             'account_id',
+                             'stat_date',
+                             'str_account_id',
+                             'account_name',
+                             'ad_ids',
+                             'customer_name',
+                             'user_name',
+                             'rebate_type',
+                             'settlement_1l',
+                             'settlement_2l',
+                             'first_valid',
+                             'cost_7day',
+                             'cost']
+            try:
+                chunk.to_sql('first_delivery_valid_materials', con=OuterConn, index=False, if_exists='append'
+                             )
+                return 0
+            except Exception as e:
+                print(e)
+                return -1
+if __name__ == '__main__':
+    UpLoadFile.uploadFile('upload.csv')