renyupeng 11 月之前
父节点
当前提交
7173b3ad3b
共有 3 个文件被更改,包括 186 次插入2 次删除
  1. 81 0
      Apietl/utlis/EtlFirstDeliveryValidMaterials.py
  2. 53 0
      Apietl/utlis/UpLoadFile.py
  3. 52 2
      Apietl/utlis/webHook.py

+ 81 - 0
Apietl/utlis/EtlFirstDeliveryValidMaterials.py

@@ -0,0 +1,81 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2022/3/24 9:53 上午
+# @Site    : 
+# @File    : EtlFirstDeliveryValidMaterials.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+
+
+class EtlFirstDeliveryValidMaterials:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.stat_date = (datetime.date.today() + datetime.timedelta(0)).strftime("%Y%m%d")
+
+    def handler(self):
+        try:
+            sql = """truncate table `jeecg-boot`.`first_delivery_valid_materials`"""
+            TidbConn.exec_sql(self.conn, sql)
+
+            sql = """
+      replace  into `jeecg-boot`.`etl_first_delivery_valid_materials`
+        select `delivery_month`,
+               `operation_label`,
+               a.`material_id`,
+               e.signature,
+               `shot_leader_id`,
+               `shot_leader_name`,
+               f.`shot_id`,
+               f.`plan_id`,
+               f.`leader_id`,
+               f.`clip_name`,
+               f.`shot_name`,
+               f.`plan_name`,
+               f.`leader_name`,
+               c.id                    as `project_id`,
+               c.project_name,
+               c.responsible_id           project_leader_id,
+               d.realname              as project_leader_name,
+               `stat_date`,
+               `customer_name`,
+               `rebate_type`,
+               `settlement_1l`,
+               `settlement_2l`,
+               `first_valid`,
+               `cost_7day`,
+               `cost`,
+            date_format(stat_date,'%Y')  as   `delivery_year`,
+               c.design_responsible_id as project_design_leader_id,
+               h.realname              as project_design_leader_name,
+               f.clip_id               as clip_id
+        from `jeecg-boot`.first_delivery_valid_materials a
+                 left join `jeecg-boot`.ctop_user_allocation b
+                           on a.account_id = b.account_id
+                 left join `jeecg-boot`.ctop_project c
+                           on b.project_id = c.id
+                 left join `jeecg-boot`.sys_user d
+                           on c.responsible_id = d.id
+                 left join application.bytedance_video_get e
+                           on a.material_id = e.material_id
+                 left join application.app_bytedance_material_info f
+                           on e.signature = f.material_id
+                 left join `jeecg-boot`.ctop_material_ascription g
+                           on e.signature = g.material_id
+                 left join `jeecg-boot`.sys_user h
+                           on c.design_responsible_id = h.id
+        where c.id is not null
+        group by a.delivery_month, a.material_id, signature, b.project_id, settlement_2l, delivery_year
+            """.format(stat_date=self.stat_date)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1

+ 53 - 0
Apietl/utlis/UpLoadFile.py

@@ -0,0 +1,53 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2024/6/27 20:28
+# @Site    : 
+# @File    : UpLoadFile.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+from urllib import parse
+
+import pandas as pd
+from sqlalchemy import create_engine
+
+
+class UpLoadFile:
+    @staticmethod
+    def uploadFile(file_name):
+        OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.101", 3390, "jeecg-boot")
+        OuterConn = create_engine(OuterStr,
+                                  pool_size=10,
+                                  max_overflow=5,
+                                  pool_timeout=10,
+                                  pool_recycle=3600)
+        chunksize = 100000
+
+        for chunk in pd.read_csv(f'/home/excel/{file_name}'.format(file_name=file_name),
+                                 chunksize=chunksize, encoding='GBK', header=None, low_memory=False, skiprows=1):
+            # 将每个块存入MySQL,'表名'需要替换为目标表名
+            chunk.columns = ['delivery_month',
+                             'operation_label',
+                             'material_id',
+                             'str_material_id',
+                             'account_id',
+                             'stat_date',
+                             'str_account_id',
+                             'account_name',
+                             'ad_ids',
+                             'customer_name',
+                             'user_name',
+                             'rebate_type',
+                             'settlement_1l',
+                             'settlement_2l',
+                             'first_valid',
+                             'cost_7day',
+                             'cost']
+            try:
+                chunk.to_sql('first_delivery_valid_materials', con=OuterConn, index=False, if_exists='append'
+                             )
+                return 0
+            except Exception as e:
+                print(e)
+                return -1

+ 52 - 2
Apietl/utlis/webHook.py

@@ -1,15 +1,19 @@
 import datetime
 import datetime
 import logging
 import logging
+import os
 import traceback
 import traceback
-
 import requests
 import requests
-from flask import json  # To read json data
+from flask import json, jsonify  # To read json data
 from flask import request  # To receive headers
 from flask import request  # To receive headers
 from flask import Flask  # To be able to start the application
 from flask import Flask  # To be able to start the application
+from werkzeug.utils import secure_filename
+
 from Apietl.constant.ConfConstant import ConfConstant
 from Apietl.constant.ConfConstant import ConfConstant
 from Apietl.kwai_etl.report.AccountReport import AccountReport
 from Apietl.kwai_etl.report.AccountReport import AccountReport
 
 
 from Apietl.bytedance_etl.report.AdvertiserReport import AdvertiserReport
 from Apietl.bytedance_etl.report.AdvertiserReport import AdvertiserReport
+from Apietl.utlis.EtlFirstDeliveryValidMaterials import EtlFirstDeliveryValidMaterials
+from Apietl.utlis.UpLoadFile import UpLoadFile
 
 
 app = Flask(__name__)
 app = Flask(__name__)
 
 
@@ -43,5 +47,51 @@ def api_webhook_bytedance_messages():
     return '200'
     return '200'
 
 
 
 
+@app.route('/webhook/get_file', methods=['POST'])
+def api_webhook_get_file():
+    payload = {}
+    headers = {
+        'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
+        'Accept': '*/*',
+        'Host': 'api.tjyourong.com.cn',
+        'Connection': 'keep-alive'
+    }
+
+    if 'file' not in request.files:
+        url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+        response = requests.request("GET", url, headers=headers, data=payload)
+        print(response.text, 'error1')
+        return '200'
+    file = request.files['file']
+    if file.filename == '':
+        url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+        response = requests.request("GET", url, headers=headers, data=payload)
+        print(response.text, 'error2')
+        return '200'
+    try:
+        file.save('/home/excel/' + 'upload.csv')
+        status = UpLoadFile.uploadFile('upload.csv')
+        if status == 0:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='success')
+            requests.request("GET", url, headers=headers, data=payload)
+        else:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+            requests.request("GET", url, headers=headers, data=payload)
+        material_status = EtlFirstDeliveryValidMaterials().handler()
+        if material_status == 0:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='success')
+            requests.request("GET", url, headers=headers, data=payload)
+        else:
+            url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+            requests.request("GET", url, headers=headers, data=payload)
+        os.remove('/data/excel/' + 'upload.csv')
+        return '200'
+    except Exception as e:
+        url = "http://api.tjyourong.com.cn/jeecg-boot/first/delivery/excelCallBack?msg={msg}".format(msg='error')
+        response = requests.request("GET", url, headers=headers, data=payload)
+        print(response.text, 'error3')
+        return '200'
+
+
 if __name__ == '__main__':
 if __name__ == '__main__':
     app.run(port=8765, host=ConfConstant.URL, debug=True)
     app.run(port=8765, host=ConfConstant.URL, debug=True)