renyupeng 2 лет назад
Родитель
Сommit
9eadd006c5

+ 2 - 0
Apietl/config.ini

@@ -17,6 +17,8 @@ tidb_password = hcst@2021
 
 [webhook]
 url = 192.168.0.202
+[livewebhook]
+liveurl=192.168.0.203
 
 [log]
 Log_home=/data/etl_logs

+ 3 - 0
Apietl/constant/ConfConstant.py

@@ -44,5 +44,8 @@ class ConfConstant:
     # webhook配置
     URL = _cfp.get('webhook', 'url')
 
+    #livewebhook'
+    LiveURL = _cfp.get('livewebhook', 'liveurl')
+
     # cdc url
     CDC_URL = _cfp.get('cdc', 'cdc_url')

+ 48 - 54
Apietl/ruixuan/report/LiveAccountDailyReport.py

@@ -29,27 +29,26 @@ group by account_id""".format(stat_date=self.stat_date)
 select report.account_id,
        account_name,
        project_id,
-       project_name,
+       acc_info.project_name,
        ks_id,
        ks_name,
-       sale_leader_id,
-       sale_leader,
-       operator_leader_id,
-       operator_leader,
-       designer_leader_id,
-       designer_leader,
-       enterprise_name,
-       collaborator_id,
-       collaborator,
-       cost_total/100,
+       account_type,
+       user_id,
+       user_name,
+      leader as lead_name,
+       pro_user_id,
+       pro_user_name,
+       sale_id,
+       sale_name,
+       cost_total / 100,
        ad_show,
-       ad_show1k_cost/100,
+       ad_show1k_cost / 100,
        impression,
        photo_click,
        photo_click_ratio,
        click,
        actionbar_click,
-       actionbar_click_cost/100,
+       actionbar_click_cost / 100,
        esp_click_ratio,
        action_ratio,
        ad_item_click_count,
@@ -73,12 +72,12 @@ select report.account_id,
        effective_play_ratio,
        ad_live_played1m_num,
        conversion_num,
-       conversion_cost_esp/100,
-       gmv/100,
-       t0_gmv/100,
-       t1_gmv/100,
-       t7_gmv/100,
-       t15_gmv/100,
+       conversion_cost_esp / 100,
+       gmv / 100,
+       t0_gmv / 100,
+       t1_gmv / 100,
+       t7_gmv / 100,
+       t15_gmv / 100,
        t30_gmv,
        roi,
        t0_roi,
@@ -89,7 +88,7 @@ select report.account_id,
        paied_order,
        order_ratio,
        t0_order_cnt,
-       t0_order_cnt_cost/100,
+       t0_order_cnt_cost / 100,
        t0_order_cnt_ratio,
        t1_order_cnt,
        t7_order_cnt,
@@ -105,63 +104,58 @@ select report.account_id,
        t15_retention_ratio,
        t30_retention_ratio,
        reservation_success,
-       reservation_cost/100,
+       reservation_cost / 100,
        standard_live_played_started,
-       live_audience_cost/100,
+       live_audience_cost / 100,
        live_event_goods_view,
        goods_click_ratio,
        direct_attr_plat_new_buyer_cnt,
        t30_attr_plat_total_buyer_cnt,
        direct_attr_seller_new_buyer_cnt,
        t30_attr_seller_total_buyer_cnt,
-       fans_t0_gmv/100,
-       fans_t1_gmv/100,
-       fans_t7_gmv/100,
-       fans_t15_gmv/100,
-       fans_t30_gmv/100,
+       fans_t0_gmv / 100,
+       fans_t1_gmv / 100,
+       fans_t7_gmv / 100,
+       fans_t15_gmv / 100,
+       fans_t30_gmv / 100,
        fans_t0_roi,
        fans_t1_roi,
        fans_t7_roi,
        fans_t15_roi,
        fans_t30_roi,
+       order_cost,
+       reco_fans_cost,
+       conversion_live_event_goods_view_cost,
        report_date
 from kuaishou_live.kuaishou_live_account_report_daily report
          left join
      (select acc.account_id,
              account_name,
              project_id,
-             project_name,
+             pro.project_name,
+             leader,
              ks_id,
              ks_name,
-             sale_leader_id,
-             sale.user_name     as sale_leader,
-             operator_leader_id,
-             user.user_name     as operator_leader,
-             designer_leader_id,
-             designer.user_name as designer_leader,
-             enterprise_name,
-             collaborator_id,
-             collaborator
-      from ruixuan.kuaishou_live_user_account acc
+             account_type,
+             user.user_id   as user_id,
+             user.nick_name as user_name,
+             dep.leader as leader_name,
+             pro.operator_id   as pro_user_id,
+             pro.operator_name as pro_user_name,
+             pro.sale_id       as sale_id,
+             pro.sale_name     as sale_name
+      from ruixuan.kuaishou_account acc
                left join
-           ruixuan.kuaishou_live_project pro
+           ruixuan.kuaishou_project pro
            on acc.project_id = pro.id
-               left join ruixuan.sys_user sale
-                         on acc.sale_leader_id = sale.user_id
-               left join ruixuan.sys_user user
-                         on acc.operator_leader_id = user.user_id
-               left join ruixuan.sys_user designer
-                         on acc.designer_leader_id = designer.user_id
-               left join(
-          select account_id,
-                 group_concat(collaborator_id) as collaborator_id,
-                 group_concat(collaborator)    as collaborator
-          from ruixuan.kuaishou_live_user_account_part
-          group by account_id
-      ) collaborator
-                        on acc.account_id = collaborator.account_id) acc_info
+         left join ruixuan.sys_user user
+         on acc.operator_id=user.user_id
+         left join ruixuan.sys_dept dep
+         on user.dept_id=dep.dept_id
+         ) acc_info
      on report.account_id = acc_info.account_id
-where report.report_date={stat_date} and report.account_id={account_id};
+where report.report_date = {stat_date}
+  and report.account_id = {account_id};
                 """.format(stat_date=self.stat_date, account_id=account_id[0])
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 48 - 54
Apietl/ruixuan/report/LiveAccountHourlyReport.py

@@ -29,27 +29,26 @@ group by account_id""".format(stat_date=self.stat_date)
 select report.account_id,
        account_name,
        project_id,
-       project_name,
+       acc_info.project_name,
        ks_id,
        ks_name,
-       sale_leader_id,
-       sale_leader,
-       operator_leader_id,
-       operator_leader,
-       designer_leader_id,
-       designer_leader,
-       enterprise_name,
-       collaborator_id,
-       collaborator,
-       cost_total/100,
+       account_type,
+       user_id,
+       user_name,
+      leader as lead_name,
+       pro_user_id,
+       pro_user_name,
+       sale_id,
+       sale_name,
+       cost_total / 100,
        ad_show,
-       ad_show1k_cost/100,
+       ad_show1k_cost / 100,
        impression,
        photo_click,
        photo_click_ratio,
        click,
        actionbar_click,
-       actionbar_click_cost/100,
+       actionbar_click_cost / 100,
        esp_click_ratio,
        action_ratio,
        ad_item_click_count,
@@ -73,12 +72,12 @@ select report.account_id,
        effective_play_ratio,
        ad_live_played1m_num,
        conversion_num,
-       conversion_cost_esp/100,
-       gmv/100,
-       t0_gmv/100,
-       t1_gmv/100,
-       t7_gmv/100,
-       t15_gmv/100,
+       conversion_cost_esp / 100,
+       gmv / 100,
+       t0_gmv / 100,
+       t1_gmv / 100,
+       t7_gmv / 100,
+       t15_gmv / 100,
        t30_gmv,
        roi,
        t0_roi,
@@ -89,7 +88,7 @@ select report.account_id,
        paied_order,
        order_ratio,
        t0_order_cnt,
-       t0_order_cnt_cost/100,
+       t0_order_cnt_cost / 100,
        t0_order_cnt_ratio,
        t1_order_cnt,
        t7_order_cnt,
@@ -105,25 +104,28 @@ select report.account_id,
        t15_retention_ratio,
        t30_retention_ratio,
        reservation_success,
-       reservation_cost/100,
+       reservation_cost / 100,
        standard_live_played_started,
-       live_audience_cost/100,
+       live_audience_cost / 100,
        live_event_goods_view,
        goods_click_ratio,
        direct_attr_plat_new_buyer_cnt,
        t30_attr_plat_total_buyer_cnt,
        direct_attr_seller_new_buyer_cnt,
        t30_attr_seller_total_buyer_cnt,
-       fans_t0_gmv/100,
-       fans_t1_gmv/100,
-       fans_t7_gmv/100,
-       fans_t15_gmv/100,
-       fans_t30_gmv/100,
+       fans_t0_gmv / 100,
+       fans_t1_gmv / 100,
+       fans_t7_gmv / 100,
+       fans_t15_gmv / 100,
+       fans_t30_gmv / 100,
        fans_t0_roi,
        fans_t1_roi,
        fans_t7_roi,
        fans_t15_roi,
        fans_t30_roi,
+       order_cost,
+       reco_fans_cost,
+       conversion_live_event_goods_view_cost,
        report_date,
        `hour`
 from kuaishou_live.kuaishou_live_account_report_hourly report
@@ -131,38 +133,30 @@ from kuaishou_live.kuaishou_live_account_report_hourly report
      (select acc.account_id,
              account_name,
              project_id,
-             project_name,
+             pro.project_name,
+             leader,
              ks_id,
              ks_name,
-             sale_leader_id,
-             sale.user_name     as sale_leader,
-             operator_leader_id,
-             user.user_name     as operator_leader,
-             designer_leader_id,
-             designer.user_name as designer_leader,
-             enterprise_name,
-             collaborator_id,
-             collaborator
-      from ruixuan.kuaishou_live_user_account acc
+             account_type,
+             user.user_id   as user_id,
+             user.nick_name as user_name,
+             dep.leader as leader_name,
+             pro.operator_id   as pro_user_id,
+             pro.operator_name as pro_user_name,
+             pro.sale_id       as sale_id,
+             pro.sale_name     as sale_name
+      from ruixuan.kuaishou_account acc
                left join
-           ruixuan.kuaishou_live_project pro
+           ruixuan.kuaishou_project pro
            on acc.project_id = pro.id
-               left join ruixuan.sys_user sale
-                         on acc.sale_leader_id = sale.user_id
-               left join ruixuan.sys_user user
-                         on acc.operator_leader_id = user.user_id
-               left join ruixuan.sys_user designer
-                         on acc.designer_leader_id = designer.user_id
-               left join(
-          select account_id,
-                 group_concat(collaborator_id) as collaborator_id,
-                 group_concat(collaborator)    as collaborator
-          from ruixuan.kuaishou_live_user_account_part
-          group by account_id
-      ) collaborator
-                        on acc.account_id = collaborator.account_id) acc_info
+         left join ruixuan.sys_user user
+         on acc.operator_id=user.user_id
+         left join ruixuan.sys_dept dep
+         on user.dept_id=dep.dept_id
+         ) acc_info
      on report.account_id = acc_info.account_id
-where report.report_date={stat_date} and report.account_id={account_id};
+where report.report_date = {stat_date}
+  and report.account_id = {account_id};
                 """.format(stat_date=self.stat_date, account_id=account_id[0])
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 313 - 0
Apietl/ruixuan/report/LiveAccountReport.py

@@ -0,0 +1,313 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2023/3/22 9:56 上午
+# @Site    : 
+# @File    : LiveAccountReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+
+
+class LiveAccountReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+
+    def daily_account(self, account_id, start_date, end_date):
+        sql = """
+        replace into ruixuan.kuaishou_live_account_report_daily_dw
+        select report.account_id,
+               account_name,
+               project_id,
+               acc_info.project_name,
+               ks_id,
+               ks_name,
+               account_type,
+               user_id,
+               user_name,
+              leader as lead_name,
+               pro_user_id,
+               pro_user_name,
+               sale_id,
+               sale_name,
+               cost_total / 100,
+               ad_show,
+               ad_show1k_cost / 100,
+               impression,
+               photo_click,
+               photo_click_ratio,
+               click,
+               actionbar_click,
+               actionbar_click_cost / 100,
+               esp_click_ratio,
+               action_ratio,
+               ad_item_click_count,
+               esp_live_played_seconds,
+               played_three_seconds,
+               play3s_ratio,
+               played_five_seconds,
+               play5s_ratio,
+               played_end,
+               play_end_ratio,
+               share,
+               comment,
+               likes,
+               report,
+               block,
+               item_negative,
+               live_share,
+               live_comment,
+               live_reward,
+               effective_play_count,
+               effective_play_ratio,
+               ad_live_played1m_num,
+               conversion_num,
+               conversion_cost_esp / 100,
+               gmv / 100,
+               t0_gmv / 100,
+               t1_gmv / 100,
+               t7_gmv / 100,
+               t15_gmv / 100,
+               t30_gmv,
+               roi,
+               t0_roi,
+               t1_roi,
+               t7_roi,
+               t15_roi,
+               t30_roi,
+               paied_order,
+               order_ratio,
+               t0_order_cnt,
+               t0_order_cnt_cost / 100,
+               t0_order_cnt_ratio,
+               t1_order_cnt,
+               t7_order_cnt,
+               t15_order_cnt,
+               t30_order_cnt,
+               merchant_reco_fans,
+               t1_retention,
+               t7_retention,
+               t15_retention,
+               t30_retention,
+               t1_retention_ratio,
+               t7_retention_ratio,
+               t15_retention_ratio,
+               t30_retention_ratio,
+               reservation_success,
+               reservation_cost / 100,
+               standard_live_played_started,
+               live_audience_cost / 100,
+               live_event_goods_view,
+               goods_click_ratio,
+               direct_attr_plat_new_buyer_cnt,
+               t30_attr_plat_total_buyer_cnt,
+               direct_attr_seller_new_buyer_cnt,
+               t30_attr_seller_total_buyer_cnt,
+               fans_t0_gmv / 100,
+               fans_t1_gmv / 100,
+               fans_t7_gmv / 100,
+               fans_t15_gmv / 100,
+               fans_t30_gmv / 100,
+               fans_t0_roi,
+               fans_t1_roi,
+               fans_t7_roi,
+               fans_t15_roi,
+               fans_t30_roi,
+               order_cost,
+               reco_fans_cost,
+               conversion_live_event_goods_view_cost,
+               report_date
+        from kuaishou_live.kuaishou_live_account_report_daily report
+                 left join
+             (select acc.account_id,
+                     account_name,
+                     project_id,
+                     pro.project_name,
+                     leader,
+                     ks_id,
+                     ks_name,
+                     account_type,
+                     user.user_id   as user_id,
+                     user.nick_name as user_name,
+                     dep.leader as leader_name,
+                     pro.operator_id   as pro_user_id,
+                     pro.operator_name as pro_user_name,
+                     pro.sale_id       as sale_id,
+                     pro.sale_name     as sale_name
+              from ruixuan.kuaishou_account acc
+                       left join
+                   ruixuan.kuaishou_project pro
+                   on acc.project_id = pro.id
+                 left join ruixuan.sys_user user
+                 on acc.operator_id=user.user_id
+                 left join ruixuan.sys_dept dep
+                 on user.dept_id=dep.dept_id
+                 ) acc_info
+             on report.account_id = acc_info.account_id
+        where report.report_date = {start_date} and report.report_date<={end_date}
+          and report.account_id = {account_id};
+                        """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+        try:
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+    def account_hourly(self, account_id, start_date, end_date):
+        sql = """
+        replace into ruixuan.kuaishou_live_account_report_hourly_dw
+        select report.account_id,
+               account_name,
+               project_id,
+               acc_info.project_name,
+               ks_id,
+               ks_name,
+               account_type,
+               user_id,
+               user_name,
+              leader as lead_name,
+               pro_user_id,
+               pro_user_name,
+               sale_id,
+               sale_name,
+               cost_total / 100,
+               ad_show,
+               ad_show1k_cost / 100,
+               impression,
+               photo_click,
+               photo_click_ratio,
+               click,
+               actionbar_click,
+               actionbar_click_cost / 100,
+               esp_click_ratio,
+               action_ratio,
+               ad_item_click_count,
+               esp_live_played_seconds,
+               played_three_seconds,
+               play3s_ratio,
+               played_five_seconds,
+               play5s_ratio,
+               played_end,
+               play_end_ratio,
+               share,
+               comment,
+               likes,
+               report,
+               block,
+               item_negative,
+               live_share,
+               live_comment,
+               live_reward,
+               effective_play_count,
+               effective_play_ratio,
+               ad_live_played1m_num,
+               conversion_num,
+               conversion_cost_esp / 100,
+               gmv / 100,
+               t0_gmv / 100,
+               t1_gmv / 100,
+               t7_gmv / 100,
+               t15_gmv / 100,
+               t30_gmv,
+               roi,
+               t0_roi,
+               t1_roi,
+               t7_roi,
+               t15_roi,
+               t30_roi,
+               paied_order,
+               order_ratio,
+               t0_order_cnt,
+               t0_order_cnt_cost / 100,
+               t0_order_cnt_ratio,
+               t1_order_cnt,
+               t7_order_cnt,
+               t15_order_cnt,
+               t30_order_cnt,
+               merchant_reco_fans,
+               t1_retention,
+               t7_retention,
+               t15_retention,
+               t30_retention,
+               t1_retention_ratio,
+               t7_retention_ratio,
+               t15_retention_ratio,
+               t30_retention_ratio,
+               reservation_success,
+               reservation_cost / 100,
+               standard_live_played_started,
+               live_audience_cost / 100,
+               live_event_goods_view,
+               goods_click_ratio,
+               direct_attr_plat_new_buyer_cnt,
+               t30_attr_plat_total_buyer_cnt,
+               direct_attr_seller_new_buyer_cnt,
+               t30_attr_seller_total_buyer_cnt,
+               fans_t0_gmv / 100,
+               fans_t1_gmv / 100,
+               fans_t7_gmv / 100,
+               fans_t15_gmv / 100,
+               fans_t30_gmv / 100,
+               fans_t0_roi,
+               fans_t1_roi,
+               fans_t7_roi,
+               fans_t15_roi,
+               fans_t30_roi,
+               order_cost,
+               reco_fans_cost,
+               conversion_live_event_goods_view_cost,
+               report_date,
+               `hour`
+        from kuaishou_live.kuaishou_live_account_report_hourly report
+                 left join
+             (select acc.account_id,
+                     account_name,
+                     project_id,
+                     pro.project_name,
+                     leader,
+                     ks_id,
+                     ks_name,
+                     account_type,
+                     user.user_id   as user_id,
+                     user.nick_name as user_name,
+                     dep.leader as leader_name,
+                     pro.operator_id   as pro_user_id,
+                     pro.operator_name as pro_user_name,
+                     pro.sale_id       as sale_id,
+                     pro.sale_name     as sale_name
+              from ruixuan.kuaishou_account acc
+                       left join
+                   ruixuan.kuaishou_project pro
+                   on acc.project_id = pro.id
+                 left join ruixuan.sys_user user
+                 on acc.operator_id=user.user_id
+                 left join ruixuan.sys_dept dep
+                 on user.dept_id=dep.dept_id
+                 ) acc_info
+             on report.account_id = acc_info.account_id
+        where report.report_date = {start_date} and report.report_date<={end_date}
+          and report.account_id = {account_id};
+                        """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+        try:
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+    def handler(self, date_type, account_id, start_date, end_date):
+        if date_type == "daily":
+            self.daily_account(account_id, start_date, end_date)
+        else:
+            self.account_hourly(account_id, start_date, end_date)

+ 35 - 0
Apietl/utlis/LiveWebHook.py

@@ -0,0 +1,35 @@
+import datetime
+import logging
+import traceback
+
+import requests
+from flask import json  # To read json data
+from flask import request  # To receive headers
+from flask import Flask  # To be able to start the application
+from Apietl.constant.ConfConstant import ConfConstant
+
+from Apietl.bytedance_etl.report.AdvertiserReport import AdvertiserReport
+from Apietl.ruixuan.report.LiveAccountReport import LiveAccountReport
+
+app = Flask(__name__)
+
+
+@app.route('/')
+def api_root():
+    return 'Welcome guys'
+
+
+@app.route('/webhook/live_account_report', methods=['POST'])
+def api_webhook_messages():
+    print(json.loads(request.data))
+    my_info = json.loads(request.data)
+    account_id = my_info["account_id"]
+    start_date = my_info["start_date"]
+    end_date = my_info["end_date"]
+    date_type = my_info["date_type"]
+    LiveAccountReport().handler(date_type, account_id, start_date, end_date)
+    return '200'
+
+
+if __name__ == '__main__':
+    app.run(port=3456, host=ConfConstant.LiveURL, debug=True)