Selaa lähdekoodia

feat(adq): 新增头条广告账户日报与时报报表同步功能

- 新增 AdqAdvertiserReport 类,支持按日和按小时生成广告主报表
- 实现 daily_report 和 hourly_report 方法,完成数据替换与更新逻辑
- 添加 handler 方法,根据日期类型调用对应报表方法
- 新增独立的日度与小时adq广告主报表类,封装数据库操作与报表生成
- 新增日度素材报表类 AdqMaterialReportDaily,支持素材相关数据同步
- 修改 webHook.py,新增 /webhook/adq_advertiser_report 接口,触发头条广告报表更新
- 完成异常捕获与日志记录,确保报表任务执行状态可跟踪
renyupeng0920 15 tuntia sitten
vanhempi
commit
7c659a68a4

+ 189 - 0
Apietl/adq/adq_advertiser_report.py

@@ -0,0 +1,189 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : AdqAdvertiserReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdqAdvertiserReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+
+    def daily_report(self, account_id, start_date, end_date):
+        try:
+            sql = """
+                replace
+                    into application.adq_advertiser_report_daily(account_id, stat_date, project_id, product_id,
+                                                                 account_name, project_name, advertiser_name, company_name, sale_id,
+                                                                 user_id, org_code,
+                                                                 cost, view_count, valid_click_count, conversions_count,
+                                                                 deep_conversions_count, activated_count,
+                                                                 reg_pv, retention_count,
+                                                                 video_outer_play_count, video_outer_play100_count,
+                                                                 key_behavior_conversions_count, purchase_pv, purchase_amount,
+                                                                 first_pay_count, income_pv_1d_pla, ad_paying_users_d1, income_val_1,
+                                                                 income_pv_pla, app_ad_paying_users, ad_monetization_amount,
+                                                                        create_time,
+                       update_time)
+                select report.advertiser_id as account_id,
+                       stat_time_day        as stat_date,
+                       project_id,
+                       product_id,
+                       account_name,
+                       project_name,
+                       advertiser_name,
+                       company_name,
+                       sale_id,
+                       user_id,
+                       org_code,
+                       cost/100 as cost,
+                       view_count,
+                       valid_click_count,
+                       conversions_count,
+                       deep_conversions_count,
+                       activated_count,
+                       reg_pv,
+                       retention_count,
+                       video_outer_play_count,
+                       video_outer_play100_count,
+                       key_behavior_conversions_count,
+                       purchase_pv,
+                       purchase_amount/100 as purchase_amount,
+                       first_pay_count,
+                       income_pv_1d_pla,
+                       ad_paying_users_d1,
+                       income_val_1/100 as income_val_1,
+                       income_pv_pla,
+                       app_ad_paying_users,
+                       ad_monetization_amount/100 as ad_monetization_amount,
+                       create_time,
+                       update_time
+                from media_api.adq_advertiser_report_daily report
+                         left join
+                     (SELECT cua.account_id  id,
+                             cua.account_id  account_id,
+                             cp.id           project_id,
+                             product.id      product_id,
+                             ca.id           advertiser_id,
+                             uc.company_id   company_id,
+                             cua.auth_name   account_name,
+                             cp.project_name project_name,
+                             ca.`name`       advertiser_name,
+                             uc.company_name company_name,
+                             cp.sale_id      sale_id,
+                             su.id           user_id,
+                             su.leader_id,
+                             su.org_code     org_code
+                      FROM `jeecg-boot`.ctop_user_allocation cua
+                               LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+                               LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+                               LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+                               LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+                               LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+                      WHERE cua.account_id IS NOT NULL) sys
+                     on report.advertiser_id = sys.account_id
+                where report.stat_time_day >= {start_date} and report.stat_time_day<={end_date}
+                and report.advertiser_id = {account_id}
+            """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            print(account_id, '头条账户日报执行完成')
+        except Exception as e:
+            logging.error(account_id, '执行失败')
+
+    def hourly_report(self, account_id, start_date, end_date):
+        try:
+            sql = """
+          replace
+    into application.adq_advertiser_report_hourly(account_id, stat_date,stat_hour, project_id, product_id,
+                                                 account_name, project_name, advertiser_name, company_name, sale_id,
+                                                 user_id, org_code,
+                                                 cost, view_count, valid_click_count, conversions_count,
+                                                 deep_conversions_count, activated_count,
+                                                 reg_pv, retention_count,
+                                                 video_outer_play_count, video_outer_play100_count,
+                                                 key_behavior_conversions_count, purchase_pv, purchase_amount,
+                                                 first_pay_count, income_pv_1d_pla, ad_paying_users_d1, income_val_1,
+                                                 income_pv_pla, app_ad_paying_users, ad_monetization_amount,
+                                                        create_time,
+       update_time)
+select report.advertiser_id as account_id,
+       stat_time_day        as stat_date,
+       `hour` as stat_hour,
+       project_id,
+       product_id,
+       account_name,
+       project_name,
+       advertiser_name,
+       company_name,
+       sale_id,
+       user_id,
+       org_code,
+       cost/100 as cost,
+       view_count,
+       valid_click_count,
+       conversions_count,
+       deep_conversions_count,
+       activated_count,
+       reg_pv,
+       retention_count,
+       video_outer_play_count,
+       video_outer_play100_count,
+       key_behavior_conversions_count,
+       purchase_pv,
+       purchase_amount/100 as purchase_amount,
+       first_pay_count,
+       income_pv_1d_pla,
+       ad_paying_users_d1,
+       income_val_1/100 as income_val_1,
+       income_pv_pla,
+       app_ad_paying_users,
+       ad_monetization_amount/100 as ad_monetization_amount,
+       create_time,
+       update_time
+from media_api.adq_advertiser_report_hour report
+         left join
+     (SELECT cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.`name`       advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.leader_id,
+             su.org_code     org_code
+      FROM `jeecg-boot`.ctop_user_allocation cua
+               LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+               LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+               LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+               LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+               LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+      WHERE cua.account_id IS NOT NULL) sys
+     on report.advertiser_id = sys.account_id
+where report.stat_time_day >= {start_date} and report.stat_time_day<={end_date}
+and report.advertiser_id = {account_id}
+
+     """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            print(account_id, '头条账户时报执行完成')
+        except Exception as e:
+            logging.error(account_id, '执行失败')
+
+    def handler(self, date_type, account_id, start_date, end_date):
+        if date_type == "daily":
+            self.daily_report(account_id, start_date, end_date)
+        else:
+            self.hourly_report(account_id, start_date, end_date)

+ 112 - 0
Apietl/adq/adq_advertiser_report_daily.py

@@ -0,0 +1,112 @@
+# Author renyupeng
+# Date 2019/10/22 14:00
+# @File    : adq_advertiser_report_daily.py
+# @Desc    : 广告主日报表
+# encoding=utf-8
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AdqAdvertiserReportDaily(object):
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def DailyReport(self):
+        try:
+            sql = """
+                replace
+                    into application.adq_advertiser_report_daily(account_id, stat_date, project_id, product_id,
+                                                                 account_name, project_name, advertiser_name, company_name, sale_id,
+                                                                 user_id, org_code,
+                                                                 cost, view_count, valid_click_count, conversions_count,
+                                                                 deep_conversions_count, activated_count,
+                                                                 reg_pv, retention_count,
+                                                                 video_outer_play_count, video_outer_play100_count,
+                                                                 key_behavior_conversions_count, purchase_pv, purchase_amount,
+                                                                 first_pay_count, income_pv_1d_pla, ad_paying_users_d1, income_val_1,
+                                                                 income_pv_pla, app_ad_paying_users, ad_monetization_amount,
+                                                                        create_time,
+                       update_time)
+                select report.advertiser_id as account_id,
+                       stat_time_day        as stat_date,
+                       project_id,
+                       product_id,
+                       account_name,
+                       project_name,
+                       advertiser_name,
+                       company_name,
+                       sale_id,
+                       user_id,
+                       org_code,
+                       cost/100 as cost,
+                       view_count,
+                       valid_click_count,
+                       conversions_count,
+                       deep_conversions_count,
+                       activated_count,
+                       reg_pv,
+                       retention_count,
+                       video_outer_play_count,
+                       video_outer_play100_count,
+                       key_behavior_conversions_count,
+                       purchase_pv,
+                       purchase_amount/100 as purchase_amount,
+                       first_pay_count,
+                       income_pv_1d_pla,
+                       ad_paying_users_d1,
+                       income_val_1/100 as income_val_1,
+                       income_pv_pla,
+                       app_ad_paying_users,
+                       ad_monetization_amount/100 as ad_monetization_amount,
+                       create_time,
+                       update_time
+                from media_api.adq_advertiser_report_daily report
+                         left join
+                     (SELECT cua.account_id  id,
+                             cua.account_id  account_id,
+                             cp.id           project_id,
+                             product.id      product_id,
+                             ca.id           advertiser_id,
+                             uc.company_id   company_id,
+                             cua.auth_name   account_name,
+                             cp.project_name project_name,
+                             ca.`name`       advertiser_name,
+                             uc.company_name company_name,
+                             cp.sale_id      sale_id,
+                             su.id           user_id,
+                             su.leader_id,
+                             su.org_code     org_code
+                      FROM `jeecg-boot`.ctop_user_allocation cua
+                               LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+                               LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+                               LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+                               LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+                               LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+                      WHERE cua.account_id IS NOT NULL) sys
+                     on report.advertiser_id = sys.account_id
+                where report.stat_time_day >= {start_date} and report.stat_time_day<={end_date}
+                              """.format(start_date=self.start_time, end_date=self.end_time)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    obj = AdqAdvertiserReportDaily()
+    status = obj.DailyReport()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 113 - 0
Apietl/adq/adq_advertiser_report_hourly.py

@@ -0,0 +1,113 @@
+# Author renyupeng
+# Date 2019/10/22 14:00
+# @File    : adq_advertiser_report_hourly.py
+# @Desc    : 广告主日报表
+# encoding=utf-8
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AdqAdvertiserReportHourly(object):
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def HourlyReport(self):
+        try:
+            sql = """
+replace
+    into application.adq_advertiser_report_hourly(account_id, stat_date,stat_hour, project_id, product_id,
+                                                 account_name, project_name, advertiser_name, company_name, sale_id,
+                                                 user_id, org_code,
+                                                 cost, view_count, valid_click_count, conversions_count,
+                                                 deep_conversions_count, activated_count,
+                                                 reg_pv, retention_count,
+                                                 video_outer_play_count, video_outer_play100_count,
+                                                 key_behavior_conversions_count, purchase_pv, purchase_amount,
+                                                 first_pay_count, income_pv_1d_pla, ad_paying_users_d1, income_val_1,
+                                                 income_pv_pla, app_ad_paying_users, ad_monetization_amount,
+                                                        create_time,
+       update_time)
+select report.advertiser_id as account_id,
+       stat_time_day        as stat_date,
+       `hour` as stat_hour,
+       project_id,
+       product_id,
+       account_name,
+       project_name,
+       advertiser_name,
+       company_name,
+       sale_id,
+       user_id,
+       org_code,
+       cost/100 as cost,
+       view_count,
+       valid_click_count,
+       conversions_count,
+       deep_conversions_count,
+       activated_count,
+       reg_pv,
+       retention_count,
+       video_outer_play_count,
+       video_outer_play100_count,
+       key_behavior_conversions_count,
+       purchase_pv,
+       purchase_amount/100 as purchase_amount,
+       first_pay_count,
+       income_pv_1d_pla,
+       ad_paying_users_d1,
+       income_val_1/100 as income_val_1,
+       income_pv_pla,
+       app_ad_paying_users,
+       ad_monetization_amount/100 as ad_monetization_amount,
+       create_time,
+       update_time
+from media_api.adq_advertiser_report_hour report
+         left join
+     (SELECT cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.`name`       advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.leader_id,
+             su.org_code     org_code
+      FROM `jeecg-boot`.ctop_user_allocation cua
+               LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+               LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+               LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+               LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+               LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+      WHERE cua.account_id IS NOT NULL) sys
+     on report.advertiser_id = sys.account_id
+where report.stat_time_day >= {start_date} and report.stat_time_day<={end_date}
+                              """.format(start_date=self.start_time, end_date=self.end_time)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    obj = AdqAdvertiserReportHourly()
+    status = obj.HourlyReport()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 157 - 0
Apietl/adq/adq_material_report_daily.py

@@ -0,0 +1,157 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialVideoDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AdqMaterialReportDaily(object):
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """
+            select advertiser_id from media_api.adq_material_report_daily where 
+            stat_time_day >= {start_time} and stat_time_day<={end_time} group by advertiser_id
+            """.format(start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """
+replace into application.adq_material_report_daily_dw(account_id,
+                                                      project_id,
+                                                      product_id,
+                                                      advertiser_id,
+                                                      company_id,
+                                                      account_name,
+                                                      project_name,
+                                                      advertiser_name,
+                                                      company_name,
+                                                      sale_id,
+                                                      user_id,
+                                                      clip_id,
+                                                      shot_id,
+                                                      plan_id,
+                                                      leader_id,
+                                                      org_code,
+                                                      photo_url,
+                                                      material_id,
+                                                      stat_date,
+                                                      material_type,
+                                                      cost,
+                                                      view_count,
+                                                      valid_click_count,
+                                                      conversions_count,
+                                                      deep_conversions_count,
+                                                      activated_count,
+                                                      reg_pv,
+                                                      retention_count,
+                                                      video_outer_play_count,
+                                                      video_outer_play100_count,
+                                                      key_behavior_conversions_count,
+                                                      purchase_pv,
+                                                      purchase_amount,
+                                                      first_pay_count,
+                                                      ad_monetization_amount,
+                                                      app_ad_paying_users,
+                                                      income_pv_pla,
+                                                      income_val_1,
+                                                      ad_paying_users_d1,
+                                                      income_pv_1d_pla,
+                                                      income_val_3,
+                                                      signature,
+                                                      channel_type,
+                                                      clip_name,
+                                                      shot_name,
+                                                      plan_name,
+                                                      leader_name,
+                                                      video_name,
+                                                      channel_name,
+                                                      clip_company_id,
+                                                      play_3s_count,
+                                                      shot_leader_id,
+                                                      shot_leader_name)
+select report.advertiser_id         as account_id,
+       cp.id                        as project_id,
+       product.id                   as product_id,
+       ca.id                        as advertiser_id,
+       uc.company_id,
+       cua.auth_name                as account_name,
+       cp.project_name              as project_name,
+       ca.name                      as advertiser_name,
+       uc.company_name,
+       cp.sale_id,
+       cua.user_id,
+       clip_id,
+       shot_id,
+       plan_id,
+       cma.leader_id,
+       leader.org_code,
+       video.preview_url            as photo_url,
+       report.material_id,
+       report.stat_time_day         as stat_date,
+       report.material_type,
+       report.cost / 100            as cost,
+       report.view_count,
+       report.valid_click_count,
+       report.conversions_count,
+       report.deep_conversions_count,
+       report.activated_count,
+       report.reg_pv,
+       report.retention_count,
+       report.video_outer_play_count,
+       report.video_outer_play100_count,
+       report.key_behavior_conversions_count,
+       report.purchase_pv,
+       report.purchase_amount / 100 as purchase_amount,
+       report.first_pay_count,
+       report.ad_monetization_amount,
+       report.app_ad_paying_users,
+       report.income_pv_pla,
+       report.income_val_1 / 100    as income_val_1,
+       report.ad_paying_users_d1,
+       report.income_pv_1d_pla,
+       report.income_val_3 / 100    as income_val_3,
+       video.signature,
+       null                           as channel_type,
+       clip.realname,
+       shot.realname,
+       plan.realname,
+       leader.realname,
+       video.description            as video_name,
+       null                              channel_name,
+       uc.company_id,
+       0                            as play_3s_count,
+       cma.shot_leader_id,
+       cma.shot_leader_name
+from media_api.adq_material_report_daily report
+         left join media_api.adq_video_info video
+                   on report.material_id = video.video_id
+         left join (select code, material_name from `jeecg-boot`.ctop_material_info where status = 1) cmi
+                   on video.signature = cmi.code
+         left join `jeecg-boot`.ctop_material_ascription cma on cma.material_id = video.signature
+         left join `jeecg-boot`.ctop_user_allocation cua on cua.account_id = report.advertiser_id
+         left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+         left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+         left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+         left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+         left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+         left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+         left join `jeecg-boot`.user_company uc on uc.user_id = clip.id
+         left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+where report.stat_time_day >= {start_time}
+  and stat_time_day <= {end_time}
+  and report.advertiser_id = {account_id}
+                """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id)
+                TidbConn.execute(self.conn, sql)
+        except Exception as e:
+            logging.error(e)

+ 13 - 5
Apietl/utlis/webHook.py

@@ -6,14 +6,11 @@ import requests
 from flask import json, jsonify  # To read json data
 from flask import request  # To receive headers
 from flask import Flask  # To be able to start the application
-from werkzeug.utils import secure_filename
-
+from Apietl.adq.adq_advertiser_report import AdqAdvertiserReport
 from Apietl.constant.ConfConstant import ConfConstant
 from Apietl.kwai_etl.report.AccountReport import AccountReport
-
 from Apietl.bytedance_etl.report.AdvertiserReport import AdvertiserReport
-from Apietl.utlis.EtlFirstDeliveryValidMaterials import EtlFirstDeliveryValidMaterials
-from Apietl.utlis.UpLoadFile import UpLoadFile
+
 
 app = Flask(__name__)
 
@@ -45,6 +42,17 @@ def api_webhook_bytedance_messages():
     date_type = my_info["date_type"]
     AdvertiserReport().handler(date_type, account_id, start_date, end_date)
     return '200'
+@app.route('/webhook/adq_advertiser_report', methods=['POST'])
+def api_webhook_adq_messages():
+    print(json.loads(request.data))
+    my_info = json.loads(request.data)
+    account_id = my_info["account_id"]
+    start_date = my_info["start_date"]
+    end_date = my_info["end_date"]
+    date_type = my_info["date_type"]
+    AdqAdvertiserReport().handler(date_type, account_id, start_date, end_date)
+    return '200'
+
 
 
 # @app.route('/webhook/get_file', methods=['POST'])