Forráskód Böngészése

feat(etl): 新增快手账户日报和小时报表的ETL作业模块

- 添加kuaishou账号日报和小时报表的SQL数据替换插入逻辑
- 实现基于时间区间的分段数据查询和处理功能
- 创建ctop_material_top_cost和ctop_history_material_catch两张分区表及数据导入脚本
- 添加相关初始化文件以支持包结构和模块调用
- 配置.idea/.gitignore忽略特定IDE及数据文件目录
- 引入TidbConn数据库连接管理及日志工具辅助开发
- 支持多维度事件指标和维度字段的全量写入数据库存储
renyupeng0920 1 napja
commit
4c1338c757
100 módosított fájl, 26858 hozzáadás és 0 törlés
  1. 10 0
      .idea/.gitignore
  2. 428 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceAdReportBusi.py
  3. 423 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceAdReportBusiMonth.py
  4. 407 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceAdvertiserReportBusi.py
  5. 400 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceAdvertiserReportBusiMonth.py
  6. 413 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceCampaignReportBusi.py
  7. 405 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceCampaignReportBusiMonth.py
  8. 670 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyDesignReportBusi.py
  9. 473 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyDesignReportBusiMonth.py
  10. 605 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyUserReportBusi.py
  11. 421 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyUserReportBusiMonth.py
  12. 209 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceFileVideoGet.py
  13. 201 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceFileVideoGetAll.py
  14. 483 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceImageReportBusi.py
  15. 467 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceImageReportBusiMonth.py
  16. 400 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceProductReportBusi.py
  17. 236 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceProductReportBusiMonth.py
  18. 415 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceProjectReportBusi.py
  19. 245 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceProjectReportBusiMonth.py
  20. 523 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceVideoReportBusi.py
  21. 520 0
      Apietl/bytedance_etl/Q4Task/AppBytedanceVideoReportBusiMonth.py
  22. 8 0
      Apietl/bytedance_etl/__init__.py
  23. 302 0
      Apietl/bytedance_etl/report/AdDailyReport.py
  24. 300 0
      Apietl/bytedance_etl/report/AdHourlyReport.py
  25. 324 0
      Apietl/bytedance_etl/report/AdvertiserDailyInventoryReport.py
  26. 320 0
      Apietl/bytedance_etl/report/AdvertiserDailyReport.py
  27. 321 0
      Apietl/bytedance_etl/report/AdvertiserHourlyReport.py
  28. 587 0
      Apietl/bytedance_etl/report/AdvertiserReport.py
  29. 322 0
      Apietl/bytedance_etl/report/AdvertiserSumDailyReport.py
  30. 127 0
      Apietl/bytedance_etl/report/AppBytedanceAgentReport.py
  31. 309 0
      Apietl/bytedance_etl/report/CampaignDailyReport.py
  32. 311 0
      Apietl/bytedance_etl/report/CampaignHourlyReport.py
  33. 309 0
      Apietl/bytedance_etl/report/CreativeDailyReport.py
  34. 8 0
      Apietl/bytedance_etl/report/CreativeHourlyReport.py
  35. 349 0
      Apietl/bytedance_etl/report/MaterialImageDailyReport.py
  36. 368 0
      Apietl/bytedance_etl/report/MaterialVideoDailyReport.py
  37. 329 0
      Apietl/bytedance_etl/report/NewMaterialImageDailyReport.py
  38. 205 0
      Apietl/bytedance_etl/report/NewMaterialVideoDailyReport.py
  39. 8 0
      Apietl/bytedance_etl/report/__init__.py
  40. 29 0
      Apietl/config.ini
  41. 118 0
      Apietl/conn/TidbConn.py
  42. 102 0
      Apietl/conn/TidbProConn.py
  43. 51 0
      Apietl/constant/ConfConstant.py
  44. 97 0
      Apietl/email/BytedanceHotMaterial.py
  45. 90 0
      Apietl/email/KuaishouClipMaterialToQingQMonth.py
  46. 89 0
      Apietl/email/KuaishouClipMaterialToQingQWeek.py
  47. 97 0
      Apietl/email/KuaishouHotMaterial.py
  48. 90 0
      Apietl/email/KuaishouMaterialToQingQMonth.py
  49. 89 0
      Apietl/email/KuaishouMaterialToQingQWeek.py
  50. 85 0
      Apietl/email/NewBytedanceAllMaterialFourD.py
  51. 91 0
      Apietl/email/NewKuaiShouAllMaterialFourD.py
  52. 72 0
      Apietl/email/OpenWindowMaterial.py
  53. 101 0
      Apietl/email/TidbKuaiShouAllMaterialEffeW.py
  54. 90 0
      Apietl/email/TouTiaoClipMaterialToQingQMonth.py
  55. 91 0
      Apietl/email/TouTiaoClipMaterialToQingQWeek.py
  56. 90 0
      Apietl/email/TouTiaoMaterialToQingQMonth.py
  57. 90 0
      Apietl/email/TouTiaoMaterialToQingQWeek.py
  58. 8 0
      Apietl/email/__init__.py
  59. 82 0
      Apietl/email/kwai_caiwu.py
  60. 115 0
      Apietl/email/send_bytedance_material_report_weekly.py
  61. 109 0
      Apietl/email/send_material_report_weekly.py
  62. 75 0
      Apietl/email/suzaomaterial.py
  63. 81 0
      Apietl/email/toutiao_caiwu.py
  64. 348 0
      Apietl/kwai_etl/Q4Task/AppKuaishouAccountReportBusi.py
  65. 343 0
      Apietl/kwai_etl/Q4Task/AppKuaishouAccountReportBusiMonth.py
  66. 354 0
      Apietl/kwai_etl/Q4Task/AppKuaishouCampaignReportBusi.py
  67. 347 0
      Apietl/kwai_etl/Q4Task/AppKuaishouCampaignReportBusiMonth.py
  68. 519 0
      Apietl/kwai_etl/Q4Task/AppKuaishouCompanyDesignReportBusi.py
  69. 375 0
      Apietl/kwai_etl/Q4Task/AppKuaishouCompanyDesignReportBusiMonth.py
  70. 515 0
      Apietl/kwai_etl/Q4Task/AppKuaishouCompanyUserReportBusi.py
  71. 367 0
      Apietl/kwai_etl/Q4Task/AppKuaishouCompanyUserReportBusiMonth.py
  72. 337 0
      Apietl/kwai_etl/Q4Task/AppKuaishouProductReportBusi.py
  73. 187 0
      Apietl/kwai_etl/Q4Task/AppKuaishouProductReportBusiMonth.py
  74. 341 0
      Apietl/kwai_etl/Q4Task/AppKuaishouProjectReportBusi.py
  75. 191 0
      Apietl/kwai_etl/Q4Task/AppKuaishouProjectReportBusiMonth.py
  76. 368 0
      Apietl/kwai_etl/Q4Task/AppKuaishouUnitReportBusi.py
  77. 360 0
      Apietl/kwai_etl/Q4Task/AppKuaishouUnitReportBusiMonth.py
  78. 258 0
      Apietl/kwai_etl/Q4Task/AppKuaishouVideoList.py
  79. 186 0
      Apietl/kwai_etl/Q4Task/AppKuaishouVideoListAll.py
  80. 377 0
      Apietl/kwai_etl/Q4Task/AppKuaishouVideoReportBusi.py
  81. 369 0
      Apietl/kwai_etl/Q4Task/AppKuaishouVideoReportBusiMonth.py
  82. 158 0
      Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceAgeReportDaily.py
  83. 168 0
      Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceCityReportDaily.py
  84. 159 0
      Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceGenderReportDaily.py
  85. 168 0
      Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceProvinceReportDaily.py
  86. 386 0
      Apietl/kwai_etl/report/AccountDailyReport.py
  87. 480 0
      Apietl/kwai_etl/report/AccountHourlyReport.py
  88. 569 0
      Apietl/kwai_etl/report/AccountHourlySumToDayReport.py
  89. 899 0
      Apietl/kwai_etl/report/AccountReport.py
  90. 366 0
      Apietl/kwai_etl/report/AppKuaiShouAccountAdAceneReport.py
  91. 164 0
      Apietl/kwai_etl/report/AppKuaiShouAgentReport.py
  92. 253 0
      Apietl/kwai_etl/report/CampaignDailyReport.py
  93. 255 0
      Apietl/kwai_etl/report/CampaignHourlyReport.py
  94. 307 0
      Apietl/kwai_etl/report/DeleteMateirialVideoDailyReport.py
  95. 316 0
      Apietl/kwai_etl/report/MateirialVideoDailyReport.py
  96. 314 0
      Apietl/kwai_etl/report/MateirialVideoHourlyReport.py
  97. 50 0
      Apietl/kwai_etl/report/ReplaceAccountInfo.py
  98. 258 0
      Apietl/kwai_etl/report/UnitDailyReport.py
  99. 253 0
      Apietl/kwai_etl/report/UnitHourlyReport.py
  100. 0 0
      Apietl/ruixuan/mgs/JyOrderItemList.py

+ 10 - 0
.idea/.gitignore

@@ -0,0 +1,10 @@
+# 默认忽略的文件
+/shelf/
+/workspace.xml
+# 基于编辑器的 HTTP 客户端请求
+/httpRequests/
+# 已忽略包含查询文件的默认文件夹
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

+ 428 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceAdReportBusi.py

@@ -0,0 +1,428 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceAdReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceAdReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select account_id from `jeecg-boot`.ctop_user_allocation where media_id=1
+group by account_id """
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+                    REPLACE INTO application.app_bytedance_ad_report_busi_{day_count}d 
+                    SELECT concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1)  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+                    report.account_id,
+                    report.campaign_id,
+                    report.ad_id,
+                    report.ad_name,
+                     bei.project_id project_id,
+                     bei.product_id product_id,
+                     bei.advertiser_id advertiser_id,
+                     bei.company_id company_id,
+                     bei.account_name account_name,
+                     bei.project_name project_name,
+                     bei.advertiser_name advertiser_name,
+                     bei.company_name company_name,
+                     bei.sale_id sale_id,
+                     bei.user_id user_id,
+                     bei.leader_id leader_id,
+                     bei.org_code org_code,
+                     report.cost,
+                     ifnull(ratio.cost,0) last_cost,
+                     ifnull(round((report.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,
+                     report. `SHOW`,
+                     report.avg_show_cost,
+                     report.click,
+                     report.avg_click_cost,
+                     report.ctr,
+                     report. `CONVERT`,
+                     report.convert_cost,
+                     report.convert_rate,
+                     report.deep_convert,
+                     report.deep_convert_cost,
+                     report.deep_convert_rate,
+                     report.attribution_convert,
+                     report.attribution_convert_cost,
+                     report.attribution_deep_convert,
+                     report.attribution_deep_convert_cost,
+                     report.download_start,
+                     report.download_start_cost,
+                     report.download_start_rate,
+                     report.download_finish,
+                     report.download_finish_cost,
+                     report.download_finish_rate,
+                     report.click_install,
+                     report.install_finish,
+                     report.install_finish_cost,
+                     report.install_finish_rate,
+                     report.active,
+                     report.active_cost,
+                     report.active_rate,
+                     report.register,
+                     report.active_register_cost,
+                     report.active_register_rate,
+                     report.next_day_open,
+                     report.next_day_open_cost,
+                     report.next_day_open_rate,
+                     report.attribution_next_day_open_cnt,
+                     report.attribution_next_day_open_cost,
+                     report.attribution_next_day_open_rate,
+                     report.game_addiction,
+                     report.game_addiction_cost,
+                     report.game_addiction_rate,
+                     report.pay_count,
+                     report.active_pay_cost,
+                     report.active_pay_rate,
+                     report.loan_completion,
+                     report.loan_completion_cost,
+                     report.loan_completion_rate,
+                     report.pre_loan_credit,
+                     report.pre_loan_credit_cost,
+                     report.loan_credit,
+                     report.loan_credit_cost,
+                     report.loan_credit_rate,
+                     report.in_app_uv,
+                     report.in_app_detail_uv,
+                     report.in_app_cart,
+                     report.in_app_pay,
+                     report.in_app_order,
+                     report.attribution_game_pay_7d_count,
+                     report.attribution_game_pay_7d_cost,
+                     report.attribution_active_pay_7d_per_count,
+                     report.game_pay_cost,
+                     report.game_pay_count,
+                     report.phone,
+                     report.form,
+                     report.map_search,
+                     report.button,
+                     report.`VIEW`,
+                     report.download,
+                     report.qq,
+                     report.lottery,
+                     report.vote,
+                     report.message,
+                     report.redirect,
+                     report.shopping,
+                     report.consult,
+                     report.wechat,
+                     report.phone_confirm,
+                     report.phone_connect,
+                     report.consult_effective,
+                     report.coupon,
+                     report.coupon_single_page,
+                     report.redirect_to_shop,
+                     report.poi_collect,
+                     report.poi_address_click,
+                     report.luban_order_cnt,
+                     report.luban_order_stat_amount,
+                     report.luban_order_roi,
+                     report.luban_live_enter_cnt,
+                     report.live_watch_one_minute_count,
+                     report.luban_live_follow_cnt,
+                     report.live_fans_club_join_cnt,
+                     report.luban_live_comment_cnt,
+                     report.luban_live_share_cnt,
+                     report.luban_live_gift_cnt,
+                     report.luban_live_gift_amount,
+                     report.luban_live_slidecart_click_cnt,
+                     report.luban_live_click_product_cnt,
+                     report.luban_live_pay_order_count,
+                     report.luban_live_pay_order_stat_cost,
+                     report.live_component_click_count,
+                     report.live_component_click_cost,
+                     report.live_component_click_rate,
+                     report.wechat_login_count,
+                     report.attribution_wechat_login_30d_count,
+                     report.wechat_login_cost,
+                     report.attribution_wechat_login_30d_cost,
+                     report.wechat_first_pay_count,
+                     report.attribution_wechat_first_pay_30d_count,
+                     report.wechat_first_pay_cost,
+                     report.attribution_wechat_first_pay_30d_cost,
+                     report.wechat_first_pay_rate,
+                     report.attribution_wechat_first_pay_30d_rate,
+                     report.wechat_pay_amount,
+                     report.attribution_wechat_pay_30d_amount,
+                     report.attribution_wechat_pay_30d_roi,
+                     report.phone_effective,
+                     report.total_play,
+                     report.valid_play,
+                     report.valid_play_cost,
+                     report.valid_play_rate,
+                     report.play_25_feed_break,
+                     report.play_50_feed_break,
+                     report.play_75_feed_break,
+                     report.play_100_feed_break,
+                     report.average_play_time_per_play,
+                     report.play_over_rate,
+                     report.wifi_play_rate,
+                     report.wifi_play,
+                     report.play_duration_sum,
+                     report.advanced_creative_phone_click,
+                     report.advanced_creative_counsel_click,
+                     report.advanced_creative_form_click,
+                     report.advanced_creative_coupon_addition,
+                     report.advanced_creative_form_submit,
+                     report.card_show,
+                     report.`SHARE`,
+                     report.`COMMENT`,
+                     report.`LIKE`,
+                     report.follow,
+                     report.home_visited,
+                     report.ies_challenge_click,
+                     report.ies_music_click,
+                     report.location_click,
+                     report.message_action,
+                     report.click_landing_page,
+                     report.click_shopwindow,
+                     report.click_website,
+                     report.click_download,
+                     report.click_call_dy,
+                     report.submit_certification_count,
+                     report.approval_count,
+                     report.first_rental_order_count,
+                     report.first_order_count,
+                     report.commute_first_pay_count
+                    FROM
+                     (
+                      SELECT
+                        advertiser_id account_id,
+                        campaign_id campaign_id,
+                        ad_id,
+                        ad_name,
+                        sum(cost) cost,
+                        sum(`show`) `show`,
+                        ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+                        sum(click)click,
+                        ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+                        ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+                        sum(`convert`)`convert`,
+                        ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+                        ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+                        sum(deep_convert) deep_convert,
+                        ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+                        ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+                        sum(attribution_convert)attribution_convert,
+                        ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+                        sum(attribution_deep_convert) attribution_deep_convert,
+                        ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+                        sum(download_start) download_start,
+                        ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+                        ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+                        sum(download_finish) download_finish,
+                        ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+                        ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+                        sum(click_install)click_install,
+                        sum(install_finish) install_finish,
+                        ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+                        ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+                        sum(active) active,
+                        ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+                        ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+                        sum(register)register,
+                        ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+                        ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+                        sum(next_day_open)next_day_open,
+                        ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+                        ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+                        sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+                        ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+                        ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+                        sum(game_addiction) game_addiction,
+                        ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+                        ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+                        sum(pay_count)pay_count,
+                        ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+                        ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+                        sum(loan_completion) loan_completion,
+                        ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+                        ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+                        ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+                        ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+                        sum(loan_credit)loan_credit,
+                        ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+                        ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+                        sum(in_app_uv)in_app_uv,
+                        sum(in_app_detail_uv) in_app_detail_uv,
+                        sum(in_app_cart)in_app_cart,
+                        sum(in_app_pay) in_app_pay,
+                        sum(in_app_order) in_app_order,
+                        ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+                        ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+                        ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+                        sum(game_pay_cost)game_pay_cost,
+                        sum(game_pay_count) game_pay_count,
+                        sum(phone)phone,
+                        sum(form) form,
+                        sum(map_search) map_search,
+                        sum(button) button,
+                        sum(`view`) `view`,
+                        sum(download)download,
+                        sum(qq)qq,
+                        sum(lottery)lottery,
+                        sum(vote) vote,
+                        sum(message)message,
+                        sum(redirect)redirect,
+                        sum(shopping)shopping,
+                        sum(consult)consult,
+                        sum(wechat) wechat,
+                        sum(phone_confirm)phone_confirm,
+                        sum(phone_connect)phone_connect,
+                        sum(consult_effective)consult_effective,
+                        sum(coupon) coupon,
+                        sum(coupon_single_page) coupon_single_page,
+                        sum(redirect_to_shop) redirect_to_shop,
+                        sum(poi_collect)poi_collect,
+                        sum(poi_address_click)poi_address_click,
+                        sum(luban_order_cnt) luban_order_cnt,
+                        sum(luban_order_stat_amount)luban_order_stat_amount,
+                        sum(luban_order_roi) luban_order_roi,
+                        sum(luban_live_enter_cnt) luban_live_enter_cnt,
+                        sum(live_watch_one_minute_count) live_watch_one_minute_count,
+                        sum(luban_live_follow_cnt) luban_live_follow_cnt,
+                        sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+                        sum(luban_live_comment_cnt) luban_live_comment_cnt,
+                        sum(luban_live_share_cnt) luban_live_share_cnt,
+                        sum(luban_live_gift_cnt)luban_live_gift_cnt,
+                        sum(luban_live_gift_amount) luban_live_gift_amount,
+                        sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+                        sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+                        sum(luban_live_pay_order_count) luban_live_pay_order_count,
+                        sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+                        sum(live_component_click_count) live_component_click_count,
+                        ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+                        ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+                        sum(wechat_login_count) wechat_login_count,
+                        sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+                        sum(wechat_first_pay_count) wechat_first_pay_count,
+                        sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+                        ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+                        ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+                        sum(wechat_pay_amount)wechat_pay_amount,
+                        sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+                        sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+                        sum(phone_effective) phone_effective,
+                        sum(total_play) total_play,
+                        sum(valid_play) valid_play,
+                        ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+                        ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+                        sum(play_25_feed_break) play_25_feed_break,
+                        sum(play_50_feed_break) play_50_feed_break,
+                        sum(play_75_feed_break) play_75_feed_break,
+                        sum(play_100_feed_break)play_100_feed_break,
+                        ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+                        ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+                        ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+                        sum(wifi_play)wifi_play,
+                        ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+                        sum(advanced_creative_phone_click) advanced_creative_phone_click,
+                        sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+                        sum(advanced_creative_form_click) advanced_creative_form_click,
+                        sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+                        sum(advanced_creative_form_submit) advanced_creative_form_submit,
+                        sum(card_show)card_show,
+                        sum(`share`)`share`,
+                        sum(`comment`)`comment`,
+                        sum(`like`) `like`,
+                        sum(follow) follow,
+                        sum(home_visited) home_visited,
+                        sum(ies_challenge_click)ies_challenge_click,
+                        sum(ies_music_click) ies_music_click,
+                        sum(location_click) location_click,
+                        sum(message_action) message_action,
+                        sum(click_landing_page) click_landing_page,
+                        sum(click_shopwindow) click_shopwindow,
+                        sum(click_website)click_website,
+                        sum(click_download) click_download,
+                        sum(click_call_dy)click_call_dy,
+                        sum(submit_certification_count) submit_certification_count,
+                        sum(approval_count) approval_count,
+                        sum(first_order_count)first_rental_order_count,
+                        sum(first_rental_order_count) first_order_count,
+                        sum(commute_first_pay_count)commute_first_pay_count
+                      FROM `media_api`.bytedance_ad_report_daily
+                      WHERE stat_datetime >= date_format(
+                       date_sub({date_time}, INTERVAL {day_count}-1 DAY),'%Y%m%d') and advertiser_id={account_id}
+                    GROUP BY
+                    advertiser_id,
+                    campaign_id,
+                    ad_id
+                     ) report
+                    LEFT JOIN (
+                     SELECT
+                      cua.account_id id,
+                      cua.account_id account_id,
+                      cp.id project_id,
+                      product.id product_id,
+                      ca.id advertiser_id,
+                      uc.company_id company_id,
+                      cua.auth_name account_name,
+                      cp.project_name project_name,
+                      ca.`name` advertiser_name,
+                      uc.company_name company_name,
+                      cp.sale_id sale_id,
+                      su.id user_id,
+                      su.leader_id,
+                      su.org_code org_code
+                     FROM
+                      `jeecg-boot`.ctop_user_allocation cua
+                     LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+                     LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+                     LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+                     LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+                     LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+                     WHERE
+                      cua.account_id IS NOT NULL
+                      and cua.account_id ={account_id}
+                    ) bei ON report.account_id = bei.account_id
+                     LEFT JOIN 
+                    (select advertiser_id account_id,campaign_id,ad_id,sum(cost) cost from `media_api`.bytedance_ad_report_daily
+                    where stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+                    and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+                    group by advertiser_id,campaign_id,ad_id) ratio
+                    on report.account_id = ratio.account_id and report.ad_id=ratio.ad_id;
+                        """.format(date_time=self.date_time,
+                                   day_count=self.day_count, account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceAdReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 423 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceAdReportBusiMonth.py

@@ -0,0 +1,423 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceAdReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceAdReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.day_count = Utils.get_days()
+        self.date_time = self.parm[0]
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select account_id from `jeecg-boot`.ctop_user_allocation where media_id=1
+group by account_id """
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+                    REPLACE INTO application.app_bytedance_ad_report_busi_m 
+                   select 
+                    date_format({date_time},'%Y%m'),
+                    report.account_id,
+                    report.campaign_id,
+                    report.ad_id,
+                    report.ad_name,
+                     bei.project_id project_id,
+                     bei.product_id product_id,
+                     bei.advertiser_id advertiser_id,
+                     bei.company_id company_id,
+                     bei.account_name account_name,
+                     bei.project_name project_name,
+                     bei.advertiser_name advertiser_name,
+                     bei.company_name company_name,
+                     bei.sale_id sale_id,
+                     bei.user_id user_id,
+                     bei.leader_id leader_id,
+                     bei.org_code org_code,
+                     report.cost,
+                     report. `SHOW`,
+                     report.avg_show_cost,
+                     report.click,
+                     report.avg_click_cost,
+                     report.ctr,
+                     report. `CONVERT`,
+                     report.convert_cost,
+                     report.convert_rate,
+                     report.deep_convert,
+                     report.deep_convert_cost,
+                     report.deep_convert_rate,
+                     report.attribution_convert,
+                     report.attribution_convert_cost,
+                     report.attribution_deep_convert,
+                     report.attribution_deep_convert_cost,
+                     report.download_start,
+                     report.download_start_cost,
+                     report.download_start_rate,
+                     report.download_finish,
+                     report.download_finish_cost,
+                     report.download_finish_rate,
+                     report.click_install,
+                     report.install_finish,
+                     report.install_finish_cost,
+                     report.install_finish_rate,
+                     report.active,
+                     report.active_cost,
+                     report.active_rate,
+                     report.register,
+                     report.active_register_cost,
+                     report.active_register_rate,
+                     report.next_day_open,
+                     report.next_day_open_cost,
+                     report.next_day_open_rate,
+                     report.attribution_next_day_open_cnt,
+                     report.attribution_next_day_open_cost,
+                     report.attribution_next_day_open_rate,
+                     report.game_addiction,
+                     report.game_addiction_cost,
+                     report.game_addiction_rate,
+                     report.pay_count,
+                     report.active_pay_cost,
+                     report.active_pay_rate,
+                     report.loan_completion,
+                     report.loan_completion_cost,
+                     report.loan_completion_rate,
+                     report.pre_loan_credit,
+                     report.pre_loan_credit_cost,
+                     report.loan_credit,
+                     report.loan_credit_cost,
+                     report.loan_credit_rate,
+                     report.in_app_uv,
+                     report.in_app_detail_uv,
+                     report.in_app_cart,
+                     report.in_app_pay,
+                     report.in_app_order,
+                     report.attribution_game_pay_7d_count,
+                     report.attribution_game_pay_7d_cost,
+                     report.attribution_active_pay_7d_per_count,
+                     report.game_pay_cost,
+                     report.game_pay_count,
+                     report.phone,
+                     report.form,
+                     report.map_search,
+                     report.button,
+                     report.`VIEW`,
+                     report.download,
+                     report.qq,
+                     report.lottery,
+                     report.vote,
+                     report.message,
+                     report.redirect,
+                     report.shopping,
+                     report.consult,
+                     report.wechat,
+                     report.phone_confirm,
+                     report.phone_connect,
+                     report.consult_effective,
+                     report.coupon,
+                     report.coupon_single_page,
+                     report.redirect_to_shop,
+                     report.poi_collect,
+                     report.poi_address_click,
+                     report.luban_order_cnt,
+                     report.luban_order_stat_amount,
+                     report.luban_order_roi,
+                     report.luban_live_enter_cnt,
+                     report.live_watch_one_minute_count,
+                     report.luban_live_follow_cnt,
+                     report.live_fans_club_join_cnt,
+                     report.luban_live_comment_cnt,
+                     report.luban_live_share_cnt,
+                     report.luban_live_gift_cnt,
+                     report.luban_live_gift_amount,
+                     report.luban_live_slidecart_click_cnt,
+                     report.luban_live_click_product_cnt,
+                     report.luban_live_pay_order_count,
+                     report.luban_live_pay_order_stat_cost,
+                     report.live_component_click_count,
+                     report.live_component_click_cost,
+                     report.live_component_click_rate,
+                     report.wechat_login_count,
+                     report.attribution_wechat_login_30d_count,
+                     report.wechat_login_cost,
+                     report.attribution_wechat_login_30d_cost,
+                     report.wechat_first_pay_count,
+                     report.attribution_wechat_first_pay_30d_count,
+                     report.wechat_first_pay_cost,
+                     report.attribution_wechat_first_pay_30d_cost,
+                     report.wechat_first_pay_rate,
+                     report.attribution_wechat_first_pay_30d_rate,
+                     report.wechat_pay_amount,
+                     report.attribution_wechat_pay_30d_amount,
+                     report.attribution_wechat_pay_30d_roi,
+                     report.phone_effective,
+                     report.total_play,
+                     report.valid_play,
+                     report.valid_play_cost,
+                     report.valid_play_rate,
+                     report.play_25_feed_break,
+                     report.play_50_feed_break,
+                     report.play_75_feed_break,
+                     report.play_100_feed_break,
+                     report.average_play_time_per_play,
+                     report.play_over_rate,
+                     report.wifi_play_rate,
+                     report.wifi_play,
+                     report.play_duration_sum,
+                     report.advanced_creative_phone_click,
+                     report.advanced_creative_counsel_click,
+                     report.advanced_creative_form_click,
+                     report.advanced_creative_coupon_addition,
+                     report.advanced_creative_form_submit,
+                     report.card_show,
+                     report.`SHARE`,
+                     report.`COMMENT`,
+                     report.`LIKE`,
+                     report.follow,
+                     report.home_visited,
+                     report.ies_challenge_click,
+                     report.ies_music_click,
+                     report.location_click,
+                     report.message_action,
+                     report.click_landing_page,
+                     report.click_shopwindow,
+                     report.click_website,
+                     report.click_download,
+                     report.click_call_dy,
+                     report.submit_certification_count,
+                     report.approval_count,
+                     report.first_rental_order_count,
+                     report.first_order_count,
+                     report.commute_first_pay_count
+                    FROM
+                     (
+                      SELECT
+                        advertiser_id account_id,
+                        campaign_id campaign_id,
+                        ad_id,
+                        ad_name,
+                        sum(cost) cost,
+                        sum(`show`) `show`,
+                        ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+                        sum(click)click,
+                        ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+                        ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+                        sum(`convert`)`convert`,
+                        ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+                        ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+                        sum(deep_convert) deep_convert,
+                        ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+                        ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+                        sum(attribution_convert)attribution_convert,
+                        ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+                        sum(attribution_deep_convert) attribution_deep_convert,
+                        ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+                        sum(download_start) download_start,
+                        ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+                        ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+                        sum(download_finish) download_finish,
+                        ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+                        ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+                        sum(click_install)click_install,
+                        sum(install_finish) install_finish,
+                        ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+                        ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+                        sum(active) active,
+                        ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+                        ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+                        sum(register)register,
+                        ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+                        ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+                        sum(next_day_open)next_day_open,
+                        ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+                        ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+                        sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+                        ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+                        ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+                        sum(game_addiction) game_addiction,
+                        ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+                        ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+                        sum(pay_count)pay_count,
+                        ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+                        ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+                        sum(loan_completion) loan_completion,
+                        ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+                        ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+                        ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+                        ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+                        sum(loan_credit)loan_credit,
+                        ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+                        ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+                        sum(in_app_uv)in_app_uv,
+                        sum(in_app_detail_uv) in_app_detail_uv,
+                        sum(in_app_cart)in_app_cart,
+                        sum(in_app_pay) in_app_pay,
+                        sum(in_app_order) in_app_order,
+                        ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+                        ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+                        ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+                        sum(game_pay_cost)game_pay_cost,
+                        sum(game_pay_count) game_pay_count,
+                        sum(phone)phone,
+                        sum(form) form,
+                        sum(map_search) map_search,
+                        sum(button) button,
+                        sum(`view`) `view`,
+                        sum(download)download,
+                        sum(qq)qq,
+                        sum(lottery)lottery,
+                        sum(vote) vote,
+                        sum(message)message,
+                        sum(redirect)redirect,
+                        sum(shopping)shopping,
+                        sum(consult)consult,
+                        sum(wechat) wechat,
+                        sum(phone_confirm)phone_confirm,
+                        sum(phone_connect)phone_connect,
+                        sum(consult_effective)consult_effective,
+                        sum(coupon) coupon,
+                        sum(coupon_single_page) coupon_single_page,
+                        sum(redirect_to_shop) redirect_to_shop,
+                        sum(poi_collect)poi_collect,
+                        sum(poi_address_click)poi_address_click,
+                        sum(luban_order_cnt) luban_order_cnt,
+                        sum(luban_order_stat_amount)luban_order_stat_amount,
+                        sum(luban_order_roi) luban_order_roi,
+                        sum(luban_live_enter_cnt) luban_live_enter_cnt,
+                        sum(live_watch_one_minute_count) live_watch_one_minute_count,
+                        sum(luban_live_follow_cnt) luban_live_follow_cnt,
+                        sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+                        sum(luban_live_comment_cnt) luban_live_comment_cnt,
+                        sum(luban_live_share_cnt) luban_live_share_cnt,
+                        sum(luban_live_gift_cnt)luban_live_gift_cnt,
+                        sum(luban_live_gift_amount) luban_live_gift_amount,
+                        sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+                        sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+                        sum(luban_live_pay_order_count) luban_live_pay_order_count,
+                        sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+                        sum(live_component_click_count) live_component_click_count,
+                        ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+                        ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+                        sum(wechat_login_count) wechat_login_count,
+                        sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+                        sum(wechat_first_pay_count) wechat_first_pay_count,
+                        sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+                        ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+                        ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+                        sum(wechat_pay_amount)wechat_pay_amount,
+                        sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+                        sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+                        sum(phone_effective) phone_effective,
+                        sum(total_play) total_play,
+                        sum(valid_play) valid_play,
+                        ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+                        ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+                        sum(play_25_feed_break) play_25_feed_break,
+                        sum(play_50_feed_break) play_50_feed_break,
+                        sum(play_75_feed_break) play_75_feed_break,
+                        sum(play_100_feed_break)play_100_feed_break,
+                        ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+                        ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+                        ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+                        sum(wifi_play)wifi_play,
+                        ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+                        sum(advanced_creative_phone_click) advanced_creative_phone_click,
+                        sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+                        sum(advanced_creative_form_click) advanced_creative_form_click,
+                        sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+                        sum(advanced_creative_form_submit) advanced_creative_form_submit,
+                        sum(card_show)card_show,
+                        sum(`share`)`share`,
+                        sum(`comment`)`comment`,
+                        sum(`like`) `like`,
+                        sum(follow) follow,
+                        sum(home_visited) home_visited,
+                        sum(ies_challenge_click)ies_challenge_click,
+                        sum(ies_music_click) ies_music_click,
+                        sum(location_click) location_click,
+                        sum(message_action) message_action,
+                        sum(click_landing_page) click_landing_page,
+                        sum(click_shopwindow) click_shopwindow,
+                        sum(click_website)click_website,
+                        sum(click_download) click_download,
+                        sum(click_call_dy)click_call_dy,
+                        sum(submit_certification_count) submit_certification_count,
+                        sum(approval_count) approval_count,
+                        sum(first_order_count)first_rental_order_count,
+                        sum(first_rental_order_count) first_order_count,
+                        sum(commute_first_pay_count)commute_first_pay_count
+                      FROM `media_api`.bytedance_ad_report_daily
+                      WHERE stat_datetime >= date_format(
+                      date_sub({date_time}, INTERVAL {day_count} DAY),
+                      '%Y%m%d') and advertiser_id={account_id}
+                       GROUP BY
+                       advertiser_id,
+                       campaign_id,
+                       ad_id,
+                       ad_name
+                       ) report
+                    LEFT JOIN (
+                     SELECT
+                      cua.account_id id,
+                      cua.account_id account_id,
+                      cp.id project_id,
+                      product.id product_id,
+                      ca.id advertiser_id,
+                      uc.company_id company_id,
+                      cua.auth_name account_name,
+                      cp.project_name project_name,
+                      ca.`name` advertiser_name,
+                      uc.company_name company_name,
+                      cp.sale_id sale_id,
+                      su.id user_id,
+                      su.leader_id,
+                      su.org_code org_code
+                     FROM
+                      `jeecg-boot`.ctop_user_allocation cua
+                     LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+                     LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+                     LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+                     LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+                     LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+                     WHERE
+                      cua.account_id IS NOT NULL
+                       and cua.account_id={account_id}
+                    ) bei ON report.account_id = bei.account_id
+                   """.format(date_time=self.date_time,
+                              day_count=self.day_count, account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceAdReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 407 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceAdvertiserReportBusi.py

@@ -0,0 +1,407 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceAdReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceAdReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+            SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_advertiser_report_busi_{day_count}d
+SELECT concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1)  DAY),'%Y%m%d'),'-',{date_time})  start_end_time,
+       report.account_id,
+       bei.project_id      project_id,
+       bei.product_id      product_id,
+       bei.advertiser_id   advertiser_id,
+       bei.company_id      company_id,
+       bei.account_name    account_name,
+       bei.project_name    project_name,
+       bei.advertiser_name advertiser_name,
+       bei.company_name    company_name,
+       bei.sale_id         sale_id,
+       bei.user_id         user_id,
+       bei.leader_id       leader_id,
+       bei.org_code        org_code,
+       report.cost,
+       ifnull(ratio.cost,0) last_cost,
+       ifnull(round((report.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,
+       report.`show`,
+       report.avg_show_cost,
+       report.click,
+       report.avg_click_cost,
+       report.ctr,
+       report.`convert`,
+       report.convert_cost,
+       report.convert_rate,
+       report.deep_convert,
+       report.deep_convert_cost,
+       report.deep_convert_rate,
+       report.attribution_convert,
+       report.attribution_convert_cost,
+       report.attribution_deep_convert,
+       report.attribution_deep_convert_cost,
+       report.download_start,
+       report.download_start_cost,
+       report.download_start_rate,
+       report.download_finish,
+       report.download_finish_cost,
+       report.download_finish_rate,
+       report.click_install,
+       report.install_finish,
+       report.install_finish_cost,
+       report.install_finish_rate,
+       report.active,
+       report.active_cost,
+       report.active_rate,
+       report.register,
+       report.active_register_cost,
+       report.active_register_rate,
+       report.next_day_open,
+       report.next_day_open_cost,
+       report.next_day_open_rate,
+       report.attribution_next_day_open_cnt,
+       report.attribution_next_day_open_cost,
+       report.attribution_next_day_open_rate,
+       report.game_addiction,
+       report.game_addiction_cost,
+       report.game_addiction_rate,
+       report.pay_count,
+       report.active_pay_cost,
+       report.active_pay_rate,
+       report.loan_completion,
+       report.loan_completion_cost,
+       report.loan_completion_rate,
+       report.pre_loan_credit,
+       report.pre_loan_credit_cost,
+       report.loan_credit,
+       report.loan_credit_cost,
+       report.loan_credit_rate,
+       report.in_app_uv,
+       report.in_app_detail_uv,
+       report.in_app_cart,
+       report.in_app_pay,
+       report.in_app_order,
+       report.attribution_game_pay_7d_count,
+       report.attribution_game_pay_7d_cost,
+       report.attribution_active_pay_7d_per_count,
+       report.game_pay_cost,
+       report.game_pay_count,
+       report.phone,
+       report.form,
+       report.map_search,
+       report.button,
+       report.`view`,
+       report.download,
+       report.qq,
+       report.lottery,
+       report.vote,
+       report.message,
+       report.redirect,
+       report.shopping,
+       report.consult,
+       report.wechat,
+       report.phone_confirm,
+       report.phone_connect,
+       report.consult_effective,
+       report.coupon,
+       report.coupon_single_page,
+       report.redirect_to_shop,
+       report.poi_collect,
+       report.poi_address_click,
+       report.luban_order_cnt,
+       report.luban_order_stat_amount,
+       report.luban_order_roi,
+       report.luban_live_enter_cnt,
+       report.live_watch_one_minute_count,
+       report.luban_live_follow_cnt,
+       report.live_fans_club_join_cnt,
+       report.luban_live_comment_cnt,
+       report.luban_live_share_cnt,
+       report.luban_live_gift_cnt,
+       report.luban_live_gift_amount,
+       report.luban_live_slidecart_click_cnt,
+       report.luban_live_click_product_cnt,
+       report.luban_live_pay_order_count,
+       report.luban_live_pay_order_stat_cost,
+       report.live_component_click_count,
+       report.live_component_click_cost,
+       report.live_component_click_rate,
+       report.wechat_login_count,
+       report.attribution_wechat_login_30d_count,
+       report.wechat_login_cost,
+       report.attribution_wechat_login_30d_cost,
+       report.wechat_first_pay_count,
+       report.attribution_wechat_first_pay_30d_count,
+       report.wechat_first_pay_cost,
+       report.attribution_wechat_first_pay_30d_cost,
+       report.wechat_first_pay_rate,
+       report.attribution_wechat_first_pay_30d_rate,
+       report.wechat_pay_amount,
+       report.attribution_wechat_pay_30d_amount,
+       report.attribution_wechat_pay_30d_roi,
+       report.phone_effective,
+       report.total_play,
+       report.valid_play,
+       report.valid_play_cost,
+       report.valid_play_rate,
+       report.play_25_feed_break,
+       report.play_50_feed_break,
+       report.play_75_feed_break,
+       report.play_100_feed_break,
+       report.average_play_time_per_play,
+       report.play_over_rate,
+       report.wifi_play_rate,
+       report.wifi_play,
+       report.play_duration_sum,
+       report.advanced_creative_phone_click,
+       report.advanced_creative_counsel_click,
+       report.advanced_creative_form_click,
+       report.advanced_creative_coupon_addition,
+       report.advanced_creative_form_submit,
+       report.card_show,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.follow,
+       report.home_visited,
+       report.ies_challenge_click,
+       report.ies_music_click,
+       report.location_click,
+       report.message_action,
+       report.click_landing_page,
+       report.click_shopwindow,
+       report.click_website,
+       report.click_download,
+       report.click_call_dy,
+       report.submit_certification_count,
+       report.approval_count,
+       report.first_rental_order_count,
+       report.first_order_count,
+       report.commute_first_pay_count
+FROM (
+       SELECT 
+               advertiser_id account_id,
+               sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click)click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`)`convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert)attribution_convert,
+               ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+               ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install)click_install,
+               sum(install_finish) install_finish,
+               ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register)register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open)next_day_open,
+               ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count)pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+               sum(loan_credit)loan_credit,
+               ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv)in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart) in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+               ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+               ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost)game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone) phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download)download,
+               sum(qq)qq,
+               sum(lottery)lottery,
+               sum(vote) vote,
+               sum(message)message,
+               sum(redirect)redirect,
+               sum(shopping)shopping,
+               sum(consult)consult,
+               sum(wechat) wechat,
+               sum(phone_confirm)phone_confirm,
+               sum(phone_connect)phone_connect,
+               sum(consult_effective)consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect)poi_collect,
+               sum(poi_address_click)poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount)luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt)luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount)wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break)play_100_feed_break,
+               ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play)wifi_play,
+               ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show)card_show,
+               sum(`share`)`share`,
+               sum(`comment`)`comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click)ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website)click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy)click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count)first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count)commute_first_pay_count
+              FROM `media_api`.bytedance_advertiser_report_daily
+              WHERE stat_datetime >= date_format(
+              date_sub({date_time}, INTERVAL {day_count} - 1 DAY),
+              '%Y%m%d'
+              )
+              GROUP BY advertiser_id
+     ) report
+         LEFT JOIN (
+    SELECT cua.account_id  id,
+           cua.account_id  account_id,
+           cp.id           project_id,
+           product.id      product_id,
+           ca.id           advertiser_id,
+           uc.company_id   company_id,
+           cua.auth_name   account_name,
+           cp.project_name project_name,
+           ca.`name`       advertiser_name,
+           uc.company_name company_name,
+           cp.sale_id      sale_id,
+           su.id           user_id,
+           su.leader_id,
+           su.org_code     org_code
+    FROM `jeecg-boot`.ctop_user_allocation cua
+             LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+             LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+             LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+             LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+             LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+    WHERE cua.account_id IS NOT NULL
+) bei ON report.account_id = bei.account_id
+left join
+(select advertiser_id account_id,sum(cost) cost from `media_api`.bytedance_advertiser_report_daily
+where stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+         and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id) ratio
+ON report.account_id = ratio.account_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceAdReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 400 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceAdvertiserReportBusiMonth.py

@@ -0,0 +1,400 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceAdvertiserReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceAdvertiserReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+            SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_advertiser_report_busi_m
+SELECT date_format({date_time},'%Y%m') as stat_mon,
+       report.account_id,
+       bei.project_id      project_id,
+       bei.product_id      product_id,
+       bei.advertiser_id   advertiser_id,
+       bei.company_id      company_id,
+       bei.account_name    account_name,
+       bei.project_name    project_name,
+       bei.advertiser_name advertiser_name,
+       bei.company_name    company_name,
+       bei.sale_id         sale_id,
+       bei.user_id         user_id,
+       bei.leader_id       leader_id,
+       bei.org_code        org_code,
+       report.cost,
+       report.`show`,
+       report.avg_show_cost,
+       report.click,
+       report.avg_click_cost,
+       report.ctr,
+       report.`convert`,
+       report.convert_cost,
+       report.convert_rate,
+       report.deep_convert,
+       report.deep_convert_cost,
+       report.deep_convert_rate,
+       report.attribution_convert,
+       report.attribution_convert_cost,
+       report.attribution_deep_convert,
+       report.attribution_deep_convert_cost,
+       report.download_start,
+       report.download_start_cost,
+       report.download_start_rate,
+       report.download_finish,
+       report.download_finish_cost,
+       report.download_finish_rate,
+       report.click_install,
+       report.install_finish,
+       report.install_finish_cost,
+       report.install_finish_rate,
+       report.active,
+       report.active_cost,
+       report.active_rate,
+       report.register,
+       report.active_register_cost,
+       report.active_register_rate,
+       report.next_day_open,
+       report.next_day_open_cost,
+       report.next_day_open_rate,
+       report.attribution_next_day_open_cnt,
+       report.attribution_next_day_open_cost,
+       report.attribution_next_day_open_rate,
+       report.game_addiction,
+       report.game_addiction_cost,
+       report.game_addiction_rate,
+       report.pay_count,
+       report.active_pay_cost,
+       report.active_pay_rate,
+       report.loan_completion,
+       report.loan_completion_cost,
+       report.loan_completion_rate,
+       report.pre_loan_credit,
+       report.pre_loan_credit_cost,
+       report.loan_credit,
+       report.loan_credit_cost,
+       report.loan_credit_rate,
+       report.in_app_uv,
+       report.in_app_detail_uv,
+       report.in_app_cart,
+       report.in_app_pay,
+       report.in_app_order,
+       report.attribution_game_pay_7d_count,
+       report.attribution_game_pay_7d_cost,
+       report.attribution_active_pay_7d_per_count,
+       report.game_pay_cost,
+       report.game_pay_count,
+       report.phone,
+       report.form,
+       report.map_search,
+       report.button,
+       report.`view`,
+       report.download,
+       report.qq,
+       report.lottery,
+       report.vote,
+       report.message,
+       report.redirect,
+       report.shopping,
+       report.consult,
+       report.wechat,
+       report.phone_confirm,
+       report.phone_connect,
+       report.consult_effective,
+       report.coupon,
+       report.coupon_single_page,
+       report.redirect_to_shop,
+       report.poi_collect,
+       report.poi_address_click,
+       report.luban_order_cnt,
+       report.luban_order_stat_amount,
+       report.luban_order_roi,
+       report.luban_live_enter_cnt,
+       report.live_watch_one_minute_count,
+       report.luban_live_follow_cnt,
+       report.live_fans_club_join_cnt,
+       report.luban_live_comment_cnt,
+       report.luban_live_share_cnt,
+       report.luban_live_gift_cnt,
+       report.luban_live_gift_amount,
+       report.luban_live_slidecart_click_cnt,
+       report.luban_live_click_product_cnt,
+       report.luban_live_pay_order_count,
+       report.luban_live_pay_order_stat_cost,
+       report.live_component_click_count,
+       report.live_component_click_cost,
+       report.live_component_click_rate,
+       report.wechat_login_count,
+       report.attribution_wechat_login_30d_count,
+       report.wechat_login_cost,
+       report.attribution_wechat_login_30d_cost,
+       report.wechat_first_pay_count,
+       report.attribution_wechat_first_pay_30d_count,
+       report.wechat_first_pay_cost,
+       report.attribution_wechat_first_pay_30d_cost,
+       report.wechat_first_pay_rate,
+       report.attribution_wechat_first_pay_30d_rate,
+       report.wechat_pay_amount,
+       report.attribution_wechat_pay_30d_amount,
+       report.attribution_wechat_pay_30d_roi,
+       report.phone_effective,
+       report.total_play,
+       report.valid_play,
+       report.valid_play_cost,
+       report.valid_play_rate,
+       report.play_25_feed_break,
+       report.play_50_feed_break,
+       report.play_75_feed_break,
+       report.play_100_feed_break,
+       report.average_play_time_per_play,
+       report.play_over_rate,
+       report.wifi_play_rate,
+       report.wifi_play,
+       report.play_duration_sum,
+       report.advanced_creative_phone_click,
+       report.advanced_creative_counsel_click,
+       report.advanced_creative_form_click,
+       report.advanced_creative_coupon_addition,
+       report.advanced_creative_form_submit,
+       report.card_show,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.follow,
+       report.home_visited,
+       report.ies_challenge_click,
+       report.ies_music_click,
+       report.location_click,
+       report.message_action,
+       report.click_landing_page,
+       report.click_shopwindow,
+       report.click_website,
+       report.click_download,
+       report.click_call_dy,
+       report.submit_certification_count,
+       report.approval_count,
+       report.first_rental_order_count,
+       report.first_order_count,
+       report.commute_first_pay_count
+FROM (
+      SELECT 
+               advertiser_id account_id,
+               sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click)click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`)`convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert)attribution_convert,
+               ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+               ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install)click_install,
+               sum(install_finish) install_finish,
+               ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register)register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open)next_day_open,
+               ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count)pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+               sum(loan_credit)loan_credit,
+               ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv)in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart)in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+               ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+               ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost)game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone)phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download)download,
+               sum(qq)qq,
+               sum(lottery)lottery,
+               sum(vote) vote,
+               sum(message)message,
+               sum(redirect)redirect,
+               sum(shopping)shopping,
+               sum(consult)consult,
+               sum(wechat) wechat,
+               sum(phone_confirm)phone_confirm,
+               sum(phone_connect)phone_connect,
+               sum(consult_effective)consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect)poi_collect,
+               sum(poi_address_click)poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount)luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt)luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount)wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break)play_100_feed_break,
+               ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play)wifi_play,
+               ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show)card_show,
+               sum(`share`)`share`,
+               sum(`comment`)`comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click)ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website)click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy)click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count)first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count)commute_first_pay_count
+              FROM `media_api`.bytedance_advertiser_report_daily
+              WHERE stat_datetime >= date_format(
+              date_sub({date_time}, INTERVAL {day_count} DAY),
+              '%Y%m%d'
+              )
+              GROUP BY advertiser_id
+     ) report
+         LEFT JOIN (
+    SELECT cua.account_id  id,
+           cua.account_id  account_id,
+           cp.id           project_id,
+           product.id      product_id,
+           ca.id           advertiser_id,
+           uc.company_id   company_id,
+           cua.auth_name   account_name,
+           cp.project_name project_name,
+           ca.`name`       advertiser_name,
+           uc.company_name company_name,
+           cp.sale_id      sale_id,
+           su.id           user_id,
+           su.leader_id,
+           su.org_code     org_code
+    FROM `jeecg-boot`.ctop_user_allocation cua
+             LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+             LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+             LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+             LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+             LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+    WHERE cua.account_id IS NOT NULL
+) bei ON report.account_id = bei.account_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceAdvertiserReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 413 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceCampaignReportBusi.py

@@ -0,0 +1,413 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceCampaignReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceCampaignReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_campaign_report_busi_{day_count}d 
+SELECT concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1)  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+report.account_id,
+ report.campaign_id,
+ report.campaign_name,
+ bei.project_id project_id,
+ bei.product_id product_id,
+ bei.advertiser_id advertiser_id,
+ bei.company_id company_id,
+ bei.account_name account_name,
+ bei.project_name project_name,
+ bei.advertiser_name advertiser_name,
+ bei.company_name company_name,
+ bei.sale_id sale_id,
+ bei.user_id user_id,
+ bei.leader_id leader_id,
+ bei.org_code org_code,
+ report.cost,
+ ifnull(ratio.cost,0) last_cost,
+ ifnull(round((report.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,
+ report. `SHOW`,
+ report.avg_show_cost,
+ report.click,
+ report.avg_click_cost,
+ report.ctr,
+ report. `CONVERT`,
+ report.convert_cost,
+ report.convert_rate,
+ report.deep_convert,
+ report.deep_convert_cost,
+ report.deep_convert_rate,
+ report.attribution_convert,
+ report.attribution_convert_cost,
+ report.attribution_deep_convert,
+ report.attribution_deep_convert_cost,
+ report.download_start,
+ report.download_start_cost,
+ report.download_start_rate,
+ report.download_finish,
+ report.download_finish_cost,
+ report.download_finish_rate,
+ report.click_install,
+ report.install_finish,
+ report.install_finish_cost,
+ report.install_finish_rate,
+ report.active,
+ report.active_cost,
+ report.active_rate,
+ report.register,
+ report.active_register_cost,
+ report.active_register_rate,
+ report.next_day_open,
+ report.next_day_open_cost,
+ report.next_day_open_rate,
+ report.attribution_next_day_open_cnt,
+ report.attribution_next_day_open_cost,
+ report.attribution_next_day_open_rate,
+ report.game_addiction,
+ report.game_addiction_cost,
+ report.game_addiction_rate,
+ report.pay_count,
+ report.active_pay_cost,
+ report.active_pay_rate,
+ report.loan_completion,
+ report.loan_completion_cost,
+ report.loan_completion_rate,
+ report.pre_loan_credit,
+ report.pre_loan_credit_cost,
+ report.loan_credit,
+ report.loan_credit_cost,
+ report.loan_credit_rate,
+ report.in_app_uv,
+ report.in_app_detail_uv,
+ report.in_app_cart,
+ report.in_app_pay,
+ report.in_app_order,
+ report.attribution_game_pay_7d_count,
+ report.attribution_game_pay_7d_cost,
+ report.attribution_active_pay_7d_per_count,
+ report.game_pay_cost,
+ report.game_pay_count,
+ report.phone,
+ report.form,
+ report.map_search,
+ report.button,
+ report.`VIEW`,
+ report.download,
+ report.qq,
+ report.lottery,
+ report.vote,
+ report.message,
+ report.redirect,
+ report.shopping,
+ report.consult,
+ report.wechat,
+ report.phone_confirm,
+ report.phone_connect,
+ report.consult_effective,
+ report.coupon,
+ report.coupon_single_page,
+ report.redirect_to_shop,
+ report.poi_collect,
+ report.poi_address_click,
+ report.luban_order_cnt,
+ report.luban_order_stat_amount,
+ report.luban_order_roi,
+ report.luban_live_enter_cnt,
+ report.live_watch_one_minute_count,
+ report.luban_live_follow_cnt,
+ report.live_fans_club_join_cnt,
+ report.luban_live_comment_cnt,
+ report.luban_live_share_cnt,
+ report.luban_live_gift_cnt,
+ report.luban_live_gift_amount,
+ report.luban_live_slidecart_click_cnt,
+ report.luban_live_click_product_cnt,
+ report.luban_live_pay_order_count,
+ report.luban_live_pay_order_stat_cost,
+ report.live_component_click_count,
+ report.live_component_click_cost,
+ report.live_component_click_rate,
+ report.wechat_login_count,
+ report.attribution_wechat_login_30d_count,
+ report.wechat_login_cost,
+ report.attribution_wechat_login_30d_cost,
+ report.wechat_first_pay_count,
+ report.attribution_wechat_first_pay_30d_count,
+ report.wechat_first_pay_cost,
+ report.attribution_wechat_first_pay_30d_cost,
+ report.wechat_first_pay_rate,
+ report.attribution_wechat_first_pay_30d_rate,
+ report.wechat_pay_amount,
+ report.attribution_wechat_pay_30d_amount,
+ report.attribution_wechat_pay_30d_roi,
+ report.phone_effective,
+ report.total_play,
+ report.valid_play,
+ report.valid_play_cost,
+ report.valid_play_rate,
+ report.play_25_feed_break,
+ report.play_50_feed_break,
+ report.play_75_feed_break,
+ report.play_100_feed_break,
+ report.average_play_time_per_play,
+ report.play_over_rate,
+ report.wifi_play_rate,
+ report.wifi_play,
+ report.play_duration_sum,
+ report.advanced_creative_phone_click,
+ report.advanced_creative_counsel_click,
+ report.advanced_creative_form_click,
+ report.advanced_creative_coupon_addition,
+ report.advanced_creative_form_submit,
+ report.card_show,
+ report.`SHARE`,
+ report.`COMMENT`,
+ report.`LIKE`,
+ report.follow,
+ report.home_visited,
+ report.ies_challenge_click,
+ report.ies_music_click,
+ report.location_click,
+ report.message_action,
+ report.click_landing_page,
+ report.click_shopwindow,
+ report.click_website,
+ report.click_download,
+ report.click_call_dy,
+ report.submit_certification_count,
+ report.approval_count,
+ report.first_rental_order_count,
+ report.first_order_count,
+ report.commute_first_pay_count
+FROM
+ (
+             SELECT
+                        advertiser_id account_id,
+                        campaign_id campaign_id,
+                        campaign_name campaign_name,
+                        sum(cost) cost,
+                        sum(`show`) `show`,
+                        ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+                        sum(click)click,
+                        ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+                        ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+                        sum(`convert`)`convert`,
+                        ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+                        ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+                        sum(deep_convert) deep_convert,
+                        ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+                        ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+                        sum(attribution_convert)attribution_convert,
+                        ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+                        sum(attribution_deep_convert) attribution_deep_convert,
+                        ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+                        sum(download_start) download_start,
+                        ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+                        ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+                        sum(download_finish) download_finish,
+                        ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+                        ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+                        sum(click_install)click_install,
+                        sum(install_finish) install_finish,
+                        ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+                        ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+                        sum(active) active,
+                        ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+                        ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+                        sum(register)register,
+                        ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+                        ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+                        sum(next_day_open)next_day_open,
+                        ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+                        ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+                        sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+                        ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+                        ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+                        sum(game_addiction) game_addiction,
+                        ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+                        ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+                        sum(pay_count)pay_count,
+                        ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+                        ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+                        sum(loan_completion) loan_completion,
+                        ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+                        ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+                        ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+                        ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+                        sum(loan_credit)loan_credit,
+                        ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+                        ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+                        sum(in_app_uv)in_app_uv,
+                        sum(in_app_detail_uv) in_app_detail_uv,
+                        sum(in_app_cart)in_app_cart,
+                        sum(in_app_pay) in_app_pay,
+                        sum(in_app_order) in_app_order,
+                        ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+                        ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+                        ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+                        sum(game_pay_cost)game_pay_cost,
+                        sum(game_pay_count) game_pay_count,
+                        sum(phone)phone,
+                        sum(form) form,
+                        sum(map_search) map_search,
+                        sum(button) button,
+                        sum(`view`) `view`,
+                        sum(download)download,
+                        sum(qq)qq,
+                        sum(lottery)lottery,
+                        sum(vote) vote,
+                        sum(message)message,
+                        sum(redirect)redirect,
+                        sum(shopping)shopping,
+                        sum(consult)consult,
+                        sum(wechat) wechat,
+                        sum(phone_confirm)phone_confirm,
+                        sum(phone_connect)phone_connect,
+                        sum(consult_effective)consult_effective,
+                        sum(coupon) coupon,
+                        sum(coupon_single_page) coupon_single_page,
+                        sum(redirect_to_shop) redirect_to_shop,
+                        sum(poi_collect)poi_collect,
+                        sum(poi_address_click)poi_address_click,
+                        sum(luban_order_cnt) luban_order_cnt,
+                        sum(luban_order_stat_amount)luban_order_stat_amount,
+                        sum(luban_order_roi) luban_order_roi,
+                        sum(luban_live_enter_cnt) luban_live_enter_cnt,
+                        sum(live_watch_one_minute_count) live_watch_one_minute_count,
+                        sum(luban_live_follow_cnt) luban_live_follow_cnt,
+                        sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+                        sum(luban_live_comment_cnt) luban_live_comment_cnt,
+                        sum(luban_live_share_cnt) luban_live_share_cnt,
+                        sum(luban_live_gift_cnt)luban_live_gift_cnt,
+                        sum(luban_live_gift_amount) luban_live_gift_amount,
+                        sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+                        sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+                        sum(luban_live_pay_order_count) luban_live_pay_order_count,
+                        sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+                        sum(live_component_click_count) live_component_click_count,
+                        ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+                        ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+                        sum(wechat_login_count) wechat_login_count,
+                        sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+                        sum(wechat_first_pay_count) wechat_first_pay_count,
+                        sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+                        ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+                        ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+                        sum(wechat_pay_amount)wechat_pay_amount,
+                        sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+                        sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+                        sum(phone_effective) phone_effective,
+                        sum(total_play) total_play,
+                        sum(valid_play) valid_play,
+                        ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+                        ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+                        sum(play_25_feed_break) play_25_feed_break,
+                        sum(play_50_feed_break) play_50_feed_break,
+                        sum(play_75_feed_break) play_75_feed_break,
+                        sum(play_100_feed_break)play_100_feed_break,
+                        ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+                        ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+                        ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+                        sum(wifi_play)wifi_play,
+                        ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+                        sum(advanced_creative_phone_click) advanced_creative_phone_click,
+                        sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+                        sum(advanced_creative_form_click) advanced_creative_form_click,
+                        sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+                        sum(advanced_creative_form_submit) advanced_creative_form_submit,
+                        sum(card_show)card_show,
+                        sum(`share`)`share`,
+                        sum(`comment`)`comment`,
+                        sum(`like`) `like`,
+                        sum(follow) follow,
+                        sum(home_visited) home_visited,
+                        sum(ies_challenge_click)ies_challenge_click,
+                        sum(ies_music_click) ies_music_click,
+                        sum(location_click) location_click,
+                        sum(message_action) message_action,
+                        sum(click_landing_page) click_landing_page,
+                        sum(click_shopwindow) click_shopwindow,
+                        sum(click_website)click_website,
+                        sum(click_download) click_download,
+                        sum(click_call_dy)click_call_dy,
+                        sum(submit_certification_count) submit_certification_count,
+                        sum(approval_count) approval_count,
+                        sum(first_order_count)first_rental_order_count,
+                        sum(first_rental_order_count) first_order_count,
+                        sum(commute_first_pay_count)commute_first_pay_count
+                      FROM `media_api`.bytedance_campaign_report_daily
+                      WHERE stat_datetime >= date_format(
+                      date_sub({date_time}, INTERVAL {day_count}-1 DAY),'%Y%m%d') 
+                    GROUP BY
+                    advertiser_id,
+                    campaign_id) report
+LEFT JOIN (
+ SELECT
+  cua.account_id id,
+  cua.account_id account_id,
+  cp.id project_id,
+  product.id product_id,
+  ca.id advertiser_id,
+  uc.company_id company_id,
+  cua.auth_name account_name,
+  cp.project_name project_name,
+  ca.`name` advertiser_name,
+  uc.company_name company_name,
+  cp.sale_id sale_id,
+  su.id user_id,
+  su.leader_id,
+  su.org_code org_code
+ FROM
+`jeecg-boot`.ctop_user_allocation cua
+ LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+ LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+ LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+ LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+ LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+ WHERE
+  cua.account_id IS NOT NULL) bei 
+  ON report.account_id = bei.account_id
+  LEFT JOIN 
+(select advertiser_id account_id,campaign_id,sum(cost) cost from `media_api`.bytedance_campaign_report_daily
+where stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+         and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id,campaign_id) ratio
+on report.account_id = ratio.account_id and report.campaign_id = ratio.campaign_id;
+""".format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceCampaignReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 405 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceCampaignReportBusiMonth.py

@@ -0,0 +1,405 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceCampaignReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceCampaignReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_campaign_report_busi_m
+SELECT date_format({date_time},'%Y%m') start_end_time,
+report.account_id,
+ report.campaign_id,
+ report.campaign_name,
+ bei.project_id project_id,
+ bei.product_id product_id,
+ bei.advertiser_id advertiser_id,
+ bei.company_id company_id,
+ bei.account_name account_name,
+ bei.project_name project_name,
+ bei.advertiser_name advertiser_name,
+ bei.company_name company_name,
+ bei.sale_id sale_id,
+ bei.user_id user_id,
+ bei.leader_id leader_id,
+ bei.org_code org_code,
+ report.cost,
+ report. `SHOW`,
+ report.avg_show_cost,
+ report.click,
+ report.avg_click_cost,
+ report.ctr,
+ report. `CONVERT`,
+ report.convert_cost,
+ report.convert_rate,
+ report.deep_convert,
+ report.deep_convert_cost,
+ report.deep_convert_rate,
+ report.attribution_convert,
+ report.attribution_convert_cost,
+ report.attribution_deep_convert,
+ report.attribution_deep_convert_cost,
+ report.download_start,
+ report.download_start_cost,
+ report.download_start_rate,
+ report.download_finish,
+ report.download_finish_cost,
+ report.download_finish_rate,
+ report.click_install,
+ report.install_finish,
+ report.install_finish_cost,
+ report.install_finish_rate,
+ report.active,
+ report.active_cost,
+ report.active_rate,
+ report.register,
+ report.active_register_cost,
+ report.active_register_rate,
+ report.next_day_open,
+ report.next_day_open_cost,
+ report.next_day_open_rate,
+ report.attribution_next_day_open_cnt,
+ report.attribution_next_day_open_cost,
+ report.attribution_next_day_open_rate,
+ report.game_addiction,
+ report.game_addiction_cost,
+ report.game_addiction_rate,
+ report.pay_count,
+ report.active_pay_cost,
+ report.active_pay_rate,
+ report.loan_completion,
+ report.loan_completion_cost,
+ report.loan_completion_rate,
+ report.pre_loan_credit,
+ report.pre_loan_credit_cost,
+ report.loan_credit,
+ report.loan_credit_cost,
+ report.loan_credit_rate,
+ report.in_app_uv,
+ report.in_app_detail_uv,
+ report.in_app_cart,
+ report.in_app_pay,
+ report.in_app_order,
+ report.attribution_game_pay_7d_count,
+ report.attribution_game_pay_7d_cost,
+ report.attribution_active_pay_7d_per_count,
+ report.game_pay_cost,
+ report.game_pay_count,
+ report.phone,
+ report.form,
+ report.map_search,
+ report.button,
+ report.`VIEW`,
+ report.download,
+ report.qq,
+ report.lottery,
+ report.vote,
+ report.message,
+ report.redirect,
+ report.shopping,
+ report.consult,
+ report.wechat,
+ report.phone_confirm,
+ report.phone_connect,
+ report.consult_effective,
+ report.coupon,
+ report.coupon_single_page,
+ report.redirect_to_shop,
+ report.poi_collect,
+ report.poi_address_click,
+ report.luban_order_cnt,
+ report.luban_order_stat_amount,
+ report.luban_order_roi,
+ report.luban_live_enter_cnt,
+ report.live_watch_one_minute_count,
+ report.luban_live_follow_cnt,
+ report.live_fans_club_join_cnt,
+ report.luban_live_comment_cnt,
+ report.luban_live_share_cnt,
+ report.luban_live_gift_cnt,
+ report.luban_live_gift_amount,
+ report.luban_live_slidecart_click_cnt,
+ report.luban_live_click_product_cnt,
+ report.luban_live_pay_order_count,
+ report.luban_live_pay_order_stat_cost,
+ report.live_component_click_count,
+ report.live_component_click_cost,
+ report.live_component_click_rate,
+ report.wechat_login_count,
+ report.attribution_wechat_login_30d_count,
+ report.wechat_login_cost,
+ report.attribution_wechat_login_30d_cost,
+ report.wechat_first_pay_count,
+ report.attribution_wechat_first_pay_30d_count,
+ report.wechat_first_pay_cost,
+ report.attribution_wechat_first_pay_30d_cost,
+ report.wechat_first_pay_rate,
+ report.attribution_wechat_first_pay_30d_rate,
+ report.wechat_pay_amount,
+ report.attribution_wechat_pay_30d_amount,
+ report.attribution_wechat_pay_30d_roi,
+ report.phone_effective,
+ report.total_play,
+ report.valid_play,
+ report.valid_play_cost,
+ report.valid_play_rate,
+ report.play_25_feed_break,
+ report.play_50_feed_break,
+ report.play_75_feed_break,
+ report.play_100_feed_break,
+ report.average_play_time_per_play,
+ report.play_over_rate,
+ report.wifi_play_rate,
+ report.wifi_play,
+ report.play_duration_sum,
+ report.advanced_creative_phone_click,
+ report.advanced_creative_counsel_click,
+ report.advanced_creative_form_click,
+ report.advanced_creative_coupon_addition,
+ report.advanced_creative_form_submit,
+ report.card_show,
+ report.`SHARE`,
+ report.`COMMENT`,
+ report.`LIKE`,
+ report.follow,
+ report.home_visited,
+ report.ies_challenge_click,
+ report.ies_music_click,
+ report.location_click,
+ report.message_action,
+ report.click_landing_page,
+ report.click_shopwindow,
+ report.click_website,
+ report.click_download,
+ report.click_call_dy,
+ report.submit_certification_count,
+ report.approval_count,
+ report.first_rental_order_count,
+ report.first_order_count,
+ report.commute_first_pay_count
+FROM
+ (
+               SELECT
+                        advertiser_id account_id,
+                        campaign_id campaign_id,
+                        campaign_name campaign_name,
+                        sum(cost) cost,
+                        sum(`show`) `show`,
+                        ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+                        sum(click)click,
+                        ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+                        ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+                        sum(`convert`)`convert`,
+                        ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+                        ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+                        sum(deep_convert) deep_convert,
+                        ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+                        ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+                        sum(attribution_convert)attribution_convert,
+                        ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+                        sum(attribution_deep_convert) attribution_deep_convert,
+                        ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+                        sum(download_start) download_start,
+                        ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+                        ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+                        sum(download_finish) download_finish,
+                        ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+                        ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+                        sum(click_install)click_install,
+                        sum(install_finish) install_finish,
+                        ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+                        ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+                        sum(active) active,
+                        ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+                        ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+                        sum(register)register,
+                        ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+                        ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+                        sum(next_day_open)next_day_open,
+                        ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+                        ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+                        sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+                        ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+                        ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+                        sum(game_addiction) game_addiction,
+                        ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+                        ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+                        sum(pay_count)pay_count,
+                        ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+                        ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+                        sum(loan_completion) loan_completion,
+                        ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+                        ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+                        ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+                        ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+                        sum(loan_credit)loan_credit,
+                        ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+                        ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+                        sum(in_app_uv)in_app_uv,
+                        sum(in_app_detail_uv) in_app_detail_uv,
+                        sum(in_app_cart)in_app_cart,
+                        sum(in_app_pay) in_app_pay,
+                        sum(in_app_order) in_app_order,
+                        ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+                        ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+                        ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+                        sum(game_pay_cost)game_pay_cost,
+                        sum(game_pay_count) game_pay_count,
+                        sum(phone)phone,
+                        sum(form) form,
+                        sum(map_search) map_search,
+                        sum(button) button,
+                        sum(`view`) `view`,
+                        sum(download)download,
+                        sum(qq)qq,
+                        sum(lottery)lottery,
+                        sum(vote) vote,
+                        sum(message)message,
+                        sum(redirect)redirect,
+                        sum(shopping)shopping,
+                        sum(consult)consult,
+                        sum(wechat) wechat,
+                        sum(phone_confirm)phone_confirm,
+                        sum(phone_connect)phone_connect,
+                        sum(consult_effective)consult_effective,
+                        sum(coupon) coupon,
+                        sum(coupon_single_page) coupon_single_page,
+                        sum(redirect_to_shop) redirect_to_shop,
+                        sum(poi_collect)poi_collect,
+                        sum(poi_address_click)poi_address_click,
+                        sum(luban_order_cnt) luban_order_cnt,
+                        sum(luban_order_stat_amount)luban_order_stat_amount,
+                        sum(luban_order_roi) luban_order_roi,
+                        sum(luban_live_enter_cnt) luban_live_enter_cnt,
+                        sum(live_watch_one_minute_count) live_watch_one_minute_count,
+                        sum(luban_live_follow_cnt) luban_live_follow_cnt,
+                        sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+                        sum(luban_live_comment_cnt) luban_live_comment_cnt,
+                        sum(luban_live_share_cnt) luban_live_share_cnt,
+                        sum(luban_live_gift_cnt)luban_live_gift_cnt,
+                        sum(luban_live_gift_amount) luban_live_gift_amount,
+                        sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+                        sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+                        sum(luban_live_pay_order_count) luban_live_pay_order_count,
+                        sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+                        sum(live_component_click_count) live_component_click_count,
+                        ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+                        ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+                        sum(wechat_login_count) wechat_login_count,
+                        sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+                        sum(wechat_first_pay_count) wechat_first_pay_count,
+                        sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+                        ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+                        ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+                        ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+                        ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+                        sum(wechat_pay_amount)wechat_pay_amount,
+                        sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+                        sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+                        sum(phone_effective) phone_effective,
+                        sum(total_play) total_play,
+                        sum(valid_play) valid_play,
+                        ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+                        ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+                        sum(play_25_feed_break) play_25_feed_break,
+                        sum(play_50_feed_break) play_50_feed_break,
+                        sum(play_75_feed_break) play_75_feed_break,
+                        sum(play_100_feed_break)play_100_feed_break,
+                        ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+                        ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+                        ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+                        sum(wifi_play)wifi_play,
+                        ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+                        sum(advanced_creative_phone_click) advanced_creative_phone_click,
+                        sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+                        sum(advanced_creative_form_click) advanced_creative_form_click,
+                        sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+                        sum(advanced_creative_form_submit) advanced_creative_form_submit,
+                        sum(card_show)card_show,
+                        sum(`share`)`share`,
+                        sum(`comment`)`comment`,
+                        sum(`like`) `like`,
+                        sum(follow) follow,
+                        sum(home_visited) home_visited,
+                        sum(ies_challenge_click)ies_challenge_click,
+                        sum(ies_music_click) ies_music_click,
+                        sum(location_click) location_click,
+                        sum(message_action) message_action,
+                        sum(click_landing_page) click_landing_page,
+                        sum(click_shopwindow) click_shopwindow,
+                        sum(click_website)click_website,
+                        sum(click_download) click_download,
+                        sum(click_call_dy)click_call_dy,
+                        sum(submit_certification_count) submit_certification_count,
+                        sum(approval_count) approval_count,
+                        sum(first_order_count)first_rental_order_count,
+                        sum(first_rental_order_count) first_order_count,
+                        sum(commute_first_pay_count)commute_first_pay_count
+                      FROM `media_api`.bytedance_campaign_report_daily
+                      WHERE stat_datetime >= date_format(
+                      date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') 
+                    GROUP BY
+                    advertiser_id,
+                    campaign_id,
+                    campaign_name) report
+LEFT JOIN (
+ SELECT
+  cua.account_id id,
+  cua.account_id account_id,
+  cp.id project_id,
+  product.id product_id,
+  ca.id advertiser_id,
+  uc.company_id company_id,
+  cua.auth_name account_name,
+  cp.project_name project_name,
+  ca.`name` advertiser_name,
+  uc.company_name company_name,
+  cp.sale_id sale_id,
+  su.id user_id,
+  su.leader_id,
+  su.org_code org_code
+ FROM
+  `jeecg-boot`.ctop_user_allocation cua
+ LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+ LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+ LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+ LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+ LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+ WHERE
+  cua.account_id IS NOT NULL
+) bei ON report.account_id = bei.account_id""".format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceCampaignReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 670 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyDesignReportBusi.py

@@ -0,0 +1,670 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceCompanyDesignReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceCompanyDesignReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_company_design_report_busi_{day_count}d
+SELECT
+new.start_end_time,
+new.company_id,
+new.company_name,
+new.org_code,
+new.org_type,
+new.dep_id,
+new.dep_name,
+new.leader_id ,
+new.leader_name ,
+new.material_count,
+new.innovate_num,
+new.cost,
+ifnull(ratio.cost,0) last_cost, 
+ifnull(round((new.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,
+new.show,
+new.avg_show_cost ,
+new.click ,
+new.avg_click_cost,
+new.ctr ,
+new.convert ,
+new.convert_cost,
+new.convert_rate,
+new.deep_convert,
+new.deep_convert_cost ,
+new.deep_convert_rate ,
+new.attribution_convert ,
+new.attribution_convert_cost,
+new.attribution_deep_convert,
+new.attribution_deep_convert_cost ,
+new.download_start,
+new.download_start_cost ,
+new.download_start_rate ,
+new.download_finish ,
+new.download_finish_cost,
+new.download_finish_rate,
+new.click_install ,
+new.install_finish,
+new.install_finish_cost ,
+new.install_finish_rate ,
+new.active,
+new.active_cost ,
+new.active_rate ,
+new.register,
+new.active_register_cost,
+new.active_register_rate,
+new.active_pay_amount ,
+new.next_day_open ,
+new.next_day_open_cost,
+new.next_day_open_rate,
+new.attribution_next_day_open_cnt ,
+new.attribution_next_day_open_cost,
+new.attribution_next_day_open_rate,
+new.game_addiction,
+new.game_addiction_cost ,
+new.game_addiction_rate ,
+new.pay_count ,
+new.active_pay_cost ,
+new.active_pay_rate ,
+new.loan_completion ,
+new.loan_completion_cost,
+new.loan_completion_rate,
+new.pre_loan_credit ,
+new.pre_loan_credit_cost,
+new.loan_credit ,
+new.loan_credit_cost,
+new.loan_credit_rate,
+new.in_app_uv ,
+new.in_app_detail_uv,
+new.in_app_cart ,
+new.in_app_pay,
+new.in_app_order,
+new.attribution_game_pay_7d_count ,
+new.attribution_game_pay_7d_cost,
+new.attribution_active_pay_7d_per_count ,
+new.game_pay_cost ,
+new.game_pay_count,
+new.phone ,
+new.form,
+new.map_search,
+new.button,
+new.view,
+new.download,
+new.qq,
+new.lottery ,
+new.vote,
+new.message ,
+new.redirect,
+new.shopping,
+new.consult ,
+new.wechat,
+new.phone_confirm ,
+new.phone_connect ,
+new.consult_effective ,
+new.coupon,
+new.coupon_single_page,
+new.redirect_to_shop,
+new.poi_collect ,
+new.poi_address_click ,
+new.luban_order_cnt ,
+new.luban_order_stat_amount ,
+new.luban_order_roi ,
+new.luban_live_enter_cnt,
+new.live_watch_one_minute_count ,
+new.luban_live_follow_cnt ,
+new.live_fans_club_join_cnt ,
+new.luban_live_comment_cnt,
+new.luban_live_share_cnt,
+new.luban_live_gift_cnt ,
+new.luban_live_gift_amount,
+new.luban_live_slidecart_click_cnt,
+new.luban_live_click_product_cnt,
+new.luban_live_pay_order_count,
+new.luban_live_pay_order_stat_cost,
+new.wechat_login_count,
+new.attribution_wechat_login_30d_count,
+new.wechat_login_cost ,
+new.attribution_wechat_login_30d_cost ,
+new.wechat_first_pay_count,
+new.attribution_wechat_first_pay_30d_count,
+new.wechat_first_pay_cost ,
+new.attribution_wechat_first_pay_30d_cost ,
+new.wechat_first_pay_rate ,
+new.attribution_wechat_first_pay_30d_rate ,
+new.wechat_pay_amount ,
+new.attribution_wechat_pay_30d_amount ,
+new.attribution_wechat_pay_30d_roi,
+new.phone_effective ,
+new.total_play,
+new.valid_play,
+new.valid_play_cost ,
+new.valid_play_rate ,
+new.play_25_feed_break,
+new.play_50_feed_break,
+new.play_75_feed_break,
+new.play_100_feed_break ,
+new.average_play_time_per_play,
+new.play_over_rate,
+new.wifi_play_rate,
+new.wifi_play ,
+new.play_duration_sum ,
+new.advanced_creative_phone_click ,
+new.advanced_creative_counsel_click ,
+new.advanced_creative_form_click,
+new.advanced_creative_coupon_addition ,
+new.advanced_creative_form_submit ,
+new.card_show ,
+new.share ,
+new.comment ,
+new.like,
+new.follow,
+new.home_visited,
+new.ies_challenge_click ,
+new.ies_music_click ,
+new.location_click,
+new.message_action,
+new.click_landing_page,
+new.click_shopwindow,
+new.click_website ,
+new.click_download,
+new.click_call_dy ,
+new.cpc ,
+new.cpm ,
+new.cpa ,
+new.interact_per_cost ,
+new.convert_show_rate ,
+new.play_duration_10s_rate,
+new.play_50_feed_break_rate ,
+new.play_25_feed_break_rate ,
+new.play_duration_5s_rate ,
+new.play_duration_2s_rate ,
+new.average_play_progress ,
+new.play_100_feed_break_rate,
+new.play_duration_3s_rate ,
+new.play_75_feed_break_rate ,
+new.play_duration ,
+new.play_duration_3s,
+new.play_duration_2s,
+new.play_duration_10s ,
+new.average_video_play,
+new.avg_rank,
+new.submit_certification_count,
+new.approval_count,
+new.first_order_count ,
+new.first_rental_order_count,
+new.commute_first_pay_count 
+from 
+(SELECT    
+          concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time})  as
+          start_end_time,
+          ifnull(company_id,'-') company_id,
+          company_name,
+          org_code,
+          org_type,
+          id dep_id,
+          depart_name dep_name,
+          ifnull(leader_id,'-') leader_id,
+          leader_name,
+          count(DISTINCT signature) material_count,
+          count(DISTINCT innovate_code) innovate_num,
+          sum(cost) cost,
+          sum(`show`) `show`,
+          ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+          sum(click) click,
+          ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+          ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+          sum(`convert`) `convert`,
+          ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+          ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+          sum(deep_convert) deep_convert,
+          ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+          ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+          sum(attribution_convert) attribution_convert,
+          ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+          sum(attribution_deep_convert) attribution_deep_convert,
+          ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+          sum(download_start) download_start,
+          ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+          ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+          sum(download_finish) download_finish,
+          ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+          ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+          sum(click_install) click_install,
+          sum(install_finish) install_finish,
+          ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+          ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+          sum(active) active,
+          ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+          ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+          sum(register) register,
+          ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+          ifnull(round(sum(active_register_rate) /{day_count} * 100,3),0) active_register_rate,
+          sum(active_pay_amount) active_pay_amount,
+          sum(next_day_open) next_day_open,
+          ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+          ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+          sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+          ifnull(round(sum(cost) / sum(	attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+          ifnull(round(sum(	attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+          sum(game_addiction) game_addiction,
+          ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+          ifnull(round(sum(game_addiction_rate) /{day_count} * 100,3),0) game_addiction_rate,
+          sum(pay_count) pay_count,
+          ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+          ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+          sum(loan_completion) loan_completion,
+          ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+          ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+          ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+          ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+          sum(loan_credit) loan_credit,
+          ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+          ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+          sum(in_app_uv) in_app_uv,
+          sum(in_app_detail_uv) in_app_detail_uv,
+          sum(in_app_cart) in_app_cart,
+          sum(in_app_pay) in_app_pay,
+          sum(in_app_order) in_app_order,
+          ifnull(round(sum(	attribution_game_pay_7d_count) /{day_count}, 3),0) attribution_game_pay_7d_count,
+          ifnull(round(sum(	attribution_game_pay_7d_cost) /{day_count}, 3),0) attribution_game_pay_7d_cost,
+          ifnull(round(sum(	attribution_active_pay_7d_per_count) /{day_count}, 3),0) attribution_active_pay_7d_per_count,
+          sum(game_pay_cost) game_pay_cost,
+          sum(game_pay_count) game_pay_count,
+          sum(phone) phone,
+          sum(form) form,
+          sum(map_search) map_search,
+          sum(button) button,
+          sum(`view`) `view`,
+          sum(download) download,
+          sum(qq) qq,
+          sum(lottery) lottery,
+          sum(vote) vote,
+          sum(message) message,
+          sum(redirect) redirect,
+          sum(shopping) shopping,
+          sum(consult) consult,
+          sum(wechat) wechat,
+          sum(phone_confirm) phone_confirm,
+          sum(phone_connect) phone_connect,
+          sum(consult_effective) consult_effective,
+          sum(coupon) coupon,
+          sum(coupon_single_page) coupon_single_page,
+          sum(redirect_to_shop) redirect_to_shop,
+          sum(poi_collect) poi_collect,
+          sum(poi_address_click) poi_address_click,
+          sum(luban_order_cnt) luban_order_cnt,
+          sum(luban_order_stat_amount) luban_order_stat_amount,
+          sum(luban_order_roi) luban_order_roi,
+          sum(luban_live_enter_cnt) luban_live_enter_cnt,
+          sum(live_watch_one_minute_count) live_watch_one_minute_count,
+          sum(luban_live_follow_cnt) luban_live_follow_cnt,
+          sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+          sum(luban_live_comment_cnt) luban_live_comment_cnt,
+          sum(luban_live_share_cnt) luban_live_share_cnt,
+          sum(luban_live_gift_cnt) luban_live_gift_cnt,
+          sum(luban_live_gift_amount) luban_live_gift_amount,
+          sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+          sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+          sum(luban_live_pay_order_count) luban_live_pay_order_count,
+          sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+          sum(wechat_login_count) wechat_login_count,
+          sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+          sum(wechat_first_pay_count) wechat_first_pay_count,
+          sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+          ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+          ifnull(round(sum(	attribution_wechat_first_pay_30d_count) / sum(	attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+          sum(wechat_pay_amount) wechat_pay_amount,
+          sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+          sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+          sum(phone_effective) phone_effective,
+          sum(total_play) total_play,
+          sum(valid_play) valid_play,
+          ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+          ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+          sum(play_25_feed_break) play_25_feed_break,
+          sum(play_50_feed_break) play_50_feed_break,
+          sum(play_75_feed_break) play_75_feed_break,
+          sum(play_100_feed_break) play_100_feed_break,
+          ifnull(round(sum(average_play_time_per_play) /{day_count}, 3),0) average_play_time_per_play,
+          ifnull(round(sum(play_over_rate) /{day_count} * 100,3),0) play_over_rate,
+          ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+          sum(wifi_play) wifi_play,
+          ifnull(round(sum(play_duration_sum) /{day_count}, 3),0) play_duration_sum,
+          sum(advanced_creative_phone_click) advanced_creative_phone_click,
+          sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+          sum(advanced_creative_form_click) advanced_creative_form_click,
+          sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+          sum(advanced_creative_form_submit) advanced_creative_form_submit,
+          sum(card_show) card_show,
+          sum(`share`) `share`,
+          sum(`comment`) `comment`,
+          sum(`like`) `like`,
+          sum(follow) follow,
+          sum(home_visited) home_visited,
+          sum(ies_challenge_click) ies_challenge_click,
+          sum(ies_music_click) ies_music_click,
+          sum(location_click) location_click,
+          sum(message_action) message_action,
+          sum(click_landing_page) click_landing_page,
+          sum(click_shopwindow) click_shopwindow,
+          sum(click_website) click_website,
+          sum(click_download) click_download,
+          sum(click_call_dy) click_call_dy,
+          sum(cpc) cpc,
+          sum(cpm) cpm,
+          sum(cpa) cpa,
+          ifnull(round(sum(interact_per_cost) /{day_count}, 3),0) interact_per_cost,
+          ifnull(round(sum(convert_show_rate) /{day_count}, 3),0) convert_show_rate,
+          ifnull(round(sum(play_duration_10s) / sum(total_play) * 100,3),0) play_duration_10s_rate,
+          ifnull(round(sum(play_50_feed_break) / sum(total_play) * 100,3),0) play_50_feed_break_rate,
+          ifnull(round(sum(play_25_feed_break) / sum(total_play) * 100,3),0) play_25_feed_break_rate,
+          ifnull(round(sum(play_duration_5s_rate) /{day_count}, 3),0) play_duration_5s_rate,
+          ifnull(round(sum(play_duration_2s) / sum(total_play) * 100,3),0) play_duration_2s_rate,
+          ifnull(round(sum(average_play_progress) /{day_count}, 3),0) average_play_progress,
+          ifnull(round(sum(play_100_feed_break) / sum(total_play) * 100,3),0) play_100_feed_break_rate,
+          ifnull(round(sum(play_duration_3s) / sum(total_play) * 100,3),0) play_duration_3s_rate,
+          ifnull(round(sum(play_75_feed_break) / sum(total_play) * 100,3),0) play_75_feed_break_rate,
+          sum(play_duration) play_duration,
+          sum(play_duration_3s) play_duration_3s,
+          sum(play_duration_2s) play_duration_2s,
+          sum(play_duration_10s) play_duration_10s,
+          ifnull(round(sum(average_video_play) /{day_count}, 3),0) average_video_play,
+          sum(avg_rank) avg_rank,
+          sum(submit_certification_count) submit_certification_count,
+          sum(approval_count) approval_count,
+          sum(first_order_count) first_rental_order_count,
+          sum(first_rental_order_count) first_order_count,
+          sum(commute_first_pay_count) commute_first_pay_count
+FROM (
+         SELECT video.signature,
+                uc.company_id,
+                CASE
+                    WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+                        '北京汇创思拓数字科技有限公司'
+                    WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+                        '广州汇创思拓数字科技有限公司'
+                    WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+                        '上海汇创思拓数字科技有限公司'
+                    ELSE
+                        uc.company_name
+                    END  company_name,
+                cma.leader_id,
+                us.realname as leader_name,
+                ur.org_type,
+                ur.org_code,
+                ur.id,
+                ur.depart_name,
+                case when cmi.material_innovate = 2 then video.signature else '' end innovate_code,
+                report.cost,
+                report.`show`,
+                report.avg_show_cost,
+                report.click,
+                report.avg_click_cost,
+                report.ctr,
+                report.`convert`,
+                report.convert_cost,
+                report.convert_rate,
+                report.deep_convert,
+                report.deep_convert_cost,
+                report.deep_convert_rate,
+                report.attribution_convert,
+                report.attribution_convert_cost,
+                report.attribution_deep_convert,
+                report.attribution_deep_convert_cost,
+                report.download_start,
+                report.download_start_cost,
+                report.download_start_rate,
+                report.download_finish,
+                report.download_finish_cost,
+                report.download_finish_rate,
+                report.click_install,
+                report.install_finish,
+                report.install_finish_cost,
+                report.install_finish_rate,
+                report.active,
+                report.active_cost,
+                report.active_rate,
+                report.register,
+                report.active_register_cost,
+                report.active_register_rate,
+                report.active_pay_amount,
+                report.next_day_open,
+                report.next_day_open_cost,
+                report.next_day_open_rate,
+                report.attribution_next_day_open_cnt,
+                report.attribution_next_day_open_cost,
+                report.attribution_next_day_open_rate,
+                report.game_addiction,
+                report.game_addiction_cost,
+                report.game_addiction_rate,
+                report.pay_count,
+                report.active_pay_cost,
+                report.active_pay_rate,
+                report.loan_completion,
+                report.loan_completion_cost,
+                report.loan_completion_rate,
+                report.pre_loan_credit,
+                report.pre_loan_credit_cost,
+                report.loan_credit,
+                report.loan_credit_cost,
+                report.loan_credit_rate,
+                report.in_app_uv,
+                report.in_app_detail_uv,
+                report.in_app_cart,
+                report.in_app_pay,
+                report.in_app_order,
+                report.attribution_game_pay_7d_count,
+                report.attribution_game_pay_7d_cost,
+                report.attribution_active_pay_7d_per_count,
+                report.game_pay_cost,
+                report.game_pay_count,
+                report.phone,
+                report.form,
+                report.map_search,
+                report.button,
+                report.`view`,
+                report.download,
+                report.qq,
+                report.lottery,
+                report.vote,
+                report.message,
+                report.redirect,
+                report.shopping,
+                report.consult,
+                report.wechat,
+                report.phone_confirm,
+                report.phone_connect,
+                report.consult_effective,
+                report.coupon,
+                report.coupon_single_page,
+                report.redirect_to_shop,
+                report.poi_collect,
+                report.poi_address_click,
+                report.luban_order_cnt,
+                report.luban_order_stat_amount,
+                report.luban_order_roi,
+                report.luban_live_enter_cnt,
+                report.live_watch_one_minute_count,
+                report.luban_live_follow_cnt,
+                report.live_fans_club_join_cnt,
+                report.luban_live_comment_cnt,
+                report.luban_live_share_cnt,
+                report.luban_live_gift_cnt,
+                report.luban_live_gift_amount,
+                report.luban_live_slidecart_click_cnt,
+                report.luban_live_click_product_cnt,
+                report.luban_live_pay_order_count,
+                report.luban_live_pay_order_stat_cost,
+                report.wechat_login_count,
+                report.attribution_wechat_login_30d_count,
+                report.wechat_login_cost,
+                report.attribution_wechat_login_30d_cost,
+                report.wechat_first_pay_count,
+                report.attribution_wechat_first_pay_30d_count,
+                report.wechat_first_pay_cost,
+                report.attribution_wechat_first_pay_30d_cost,
+                report.wechat_first_pay_rate,
+                report.attribution_wechat_first_pay_30d_rate,
+                report.wechat_pay_amount,
+                report.attribution_wechat_pay_30d_amount,
+                report.attribution_wechat_pay_30d_roi,
+                report.phone_effective,
+                report.total_play,
+                report.valid_play,
+                report.valid_play_cost,
+                report.valid_play_rate,
+                report.play_25_feed_break,
+                report.play_50_feed_break,
+                report.play_75_feed_break,
+                report.play_100_feed_break,
+                report.average_play_time_per_play,
+                report.play_over_rate,
+                report.wifi_play_rate,
+                report.wifi_play,
+                report.play_duration_sum,
+                report.advanced_creative_phone_click,
+                report.advanced_creative_counsel_click,
+                report.advanced_creative_form_click,
+                report.advanced_creative_coupon_addition,
+                report.advanced_creative_form_submit,
+                report.card_show,
+                report.`share`,
+                report.`comment`,
+                report.`like`,
+                report.follow,
+                report.home_visited,
+                report.ies_challenge_click,
+                report.ies_music_click,
+                report.location_click,
+                report.message_action,
+                report.click_landing_page,
+                report.click_shopwindow,
+                report.click_website,
+                report.click_download,
+                report.click_call_dy,
+                report.`cpc`,
+                report.`cpm`,
+                report.`cpa`,
+                report.`interact_per_cost`,
+                report.`convert_show_rate`,
+                report.`play_duration_10s_rate`,
+                report.`play_50_feed_break_rate`,
+                report.`play_25_feed_break_rate`,
+                report.`play_duration_5s_rate`,
+                report.`play_duration_2s_rate`,
+                report.`average_play_progress`,
+                report.`play_100_feed_break_rate`,
+                report.`play_duration_3s_rate`,
+                report.`play_75_feed_break_rate`,
+                report.`play_duration`,
+                report.`play_duration_3s`,
+                report.`play_duration_2s`,
+                report.`play_duration_10s`,
+                report.`average_video_play`,
+                report.`avg_rank`,
+                report.submit_certification_count,
+                report.approval_count,
+                report.first_rental_order_count,
+                report.first_order_count,
+                report.commute_first_pay_count
+         FROM media_api.bytedance_material_report_daily report
+                  LEFT JOIN (
+             SELECT advertiser_id,
+                    signature,
+                    material_id
+             FROM `media_api`.bytedance_file_video_get
+             GROUP BY advertiser_id,
+                      signature,
+                      material_id
+         ) video ON report.advertiser_id = video.advertiser_id
+             AND report.material_id = video.material_id
+                  LEFT JOIN (
+             SELECT CODE,
+                    material_innovate,
+                    cover_url
+             FROM `jeecg-boot`.ctop_material_info
+             GROUP BY CODE,
+                      material_innovate,
+                      cover_url
+         ) cmi ON video.signature = cmi.CODE
+                  LEFT JOIN (
+             SELECT material_id,
+                    leader_id
+             FROM (
+                      SELECT material_id,
+                             leader_id,
+                             row_number() over (
+                                 PARTITION BY material_id
+                                 ORDER BY
+                                     create_time
+                                 ) rn
+                      FROM `jeecg-boot`.ctop_material_ascription
+                  ) m
+             WHERE rn = 1
+         ) cma ON video.signature = cma.material_id
+                  LEFT JOIN `jeecg-boot`.user_company uc ON cma.leader_id = uc.user_id
+                  LEFT JOIN `jeecg-boot`.sys_user_depart de ON cma.leader_id = de.user_id
+                  LEFT JOIN `jeecg-boot`.sys_depart ur ON de.dep_id = ur.id
+                  LEFT JOIN  `jeecg-boot`.sys_user us on us.id =cma.leader_id
+         WHERE image_mode IN (
+                              'CREATIVE_IMAGE_MODE_VIDEO',
+                              'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
+                              'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+          AND stat_datetime >= date_format( date_sub({date_time}, INTERVAL {day_count}-1 DAY),'%Y%m%d')) info
+          GROUP BY 
+          company_id,
+          leader_id) new
+          left join 
+          (select  ifnull(cma.leader_id,'-') leader_id,round(sum(cost),3) cost
+           FROM media_api.bytedance_material_report_daily report
+           LEFT JOIN (SELECT advertiser_id,signature,material_id FROM `media_api`.bytedance_file_video_get
+           GROUP BY advertiser_id,signature,material_id) video
+           ON report.advertiser_id = video.advertiser_id AND report.material_id = video.material_id
+           LEFT JOIN  (SELECT material_id,leader_id FROM (
+           SELECT material_id,leader_id,row_number() over (PARTITION BY material_id ORDER BY create_time ) rn
+           FROM `jeecg-boot`.ctop_material_ascription ) m WHERE rn = 1 )  cma
+           ON video.signature = cma.material_id
+           WHERE image_mode IN ('CREATIVE_IMAGE_MODE_VIDEO','CREATIVE_IMAGE_MODE_VIDEO_VERTICAL','CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+           and  stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+           and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+           group by cma.leader_id) ratio
+           on new.leader_id=ratio.leader_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceCompanyDesignReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 473 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyDesignReportBusiMonth.py

@@ -0,0 +1,473 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceCompanyDesignReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceCompanyDesignReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+delete from application.app_bytedance_company_design_report_busi_m where stat_mon=date_format({date_time},'%Y%m');
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_company_design_report_busi_m
+SELECT 
+       date_format({date_time},'%Y%m')    as  stat_mon,
+          ifnull(company_id,'-') company_id,
+          company_name,
+          org_code,
+          org_type,
+          id,
+          depart_name,
+          ifnull(leader_id,'-') leader_id,
+          leader_name,
+          count(DISTINCT signature) num,
+          count(DISTINCT innovate_code) innovate_num,
+          sum(cost) cost,
+          sum(`show`) `show`,
+          ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+          sum(click) click,
+          ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+          ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+          sum(`convert`) `convert`,
+          ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+          ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+          sum(deep_convert) deep_convert,
+          ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+          ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+          sum(attribution_convert) attribution_convert,
+          ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+          sum(attribution_deep_convert) attribution_deep_convert,
+          ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+          sum(download_start) download_start,
+          ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+          ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+          sum(download_finish) download_finish,
+          ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+          ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+          sum(click_install) click_install,
+          sum(install_finish) install_finish,
+          ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+          ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+          sum(active) active,
+          ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+          ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+          sum(register) register,
+          ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+          ifnull(round(sum(active_register_rate) /{day_count} * 100,3),0) active_register_rate,
+          sum(active_pay_amount) active_pay_amount,
+          sum(next_day_open) next_day_open,
+          ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+          ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+          sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+          ifnull(round(sum(cost) / sum(	attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+          ifnull(round(sum(	attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+          sum(game_addiction) game_addiction,
+          ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+          ifnull(round(sum(game_addiction_rate) /{day_count} * 100,3),0) game_addiction_rate,
+          sum(pay_count) pay_count,
+          ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+          ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+          sum(loan_completion) loan_completion,
+          ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+          ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+          ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+          ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+          sum(loan_credit) loan_credit,
+          ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+          ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+          sum(in_app_uv) in_app_uv,
+          sum(in_app_detail_uv) in_app_detail_uv,
+          sum(in_app_cart) in_app_cart,
+          sum(in_app_pay) in_app_pay,
+          sum(in_app_order) in_app_order,
+          ifnull(round(sum(	attribution_game_pay_7d_count) /{day_count}, 3),0) attribution_game_pay_7d_count,
+          ifnull(round(sum(	attribution_game_pay_7d_cost) /{day_count}, 3),0) attribution_game_pay_7d_cost,
+          ifnull(round(sum(	attribution_active_pay_7d_per_count) /{day_count}, 3),0) attribution_active_pay_7d_per_count,
+          sum(game_pay_cost) game_pay_cost,
+          sum(game_pay_count) game_pay_count,
+          sum(phone) phone,
+          sum(form) form,
+          sum(map_search) map_search,
+          sum(button) button,
+          sum(`view`) `view`,
+          sum(download) download,
+          sum(qq) qq,
+          sum(lottery) lottery,
+          sum(vote) vote,
+          sum(message) message,
+          sum(redirect) redirect,
+          sum(shopping) shopping,
+          sum(consult) consult,
+          sum(wechat) wechat,
+          sum(phone_confirm) phone_confirm,
+          sum(phone_connect) phone_connect,
+          sum(consult_effective) consult_effective,
+          sum(coupon) coupon,
+          sum(coupon_single_page) coupon_single_page,
+          sum(redirect_to_shop) redirect_to_shop,
+          sum(poi_collect) poi_collect,
+          sum(poi_address_click) poi_address_click,
+          sum(luban_order_cnt) luban_order_cnt,
+          sum(luban_order_stat_amount) luban_order_stat_amount,
+          sum(luban_order_roi) luban_order_roi,
+          sum(luban_live_enter_cnt) luban_live_enter_cnt,
+          sum(live_watch_one_minute_count) live_watch_one_minute_count,
+          sum(luban_live_follow_cnt) luban_live_follow_cnt,
+          sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+          sum(luban_live_comment_cnt) luban_live_comment_cnt,
+          sum(luban_live_share_cnt) luban_live_share_cnt,
+          sum(luban_live_gift_cnt) luban_live_gift_cnt,
+          sum(luban_live_gift_amount) luban_live_gift_amount,
+          sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+          sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+          sum(luban_live_pay_order_count) luban_live_pay_order_count,
+          sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+          sum(wechat_login_count) wechat_login_count,
+          sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+          sum(wechat_first_pay_count) wechat_first_pay_count,
+          sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+          ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+          ifnull(round(sum(	attribution_wechat_first_pay_30d_count) / sum(	attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+          sum(wechat_pay_amount) wechat_pay_amount,
+          sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+          sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+          sum(phone_effective) phone_effective,
+          sum(total_play) total_play,
+          sum(valid_play) valid_play,
+          ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+          ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+          sum(play_25_feed_break) play_25_feed_break,
+          sum(play_50_feed_break) play_50_feed_break,
+          sum(play_75_feed_break) play_75_feed_break,
+          sum(play_100_feed_break) play_100_feed_break,
+          ifnull(round(sum(average_play_time_per_play) /{day_count}, 3),0) average_play_time_per_play,
+          ifnull(round(sum(play_over_rate) /{day_count} * 100,3),0) play_over_rate,
+          ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+          sum(wifi_play) wifi_play,
+          ifnull(round(sum(play_duration_sum) /{day_count}, 3),0) play_duration_sum,
+          sum(advanced_creative_phone_click) advanced_creative_phone_click,
+          sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+          sum(advanced_creative_form_click) advanced_creative_form_click,
+          sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+          sum(advanced_creative_form_submit) advanced_creative_form_submit,
+          sum(card_show) card_show,
+          sum(`share`) `share`,
+          sum(`comment`) `comment`,
+          sum(`like`) `like`,
+          sum(follow) follow,
+          sum(home_visited) home_visited,
+          sum(ies_challenge_click) ies_challenge_click,
+          sum(ies_music_click) ies_music_click,
+          sum(location_click) location_click,
+          sum(message_action) message_action,
+          sum(click_landing_page) click_landing_page,
+          sum(click_shopwindow) click_shopwindow,
+          sum(click_website) click_website,
+          sum(click_download) click_download,
+          sum(click_call_dy) click_call_dy,
+          sum(cpc) cpc,
+          sum(cpm) cpm,
+          sum(cpa) cpa,
+          ifnull(round(sum(interact_per_cost) /{day_count}, 3),0) interact_per_cost,
+          ifnull(round(sum(convert_show_rate) /{day_count}, 3),0) convert_show_rate,
+          ifnull(round(sum(play_duration_10s) / sum(total_play) * 100,3),0) play_duration_10s_rate,
+          ifnull(round(sum(play_50_feed_break) / sum(total_play) * 100,3),0) play_50_feed_break_rate,
+          ifnull(round(sum(play_25_feed_break) / sum(total_play) * 100,3),0) play_25_feed_break_rate,
+          ifnull(round(sum(play_duration_5s_rate) /{day_count}, 3),0) play_duration_5s_rate,
+          ifnull(round(sum(play_duration_2s) / sum(total_play) * 100,3),0) play_duration_2s_rate,
+          ifnull(round(sum(average_play_progress) /{day_count}, 3),0) average_play_progress,
+          ifnull(round(sum(play_100_feed_break) / sum(total_play) * 100,3),0) play_100_feed_break_rate,
+          ifnull(round(sum(play_duration_3s) / sum(total_play) * 100,3),0) play_duration_3s_rate,
+          ifnull(round(sum(play_75_feed_break) / sum(total_play) * 100,3),0) play_75_feed_break_rate,
+          sum(play_duration) play_duration,
+          sum(play_duration_3s) play_duration_3s,
+          sum(play_duration_2s) play_duration_2s,
+          sum(play_duration_10s) play_duration_10s,
+          ifnull(round(sum(average_video_play) /{day_count}, 3),0) average_video_play,
+          sum(avg_rank) avg_rank,
+          sum(submit_certification_count) submit_certification_count,
+          sum(approval_count) approval_count,
+          sum(first_order_count) first_rental_order_count,
+          sum(first_rental_order_count) first_order_count,
+          sum(commute_first_pay_count) commute_first_pay_count
+FROM (
+         SELECT video.signature,
+                uc.company_id,
+                CASE
+                    WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+                        '北京汇创思拓数字科技有限公司'
+                    WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+                        '广州汇创思拓数字科技有限公司'
+                    WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+                        '上海汇创思拓数字科技有限公司'
+                    ELSE
+                        uc.company_name
+                    END                                                              company_name,
+                cma.leader_id,
+                us.realname as leader_name,
+                ur.org_type,
+                ur.org_code,
+                ur.id,
+                ur.depart_name,
+                case when cmi.material_innovate = 2 then video.signature else '' end innovate_code,
+                report.cost,
+                report.`show`,
+                report.avg_show_cost,
+                report.click,
+                report.avg_click_cost,
+                report.ctr,
+                report.`convert`,
+                report.convert_cost,
+                report.convert_rate,
+                report.deep_convert,
+                report.deep_convert_cost,
+                report.deep_convert_rate,
+                report.attribution_convert,
+                report.attribution_convert_cost,
+                report.attribution_deep_convert,
+                report.attribution_deep_convert_cost,
+                report.download_start,
+                report.download_start_cost,
+                report.download_start_rate,
+                report.download_finish,
+                report.download_finish_cost,
+                report.download_finish_rate,
+                report.click_install,
+                report.install_finish,
+                report.install_finish_cost,
+                report.install_finish_rate,
+                report.active,
+                report.active_cost,
+                report.active_rate,
+                report.register,
+                report.active_register_cost,
+                report.active_register_rate,
+                report.active_pay_amount,
+                report.next_day_open,
+                report.next_day_open_cost,
+                report.next_day_open_rate,
+                report.attribution_next_day_open_cnt,
+                report.attribution_next_day_open_cost,
+                report.attribution_next_day_open_rate,
+                report.game_addiction,
+                report.game_addiction_cost,
+                report.game_addiction_rate,
+                report.pay_count,
+                report.active_pay_cost,
+                report.active_pay_rate,
+                report.loan_completion,
+                report.loan_completion_cost,
+                report.loan_completion_rate,
+                report.pre_loan_credit,
+                report.pre_loan_credit_cost,
+                report.loan_credit,
+                report.loan_credit_cost,
+                report.loan_credit_rate,
+                report.in_app_uv,
+                report.in_app_detail_uv,
+                report.in_app_cart,
+                report.in_app_pay,
+                report.in_app_order,
+                report.attribution_game_pay_7d_count,
+                report.attribution_game_pay_7d_cost,
+                report.attribution_active_pay_7d_per_count,
+                report.game_pay_cost,
+                report.game_pay_count,
+                report.phone,
+                report.form,
+                report.map_search,
+                report.button,
+                report.`view`,
+                report.download,
+                report.qq,
+                report.lottery,
+                report.vote,
+                report.message,
+                report.redirect,
+                report.shopping,
+                report.consult,
+                report.wechat,
+                report.phone_confirm,
+                report.phone_connect,
+                report.consult_effective,
+                report.coupon,
+                report.coupon_single_page,
+                report.redirect_to_shop,
+                report.poi_collect,
+                report.poi_address_click,
+                report.luban_order_cnt,
+                report.luban_order_stat_amount,
+                report.luban_order_roi,
+                report.luban_live_enter_cnt,
+                report.live_watch_one_minute_count,
+                report.luban_live_follow_cnt,
+                report.live_fans_club_join_cnt,
+                report.luban_live_comment_cnt,
+                report.luban_live_share_cnt,
+                report.luban_live_gift_cnt,
+                report.luban_live_gift_amount,
+                report.luban_live_slidecart_click_cnt,
+                report.luban_live_click_product_cnt,
+                report.luban_live_pay_order_count,
+                report.luban_live_pay_order_stat_cost,
+                report.wechat_login_count,
+                report.attribution_wechat_login_30d_count,
+                report.wechat_login_cost,
+                report.attribution_wechat_login_30d_cost,
+                report.wechat_first_pay_count,
+                report.attribution_wechat_first_pay_30d_count,
+                report.wechat_first_pay_cost,
+                report.attribution_wechat_first_pay_30d_cost,
+                report.wechat_first_pay_rate,
+                report.attribution_wechat_first_pay_30d_rate,
+                report.wechat_pay_amount,
+                report.attribution_wechat_pay_30d_amount,
+                report.attribution_wechat_pay_30d_roi,
+                report.phone_effective,
+                report.total_play,
+                report.valid_play,
+                report.valid_play_cost,
+                report.valid_play_rate,
+                report.play_25_feed_break,
+                report.play_50_feed_break,
+                report.play_75_feed_break,
+                report.play_100_feed_break,
+                report.average_play_time_per_play,
+                report.play_over_rate,
+                report.wifi_play_rate,
+                report.wifi_play,
+                report.play_duration_sum,
+                report.advanced_creative_phone_click,
+                report.advanced_creative_counsel_click,
+                report.advanced_creative_form_click,
+                report.advanced_creative_coupon_addition,
+                report.advanced_creative_form_submit,
+                report.card_show,
+                report.`share`,
+                report.`comment`,
+                report.`like`,
+                report.follow,
+                report.home_visited,
+                report.ies_challenge_click,
+                report.ies_music_click,
+                report.location_click,
+                report.message_action,
+                report.click_landing_page,
+                report.click_shopwindow,
+                report.click_website,
+                report.click_download,
+                report.click_call_dy,
+                report.`cpc`,
+                report.`cpm`,
+                report.`cpa`,
+                report.`interact_per_cost`,
+                report.`convert_show_rate`,
+                report.`play_duration_10s_rate`,
+                report.`play_50_feed_break_rate`,
+                report.`play_25_feed_break_rate`,
+                report.`play_duration_5s_rate`,
+                report.`play_duration_2s_rate`,
+                report.`average_play_progress`,
+                report.`play_100_feed_break_rate`,
+                report.`play_duration_3s_rate`,
+                report.`play_75_feed_break_rate`,
+                report.`play_duration`,
+                report.`play_duration_3s`,
+                report.`play_duration_2s`,
+                report.`play_duration_10s`,
+                report.`average_video_play`,
+                report.`avg_rank`,
+                report.submit_certification_count,
+                report.approval_count,
+                report.first_rental_order_count,
+                report.first_order_count,
+                report.commute_first_pay_count
+         FROM media_api.bytedance_material_report_daily report
+                  LEFT JOIN (
+             SELECT advertiser_id,
+                    signature,
+                    material_id
+             FROM `media_api`.bytedance_file_video_get
+             GROUP BY advertiser_id,
+                      signature,
+                      material_id
+         ) video ON report.advertiser_id = video.advertiser_id
+             AND report.material_id = video.material_id
+                  LEFT JOIN (
+             SELECT CODE,
+                    material_innovate,
+                    cover_url
+             FROM `jeecg-boot`.ctop_material_info
+             GROUP BY CODE,
+                      material_innovate,
+                      cover_url
+         ) cmi ON video.signature = cmi.CODE
+                  LEFT JOIN (
+             SELECT material_id,
+                    leader_id
+             FROM (
+                      SELECT material_id,
+                             leader_id,
+                             row_number() over (
+                                 PARTITION BY material_id
+                                 ORDER BY
+                                     create_time
+                                 ) rn
+                      FROM `jeecg-boot`.ctop_material_ascription
+                  ) m
+             WHERE rn = 1
+         ) cma ON video.signature = cma.material_id
+                  LEFT JOIN `jeecg-boot`.user_company uc ON cma.leader_id = uc.user_id
+                  LEFT JOIN `jeecg-boot`.sys_user_depart de ON cma.leader_id = de.user_id
+                  LEFT JOIN `jeecg-boot`.sys_depart ur ON de.dep_id = ur.id
+                  LEFT JOIN  `jeecg-boot`.sys_user us on us.id =cma.leader_id
+         WHERE image_mode IN (
+                              'CREATIVE_IMAGE_MODE_VIDEO',
+                              'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
+                              'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+          AND stat_datetime >= date_format( date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')) info
+          GROUP BY company_id,
+          company_name,
+          leader_id,
+          org_type,
+          org_code,
+          id,
+          depart_name;
+    """.format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceCompanyDesignReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 605 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyUserReportBusi.py

@@ -0,0 +1,605 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceCompanyUserReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceCompanyUserReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_company_user_report_busi_{day_count}d 
+SELECT 
+new.start_end_time ,
+new.company_id ,
+new.company_name ,
+new.org_code ,
+new.org_type ,
+new.dep_id ,
+new.dep_name ,
+new.leader_id,
+new.leader_name,
+new.account_num,
+new.cost ,
+ifnull(ratio.cost,0) last_cost,                                    
+ifnull(round((new.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,
+new.show ,
+new.avg_show_cost,
+new.click,
+new.avg_click_cost ,
+new.ctr,
+new.convert,
+new.convert_cost ,
+new.convert_rate ,
+new.deep_convert ,
+new.deep_convert_cost,
+new.deep_convert_rate,
+new.attribution_convert,
+new.attribution_convert_cost ,
+new.attribution_deep_convert ,
+new.attribution_deep_convert_cost,
+new.download_start ,
+new.download_start_cost,
+new.download_start_rate,
+new.download_finish,
+new.download_finish_cost ,
+new.download_finish_rate ,
+new.click_install,
+new.install_finish ,
+new.install_finish_cost,
+new.install_finish_rate,
+new.active ,
+new.active_cost,
+new.active_rate,
+new.register ,
+new.active_register_cost ,
+new.active_register_rate ,
+new.next_day_open,
+new.next_day_open_cost ,
+new.next_day_open_rate ,
+new.attribution_next_day_open_cnt,
+new.attribution_next_day_open_cost ,
+new.attribution_next_day_open_rate ,
+new.game_addiction ,
+new.game_addiction_cost,
+new.game_addiction_rate,
+new.pay_count,
+new.active_pay_cost,
+new.active_pay_rate,
+new.loan_completion,
+new.loan_completion_cost ,
+new.loan_completion_rate ,
+new.pre_loan_credit,
+new.pre_loan_credit_cost ,
+new.loan_credit,
+new.loan_credit_cost ,
+new.loan_credit_rate ,
+new.in_app_uv,
+new.in_app_detail_uv ,
+new.in_app_cart,
+new.in_app_pay ,
+new.in_app_order ,
+new.attribution_game_pay_7d_count,
+new.attribution_game_pay_7d_cost ,
+new.attribution_active_pay_7d_per_count,
+new.game_pay_cost,
+new.game_pay_count ,
+new.phone,
+new.form ,
+new.map_search ,
+new.button ,
+new.view ,
+new.download ,
+new.qq ,
+new.lottery,
+new.vote ,
+new.message,
+new.redirect ,
+new.shopping ,
+new.consult,
+new.wechat ,
+new.phone_confirm,
+new.phone_connect,
+new.consult_effective,
+new.coupon ,
+new.coupon_single_page ,
+new.redirect_to_shop ,
+new.poi_collect,
+new.poi_address_click,
+new.luban_order_cnt,
+new.luban_order_stat_amount,
+new.luban_order_roi,
+new.luban_live_enter_cnt ,
+new.live_watch_one_minute_count,
+new.luban_live_follow_cnt,
+new.live_fans_club_join_cnt,
+new.luban_live_comment_cnt ,
+new.luban_live_share_cnt ,
+new.luban_live_gift_cnt,
+new.luban_live_gift_amount ,
+new.luban_live_slidecart_click_cnt ,
+new.luban_live_click_product_cnt ,
+new.luban_live_pay_order_count ,
+new.luban_live_pay_order_stat_cost ,
+new.live_component_click_count ,
+new.live_component_click_cost,
+new.live_component_click_rate,
+new.wechat_login_count ,
+new.attribution_wechat_login_30d_count ,
+new.wechat_login_cost,
+new.attribution_wechat_login_30d_cost,
+new.wechat_first_pay_count ,
+new.attribution_wechat_first_pay_30d_count ,
+new.wechat_first_pay_cost,
+new.attribution_wechat_first_pay_30d_cost,
+new.wechat_first_pay_rate,
+new.attribution_wechat_first_pay_30d_rate,
+new.wechat_pay_amount,
+new.attribution_wechat_pay_30d_amount,
+new.attribution_wechat_pay_30d_roi ,
+new.phone_effective,
+new.total_play ,
+new.valid_play ,
+new.valid_play_cost,
+new.valid_play_rate,
+new.play_25_feed_break ,
+new.play_50_feed_break ,
+new.play_75_feed_break ,
+new.play_100_feed_break,
+new.average_play_time_per_play ,
+new.play_over_rate ,
+new.wifi_play_rate ,
+new.wifi_play,
+new.play_duration_sum,
+new.advanced_creative_phone_click,
+new.advanced_creative_counsel_click,
+new.advanced_creative_form_click ,
+new.advanced_creative_coupon_addition,
+new.advanced_creative_form_submit,
+new.card_show,
+new.share,
+new.comment,
+new.like ,
+new.follow ,
+new.home_visited ,
+new.ies_challenge_click,
+new.ies_music_click,
+new.location_click ,
+new.message_action ,
+new.click_landing_page ,
+new.click_shopwindow ,
+new.click_website,
+new.click_download ,
+new.click_call_dy,
+new.submit_certification_count ,
+new.approval_count ,
+new.first_rental_order_count ,
+new.first_order_count,
+new.commute_first_pay_count
+from (
+SELECT
+concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time})  start_end_time,
+               ifnull(company_id,'-') company_id,
+               company_name,
+               org_code,
+               org_type,
+               dep_id,
+               depart_name dep_name,
+               ifnull(leader_id,'-') leader_id,
+               trim(leader_name) leader_name,
+               count(DISTINCT account_id) account_num,
+               sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click) click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`) `convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert) attribution_convert,
+               ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+               ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install) click_install,
+               sum(install_finish) install_finish,
+               ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register) register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate)/{day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open) next_day_open,
+               ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round(sum(cost) / sum(attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate)/{day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count) pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+               sum(loan_credit) loan_credit,
+               ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv) in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart) in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round(sum(attribution_game_pay_7d_count)/{day_count},3),0) attribution_game_pay_7d_count,
+               ifnull(round(sum(attribution_game_pay_7d_cost)/{day_count},3),0) attribution_game_pay_7d_cost,
+               ifnull(round(sum(attribution_active_pay_7d_per_count)/{day_count},3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost) game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone) phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download) download,
+               sum(qq) qq,
+               sum(lottery) lottery,
+               sum(vote) vote,
+               sum(message) message,
+               sum(redirect) redirect,
+               sum(shopping) shopping,
+               sum(consult) consult,
+               sum(wechat) wechat,
+               sum(phone_confirm) phone_confirm,
+               sum(phone_connect) phone_connect,
+               sum(consult_effective) consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect) poi_collect,
+               sum(poi_address_click) poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount) luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt) luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round(sum(cost) / sum(live_component_click_count),3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count)/{day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+               ifnull(round(sum(cost) / sum(attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+               ifnull(round(sum(cost) / sum(attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount) wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break) play_100_feed_break,
+               ifnull(round(sum(average_play_time_per_play)/{day_count},3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate)/{day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play) wifi_play,
+               ifnull(round(sum(play_duration_sum)/{day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show) card_show,
+               sum(`share`) `share`,
+               sum(`comment`) `comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click) ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website) click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy) click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count) first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count) commute_first_pay_count
+               FROM
+(
+SELECT
+report.advertiser_id account_id,
+dep.company_id,
+CASE
+WHEN dep.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN dep.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN dep.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+dep.company_name
+END company_name,
+dep.org_code,
+dep.org_type,
+dep.dep_id,
+dep.depart_name,
+dep.user_id,
+dep.realname,
+dep.leader_id,
+dep.leader_name,
+report.cost,
+report.`show`,
+report.avg_show_cost,
+report.click,
+report.avg_click_cost,
+report.ctr,
+report.`convert`,
+report.convert_cost,
+report.convert_rate,
+report.deep_convert,
+report.deep_convert_cost,
+report.deep_convert_rate,
+report.attribution_convert,
+report.attribution_convert_cost,
+report.attribution_deep_convert,
+report.attribution_deep_convert_cost,
+report.download_start,
+report.download_start_cost,
+report.download_start_rate,
+report.download_finish,
+report.download_finish_cost,
+report.download_finish_rate,
+report.click_install,
+report.install_finish,
+report.install_finish_cost,
+report.install_finish_rate,
+report.active,
+report.active_cost,
+report.active_rate,
+report.register,
+report.active_register_cost,
+report.active_register_rate,
+report.next_day_open,
+report.next_day_open_cost,
+report.next_day_open_rate,
+report.attribution_next_day_open_cnt,
+report.attribution_next_day_open_cost,
+report.attribution_next_day_open_rate,
+report.game_addiction,
+report.game_addiction_cost,
+report.game_addiction_rate,
+report.pay_count,
+report.active_pay_cost,
+report.active_pay_rate,
+report.loan_completion,
+report.loan_completion_cost,
+report.loan_completion_rate,
+report.pre_loan_credit,
+report.pre_loan_credit_cost,
+report.loan_credit,
+report.loan_credit_cost,
+report.loan_credit_rate,
+report.in_app_uv,
+report.in_app_detail_uv,
+report.in_app_cart,
+report.in_app_pay,
+report.in_app_order,
+report.attribution_game_pay_7d_count,
+report.attribution_game_pay_7d_cost,
+report.attribution_active_pay_7d_per_count,
+report.game_pay_cost,
+report.game_pay_count,
+report.phone,
+report.form,
+report.map_search,
+report.button,
+report.`view`,
+report.download,
+report.qq,
+report.lottery,
+report.vote,
+report.message,
+report.redirect,
+report.shopping,
+report.consult,
+report.wechat,
+report.phone_confirm,
+report.phone_connect,
+report.consult_effective,
+report.coupon,
+report.coupon_single_page,
+report.redirect_to_shop,
+report.poi_collect,
+report.poi_address_click,
+report.luban_order_cnt,
+report.luban_order_stat_amount,
+report.luban_order_roi,
+report.luban_live_enter_cnt,
+report.live_watch_one_minute_count,
+report.luban_live_follow_cnt,
+report.live_fans_club_join_cnt,
+report.luban_live_comment_cnt,
+report.luban_live_share_cnt,
+report.luban_live_gift_cnt,
+report.luban_live_gift_amount,
+report.luban_live_slidecart_click_cnt,
+report.luban_live_click_product_cnt,
+report.luban_live_pay_order_count,
+report.luban_live_pay_order_stat_cost,
+report.live_component_click_count,
+report.live_component_click_cost,
+report.live_component_click_rate,
+report.wechat_login_count,
+report.attribution_wechat_login_30d_count,
+report.wechat_login_cost,
+report.attribution_wechat_login_30d_cost,
+report.wechat_first_pay_count,
+report.attribution_wechat_first_pay_30d_count,
+report.wechat_first_pay_cost,
+report.attribution_wechat_first_pay_30d_cost,
+report.wechat_first_pay_rate,
+report.attribution_wechat_first_pay_30d_rate,
+report.wechat_pay_amount,
+report.attribution_wechat_pay_30d_amount,
+report.attribution_wechat_pay_30d_roi,
+report.phone_effective,
+report.total_play,
+report.valid_play,
+report.valid_play_cost,
+report.valid_play_rate,
+report.play_25_feed_break,
+report.play_50_feed_break,
+report.play_75_feed_break,
+report.play_100_feed_break,
+report.average_play_time_per_play,
+report.play_over_rate,
+report.wifi_play_rate,
+report.wifi_play,
+report.play_duration_sum,
+report.advanced_creative_phone_click,
+report.advanced_creative_counsel_click,
+report.advanced_creative_form_click,
+report.advanced_creative_coupon_addition,
+report.advanced_creative_form_submit,
+report.card_show,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.home_visited,
+report.ies_challenge_click,
+report.ies_music_click,
+report.location_click,
+report.message_action,
+report.click_landing_page,
+report.click_shopwindow,
+report.click_website,
+report.click_download,
+report.click_call_dy,
+report.submit_certification_count,
+report.approval_count,
+report.first_rental_order_count,
+report.first_order_count,
+report.commute_first_pay_count
+FROM
+(
+SELECT
+*
+FROM
+media_api.bytedance_advertiser_report_daily
+WHERE
+stat_datetime >= date_format(
+date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),
+'%Y%m%d'
+)
+) report
+LEFT JOIN (
+SELECT
+cua.account_id,
+uc.company_id,
+uc.company_name,
+pa.org_code,
+pa.org_type,
+pa.id dep_id,
+pa.depart_name,
+cua.user_id,
+su.realname ,
+cp.sale_id,
+su.leader_id,
+se.realname as leader_name
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user se ON se.id = su.leader_id
+LEFT JOIN `jeecg-boot`.sys_user_depart de ON su.leader_id = de.user_id
+LEFT JOIN `jeecg-boot`.sys_depart pa ON pa.id = de.dep_id
+) dep ON report.advertiser_id = dep.account_id
+) info
+GROUP BY
+company_id,
+leader_id) new
+LEFT JOIN
+(select ifnull(su.leader_id,'-') leader_id,sum(report.cost) cost  from
+(select advertiser_id account_id,sum(cost) cost from media_api.bytedance_advertiser_report_daily
+where stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id) report
+LEFT JOIN  `jeecg-boot`.ctop_user_allocation cua
+ON report.account_id = cua.account_id
+LEFT JOIN `jeecg-boot`.sys_user su
+ON su.id = cua.user_id
+group by su.leader_id) ratio
+ON new.leader_id=ratio.leader_id ;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceCompanyUserReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 421 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyUserReportBusiMonth.py

@@ -0,0 +1,421 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceCompanyUserReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceCompanyUserReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_company_user_report_busi_m 
+SELECT
+               date_format({date_time},'%Y%m') stat_mon,
+               ifnull(company_id,'-') company_id,
+               company_name,
+               org_code,
+               org_type,
+               dep_id,
+               depart_name,
+               ifnull(leader_id,'-') leader_id,
+               trim(leader_name) leader_name,
+               count(DISTINCT account_id),
+               sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click) click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`) `convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert) attribution_convert,
+               ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+               ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install) click_install,
+               sum(install_finish) install_finish,
+               ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register) register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate)/{day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open) next_day_open,
+               ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round(sum(cost) / sum(attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate)/{day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count) pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+               sum(loan_credit) loan_credit,
+               ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv) in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart) in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round(sum(attribution_game_pay_7d_count)/{day_count},3),0) attribution_game_pay_7d_count,
+               ifnull(round(sum(attribution_game_pay_7d_cost)/{day_count},3),0) attribution_game_pay_7d_cost,
+               ifnull(round(sum(attribution_active_pay_7d_per_count)/{day_count},3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost) game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone) phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download) download,
+               sum(qq) qq,
+               sum(lottery) lottery,
+               sum(vote) vote,
+               sum(message) message,
+               sum(redirect) redirect,
+               sum(shopping) shopping,
+               sum(consult) consult,
+               sum(wechat) wechat,
+               sum(phone_confirm) phone_confirm,
+               sum(phone_connect) phone_connect,
+               sum(consult_effective) consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect) poi_collect,
+               sum(poi_address_click) poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount) luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt) luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round(sum(cost) / sum(live_component_click_count),3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count)/{day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+               ifnull(round(sum(cost) / sum(attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+               ifnull(round(sum(cost) / sum(attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount) wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break) play_100_feed_break,
+               ifnull(round(sum(average_play_time_per_play)/{day_count},3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate)/{day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play) wifi_play,
+               ifnull(round(sum(play_duration_sum)/{day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show) card_show,
+               sum(`share`) `share`,
+               sum(`comment`) `comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click) ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website) click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy) click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count) first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count) commute_first_pay_count
+               FROM
+(
+SELECT
+report.advertiser_id account_id,
+dep.company_id,
+CASE
+WHEN dep.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN dep.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN dep.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+dep.company_name
+END company_name,
+dep.org_code,
+dep.org_type,
+dep.dep_id,
+dep.depart_name,
+dep.user_id,
+dep.realname,
+dep.leader_id,
+dep.leader_name,
+report.cost,
+report.`show`,
+report.avg_show_cost,
+report.click,
+report.avg_click_cost,
+report.ctr,
+report.`convert`,
+report.convert_cost,
+report.convert_rate,
+report.deep_convert,
+report.deep_convert_cost,
+report.deep_convert_rate,
+report.attribution_convert,
+report.attribution_convert_cost,
+report.attribution_deep_convert,
+report.attribution_deep_convert_cost,
+report.download_start,
+report.download_start_cost,
+report.download_start_rate,
+report.download_finish,
+report.download_finish_cost,
+report.download_finish_rate,
+report.click_install,
+report.install_finish,
+report.install_finish_cost,
+report.install_finish_rate,
+report.active,
+report.active_cost,
+report.active_rate,
+report.register,
+report.active_register_cost,
+report.active_register_rate,
+report.next_day_open,
+report.next_day_open_cost,
+report.next_day_open_rate,
+report.attribution_next_day_open_cnt,
+report.attribution_next_day_open_cost,
+report.attribution_next_day_open_rate,
+report.game_addiction,
+report.game_addiction_cost,
+report.game_addiction_rate,
+report.pay_count,
+report.active_pay_cost,
+report.active_pay_rate,
+report.loan_completion,
+report.loan_completion_cost,
+report.loan_completion_rate,
+report.pre_loan_credit,
+report.pre_loan_credit_cost,
+report.loan_credit,
+report.loan_credit_cost,
+report.loan_credit_rate,
+report.in_app_uv,
+report.in_app_detail_uv,
+report.in_app_cart,
+report.in_app_pay,
+report.in_app_order,
+report.attribution_game_pay_7d_count,
+report.attribution_game_pay_7d_cost,
+report.attribution_active_pay_7d_per_count,
+report.game_pay_cost,
+report.game_pay_count,
+report.phone,
+report.form,
+report.map_search,
+report.button,
+report.`view`,
+report.download,
+report.qq,
+report.lottery,
+report.vote,
+report.message,
+report.redirect,
+report.shopping,
+report.consult,
+report.wechat,
+report.phone_confirm,
+report.phone_connect,
+report.consult_effective,
+report.coupon,
+report.coupon_single_page,
+report.redirect_to_shop,
+report.poi_collect,
+report.poi_address_click,
+report.luban_order_cnt,
+report.luban_order_stat_amount,
+report.luban_order_roi,
+report.luban_live_enter_cnt,
+report.live_watch_one_minute_count,
+report.luban_live_follow_cnt,
+report.live_fans_club_join_cnt,
+report.luban_live_comment_cnt,
+report.luban_live_share_cnt,
+report.luban_live_gift_cnt,
+report.luban_live_gift_amount,
+report.luban_live_slidecart_click_cnt,
+report.luban_live_click_product_cnt,
+report.luban_live_pay_order_count,
+report.luban_live_pay_order_stat_cost,
+report.live_component_click_count,
+report.live_component_click_cost,
+report.live_component_click_rate,
+report.wechat_login_count,
+report.attribution_wechat_login_30d_count,
+report.wechat_login_cost,
+report.attribution_wechat_login_30d_cost,
+report.wechat_first_pay_count,
+report.attribution_wechat_first_pay_30d_count,
+report.wechat_first_pay_cost,
+report.attribution_wechat_first_pay_30d_cost,
+report.wechat_first_pay_rate,
+report.attribution_wechat_first_pay_30d_rate,
+report.wechat_pay_amount,
+report.attribution_wechat_pay_30d_amount,
+report.attribution_wechat_pay_30d_roi,
+report.phone_effective,
+report.total_play,
+report.valid_play,
+report.valid_play_cost,
+report.valid_play_rate,
+report.play_25_feed_break,
+report.play_50_feed_break,
+report.play_75_feed_break,
+report.play_100_feed_break,
+report.average_play_time_per_play,
+report.play_over_rate,
+report.wifi_play_rate,
+report.wifi_play,
+report.play_duration_sum,
+report.advanced_creative_phone_click,
+report.advanced_creative_counsel_click,
+report.advanced_creative_form_click,
+report.advanced_creative_coupon_addition,
+report.advanced_creative_form_submit,
+report.card_show,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.home_visited,
+report.ies_challenge_click,
+report.ies_music_click,
+report.location_click,
+report.message_action,
+report.click_landing_page,
+report.click_shopwindow,
+report.click_website,
+report.click_download,
+report.click_call_dy,
+report.submit_certification_count,
+report.approval_count,
+report.first_rental_order_count,
+report.first_order_count,
+report.commute_first_pay_count
+FROM
+(
+SELECT
+*
+FROM
+media_api.bytedance_advertiser_report_daily
+WHERE
+stat_datetime >= date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+) report
+LEFT JOIN (
+SELECT
+cua.account_id,
+uc.company_id,
+uc.company_name,
+pa.org_code,
+pa.org_type,
+pa.id dep_id,
+pa.depart_name,
+cua.user_id,
+su.realname ,
+cp.sale_id,
+su.leader_id,
+se.realname as leader_name
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user se ON se.id = su.leader_id
+LEFT JOIN `jeecg-boot`.sys_user_depart de ON su.leader_id = de.user_id
+LEFT JOIN `jeecg-boot`.sys_depart pa ON pa.id = de.dep_id
+) dep ON report.advertiser_id = dep.account_id
+) info
+GROUP BY
+company_id,
+leader_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceCompanyUserReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 209 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceFileVideoGet.py

@@ -0,0 +1,209 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/12 10:20 上午
+# @Site    : 
+# @File    : AppBytedanceFileVideoGet.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+import logging
+from math import ceil
+
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AppBytedanceFileVideoGet:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.date_time = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y-%m-%d")
+
+    def execSql(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_file_video_get where  date_format(create_time, '%Y-%m-%d') >= date_sub('{date_time}', INTERVAL 15 DAY)
+                """.format(date_time=self.date_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 10000)
+            for i in range(pageCount):
+                try:
+                    sql = """
+                    replace into application.app_bytedance_file_video_get
+select advertiser_id,
+       id,
+       size,
+       width,
+       height,
+       url,
+       format,
+       case when v.signature is null then '' else v.signature end as signature,
+       poster_url,
+       bit_rate,
+       duration,
+       material_id,
+       source,
+       create_time,
+       filename,
+       labels,
+       video_url,
+       case when tag_category_id is null then '' else tag_category_id end as tag_category_id,
+       tag_category_name,
+       case when parent_id is null then '' else parent_id end as parent_id,
+       parent_name,
+      case when tag_code is null then '' else tag_code end as tag_code,
+       tag_name,
+       clip_id,
+       clip_name,
+       shot_id,
+       shot_name,
+       plan_id,
+       plan_name,
+       leader_id,
+       leader_name,
+       material_innovate,
+       hot.stat_datetime,
+       effect.stat_datetime
+
+from (
+         select advertiser_id,
+                id,
+                size,
+                width,
+                height,
+                url,
+                format,
+                signature,
+                poster_url,
+                bit_rate,
+                duration,
+                a.material_id,
+                source,
+                create_time,
+                case when t.material_name is null then a.filename else t.material_name end as filename,
+                labels,
+                video_url,
+                clip_id,
+                clip_name,
+                shot_id,
+                shot_name,
+                plan_id,
+                plan_name,
+                leader_id,
+                leader_name,
+                material_innovate
+
+         from (select advertiser_id,
+                      id,
+                      size,
+                      width,
+                      height,
+                      url,
+                      format,
+                      signature,
+                      poster_url,
+                      bit_rate,
+                      duration,
+                      material_id,
+                      source,
+                      create_time,
+                      filename,
+                      labels,
+                      video_url
+               from media_api.bytedance_file_video_get  
+               where  date_format(create_time, '%Y-%m-%d') >= date_sub('{date_time}', INTERVAL 13 DAY)
+              ) a
+                  left join
+              (select material_id,
+                      clip_id,
+                      clip_name,
+                      shot_id,
+                      shot_name,
+                      plan_id,
+                      plan_name,
+                      leader_id,
+                      leader_name,
+                      material_name,
+                      material_innovate
+               from (select cma.material_id,
+                            clip_id,
+                            clip.realname   as clip_name,
+                            shot_id,
+                            shot.realname   as shot_name,
+                            plan_id,
+                            plan.realname   as plan_name,
+                            cma.leader_id,
+                            leader.realname as leader_name,
+                            material_name,
+                            material_innovate
+                     from `jeecg-boot`.ctop_material_ascription cma
+                              left join
+                          `jeecg-boot`.ctop_material_info b
+                          on cma.material_id = b.id
+                              left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+                              left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+                              left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+                              left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+                    ) v) t
+              on a.signature = t.material_id) v
+         left join
+     (select code,
+             tag_category_id,
+             tag_category_name,
+             parent_id,
+             parent_name,
+             group_concat(tag_code order by tag_code separator '-') tag_code,
+             group_concat(tag_name order by tag_code separator '-') tag_name
+      from (select a.code, b.tag_category_id, b.tag_category_name, b.parent_id, b.parent_name, b.tag_code, b.tag_name
+            from `jeecg-boot`.ctop_material_tag_info a
+                     left join
+                 (select tag_category_id,
+                         tag_category_name,
+                         case
+                             when parent_id is not null then parent_id
+                             when tag_category_id = 2 then ''
+                             else tag_code end parent_id,
+                         case
+                             when parent_id = 60155 then '真人'
+                             when parent_id = 60156 then '制作'
+                             when parent_id = 60160 then '外拍场景'
+                             when parent_id = 60161 then '公司内景'
+                             when tag_category_id = 2 then ''
+                             else '其他' end     parent_name,
+                         tag_code,
+                         tag_name
+                  from `jeecg-boot`.ctop_tag_info) b
+                 on a.tag_code = b.tag_code) c
+      group by code, tag_category_id, tag_category_name, parent_id, parent_name) m
+     on v.signature = m.code
+         left join (
+   select signature,effect_date as stat_datetime  from application.effect_material where media_id =1 
+) effect
+                   on v.signature = effect.signature
+         left join (
+  select signature,hot_date as stat_datetime from application.hot_material where media_id =1 
+
+) hot
+                   on v.signature = hot.signature
+group by advertiser_id, id, v.signature, material_id, tag_category_id, parent_id, tag_code  
+order by advertiser_id, id, v.signature, material_id, tag_category_id, parent_id, tag_code
+limit {num},10000;
+                    """.format(date_time=self.date_time, num=10000 * i)
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceFileVideoGet().execSql()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 201 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceFileVideoGetAll.py

@@ -0,0 +1,201 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/12 10:20 上午
+# @Site    : 
+# @File    : AppBytedanceFileVideoGetAll.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AppBytedanceFileVideoGetAll:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+
+    def execSql(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_file_video_get  
+                group by advertiser_id"""
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                try:
+                    sql = """
+replace into application.app_bytedance_file_video_get
+select advertiser_id,
+       id,
+       size,
+       width,
+       height,
+       url,
+       format,
+       case when v.signature is null then '' else v.signature end as signature,
+       poster_url,
+       bit_rate,
+       duration,
+       material_id,
+       source,
+       create_time,
+       filename,
+       labels,
+       video_url,
+       case when tag_category_id is null then '' else tag_category_id end as tag_category_id,
+       tag_category_name,
+       case when parent_id is null then '' else parent_id end as parent_id,
+       parent_name,
+      case when tag_code is null then '' else tag_code end as tag_code,
+       tag_name,
+       clip_id,
+       clip_name,
+       shot_id,
+       shot_name,
+       plan_id,
+       plan_name,
+       leader_id,
+       leader_name,
+       material_innovate,
+       hot.stat_datetime,
+       effect.stat_datetime
+
+from (
+         select advertiser_id,
+                id,
+                size,
+                width,
+                height,
+                url,
+                format,
+                signature,
+                poster_url,
+                bit_rate,
+                duration,
+                a.material_id,
+                source,
+                create_time,
+                case when t.material_name is null then a.filename else t.material_name end as filename,
+                labels,
+                video_url,
+                clip_id,
+                clip_name,
+                shot_id,
+                shot_name,
+                plan_id,
+                plan_name,
+                leader_id,
+                leader_name,
+                material_innovate
+
+         from (select advertiser_id,
+                      id,
+                      size,
+                      width,
+                      height,
+                      url,
+                      format,
+                      signature,
+                      poster_url,
+                      bit_rate,
+                      duration,
+                      material_id,
+                      source,
+                      create_time,
+                      filename,
+                      labels,
+                      video_url
+               from media_api.bytedance_file_video_get where advertiser_id={account_id}
+              ) a
+                  left join
+              (select material_id,
+                      clip_id,
+                      clip_name,
+                      shot_id,
+                      shot_name,
+                      plan_id,
+                      plan_name,
+                      leader_id,
+                      leader_name,
+                      material_name,
+                      material_innovate
+               from (select cma.material_id,
+                            clip_id,
+                            clip.realname   as clip_name,
+                            shot_id,
+                            shot.realname   as shot_name,
+                            plan_id,
+                            plan.realname   as plan_name,
+                            cma.leader_id,
+                            leader.realname as leader_name,
+                            material_name,
+                            material_innovate
+                     from `jeecg-boot`.ctop_material_ascription cma
+                              left join
+                          `jeecg-boot`.ctop_material_info b
+                          on cma.material_id = b.id
+                              left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+                              left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+                              left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+                              left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+                    ) v) t
+              on a.signature = t.material_id) v
+         left join
+     (select code,
+             tag_category_id,
+             tag_category_name,
+             parent_id,
+             parent_name,
+             group_concat(tag_code order by tag_code separator '-') tag_code,
+             group_concat(tag_name order by tag_code separator '-') tag_name
+      from (select a.code, b.tag_category_id, b.tag_category_name, b.parent_id, b.parent_name, b.tag_code, b.tag_name
+            from `jeecg-boot`.ctop_material_tag_info a
+                     left join
+                 (select if(tag_category_id is null,'',tag_category_id) as tag_category_id,
+                         tag_category_name,
+                         case
+                             when parent_id is not null then parent_id
+                             when tag_category_id = 2 then ''
+                             else tag_code end parent_id,
+                         case
+                             when parent_id = 60155 then '真人'
+                             when parent_id = 60156 then '制作'
+                             when parent_id = 60160 then '外拍场景'
+                             when parent_id = 60161 then '公司内景'
+                             when tag_category_id = 2 then ''
+                             else '其他' end     parent_name,
+                         tag_code,
+                         tag_name
+                  from `jeecg-boot`.ctop_tag_info) b
+                 on a.tag_code = b.tag_code) c
+      group by code, tag_category_id, tag_category_name, parent_id, parent_name) m
+     on v.signature = m.code
+         left join (
+   select signature,effect_date as stat_datetime  from application.effect_material where media_id =1 
+) effect
+                   on v.signature = effect.signature
+         left join (
+  select signature,hot_date as stat_datetime from application.hot_material where media_id =1 
+) hot
+                   on v.signature = hot.signature
+group by advertiser_id, id,  v.signature, material_id, tag_category_id, parent_id, tag_code       
+                """.format(account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+
+
+if __name__ == '__main__':
+    status = AppBytedanceFileVideoGetAll().execSql()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 483 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceImageReportBusi.py

@@ -0,0 +1,483 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceImageReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceImageReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select project_id from `jeecg-boot`.ctop_user_allocation where media_id=1
+group by project_id """
+            project_list = TidbConn.quary(self.conn,sql_porject)
+            for project_id in project_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_image_report_busi_{day_count}d
+SELECT concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time}) time,
+       report.account_id,
+       #'账户ID'
+       report.material_id,
+       #头条素材id
+       report.image_mode,
+       #素材类型
+       report.inventory,
+       #广告位
+       cua.project_id,
+       #项目id
+       product.id    product_id,
+       #项目中产品id
+       cua.advertiser_id,
+       #广告主id
+       uc.company_id,
+       #运营人的公司id
+       cua.auth_name account_name,
+       #账户名称
+       cp.project_name,
+       #项目名称
+       ca.`name`     advertiser_name,
+       #广告主名称
+       uc.company_name,
+       #运营人的公司名称
+       cp.sale_id,
+       #项目里的销售id
+       cua.user_id,
+       #运营id
+       su.org_code,
+       #机构编码
+       video.signature    signature,
+       #素材id
+       video.filename  as video_name,
+       report.cost,
+       ifnull(ratio.cost,0) last_cost,
+       ifnull(round((report.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,
+       report.`show`,
+       report.avg_show_cost,
+       report.click,
+       report.avg_click_cost,
+       report.ctr,
+       report.`convert`,
+       report.convert_cost,
+       report.convert_rate,
+       report.deep_convert,
+       report.deep_convert_cost,
+       report.deep_convert_rate,
+       report.attribution_convert,
+       report.attribution_convert_cost,
+       report.attribution_deep_convert,
+       report.attribution_deep_convert_cost,
+       report.download_start,
+       report.download_start_cost,
+       report.download_start_rate,
+       report.download_finish,
+       report.download_finish_cost,
+       report.download_finish_rate,
+       report.click_install,
+       report.install_finish,
+       report.install_finish_cost,
+       report.install_finish_rate,
+       report.active,
+       report.active_cost,
+       report.active_rate,
+       report.register,
+       report.active_register_cost,
+       report.active_register_rate,
+       report.active_pay_amount,
+       report.next_day_open,
+       report.next_day_open_cost,
+       report.next_day_open_rate,
+       report.attribution_next_day_open_cnt,
+       report.attribution_next_day_open_cost,
+       report.attribution_next_day_open_rate,
+       report.game_addiction,
+       report.game_addiction_cost,
+       report.game_addiction_rate,
+       report.pay_count,
+       report.active_pay_cost,
+       report.active_pay_rate,
+       report.loan_completion,
+       report.loan_completion_cost,
+       report.loan_completion_rate,
+       report.pre_loan_credit,
+       report.pre_loan_credit_cost,
+       report.loan_credit,
+       report.loan_credit_cost,
+       report.loan_credit_rate,
+       report.in_app_uv,
+       report.in_app_detail_uv,
+       report.in_app_cart,
+       report.in_app_pay,
+       report.in_app_order,
+       report.attribution_game_pay_7d_count,
+       report.attribution_game_pay_7d_cost,
+       report.attribution_active_pay_7d_per_count,
+       report.game_pay_cost,
+       report.game_pay_count,
+       report.phone,
+       report.form,
+       report.map_search,
+       report.button,
+       report.`view`,
+       report.download,
+       report.qq,
+       report.lottery,
+       report.vote,
+       report.message,
+       report.redirect,
+       report.shopping,
+       report.consult,
+       report.wechat,
+       report.phone_confirm,
+       report.phone_connect,
+       report.consult_effective,
+       report.coupon,
+       report.coupon_single_page,
+       report.redirect_to_shop,
+       report.poi_collect,
+       report.poi_address_click,
+       report.luban_order_cnt,
+       report.luban_order_stat_amount,
+       report.luban_order_roi,
+       report.luban_live_enter_cnt,
+       report.live_watch_one_minute_count,
+       report.luban_live_follow_cnt,
+       report.live_fans_club_join_cnt,
+       report.luban_live_comment_cnt,
+       report.luban_live_share_cnt,
+       report.luban_live_gift_cnt,
+       report.luban_live_gift_amount,
+       report.luban_live_slidecart_click_cnt,
+       report.luban_live_click_product_cnt,
+       report.luban_live_pay_order_count,
+       report.luban_live_pay_order_stat_cost,
+       report.wechat_login_count,
+       report.attribution_wechat_login_30d_count,
+       report.wechat_login_cost,
+       report.attribution_wechat_login_30d_cost,
+       report.wechat_first_pay_count,
+       report.attribution_wechat_first_pay_30d_count,
+       report.wechat_first_pay_cost,
+       report.attribution_wechat_first_pay_30d_cost,
+       report.wechat_first_pay_rate,
+       report.attribution_wechat_first_pay_30d_rate,
+       report.wechat_pay_amount,
+       report.attribution_wechat_pay_30d_amount,
+       report.attribution_wechat_pay_30d_roi,
+       report.phone_effective,
+       report.total_play,
+       report.valid_play,
+       report.valid_play_cost,
+       report.valid_play_rate,
+       report.play_25_feed_break,
+       report.play_50_feed_break,
+       report.play_75_feed_break,
+       report.play_100_feed_break,
+       report.average_play_time_per_play,
+       report.play_over_rate,
+       report.wifi_play_rate,
+       report.wifi_play,
+       report.play_duration_sum,
+       report.advanced_creative_phone_click,
+       report.advanced_creative_counsel_click,
+       report.advanced_creative_form_click,
+       report.advanced_creative_coupon_addition,
+       report.advanced_creative_form_submit,
+       report.card_show,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.follow,
+       report.home_visited,
+       report.ies_challenge_click,
+       report.ies_music_click,
+       report.location_click,
+       report.message_action,
+       report.click_landing_page,
+       report.click_shopwindow,
+       report.click_website,
+       report.click_download,
+       report.click_call_dy,
+       report.`cpc`,
+       report.`cpm`,
+       report.`cpa`,
+       report.`interact_per_cost`,
+       report.`convert_show_rate`,
+       report.`play_duration_10s_rate`,
+       report.`play_50_feed_break_rate`,
+       report.`play_25_feed_break_rate`,
+       report.`play_duration_5s_rate`,
+       report.`play_duration_2s_rate`,
+       report.`average_play_progress`,
+       report.`play_100_feed_break_rate`,
+       report.`play_duration_3s_rate`,
+       report.`play_75_feed_break_rate`,
+       report.`play_duration`,
+       report.`play_duration_3s`,
+       report.`play_duration_2s`,
+       report.`play_duration_10s`,
+       report.`average_video_play`,
+       report.`avg_rank`,
+       report.submit_certification_count,
+       report.approval_count,
+       report.first_rental_order_count,
+       report.first_order_count,
+       report.commute_first_pay_count,
+       video.`image_url`
+FROM ( SELECT advertiser_id    account_id,
+                material_id,
+                image_mode,
+                inventory,
+          sum(cost) cost,
+          sum(`show`) `show`,
+          ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+          sum(click) click,
+          ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+          ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+          sum(`convert`) `convert`,
+          ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+          ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+          sum(deep_convert) deep_convert,
+          ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+          ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+          sum(attribution_convert) attribution_convert,
+          ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+          sum(attribution_deep_convert) attribution_deep_convert,
+          ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+          sum(download_start) download_start,
+          ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+          ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+          sum(download_finish) download_finish,
+          ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+          ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+          sum(click_install) click_install,
+          sum(install_finish) install_finish,
+          ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+          ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+          sum(active) active,
+          ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+          ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+          sum(register) register,
+          ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+          ifnull(round(sum(active_register_rate) /{day_count} * 100,3),0) active_register_rate,
+          sum(active_pay_amount) active_pay_amount,
+          sum(next_day_open) next_day_open,
+          ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+          ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+          sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+          ifnull(round(sum(cost) / sum(	attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+          ifnull(round(sum(	attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+          sum(game_addiction) game_addiction,
+          ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+          ifnull(round(sum(game_addiction_rate) /{day_count} * 100,3),0) game_addiction_rate,
+          sum(pay_count) pay_count,
+          ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+          ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+          sum(loan_completion) loan_completion,
+          ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+          ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+          ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+          ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+          sum(loan_credit) loan_credit,
+          ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+          ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+          sum(in_app_uv) in_app_uv,
+          sum(in_app_detail_uv) in_app_detail_uv,
+          sum(in_app_cart) in_app_cart,
+          sum(in_app_pay) in_app_pay,
+          sum(in_app_order) in_app_order,
+          ifnull(round(sum(	attribution_game_pay_7d_count) /{day_count}, 3),0) attribution_game_pay_7d_count,
+          ifnull(round(sum(	attribution_game_pay_7d_cost) /{day_count}, 3),0) attribution_game_pay_7d_cost,
+          ifnull(round(sum(	attribution_active_pay_7d_per_count) /{day_count}, 3),0) attribution_active_pay_7d_per_count,
+          sum(game_pay_cost) game_pay_cost,
+          sum(game_pay_count) game_pay_count,
+          sum(phone) phone,
+          sum(form) form,
+          sum(map_search) map_search,
+          sum(button) button,
+          sum(`view`) `view`,
+          sum(download) download,
+          sum(qq) qq,
+          sum(lottery) lottery,
+          sum(vote) vote,
+          sum(message) message,
+          sum(redirect) redirect,
+          sum(shopping) shopping,
+          sum(consult) consult,
+          sum(wechat) wechat,
+          sum(phone_confirm) phone_confirm,
+          sum(phone_connect) phone_connect,
+          sum(consult_effective) consult_effective,
+          sum(coupon) coupon,
+          sum(coupon_single_page) coupon_single_page,
+          sum(redirect_to_shop) redirect_to_shop,
+          sum(poi_collect) poi_collect,
+          sum(poi_address_click) poi_address_click,
+          sum(luban_order_cnt) luban_order_cnt,
+          sum(luban_order_stat_amount) luban_order_stat_amount,
+          sum(luban_order_roi) luban_order_roi,
+          sum(luban_live_enter_cnt) luban_live_enter_cnt,
+          sum(live_watch_one_minute_count) live_watch_one_minute_count,
+          sum(luban_live_follow_cnt) luban_live_follow_cnt,
+          sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+          sum(luban_live_comment_cnt) luban_live_comment_cnt,
+          sum(luban_live_share_cnt) luban_live_share_cnt,
+          sum(luban_live_gift_cnt) luban_live_gift_cnt,
+          sum(luban_live_gift_amount) luban_live_gift_amount,
+          sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+          sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+          sum(luban_live_pay_order_count) luban_live_pay_order_count,
+          sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+          sum(wechat_login_count) wechat_login_count,
+          sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+          sum(wechat_first_pay_count) wechat_first_pay_count,
+          sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+          ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+          ifnull(round(sum(	attribution_wechat_first_pay_30d_count) / sum(	attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+          sum(wechat_pay_amount) wechat_pay_amount,
+          sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+          sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+          sum(phone_effective) phone_effective,
+          sum(total_play) total_play,
+          sum(valid_play) valid_play,
+          ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+          ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+          sum(play_25_feed_break) play_25_feed_break,
+          sum(play_50_feed_break) play_50_feed_break,
+          sum(play_75_feed_break) play_75_feed_break,
+          sum(play_100_feed_break) play_100_feed_break,
+          ifnull(round(sum(average_play_time_per_play) /{day_count}, 3),0) average_play_time_per_play,
+          ifnull(round(sum(play_over_rate) /{day_count} * 100,3),0) play_over_rate,
+          ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+          sum(wifi_play) wifi_play,
+          ifnull(round(sum(play_duration_sum) /{day_count}, 3),0) play_duration_sum,
+          sum(advanced_creative_phone_click) advanced_creative_phone_click,
+          sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+          sum(advanced_creative_form_click) advanced_creative_form_click,
+          sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+          sum(advanced_creative_form_submit) advanced_creative_form_submit,
+          sum(card_show) card_show,
+          sum(`share`) `share`,
+          sum(`comment`) `comment`,
+          sum(`like`) `like`,
+          sum(follow) follow,
+          sum(home_visited) home_visited,
+          sum(ies_challenge_click) ies_challenge_click,
+          sum(ies_music_click) ies_music_click,
+          sum(location_click) location_click,
+          sum(message_action) message_action,
+          sum(click_landing_page) click_landing_page,
+          sum(click_shopwindow) click_shopwindow,
+          sum(click_website) click_website,
+          sum(click_download) click_download,
+          sum(click_call_dy) click_call_dy,
+          sum(cpc) cpc,
+          sum(cpm) cpm,
+          sum(cpa) cpa,
+          ifnull(round(sum(interact_per_cost) /{day_count}, 3),0) interact_per_cost,
+          ifnull(round(sum(convert_show_rate) /{day_count}, 3),0) convert_show_rate,
+          ifnull(round(sum(play_duration_10s) / sum(total_play) * 100,3),0) play_duration_10s_rate,
+          ifnull(round(sum(play_50_feed_break) / sum(total_play) * 100,3),0) play_50_feed_break_rate,
+          ifnull(round(sum(play_25_feed_break) / sum(total_play) * 100,3),0) play_25_feed_break_rate,
+          ifnull(round(sum(play_duration_5s_rate) /{day_count}, 3),0) play_duration_5s_rate,
+          ifnull(round(sum(play_duration_2s) / sum(total_play) * 100,3),0) play_duration_2s_rate,
+          ifnull(round(sum(average_play_progress) /{day_count}, 3),0) average_play_progress,
+          ifnull(round(sum(play_100_feed_break) / sum(total_play) * 100,3),0) play_100_feed_break_rate,
+          ifnull(round(sum(play_duration_3s) / sum(total_play) * 100,3),0) play_duration_3s_rate,
+          ifnull(round(sum(play_75_feed_break) / sum(total_play) * 100,3),0) play_75_feed_break_rate,
+          sum(play_duration) play_duration,
+          sum(play_duration_3s) play_duration_3s,
+          sum(play_duration_2s) play_duration_2s,
+          sum(play_duration_10s) play_duration_10s,
+          ifnull(round(sum(average_video_play) /{day_count}, 3),0) average_video_play,
+          sum(avg_rank) avg_rank,
+          sum(submit_certification_count) submit_certification_count,
+          sum(approval_count) approval_count,
+          sum(first_order_count) first_rental_order_count,
+          sum(first_rental_order_count) first_order_count,
+          sum(commute_first_pay_count) commute_first_pay_count
+         FROM media_api.bytedance_material_report_daily zhubiao
+         WHERE image_mode IN (
+    'CREATIVE_IMAGE_MODE_SMALL',
+    'CREATIVE_IMAGE_MODE_LARGE',
+    'CREATIVE_IMAGE_MODE_LARGE_VERTICAL',
+    'CREATIVE_IMAGE_MODE_GROUP',
+    'CREATIVE_IMAGE_MODE_GIF',
+    'TOUTIAO_SEARCH_AD_IMAGE',
+    'SEARCH_AD_SMALL_IMAGE',
+    'CREATIVE_IMAGE_MODE_UNION_SPLASH')
+       AND stat_datetime >= date_format(date_sub({date_time}, INTERVAL ({day_count} -1)  DAY),'%Y%m%d')
+       GROUP BY advertiser_id,
+                  material_id,
+                  image_mode,
+                  inventory) report
+LEFT JOIN (select advertiser_id,signature,material_id,filename,image_url
+from  `media_api`.bytedance_file_image_get group by advertiser_id,signature,material_id,filename,image_url)  video 
+ON report.account_id = video.advertiser_id AND report.material_id = video.material_id
+          LEFT JOIN `jeecg-boot`.ctop_user_allocation cua
+                   ON report.account_id = cua.account_id
+          LEFT JOIN `jeecg-boot`.ctop_advertiser ca
+                   ON cua.advertiser_id = ca.id
+          LEFT JOIN `jeecg-boot`.ctop_project cp
+                   ON cua.project_id = cp.id
+          LEFT JOIN `jeecg-boot`.user_company uc
+                   ON cua.user_id = uc.user_id
+          LEFT JOIN `jeecg-boot`.ctop_product product
+                   ON cp.product_id = product.id
+         LEFT JOIN `jeecg-boot`.sys_user su
+                   ON cua.user_id = su.id
+         LEFT JOIN 
+         (select advertiser_id account_id,material_id,sum(cost) cost  FROM media_api.bytedance_material_report_daily
+    WHERE image_mode IN ('CREATIVE_IMAGE_MODE_SMALL',
+     'CREATIVE_IMAGE_MODE_LARGE',
+     'CREATIVE_IMAGE_MODE_LARGE_VERTICAL',
+     'CREATIVE_IMAGE_MODE_GROUP',
+     'CREATIVE_IMAGE_MODE_GIF',
+     'TOUTIAO_SEARCH_AD_IMAGE',
+     'SEARCH_AD_SMALL_IMAGE',
+     'CREATIVE_IMAGE_MODE_UNION_SPLASH')
+         and  stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+         and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+         group by advertiser_id,material_id) ratio
+         on report.account_id = ratio.account_id and report.material_id = ratio.material_id 
+        where cua.project_id={project_id};       
+                   
+""".format(date_time=self.date_time, day_count=self.day_count, project_id=project_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceImageReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 467 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceImageReportBusiMonth.py

@@ -0,0 +1,467 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceImageReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceImageReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select project_id from `jeecg-boot`.ctop_user_allocation where media_id=1
+group by project_id """
+            project_list = TidbConn.quary(self.conn,sql_porject)
+            for project_id in project_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_image_report_busi_m
+SELECT date_format({date_time},'%Y%m')          stat_non,
+     report.account_id,
+       #'账户ID'
+       report.material_id,
+       #头条素材id
+       report.image_mode,
+       #素材类型
+       report.inventory,
+       #广告位
+       cua.project_id,
+       #项目id
+       product.id    product_id,
+       #项目中产品id
+       cua.advertiser_id,
+       #广告主id
+       uc.company_id,
+       #运营人的公司id
+       cua.auth_name account_name,
+       #账户名称
+       cp.project_name,
+       #项目名称
+       ca.`name`     advertiser_name,
+       #广告主名称
+       uc.company_name,
+       #运营人的公司名称
+       cp.sale_id,
+       #项目里的销售id
+       cua.user_id,
+       #运营id
+       su.org_code,
+       #机构编码
+       video.signature    signature,
+       #素材id
+       video.filename  as video_name,
+       report.cost,
+       report.`show`,
+       report.avg_show_cost,
+       report.click,
+       report.avg_click_cost,
+       report.ctr,
+       report.`convert`,
+       report.convert_cost,
+       report.convert_rate,
+       report.deep_convert,
+       report.deep_convert_cost,
+       report.deep_convert_rate,
+       report.attribution_convert,
+       report.attribution_convert_cost,
+       report.attribution_deep_convert,
+       report.attribution_deep_convert_cost,
+       report.download_start,
+       report.download_start_cost,
+       report.download_start_rate,
+       report.download_finish,
+       report.download_finish_cost,
+       report.download_finish_rate,
+       report.click_install,
+       report.install_finish,
+       report.install_finish_cost,
+       report.install_finish_rate,
+       report.active,
+       report.active_cost,
+       report.active_rate,
+       report.register,
+       report.active_register_cost,
+       report.active_register_rate,
+       report.active_pay_amount,
+       report.next_day_open,
+       report.next_day_open_cost,
+       report.next_day_open_rate,
+       report.attribution_next_day_open_cnt,
+       report.attribution_next_day_open_cost,
+       report.attribution_next_day_open_rate,
+       report.game_addiction,
+       report.game_addiction_cost,
+       report.game_addiction_rate,
+       report.pay_count,
+       report.active_pay_cost,
+       report.active_pay_rate,
+       report.loan_completion,
+       report.loan_completion_cost,
+       report.loan_completion_rate,
+       report.pre_loan_credit,
+       report.pre_loan_credit_cost,
+       report.loan_credit,
+       report.loan_credit_cost,
+       report.loan_credit_rate,
+       report.in_app_uv,
+       report.in_app_detail_uv,
+       report.in_app_cart,
+       report.in_app_pay,
+       report.in_app_order,
+       report.attribution_game_pay_7d_count,
+       report.attribution_game_pay_7d_cost,
+       report.attribution_active_pay_7d_per_count,
+       report.game_pay_cost,
+       report.game_pay_count,
+       report.phone,
+       report.form,
+       report.map_search,
+       report.button,
+       report.`view`,
+       report.download,
+       report.qq,
+       report.lottery,
+       report.vote,
+       report.message,
+       report.redirect,
+       report.shopping,
+       report.consult,
+       report.wechat,
+       report.phone_confirm,
+       report.phone_connect,
+       report.consult_effective,
+       report.coupon,
+       report.coupon_single_page,
+       report.redirect_to_shop,
+       report.poi_collect,
+       report.poi_address_click,
+       report.luban_order_cnt,
+       report.luban_order_stat_amount,
+       report.luban_order_roi,
+       report.luban_live_enter_cnt,
+       report.live_watch_one_minute_count,
+       report.luban_live_follow_cnt,
+       report.live_fans_club_join_cnt,
+       report.luban_live_comment_cnt,
+       report.luban_live_share_cnt,
+       report.luban_live_gift_cnt,
+       report.luban_live_gift_amount,
+       report.luban_live_slidecart_click_cnt,
+       report.luban_live_click_product_cnt,
+       report.luban_live_pay_order_count,
+       report.luban_live_pay_order_stat_cost,
+       report.wechat_login_count,
+       report.attribution_wechat_login_30d_count,
+       report.wechat_login_cost,
+       report.attribution_wechat_login_30d_cost,
+       report.wechat_first_pay_count,
+       report.attribution_wechat_first_pay_30d_count,
+       report.wechat_first_pay_cost,
+       report.attribution_wechat_first_pay_30d_cost,
+       report.wechat_first_pay_rate,
+       report.attribution_wechat_first_pay_30d_rate,
+       report.wechat_pay_amount,
+       report.attribution_wechat_pay_30d_amount,
+       report.attribution_wechat_pay_30d_roi,
+       report.phone_effective,
+       report.total_play,
+       report.valid_play,
+       report.valid_play_cost,
+       report.valid_play_rate,
+       report.play_25_feed_break,
+       report.play_50_feed_break,
+       report.play_75_feed_break,
+       report.play_100_feed_break,
+       report.average_play_time_per_play,
+       report.play_over_rate,
+       report.wifi_play_rate,
+       report.wifi_play,
+       report.play_duration_sum,
+       report.advanced_creative_phone_click,
+       report.advanced_creative_counsel_click,
+       report.advanced_creative_form_click,
+       report.advanced_creative_coupon_addition,
+       report.advanced_creative_form_submit,
+       report.card_show,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.follow,
+       report.home_visited,
+       report.ies_challenge_click,
+       report.ies_music_click,
+       report.location_click,
+       report.message_action,
+       report.click_landing_page,
+       report.click_shopwindow,
+       report.click_website,
+       report.click_download,
+       report.click_call_dy,
+       report.`cpc`,
+       report.`cpm`,
+       report.`cpa`,
+       report.`interact_per_cost`,
+       report.`convert_show_rate`,
+       report.`play_duration_10s_rate`,
+       report.`play_50_feed_break_rate`,
+       report.`play_25_feed_break_rate`,
+       report.`play_duration_5s_rate`,
+       report.`play_duration_2s_rate`,
+       report.`average_play_progress`,
+       report.`play_100_feed_break_rate`,
+       report.`play_duration_3s_rate`,
+       report.`play_75_feed_break_rate`,
+       report.`play_duration`,
+       report.`play_duration_3s`,
+       report.`play_duration_2s`,
+       report.`play_duration_10s`,
+       report.`average_video_play`,
+       report.`avg_rank`,
+       report.submit_certification_count,
+       report.approval_count,
+       report.first_rental_order_count,
+       report.first_order_count,
+       report.commute_first_pay_count,
+       video.`image_url`,
+       ''
+FROM ( SELECT advertiser_id    account_id,
+                material_id,
+                image_mode,
+                inventory,
+          sum(cost) cost,
+          sum(`show`) `show`,
+          ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+          sum(click) click,
+          ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+          ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+          sum(`convert`) `convert`,
+          ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+          ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+          sum(deep_convert) deep_convert,
+          ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+          ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+          sum(attribution_convert) attribution_convert,
+          ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+          sum(attribution_deep_convert) attribution_deep_convert,
+          ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+          sum(download_start) download_start,
+          ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+          ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+          sum(download_finish) download_finish,
+          ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+          ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+          sum(click_install) click_install,
+          sum(install_finish) install_finish,
+          ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+          ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+          sum(active) active,
+          ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+          ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+          sum(register) register,
+          ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+          ifnull(round(sum(active_register_rate) /{day_count} * 100,3),0) active_register_rate,
+          sum(active_pay_amount) active_pay_amount,
+          sum(next_day_open) next_day_open,
+          ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+          ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+          sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+          ifnull(round(sum(cost) / sum(	attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+          ifnull(round(sum(	attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+          sum(game_addiction) game_addiction,
+          ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+          ifnull(round(sum(game_addiction_rate) /{day_count} * 100,3),0) game_addiction_rate,
+          sum(pay_count) pay_count,
+          ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+          ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+          sum(loan_completion) loan_completion,
+          ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+          ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+          ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+          ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+          sum(loan_credit) loan_credit,
+          ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+          ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+          sum(in_app_uv) in_app_uv,
+          sum(in_app_detail_uv) in_app_detail_uv,
+          sum(in_app_cart) in_app_cart,
+          sum(in_app_pay) in_app_pay,
+          sum(in_app_order) in_app_order,
+          ifnull(round(sum(	attribution_game_pay_7d_count) /{day_count}, 3),0) attribution_game_pay_7d_count,
+          ifnull(round(sum(	attribution_game_pay_7d_cost) /{day_count}, 3),0) attribution_game_pay_7d_cost,
+          ifnull(round(sum(	attribution_active_pay_7d_per_count) /{day_count}, 3),0) attribution_active_pay_7d_per_count,
+          sum(game_pay_cost) game_pay_cost,
+          sum(game_pay_count) game_pay_count,
+          sum(phone) phone,
+          sum(form) form,
+          sum(map_search) map_search,
+          sum(button) button,
+          sum(`view`) `view`,
+          sum(download) download,
+          sum(qq) qq,
+          sum(lottery) lottery,
+          sum(vote) vote,
+          sum(message) message,
+          sum(redirect) redirect,
+          sum(shopping) shopping,
+          sum(consult) consult,
+          sum(wechat) wechat,
+          sum(phone_confirm) phone_confirm,
+          sum(phone_connect) phone_connect,
+          sum(consult_effective) consult_effective,
+          sum(coupon) coupon,
+          sum(coupon_single_page) coupon_single_page,
+          sum(redirect_to_shop) redirect_to_shop,
+          sum(poi_collect) poi_collect,
+          sum(poi_address_click) poi_address_click,
+          sum(luban_order_cnt) luban_order_cnt,
+          sum(luban_order_stat_amount) luban_order_stat_amount,
+          sum(luban_order_roi) luban_order_roi,
+          sum(luban_live_enter_cnt) luban_live_enter_cnt,
+          sum(live_watch_one_minute_count) live_watch_one_minute_count,
+          sum(luban_live_follow_cnt) luban_live_follow_cnt,
+          sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+          sum(luban_live_comment_cnt) luban_live_comment_cnt,
+          sum(luban_live_share_cnt) luban_live_share_cnt,
+          sum(luban_live_gift_cnt) luban_live_gift_cnt,
+          sum(luban_live_gift_amount) luban_live_gift_amount,
+          sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+          sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+          sum(luban_live_pay_order_count) luban_live_pay_order_count,
+          sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+          sum(wechat_login_count) wechat_login_count,
+          sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+          sum(wechat_first_pay_count) wechat_first_pay_count,
+          sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+          ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+          ifnull(round(sum(	attribution_wechat_first_pay_30d_count) / sum(	attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+          sum(wechat_pay_amount) wechat_pay_amount,
+          sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+          sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+          sum(phone_effective) phone_effective,
+          sum(total_play) total_play,
+          sum(valid_play) valid_play,
+          ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+          ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+          sum(play_25_feed_break) play_25_feed_break,
+          sum(play_50_feed_break) play_50_feed_break,
+          sum(play_75_feed_break) play_75_feed_break,
+          sum(play_100_feed_break) play_100_feed_break,
+          ifnull(round(sum(average_play_time_per_play) /{day_count}, 3),0) average_play_time_per_play,
+          ifnull(round(sum(play_over_rate) /{day_count} * 100,3),0) play_over_rate,
+          ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+          sum(wifi_play) wifi_play,
+          ifnull(round(sum(play_duration_sum) /{day_count}, 3),0) play_duration_sum,
+          sum(advanced_creative_phone_click) advanced_creative_phone_click,
+          sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+          sum(advanced_creative_form_click) advanced_creative_form_click,
+          sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+          sum(advanced_creative_form_submit) advanced_creative_form_submit,
+          sum(card_show) card_show,
+          sum(`share`) `share`,
+          sum(`comment`) `comment`,
+          sum(`like`) `like`,
+          sum(follow) follow,
+          sum(home_visited) home_visited,
+          sum(ies_challenge_click) ies_challenge_click,
+          sum(ies_music_click) ies_music_click,
+          sum(location_click) location_click,
+          sum(message_action) message_action,
+          sum(click_landing_page) click_landing_page,
+          sum(click_shopwindow) click_shopwindow,
+          sum(click_website) click_website,
+          sum(click_download) click_download,
+          sum(click_call_dy) click_call_dy,
+          sum(cpc) cpc,
+          sum(cpm) cpm,
+          sum(cpa) cpa,
+          ifnull(round(sum(interact_per_cost) /{day_count}, 3),0) interact_per_cost,
+          ifnull(round(sum(convert_show_rate) /{day_count}, 3),0) convert_show_rate,
+          ifnull(round(sum(play_duration_10s) / sum(total_play) * 100,3),0) play_duration_10s_rate,
+          ifnull(round(sum(play_50_feed_break) / sum(total_play) * 100,3),0) play_50_feed_break_rate,
+          ifnull(round(sum(play_25_feed_break) / sum(total_play) * 100,3),0) play_25_feed_break_rate,
+          ifnull(round(sum(play_duration_5s_rate) /{day_count}, 3),0) play_duration_5s_rate,
+          ifnull(round(sum(play_duration_2s) / sum(total_play) * 100,3),0) play_duration_2s_rate,
+          ifnull(round(sum(average_play_progress) /{day_count}, 3),0) average_play_progress,
+          ifnull(round(sum(play_100_feed_break) / sum(total_play) * 100,3),0) play_100_feed_break_rate,
+          ifnull(round(sum(play_duration_3s) / sum(total_play) * 100,3),0) play_duration_3s_rate,
+          ifnull(round(sum(play_75_feed_break) / sum(total_play) * 100,3),0) play_75_feed_break_rate,
+          sum(play_duration) play_duration,
+          sum(play_duration_3s) play_duration_3s,
+          sum(play_duration_2s) play_duration_2s,
+          sum(play_duration_10s) play_duration_10s,
+          ifnull(round(sum(average_video_play) /{day_count}, 3),0) average_video_play,
+          sum(avg_rank) avg_rank,
+          sum(submit_certification_count) submit_certification_count,
+          sum(approval_count) approval_count,
+          sum(first_order_count) first_rental_order_count,
+          sum(first_rental_order_count) first_order_count,
+          sum(commute_first_pay_count) commute_first_pay_count
+         FROM media_api.bytedance_material_report_daily zhubiao
+         WHERE image_mode IN (
+    'CREATIVE_IMAGE_MODE_SMALL',
+    'CREATIVE_IMAGE_MODE_LARGE',
+    'CREATIVE_IMAGE_MODE_LARGE_VERTICAL',
+    'CREATIVE_IMAGE_MODE_GROUP',
+    'CREATIVE_IMAGE_MODE_GIF',
+    'TOUTIAO_SEARCH_AD_IMAGE',
+    'SEARCH_AD_SMALL_IMAGE',
+    'CREATIVE_IMAGE_MODE_UNION_SPLASH')
+       AND stat_datetime >= date_format(date_sub({date_time}, INTERVAL ({day_count})  DAY),'%Y%m%d')
+       GROUP BY advertiser_id,
+                  material_id,
+                  image_mode,
+                  inventory) report
+LEFT JOIN (select advertiser_id,signature,material_id,filename,image_url
+from  `media_api`.bytedance_file_image_get group by advertiser_id,signature,material_id,filename,image_url)  video 
+ON report.account_id = video.advertiser_id AND report.material_id = video.material_id
+          LEFT JOIN `jeecg-boot`.ctop_user_allocation cua
+                   ON report.account_id = cua.account_id
+          LEFT JOIN `jeecg-boot`.ctop_advertiser ca
+                   ON cua.advertiser_id = ca.id
+          LEFT JOIN `jeecg-boot`.ctop_project cp
+                   ON cua.project_id = cp.id
+          LEFT JOIN `jeecg-boot`.user_company uc
+                   ON cua.user_id = uc.user_id
+          LEFT JOIN `jeecg-boot`.ctop_product product
+                   ON cp.product_id = product.id
+         LEFT JOIN `jeecg-boot`.sys_user su
+                   ON cua.user_id = su.id
+         where cua.project_id={project_id}          
+""".format(date_time=self.date_time, day_count=self.day_count, project_id=project_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceImageReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 400 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceProductReportBusi.py

@@ -0,0 +1,400 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceProductReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceProductReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_product_report_busi_{day_count}d
+SELECT
+new.start_end_time,
+new.company_id,
+new.company_name,
+ifnull(new.product_id,0) product_id,
+new.product_name,
+new.account_num ,
+new.cost,
+ifnull(ratio.cost,0) last_cost,                                       
+ifnull(round((new.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,   
+new.show,
+new.avg_show_cost ,
+new.click ,
+new.avg_click_cost,
+new.ctr ,
+new.convert ,
+new.convert_cost,
+new.convert_rate,
+new.deep_convert,
+new.deep_convert_cost ,
+new.deep_convert_rate ,
+new.attribution_convert ,
+new.attribution_convert_cost,
+new.attribution_deep_convert,
+new.attribution_deep_convert_cost ,
+new.download_start,
+new.download_start_cost ,
+new.download_start_rate ,
+new.download_finish ,
+new.download_finish_cost,
+new.download_finish_rate,
+new.click_install ,
+new.install_finish,
+new.install_finish_cost ,
+new.install_finish_rate ,
+new.active,
+new.active_cost ,
+new.active_rate ,
+new.register,
+new.active_register_cost,
+new.active_register_rate,
+new.next_day_open ,
+new.next_day_open_cost,
+new.next_day_open_rate,
+new.attribution_next_day_open_cnt ,
+new.attribution_next_day_open_cost,
+new.attribution_next_day_open_rate,
+new.game_addiction,
+new.game_addiction_cost ,
+new.game_addiction_rate ,
+new.pay_count ,
+new.active_pay_cost ,
+new.active_pay_rate ,
+new.loan_completion ,
+new.loan_completion_cost,
+new.loan_completion_rate,
+new.pre_loan_credit ,
+new.pre_loan_credit_cost,
+new.loan_credit ,
+new.loan_credit_cost,
+new.loan_credit_rate,
+new.in_app_uv ,
+new.in_app_detail_uv,
+new.in_app_cart ,
+new.in_app_pay,
+new.in_app_order,
+new.attribution_game_pay_7d_count ,
+new.attribution_game_pay_7d_cost,
+new.attribution_active_pay_7d_per_count ,
+new.game_pay_cost ,
+new.game_pay_count,
+new.phone ,
+new.form,
+new.map_search,
+new.button,
+new.view,
+new.download,
+new.qq,
+new.lottery ,
+new.vote,
+new.message ,
+new.redirect,
+new.shopping,
+new.consult ,
+new.wechat,
+new.phone_confirm ,
+new.phone_connect ,
+new.consult_effective ,
+new.coupon,
+new.coupon_single_page,
+new.redirect_to_shop,
+new.poi_collect ,
+new.poi_address_click ,
+new.luban_order_cnt ,
+new.luban_order_stat_amount ,
+new.luban_order_roi ,
+new.luban_live_enter_cnt,
+new.live_watch_one_minute_count ,
+new.luban_live_follow_cnt ,
+new.live_fans_club_join_cnt ,
+new.luban_live_comment_cnt,
+new.luban_live_share_cnt,
+new.luban_live_gift_cnt ,
+new.luban_live_gift_amount,
+new.luban_live_slidecart_click_cnt,
+new.luban_live_click_product_cnt,
+new.luban_live_pay_order_count,
+new.luban_live_pay_order_stat_cost,
+new.live_component_click_count,
+new.live_component_click_cost ,
+new.live_component_click_rate ,
+new.wechat_login_count,
+new.attribution_wechat_login_30d_count,
+new.wechat_login_cost ,
+new.attribution_wechat_login_30d_cost ,
+new.wechat_first_pay_count,
+new.attribution_wechat_first_pay_30d_count,
+new.wechat_first_pay_cost ,
+new.attribution_wechat_first_pay_30d_cost ,
+new.wechat_first_pay_rate ,
+new.attribution_wechat_first_pay_30d_rate ,
+new.wechat_pay_amount ,
+new.attribution_wechat_pay_30d_amount ,
+new.attribution_wechat_pay_30d_roi,
+new.phone_effective ,
+new.total_play,
+new.valid_play,
+new.valid_play_cost ,
+new.valid_play_rate ,
+new.play_25_feed_break,
+new.play_50_feed_break,
+new.play_75_feed_break,
+new.play_100_feed_break ,
+new.average_play_time_per_play,
+new.play_over_rate,
+new.wifi_play_rate,
+new.wifi_play ,
+new.play_duration_sum ,
+new.advanced_creative_phone_click ,
+new.advanced_creative_counsel_click ,
+new.advanced_creative_form_click,
+new.advanced_creative_coupon_addition ,
+new.advanced_creative_form_submit ,
+new.card_show ,
+new.share ,
+new.comment ,
+new.like,
+new.follow,
+new.home_visited,
+new.ies_challenge_click ,
+new.ies_music_click ,
+new.location_click,
+new.message_action,
+new.click_landing_page,
+new.click_shopwindow,
+new.click_website ,
+new.click_download,
+new.click_call_dy ,
+new.submit_certification_count,
+new.approval_count,
+new.first_rental_order_count,
+new.first_order_count ,
+new.commute_first_pay_count 
+from (
+SELECT
+concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+uc.company_id,
+CASE
+WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+uc.company_name
+END company_name,
+ cp.product_id,
+ cn.product_name,
+ count(report.advertiser_id) account_num,
+ sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click)click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`)`convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert)attribution_convert,
+               ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+               ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install)click_install,
+               sum(install_finish) install_finish,
+               ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register)register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open)next_day_open,
+               ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count)pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+               sum(loan_credit)loan_credit,
+               ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv)in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart) in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+               ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+               ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost)game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone) phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download)download,
+               sum(qq)qq,
+               sum(lottery)lottery,
+               sum(vote) vote,
+               sum(message)message,
+               sum(redirect)redirect,
+               sum(shopping)shopping,
+               sum(consult)consult,
+               sum(wechat) wechat,
+               sum(phone_confirm)phone_confirm,
+               sum(phone_connect)phone_connect,
+               sum(consult_effective)consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect)poi_collect,
+               sum(poi_address_click)poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount)luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt)luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount)wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break)play_100_feed_break,
+               ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play)wifi_play,
+               ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show)card_show,
+               sum(`share`)`share`,
+               sum(`comment`)`comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click)ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website)click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy)click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count)first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count)commute_first_pay_count
+FROM
+(SELECT * FROM media_api.bytedance_advertiser_report_daily
+WHERE stat_datetime >= date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d')) report
+LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_product cn ON cp.product_id = cn.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+GROUP BY
+uc.company_id,
+cp.product_id) new
+LEFT JOIN 
+(select ifnull(cp.product_id,'0') product_id,sum(report.cost) cost  from
+(select advertiser_id account_id,sum(cost) cost from media_api.bytedance_advertiser_report_daily
+where stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id) report
+LEFT JOIN  `jeecg-boot`.ctop_user_allocation cua
+ON report.account_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_project cp
+ON cua.project_id = cp.id
+group by cp.product_id) ratio
+ON new.product_id=ratio.product_id;
+""".format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceProductReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 236 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceProductReportBusiMonth.py

@@ -0,0 +1,236 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceProductReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceProductReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_product_report_busi_m
+               SELECT 
+               date_format({date_time},'%Y%m')  as  stat_mon,
+               uc.company_id,
+               CASE
+               WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+               '北京汇创思拓数字科技有限公司'
+               WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+               '广州汇创思拓数字科技有限公司'
+               WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+               '上海汇创思拓数字科技有限公司'
+               ELSE
+               uc.company_name
+               END   company_name,
+               cn.id   product_id,
+               cn.product_name,
+               count(report.advertiser_id)  account_count,
+               sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click)click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`)`convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert)attribution_convert,
+               ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+               ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install)click_install,
+               sum(install_finish) install_finish,
+               ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register)register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open)next_day_open,
+               ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count)pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+               sum(loan_credit)loan_credit,
+               ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv)in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart) in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+               ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+               ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost)game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone) phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download)download,
+               sum(qq)qq,
+               sum(lottery)lottery,
+               sum(vote) vote,
+               sum(message)message,
+               sum(redirect)redirect,
+               sum(shopping)shopping,
+               sum(consult)consult,
+               sum(wechat) wechat,
+               sum(phone_confirm)phone_confirm,
+               sum(phone_connect)phone_connect,
+               sum(consult_effective)consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect)poi_collect,
+               sum(poi_address_click)poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount)luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt)luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount)wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break)play_100_feed_break,
+               ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play)wifi_play,
+               ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show)card_show,
+               sum(`share`)`share`,
+               sum(`comment`)`comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click)ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website)click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy)click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count)first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count)commute_first_pay_count
+FROM (
+         SELECT *
+         FROM media_api.bytedance_advertiser_report_daily
+         WHERE stat_datetime >= date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d')) report
+         LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id = cua.account_id
+         LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+         LEFT JOIN `jeecg-boot`.ctop_product cn ON cp.product_id = cn.id
+         LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+where cn.id is not null
+GROUP BY uc.company_id,
+         CASE
+             WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+                 '北京汇创思拓数字科技有限公司'
+             WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+                 '广州汇创思拓数字科技有限公司'
+             WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+                 '上海汇创思拓数字科技有限公司'
+             ELSE
+                 uc.company_name
+             END,
+         cn.id,
+         cn.product_name;
+""".format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceProductReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 415 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceProjectReportBusi.py

@@ -0,0 +1,415 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceCompanyUserReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceCompanyUserReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_project_report_busi_{day_count}d 
+SELECT
+new.start_end_time,
+new.company_id,
+new.company_name,
+ifnull(new.product_id,0) product_id,
+new.product_name,
+ifnull(new.project_id,0) project_id,
+new.project_name,
+new.account_num ,
+new.cost,
+ifnull(ratio.cost,0) last_cost,                                       
+ifnull(round((new.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,   
+new.show,
+new.avg_show_cost ,
+new.click ,
+new.avg_click_cost,
+new.ctr ,
+new.convert ,
+new.convert_cost,
+new.convert_rate,
+new.deep_convert,
+new.deep_convert_cost ,
+new.deep_convert_rate ,
+new.attribution_convert ,
+new.attribution_convert_cost,
+new.attribution_deep_convert,
+new.attribution_deep_convert_cost ,
+new.download_start,
+new.download_start_cost ,
+new.download_start_rate ,
+new.download_finish ,
+new.download_finish_cost,
+new.download_finish_rate,
+new.click_install ,
+new.install_finish,
+new.install_finish_cost ,
+new.install_finish_rate ,
+new.active,
+new.active_cost ,
+new.active_rate ,
+new.register,
+new.active_register_cost,
+new.active_register_rate,
+new.next_day_open ,
+new.next_day_open_cost,
+new.next_day_open_rate,
+new.attribution_next_day_open_cnt ,
+new.attribution_next_day_open_cost,
+new.attribution_next_day_open_rate,
+new.game_addiction,
+new.game_addiction_cost ,
+new.game_addiction_rate ,
+new.pay_count ,
+new.active_pay_cost ,
+new.active_pay_rate ,
+new.loan_completion ,
+new.loan_completion_cost,
+new.loan_completion_rate,
+new.pre_loan_credit ,
+new.pre_loan_credit_cost,
+new.loan_credit ,
+new.loan_credit_cost,
+new.loan_credit_rate,
+new.in_app_uv ,
+new.in_app_detail_uv,
+new.in_app_cart ,
+new.in_app_pay,
+new.in_app_order,
+new.attribution_game_pay_7d_count ,
+new.attribution_game_pay_7d_cost,
+new.attribution_active_pay_7d_per_count ,
+new.game_pay_cost ,
+new.game_pay_count,
+new.phone ,
+new.form,
+new.map_search,
+new.button,
+new.view,
+new.download,
+new.qq,
+new.lottery ,
+new.vote,
+new.message ,
+new.redirect,
+new.shopping,
+new.consult ,
+new.wechat,
+new.phone_confirm ,
+new.phone_connect ,
+new.consult_effective ,
+new.coupon,
+new.coupon_single_page,
+new.redirect_to_shop,
+new.poi_collect ,
+new.poi_address_click ,
+new.luban_order_cnt ,
+new.luban_order_stat_amount ,
+new.luban_order_roi ,
+new.luban_live_enter_cnt,
+new.live_watch_one_minute_count ,
+new.luban_live_follow_cnt ,
+new.live_fans_club_join_cnt ,
+new.luban_live_comment_cnt,
+new.luban_live_share_cnt,
+new.luban_live_gift_cnt ,
+new.luban_live_gift_amount,
+new.luban_live_slidecart_click_cnt,
+new.luban_live_click_product_cnt,
+new.luban_live_pay_order_count,
+new.luban_live_pay_order_stat_cost,
+new.live_component_click_count,
+new.live_component_click_cost ,
+new.live_component_click_rate ,
+new.wechat_login_count,
+new.attribution_wechat_login_30d_count,
+new.wechat_login_cost ,
+new.attribution_wechat_login_30d_cost ,
+new.wechat_first_pay_count,
+new.attribution_wechat_first_pay_30d_count,
+new.wechat_first_pay_cost ,
+new.attribution_wechat_first_pay_30d_cost ,
+new.wechat_first_pay_rate ,
+new.attribution_wechat_first_pay_30d_rate ,
+new.wechat_pay_amount ,
+new.attribution_wechat_pay_30d_amount ,
+new.attribution_wechat_pay_30d_roi,
+new.phone_effective ,
+new.total_play,
+new.valid_play,
+new.valid_play_cost ,
+new.valid_play_rate ,
+new.play_25_feed_break,
+new.play_50_feed_break,
+new.play_75_feed_break,
+new.play_100_feed_break ,
+new.average_play_time_per_play,
+new.play_over_rate,
+new.wifi_play_rate,
+new.wifi_play ,
+new.play_duration_sum ,
+new.advanced_creative_phone_click ,
+new.advanced_creative_counsel_click ,
+new.advanced_creative_form_click,
+new.advanced_creative_coupon_addition ,
+new.advanced_creative_form_submit ,
+new.card_show ,
+new.share ,
+new.comment ,
+new.like,
+new.follow,
+new.home_visited,
+new.ies_challenge_click ,
+new.ies_music_click ,
+new.location_click,
+new.message_action,
+new.click_landing_page,
+new.click_shopwindow,
+new.click_website ,
+new.click_download,
+new.click_call_dy ,
+new.submit_certification_count,
+new.approval_count,
+new.first_rental_order_count,
+new.first_order_count ,
+new.commute_first_pay_count 
+from (
+SELECT
+concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+uc.company_id,
+CASE
+WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+uc.company_name
+END company_name,
+ cp.product_id,
+ cn.product_name,
+ cua.project_id,
+ cp.project_name,
+ count(report.advertiser_id) account_num,
+ sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click)click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`)`convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert)attribution_convert,
+               ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+               ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install)click_install,
+               sum(install_finish) install_finish,
+               ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register)register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open)next_day_open,
+               ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count)pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+               sum(loan_credit)loan_credit,
+               ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv)in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart) in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+               ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+               ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost)game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone) phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download)download,
+               sum(qq)qq,
+               sum(lottery)lottery,
+               sum(vote) vote,
+               sum(message)message,
+               sum(redirect)redirect,
+               sum(shopping)shopping,
+               sum(consult)consult,
+               sum(wechat) wechat,
+               sum(phone_confirm)phone_confirm,
+               sum(phone_connect)phone_connect,
+               sum(consult_effective)consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect)poi_collect,
+               sum(poi_address_click)poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount)luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt)luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount)wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break)play_100_feed_break,
+               ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play)wifi_play,
+               ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show)card_show,
+               sum(`share`)`share`,
+               sum(`comment`)`comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click)ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website)click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy)click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count)first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count)commute_first_pay_count
+FROM
+(SELECT * FROM media_api.bytedance_advertiser_report_daily
+WHERE stat_datetime >= date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d')) report
+LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_product cn ON cp.product_id = cn.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+GROUP BY
+uc.company_id,
+CASE
+WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+uc.company_name
+END,cua.project_id,
+cp.project_name) new
+LEFT JOIN 
+(select ifnull(cp.product_id,'0') product_id,ifnull(cua.project_id,'0') project_id,sum(report.cost) cost  from
+(select advertiser_id account_id,sum(cost) cost from media_api.bytedance_advertiser_report_daily
+where stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id) report
+LEFT JOIN  `jeecg-boot`.ctop_user_allocation cua
+ON report.account_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_project cp
+ON cua.project_id = cp.id
+group by cua.project_id) ratio
+ON new.project_id=ratio.project_id;
+""".format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceCompanyUserReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 245 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceProjectReportBusiMonth.py

@@ -0,0 +1,245 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppBytedanceProjectReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceProjectReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_project_report_busi_m
+SELECT
+date_format({date_time},'%Y%m') stat_mon,
+uc.company_id,
+CASE
+WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+uc.company_name
+END company_name,
+ cp.product_id,
+ cn.product_name,
+ cua.project_id,
+ cp.project_name,
+ count(report.advertiser_id) account_num,
+ sum(cost) cost,
+               sum(`show`) `show`,
+               ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+               sum(click)click,
+               ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+               ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+               sum(`convert`)`convert`,
+               ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+               ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+               sum(deep_convert) deep_convert,
+               ifnull(round( sum(cost) / sum(deep_convert), 3),0) deep_convert_cost,
+               ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+               sum(attribution_convert)attribution_convert,
+               ifnull(round( sum(cost) / sum(attribution_convert), 3),0) attribution_convert_cost,
+               sum(attribution_deep_convert) attribution_deep_convert,
+               ifnull(round( sum(cost) / sum(attribution_deep_convert), 3),0) attribution_deep_convert_cost,
+               sum(download_start) download_start,
+               ifnull(round( sum(cost) / sum(download_start), 3),0) download_start_cost,
+               ifnull(round( sum(download_start) / sum(click), 3),0) download_start_rate,
+               sum(download_finish) download_finish,
+               ifnull(round( sum(cost) / sum(download_finish), 3),0) download_finish_cost,
+               ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+               sum(click_install)click_install,
+               sum(install_finish) install_finish,
+               ifnull(round( sum(cost) / sum(install_finish), 3),0) install_finish_cost,
+               ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+               sum(active) active,
+               ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+               ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+               sum(register)register,
+               ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+               ifnull(round(sum(active_register_rate) / {day_count} * 100,3),0) active_register_rate,
+               sum(next_day_open)next_day_open,
+               ifnull(round( sum(cost) / sum(next_day_open), 3),0) next_day_open_cost,
+               ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+               sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+               ifnull(round( sum(cost) / sum(attribution_next_day_open_cnt ), 3),0) attribution_next_day_open_cost,
+               ifnull(round(sum(attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+               sum(game_addiction) game_addiction,
+               ifnull(round( sum(cost) / sum(game_addiction), 3),0) game_addiction_cost,
+               ifnull(round(sum(game_addiction_rate) / {day_count} * 100,3),0) game_addiction_rate,
+               sum(pay_count)pay_count,
+               ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+               ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+               sum(loan_completion) loan_completion,
+               ifnull(round( sum(cost) / sum(loan_completion), 3),0) loan_completion_cost,
+               ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+               ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+               ifnull(round( sum(cost) / sum(pre_loan_credit), 3),0) pre_loan_credit_cost,
+               sum(loan_credit)loan_credit,
+               ifnull(round( sum(cost) / sum(loan_credit), 3),0) loan_credit_cost,
+               ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+               sum(in_app_uv)in_app_uv,
+               sum(in_app_detail_uv) in_app_detail_uv,
+               sum(in_app_cart) in_app_cart,
+               sum(in_app_pay) in_app_pay,
+               sum(in_app_order) in_app_order,
+               ifnull(round( sum(attribution_game_pay_7d_count) / {day_count}, 3),0) attribution_game_pay_7d_count,
+               ifnull(round( sum(attribution_game_pay_7d_cost) / {day_count}, 3),0) attribution_game_pay_7d_cost,
+               ifnull(round( sum(attribution_active_pay_7d_per_count) / {day_count}, 3),0) attribution_active_pay_7d_per_count,
+               sum(game_pay_cost)game_pay_cost,
+               sum(game_pay_count) game_pay_count,
+               sum(phone) phone,
+               sum(form) form,
+               sum(map_search) map_search,
+               sum(button) button,
+               sum(`view`) `view`,
+               sum(download)download,
+               sum(qq)qq,
+               sum(lottery)lottery,
+               sum(vote) vote,
+               sum(message)message,
+               sum(redirect)redirect,
+               sum(shopping)shopping,
+               sum(consult)consult,
+               sum(wechat) wechat,
+               sum(phone_confirm)phone_confirm,
+               sum(phone_connect)phone_connect,
+               sum(consult_effective)consult_effective,
+               sum(coupon) coupon,
+               sum(coupon_single_page) coupon_single_page,
+               sum(redirect_to_shop) redirect_to_shop,
+               sum(poi_collect)poi_collect,
+               sum(poi_address_click)poi_address_click,
+               sum(luban_order_cnt) luban_order_cnt,
+               sum(luban_order_stat_amount)luban_order_stat_amount,
+               sum(luban_order_roi) luban_order_roi,
+               sum(luban_live_enter_cnt) luban_live_enter_cnt,
+               sum(live_watch_one_minute_count) live_watch_one_minute_count,
+               sum(luban_live_follow_cnt) luban_live_follow_cnt,
+               sum(live_fans_club_join_cnt)live_fans_club_join_cnt,
+               sum(luban_live_comment_cnt) luban_live_comment_cnt,
+               sum(luban_live_share_cnt) luban_live_share_cnt,
+               sum(luban_live_gift_cnt)luban_live_gift_cnt,
+               sum(luban_live_gift_amount) luban_live_gift_amount,
+               sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+               sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+               sum(luban_live_pay_order_count) luban_live_pay_order_count,
+               sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+               sum(live_component_click_count) live_component_click_count,
+               ifnull(round( sum(cost) / sum(live_component_click_count), 3),0) live_component_click_cost,
+               ifnull(round(sum(live_component_click_count) / {day_count} * 100,3),0) live_component_click_rate,
+               sum(wechat_login_count) wechat_login_count,
+               sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_login_count), 3),0) wechat_login_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_login_30d_count ), 3),0) attribution_wechat_login_30d_cost,
+               sum(wechat_first_pay_count) wechat_first_pay_count,
+               sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+               ifnull(round( sum(cost) / sum(wechat_first_pay_count), 3),0) wechat_first_pay_cost,
+               ifnull(round( sum(cost) / sum(attribution_wechat_first_pay_30d_count ), 3),0) attribution_wechat_first_pay_30d_cost,
+               ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+               ifnull(round(sum(attribution_wechat_first_pay_30d_count) / sum(attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+               sum(wechat_pay_amount)wechat_pay_amount,
+               sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+               sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+               sum(phone_effective) phone_effective,
+               sum(total_play) total_play,
+               sum(valid_play) valid_play,
+               ifnull(round( sum(cost) / sum(valid_play), 3),0) valid_play_cost,
+               ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+               sum(play_25_feed_break) play_25_feed_break,
+               sum(play_50_feed_break) play_50_feed_break,
+               sum(play_75_feed_break) play_75_feed_break,
+               sum(play_100_feed_break)play_100_feed_break,
+               ifnull(round( sum(average_play_time_per_play) / {day_count}, 3),0) average_play_time_per_play,
+               ifnull(round(sum(play_over_rate) / {day_count} * 100,3),0) play_over_rate,
+               ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+               sum(wifi_play)wifi_play,
+               ifnull(round(sum(play_duration_sum) / {day_count}, 3),0) play_duration_sum,
+               sum(advanced_creative_phone_click) advanced_creative_phone_click,
+               sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+               sum(advanced_creative_form_click) advanced_creative_form_click,
+               sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+               sum(advanced_creative_form_submit) advanced_creative_form_submit,
+               sum(card_show)card_show,
+               sum(`share`)`share`,
+               sum(`comment`)`comment`,
+               sum(`like`) `like`,
+               sum(follow) follow,
+               sum(home_visited) home_visited,
+               sum(ies_challenge_click)ies_challenge_click,
+               sum(ies_music_click) ies_music_click,
+               sum(location_click) location_click,
+               sum(message_action) message_action,
+               sum(click_landing_page) click_landing_page,
+               sum(click_shopwindow) click_shopwindow,
+               sum(click_website)click_website,
+               sum(click_download) click_download,
+               sum(click_call_dy)click_call_dy,
+               sum(submit_certification_count) submit_certification_count,
+               sum(approval_count) approval_count,
+               sum(first_order_count)first_rental_order_count,
+               sum(first_rental_order_count) first_order_count,
+               sum(commute_first_pay_count)commute_first_pay_count
+FROM
+(
+SELECT
+*
+FROM
+media_api.bytedance_advertiser_report_daily
+WHERE
+stat_datetime >= date_format(
+date_sub({date_time}, INTERVAL {day_count}  DAY),
+'%Y%m%d'
+)
+) report
+LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_product cn ON cp.product_id = cn.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+GROUP BY
+uc.company_id,
+CASE
+WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+uc.company_name
+END,cua.project_id,
+cp.project_name
+having sum(cost)>0""".format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceProjectReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 523 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceVideoReportBusi.py

@@ -0,0 +1,523 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceVideoReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceVideoReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select advertiser_id from media_api.bytedance_material_report_daily zhubiao
+         WHERE image_mode IN (
+                              'CREATIVE_IMAGE_MODE_VIDEO',
+                              'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
+                              'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO'
+             )
+           AND stat_datetime >= date_format( date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d') 
+group by advertiser_id """.format(date_time=self.date_time, day_count=self.day_count)
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_video_report_busi_{day_count}d
+SELECT concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time})           time,
+       report.account_id,
+       #'账户ID'                                                                                                                                                                                                                                                                    
+       report.material_id,
+       #头条素材id                                                                                                                                                                                                                                
+       report.image_mode,
+       #素材类型                                                                                                                                                                                                                               
+       report.inventory,
+       #广告位                                                                                                                                                                                                                                
+       cua.project_id,
+       #项目id  
+       product.id      product_id,
+       #项目中产品id
+       cua.advertiser_id,
+       #广告主id
+       uc.company_id,
+       #运营人的公司id                      
+       cua.auth_name   account_name,
+       #账户名称
+       cp.project_name,
+       #项目名称  
+       ca.`name`       advertiser_name,
+       #广告主名称
+       uc.company_name,
+       #运营人的公司名称
+       cp.sale_id,
+       #项目里的销售id
+       cua.user_id,
+       #运营id                                                                                                          
+       cma.clip_id,
+       #剪辑id
+       cma.shot_id,
+       #拍摄id
+       cma.plan_id,
+       #编导id
+       cma.leader_id,
+       #负责人id
+       su.org_code,
+       #机构编码
+       video.signature,
+       #素材id  
+       NULL            chl_type,
+       #渠道类型
+       clip.realname   clip_name,
+       #剪辑名称
+       shot.realname   shot_name,
+       #拍摄名称
+       plan.realname   plan_name,
+       #编导名称
+       leader.realname lead_name,
+       #负责人名称
+       CASE
+           WHEN cmi.material_name IS NOT NULL THEN
+               cmi.material_name
+           ELSE
+               video.filename
+           END AS      video_name,
+       ##视频名称
+       NULL            chl_name,
+       #渠道名称
+       up.company_id   clip_company_id,
+       #剪辑公司
+       report.cost,
+       ifnull(ratio.cost,0) last_cost,
+       ifnull(round((report.cost-ratio.cost)/ratio.cost*100,3),0) cost_ratio,
+       report.`show`,
+       report.avg_show_cost,
+       report.click,
+       report.avg_click_cost,
+       report.ctr,
+       report.`convert`,
+       report.convert_cost,
+       report.convert_rate,
+       report.deep_convert,
+       report.deep_convert_cost,
+       report.deep_convert_rate,
+       report.attribution_convert,
+       report.attribution_convert_cost,
+       report.attribution_deep_convert,
+       report.attribution_deep_convert_cost,
+       report.download_start,
+       report.download_start_cost,
+       report.download_start_rate,
+       report.download_finish,
+       report.download_finish_cost,
+       report.download_finish_rate,
+       report.click_install,
+       report.install_finish,
+       report.install_finish_cost,
+       report.install_finish_rate,
+       report.active,
+       report.active_cost,
+       report.active_rate,
+       report.register,
+       report.active_register_cost,
+       report.active_register_rate,
+       report.active_pay_amount,
+       report.next_day_open,
+       report.next_day_open_cost,
+       report.next_day_open_rate,
+       report.attribution_next_day_open_cnt,
+       report.attribution_next_day_open_cost,
+       report.attribution_next_day_open_rate,
+       report.game_addiction,
+       report.game_addiction_cost,
+       report.game_addiction_rate,
+       report.pay_count,
+       report.active_pay_cost,
+       report.active_pay_rate,
+       report.loan_completion,
+       report.loan_completion_cost,
+       report.loan_completion_rate,
+       report.pre_loan_credit,
+       report.pre_loan_credit_cost,
+       report.loan_credit,
+       report.loan_credit_cost,
+       report.loan_credit_rate,
+       report.in_app_uv,
+       report.in_app_detail_uv,
+       report.in_app_cart,
+       report.in_app_pay,
+       report.in_app_order,
+       report.attribution_game_pay_7d_count,
+       report.attribution_game_pay_7d_cost,
+       report.attribution_active_pay_7d_per_count,
+       report.game_pay_cost,
+       report.game_pay_count,
+       report.phone,
+       report.form,
+       report.map_search,
+       report.button,
+       report.`view`,
+       report.download,
+       report.qq,
+       report.lottery,
+       report.vote,
+       report.message,
+       report.redirect,
+       report.shopping,
+       report.consult,
+       report.wechat,
+       report.phone_confirm,
+       report.phone_connect,
+       report.consult_effective,
+       report.coupon,
+       report.coupon_single_page,
+       report.redirect_to_shop,
+       report.poi_collect,
+       report.poi_address_click,
+       report.luban_order_cnt,
+       report.luban_order_stat_amount,
+       report.luban_order_roi,
+       report.luban_live_enter_cnt,
+       report.live_watch_one_minute_count,
+       report.luban_live_follow_cnt,
+       report.live_fans_club_join_cnt,
+       report.luban_live_comment_cnt,
+       report.luban_live_share_cnt,
+       report.luban_live_gift_cnt,
+       report.luban_live_gift_amount,
+       report.luban_live_slidecart_click_cnt,
+       report.luban_live_click_product_cnt,
+       report.luban_live_pay_order_count,
+       report.luban_live_pay_order_stat_cost,
+       report.wechat_login_count,
+       report.attribution_wechat_login_30d_count,
+       report.wechat_login_cost,
+       report.attribution_wechat_login_30d_cost,
+       report.wechat_first_pay_count,
+       report.attribution_wechat_first_pay_30d_count,
+       report.wechat_first_pay_cost,
+       report.attribution_wechat_first_pay_30d_cost,
+       report.wechat_first_pay_rate,
+       report.attribution_wechat_first_pay_30d_rate,
+       report.wechat_pay_amount,
+       report.attribution_wechat_pay_30d_amount,
+       report.attribution_wechat_pay_30d_roi,
+       report.phone_effective,
+       report.total_play,
+       report.valid_play,
+       report.valid_play_cost,
+       report.valid_play_rate,
+       report.play_25_feed_break,
+       report.play_50_feed_break,
+       report.play_75_feed_break,
+       report.play_100_feed_break,
+       report.average_play_time_per_play,
+       report.play_over_rate,
+       report.wifi_play_rate,
+       report.wifi_play,
+       report.play_duration_sum,
+       report.advanced_creative_phone_click,
+       report.advanced_creative_counsel_click,
+       report.advanced_creative_form_click,
+       report.advanced_creative_coupon_addition,
+       report.advanced_creative_form_submit,
+       report.card_show,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.follow,
+       report.home_visited,
+       report.ies_challenge_click,
+       report.ies_music_click,
+       report.location_click,
+       report.message_action,
+       report.click_landing_page,
+       report.click_shopwindow,
+       report.click_website,
+       report.click_download,
+       report.click_call_dy,
+       report.`cpc`,
+       report.`cpm`,
+       report.`cpa`,
+       report.`interact_per_cost`,
+       report.`convert_show_rate`,
+       report.`play_duration_10s_rate`,
+       report.`play_50_feed_break_rate`,
+       report.`play_25_feed_break_rate`,
+       report.`play_duration_5s_rate`,
+       report.`play_duration_2s_rate`,
+       report.`average_play_progress`,
+       report.`play_100_feed_break_rate`,
+       report.`play_duration_3s_rate`,
+       report.`play_75_feed_break_rate`,
+       report.`play_duration`,
+       report.`play_duration_3s`,
+       report.`play_duration_2s`,
+       report.`play_duration_10s`,
+       report.`average_video_play`,
+       report.`avg_rank`,
+       report.submit_certification_count,
+       report.approval_count,
+       report.first_rental_order_count,
+       report.first_order_count,
+       report.commute_first_pay_count,
+       video.`video_url`,
+       '' `cover_url`
+FROM (
+         SELECT advertiser_id    account_id,
+                material_id,
+                image_mode,
+                inventory,
+          sum(cost) cost,
+          sum(`show`) `show`,
+          ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+          sum(click) click,
+          ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+          ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+          sum(`convert`) `convert`,
+          ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+          ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+          sum(deep_convert) deep_convert,
+          ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+          ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+          sum(attribution_convert) attribution_convert,
+          ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+          sum(attribution_deep_convert) attribution_deep_convert,
+          ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+          sum(download_start) download_start,
+          ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+          ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+          sum(download_finish) download_finish,
+          ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+          ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+          sum(click_install) click_install,
+          sum(install_finish) install_finish,
+          ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+          ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+          sum(active) active,
+          ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+          ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+          sum(register) register,
+          ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+          ifnull(round(sum(active_register_rate) /{day_count} * 100,3),0) active_register_rate,
+          sum(active_pay_amount) active_pay_amount,
+          sum(next_day_open) next_day_open,
+          ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+          ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+          sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+          ifnull(round(sum(cost) / sum(	attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+          ifnull(round(sum(	attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+          sum(game_addiction) game_addiction,
+          ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+          ifnull(round(sum(game_addiction_rate) /{day_count} * 100,3),0) game_addiction_rate,
+          sum(pay_count) pay_count,
+          ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+          ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+          sum(loan_completion) loan_completion,
+          ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+          ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+          ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+          ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+          sum(loan_credit) loan_credit,
+          ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+          ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+          sum(in_app_uv) in_app_uv,
+          sum(in_app_detail_uv) in_app_detail_uv,
+          sum(in_app_cart) in_app_cart,
+          sum(in_app_pay) in_app_pay,
+          sum(in_app_order) in_app_order,
+          ifnull(round(sum(	attribution_game_pay_7d_count) /{day_count}, 3),0) attribution_game_pay_7d_count,
+          ifnull(round(sum(	attribution_game_pay_7d_cost) /{day_count}, 3),0) attribution_game_pay_7d_cost,
+          ifnull(round(sum(	attribution_active_pay_7d_per_count) /{day_count}, 3),0) attribution_active_pay_7d_per_count,
+          sum(game_pay_cost) game_pay_cost,
+          sum(game_pay_count) game_pay_count,
+          sum(phone) phone,
+          sum(form) form,
+          sum(map_search) map_search,
+          sum(button) button,
+          sum(`view`) `view`,
+          sum(download) download,
+          sum(qq) qq,
+          sum(lottery) lottery,
+          sum(vote) vote,
+          sum(message) message,
+          sum(redirect) redirect,
+          sum(shopping) shopping,
+          sum(consult) consult,
+          sum(wechat) wechat,
+          sum(phone_confirm) phone_confirm,
+          sum(phone_connect) phone_connect,
+          sum(consult_effective) consult_effective,
+          sum(coupon) coupon,
+          sum(coupon_single_page) coupon_single_page,
+          sum(redirect_to_shop) redirect_to_shop,
+          sum(poi_collect) poi_collect,
+          sum(poi_address_click) poi_address_click,
+          sum(luban_order_cnt) luban_order_cnt,
+          sum(luban_order_stat_amount) luban_order_stat_amount,
+          sum(luban_order_roi) luban_order_roi,
+          sum(luban_live_enter_cnt) luban_live_enter_cnt,
+          sum(live_watch_one_minute_count) live_watch_one_minute_count,
+          sum(luban_live_follow_cnt) luban_live_follow_cnt,
+          sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+          sum(luban_live_comment_cnt) luban_live_comment_cnt,
+          sum(luban_live_share_cnt) luban_live_share_cnt,
+          sum(luban_live_gift_cnt) luban_live_gift_cnt,
+          sum(luban_live_gift_amount) luban_live_gift_amount,
+          sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+          sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+          sum(luban_live_pay_order_count) luban_live_pay_order_count,
+          sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+          sum(wechat_login_count) wechat_login_count,
+          sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+          sum(wechat_first_pay_count) wechat_first_pay_count,
+          sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+          ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+          ifnull(round(sum(	attribution_wechat_first_pay_30d_count) / sum(	attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+          sum(wechat_pay_amount) wechat_pay_amount,
+          sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+          sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+          sum(phone_effective) phone_effective,
+          sum(total_play) total_play,
+          sum(valid_play) valid_play,
+          ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+          ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+          sum(play_25_feed_break) play_25_feed_break,
+          sum(play_50_feed_break) play_50_feed_break,
+          sum(play_75_feed_break) play_75_feed_break,
+          sum(play_100_feed_break) play_100_feed_break,
+          ifnull(round(sum(average_play_time_per_play) /{day_count}, 3),0) average_play_time_per_play,
+          ifnull(round(sum(play_over_rate) /{day_count} * 100,3),0) play_over_rate,
+          ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+          sum(wifi_play) wifi_play,
+          ifnull(round(sum(play_duration_sum) /{day_count}, 3),0) play_duration_sum,
+          sum(advanced_creative_phone_click) advanced_creative_phone_click,
+          sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+          sum(advanced_creative_form_click) advanced_creative_form_click,
+          sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+          sum(advanced_creative_form_submit) advanced_creative_form_submit,
+          sum(card_show) card_show,
+          sum(`share`) `share`,
+          sum(`comment`) `comment`,
+          sum(`like`) `like`,
+          sum(follow) follow,
+          sum(home_visited) home_visited,
+          sum(ies_challenge_click) ies_challenge_click,
+          sum(ies_music_click) ies_music_click,
+          sum(location_click) location_click,
+          sum(message_action) message_action,
+          sum(click_landing_page) click_landing_page,
+          sum(click_shopwindow) click_shopwindow,
+          sum(click_website) click_website,
+          sum(click_download) click_download,
+          sum(click_call_dy) click_call_dy,
+          sum(cpc) cpc,
+          sum(cpm) cpm,
+          sum(cpa) cpa,
+          ifnull(round(sum(interact_per_cost) /{day_count}, 3),0) interact_per_cost,
+          ifnull(round(sum(convert_show_rate) /{day_count}, 3),0) convert_show_rate,
+          ifnull(round(sum(play_duration_10s) / sum(total_play) * 100,3),0) play_duration_10s_rate,
+          ifnull(round(sum(play_50_feed_break) / sum(total_play) * 100,3),0) play_50_feed_break_rate,
+          ifnull(round(sum(play_25_feed_break) / sum(total_play) * 100,3),0) play_25_feed_break_rate,
+          ifnull(round(sum(play_duration_5s_rate) /{day_count}, 3),0) play_duration_5s_rate,
+          ifnull(round(sum(play_duration_2s) / sum(total_play) * 100,3),0) play_duration_2s_rate,
+          ifnull(round(sum(average_play_progress) /{day_count}, 3),0) average_play_progress,
+          ifnull(round(sum(play_100_feed_break) / sum(total_play) * 100,3),0) play_100_feed_break_rate,
+          ifnull(round(sum(play_duration_3s) / sum(total_play) * 100,3),0) play_duration_3s_rate,
+          ifnull(round(sum(play_75_feed_break) / sum(total_play) * 100,3),0) play_75_feed_break_rate,
+          sum(play_duration) play_duration,
+          sum(play_duration_3s) play_duration_3s,
+          sum(play_duration_2s) play_duration_2s,
+          sum(play_duration_10s) play_duration_10s,
+          ifnull(round(sum(average_video_play) /{day_count}, 3),0) average_video_play,
+          sum(avg_rank) avg_rank,
+          sum(submit_certification_count) submit_certification_count,
+          sum(approval_count) approval_count,
+          sum(first_order_count) first_rental_order_count,
+          sum(first_rental_order_count) first_order_count,
+          sum(commute_first_pay_count) commute_first_pay_count
+         FROM media_api.bytedance_material_report_daily zhubiao
+         WHERE image_mode IN (
+                              'CREATIVE_IMAGE_MODE_VIDEO',
+                              'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
+                              'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO'
+             )
+           AND stat_datetime >= date_format( date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d') 
+           and advertiser_id={account_id}
+         GROUP BY advertiser_id,
+                  material_id,
+                  image_mode,
+                  inventory
+     ) report
+         LEFT JOIN (select advertiser_id, signature, material_id, filename, video_url
+                    from `media_api`.bytedance_file_video_get where advertiser_id={account_id}
+                    group by advertiser_id, signature, material_id, filename, video_url) video
+                   ON report.account_id = video.advertiser_id AND report.material_id = video.material_id
+         LEFT JOIN (select  code,material_name,cover_url from `jeecg-boot`.ctop_material_info where status=1) cmi 
+         ON video.signature = cmi.code
+         LEFT JOIN (select material_id, clip_id, shot_id, plan_id, leader_id
+                    from (
+                             select material_id,
+                                    clip_id,
+                                    shot_id,
+                                    plan_id,
+                                    leader_id,
+                                    row_number() over (partition by material_id order by create_time) rn
+                             from `jeecg-boot`.ctop_material_ascription) m
+                    where rn = 1) cma
+                   ON video.signature = cma.material_id
+         LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.account_id = cua.account_id
+         LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON cua.advertiser_id = ca.id
+         LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+         LEFT JOIN `jeecg-boot`.user_company uc ON cua.user_id = uc.user_id
+         LEFT JOIN `jeecg-boot`.ctop_product product ON cp.product_id = product.id
+         LEFT JOIN `jeecg-boot`.sys_user su ON cua.user_id = su.id
+         LEFT JOIN `jeecg-boot`.sys_user clip ON clip.id = cma.clip_id
+         LEFT JOIN `jeecg-boot`.sys_user shot ON shot.id = cma.shot_id
+         LEFT JOIN `jeecg-boot`.sys_user plan ON plan.id = cma.plan_id
+         LEFT JOIN `jeecg-boot`.sys_user leader ON leader.id = cma.leader_id
+         LEFT JOIN `jeecg-boot`.user_company up ON up.user_id = cma.clip_id
+         LEFT JOIN 
+         (select advertiser_id account_id,material_id,sum(cost) cost  FROM media_api.bytedance_material_report_daily
+         WHERE advertiser_id ={account_id} and image_mode IN ('CREATIVE_IMAGE_MODE_VIDEO','CREATIVE_IMAGE_MODE_VIDEO_VERTICAL','CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+         and  stat_datetime between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+         and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+         group by advertiser_id,material_id) ratio
+         on report.account_id = ratio.account_id and report.material_id = ratio.material_id 
+         where cua.account_id = {account_id};
+         """.format(date_time=self.date_time,
+                    day_count=self.day_count,
+                    account_id=account_id[0])
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceVideoReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 520 - 0
Apietl/bytedance_etl/Q4Task/AppBytedanceVideoReportBusiMonth.py

@@ -0,0 +1,520 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    : 
+# @File    : AppBytedanceVideoReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceVideoReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select advertiser_id from media_api.bytedance_material_report_daily zhubiao
+                     WHERE image_mode IN (
+                                          'CREATIVE_IMAGE_MODE_VIDEO',
+                                          'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
+                                          'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO'
+                         )
+                       AND stat_datetime >= date_format(
+                 date_sub({date_time}, INTERVAL {day_count}  DAY),
+                 '%Y%m%d'
+             )
+            group by advertiser_id """.format(date_time=self.date_time, day_count=self.day_count)
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_bytedance_video_report_busi_m
+SELECT date_format({date_time},'%Y%m')         stat_mon,
+       report.account_id,
+       #'账户ID'                                                                                                                                                                                                                                                                    
+       report.material_id,
+       #头条素材id                                                                                                                                                                                                                                
+       report.image_mode,
+       #素材类型                                                                                                                                                                                                                               
+       report.inventory,
+       #广告位                                                                                                                                                                                                                                
+       cua.project_id,
+       #项目id  
+       product.id      product_id,
+       #项目中产品id
+       cua.advertiser_id,
+       #广告主id
+       uc.company_id,
+       #运营人的公司id                      
+       cua.auth_name   account_name,
+       #账户名称
+       cp.project_name,
+       #项目名称  
+       ca.`name`       advertiser_name,
+       #广告主名称
+       uc.company_name,
+       #运营人的公司名称
+       cp.sale_id,
+       #项目里的销售id
+       cua.user_id,
+       #运营id                                                                                                          
+       cma.clip_id,
+       #剪辑id
+       cma.shot_id,
+       #拍摄id
+       cma.plan_id,
+       #编导id
+       cma.leader_id,
+       #负责人id
+       su.org_code,
+       #机构编码
+       video.signature,
+       #素材id  
+       NULL            chl_type,
+       #渠道类型
+       clip.realname   clip_name,
+       #剪辑名称
+       shot.realname   shot_name,
+       #拍摄名称
+       plan.realname   plan_name,
+       #编导名称
+       leader.realname lead_name,
+       #负责人名称
+       CASE
+           WHEN cmi.material_name IS NOT NULL THEN
+               cmi.material_name
+           ELSE
+               video.filename
+           END AS      video_name,
+       ##视频名称
+       NULL            chl_name,
+       #渠道名称
+       up.company_id   clip_company_id,
+       #剪辑公司
+       report.cost,
+       report.`show`,
+       report.avg_show_cost,
+       report.click,
+       report.avg_click_cost,
+       report.ctr,
+       report.`convert`,
+       report.convert_cost,
+       report.convert_rate,
+       report.deep_convert,
+       report.deep_convert_cost,
+       report.deep_convert_rate,
+       report.attribution_convert,
+       report.attribution_convert_cost,
+       report.attribution_deep_convert,
+       report.attribution_deep_convert_cost,
+       report.download_start,
+       report.download_start_cost,
+       report.download_start_rate,
+       report.download_finish,
+       report.download_finish_cost,
+       report.download_finish_rate,
+       report.click_install,
+       report.install_finish,
+       report.install_finish_cost,
+       report.install_finish_rate,
+       report.active,
+       report.active_cost,
+       report.active_rate,
+       report.register,
+       report.active_register_cost,
+       report.active_register_rate,
+       report.active_pay_amount,
+       report.next_day_open,
+       report.next_day_open_cost,
+       report.next_day_open_rate,
+       report.attribution_next_day_open_cnt,
+       report.attribution_next_day_open_cost,
+       report.attribution_next_day_open_rate,
+       report.game_addiction,
+       report.game_addiction_cost,
+       report.game_addiction_rate,
+       report.pay_count,
+       report.active_pay_cost,
+       report.active_pay_rate,
+       report.loan_completion,
+       report.loan_completion_cost,
+       report.loan_completion_rate,
+       report.pre_loan_credit,
+       report.pre_loan_credit_cost,
+       report.loan_credit,
+       report.loan_credit_cost,
+       report.loan_credit_rate,
+       report.in_app_uv,
+       report.in_app_detail_uv,
+       report.in_app_cart,
+       report.in_app_pay,
+       report.in_app_order,
+       report.attribution_game_pay_7d_count,
+       report.attribution_game_pay_7d_cost,
+       report.attribution_active_pay_7d_per_count,
+       report.game_pay_cost,
+       report.game_pay_count,
+       report.phone,
+       report.form,
+       report.map_search,
+       report.button,
+       report.`view`,
+       report.download,
+       report.qq,
+       report.lottery,
+       report.vote,
+       report.message,
+       report.redirect,
+       report.shopping,
+       report.consult,
+       report.wechat,
+       report.phone_confirm,
+       report.phone_connect,
+       report.consult_effective,
+       report.coupon,
+       report.coupon_single_page,
+       report.redirect_to_shop,
+       report.poi_collect,
+       report.poi_address_click,
+       report.luban_order_cnt,
+       report.luban_order_stat_amount,
+       report.luban_order_roi,
+       report.luban_live_enter_cnt,
+       report.live_watch_one_minute_count,
+       report.luban_live_follow_cnt,
+       report.live_fans_club_join_cnt,
+       report.luban_live_comment_cnt,
+       report.luban_live_share_cnt,
+       report.luban_live_gift_cnt,
+       report.luban_live_gift_amount,
+       report.luban_live_slidecart_click_cnt,
+       report.luban_live_click_product_cnt,
+       report.luban_live_pay_order_count,
+       report.luban_live_pay_order_stat_cost,
+       report.wechat_login_count,
+       report.attribution_wechat_login_30d_count,
+       report.wechat_login_cost,
+       report.attribution_wechat_login_30d_cost,
+       report.wechat_first_pay_count,
+       report.attribution_wechat_first_pay_30d_count,
+       report.wechat_first_pay_cost,
+       report.attribution_wechat_first_pay_30d_cost,
+       report.wechat_first_pay_rate,
+       report.attribution_wechat_first_pay_30d_rate,
+       report.wechat_pay_amount,
+       report.attribution_wechat_pay_30d_amount,
+       report.attribution_wechat_pay_30d_roi,
+       report.phone_effective,
+       report.total_play,
+       report.valid_play,
+       report.valid_play_cost,
+       report.valid_play_rate,
+       report.play_25_feed_break,
+       report.play_50_feed_break,
+       report.play_75_feed_break,
+       report.play_100_feed_break,
+       report.average_play_time_per_play,
+       report.play_over_rate,
+       report.wifi_play_rate,
+       report.wifi_play,
+       report.play_duration_sum,
+       report.advanced_creative_phone_click,
+       report.advanced_creative_counsel_click,
+       report.advanced_creative_form_click,
+       report.advanced_creative_coupon_addition,
+       report.advanced_creative_form_submit,
+       report.card_show,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.follow,
+       report.home_visited,
+       report.ies_challenge_click,
+       report.ies_music_click,
+       report.location_click,
+       report.message_action,
+       report.click_landing_page,
+       report.click_shopwindow,
+       report.click_website,
+       report.click_download,
+       report.click_call_dy,
+       report.`cpc`,
+       report.`cpm`,
+       report.`cpa`,
+       report.`interact_per_cost`,
+       report.`convert_show_rate`,
+       report.`play_duration_10s_rate`,
+       report.`play_50_feed_break_rate`,
+       report.`play_25_feed_break_rate`,
+       report.`play_duration_5s_rate`,
+       report.`play_duration_2s_rate`,
+       report.`average_play_progress`,
+       report.`play_100_feed_break_rate`,
+       report.`play_duration_3s_rate`,
+       report.`play_75_feed_break_rate`,
+       report.`play_duration`,
+       report.`play_duration_3s`,
+       report.`play_duration_2s`,
+       report.`play_duration_10s`,
+       report.`average_video_play`,
+       report.`avg_rank`,
+       report.submit_certification_count,
+       report.approval_count,
+       report.first_rental_order_count,
+       report.first_order_count,
+       report.commute_first_pay_count,
+       video.`video_url`,
+       cmi.`cover_url`
+FROM (
+          SELECT advertiser_id    account_id,
+                material_id,
+                image_mode,
+                inventory,
+          sum(cost) cost,
+          sum(`show`) `show`,
+          ifnull(round(sum(cost) / sum(`show`), 3),0) avg_show_cost,
+          sum(click) click,
+          ifnull(round(sum(cost) / sum(`click`), 3),0) avg_click_cost,
+          ifnull(round(sum(click) / sum(`show`) * 100,3),0) ctr,
+          sum(`convert`) `convert`,
+          ifnull(round(sum(cost) / sum(`convert`), 3),0) convert_cost,
+          ifnull(round(sum(`convert`) / sum(click) * 100,3),0) convert_rate,
+          sum(deep_convert) deep_convert,
+          ifnull(round(sum(cost) / sum(deep_convert),3),0) deep_convert_cost,
+          ifnull(round(sum(deep_convert) / sum(`convert`) * 100,3),0) deep_convert_rate,
+          sum(attribution_convert) attribution_convert,
+          ifnull(round(sum(cost) / sum(attribution_convert),3),0) attribution_convert_cost,
+          sum(attribution_deep_convert) attribution_deep_convert,
+          ifnull(round(sum(cost) / sum(attribution_deep_convert),3),0) attribution_deep_convert_cost,
+          sum(download_start) download_start,
+          ifnull(round(sum(cost) / sum(download_start),3),0) download_start_cost,
+          ifnull(round(sum(download_start) / sum(click),3),0) download_start_rate,
+          sum(download_finish) download_finish,
+          ifnull(round(sum(cost) / sum(download_finish),3),0) download_finish_cost,
+          ifnull(round(sum(download_finish) / sum(download_start) * 100,3),0) download_finish_rate,
+          sum(click_install) click_install,
+          sum(install_finish) install_finish,
+          ifnull(round(sum(cost) / sum(install_finish),3),0) install_finish_cost,
+          ifnull(round(sum(install_finish) / sum(download_finish) * 100,3),0) install_finish_rate,
+          sum(active) active,
+          ifnull(round(sum(cost) / sum(active), 3),0) active_cost,
+          ifnull(round(sum(active) / sum(click) * 100,3),0) active_rate,
+          sum(register) register,
+          ifnull(round(sum(cost) / sum(register), 3),0) active_register_cost,
+          ifnull(round(sum(active_register_rate) /{day_count} * 100,3),0) active_register_rate,
+          sum(active_pay_amount) active_pay_amount,
+          sum(next_day_open) next_day_open,
+          ifnull(round(sum(cost) / sum(next_day_open),3),0) next_day_open_cost,
+          ifnull(round(sum(next_day_open) / sum(active) * 100,3),0) next_day_open_rate,
+          sum(attribution_next_day_open_cnt) attribution_next_day_open_cnt,
+          ifnull(round(sum(cost) / sum(	attribution_next_day_open_cnt),3),0) attribution_next_day_open_cost,
+          ifnull(round(sum(	attribution_next_day_open_cnt) / sum(active) * 100,3),0) attribution_next_day_open_rate,
+          sum(game_addiction) game_addiction,
+          ifnull(round(sum(cost) / sum(game_addiction),3),0) game_addiction_cost,
+          ifnull(round(sum(game_addiction_rate) /{day_count} * 100,3),0) game_addiction_rate,
+          sum(pay_count) pay_count,
+          ifnull(round(sum(cost) / sum(pay_count), 3),0) active_pay_cost,
+          ifnull(round(sum(pay_count) / sum(active) * 100,3),0) active_pay_rate,
+          sum(loan_completion) loan_completion,
+          ifnull(round(sum(cost) / sum(loan_completion),3),0) loan_completion_cost,
+          ifnull(round(sum(loan_completion) / sum(register) * 100,3),0) loan_completion_rate,
+          ifnull(round(sum(pre_loan_credit), 3),0) pre_loan_credit,
+          ifnull(round(sum(cost) / sum(pre_loan_credit),3),0) pre_loan_credit_cost,
+          sum(loan_credit) loan_credit,
+          ifnull(round(sum(cost) / sum(loan_credit),3),0) loan_credit_cost,
+          ifnull(round(sum(loan_credit) / sum(loan_completion) * 100,3),0) loan_credit_rate,
+          sum(in_app_uv) in_app_uv,
+          sum(in_app_detail_uv) in_app_detail_uv,
+          sum(in_app_cart) in_app_cart,
+          sum(in_app_pay) in_app_pay,
+          sum(in_app_order) in_app_order,
+          ifnull(round(sum(	attribution_game_pay_7d_count) /{day_count}, 3),0) attribution_game_pay_7d_count,
+          ifnull(round(sum(	attribution_game_pay_7d_cost) /{day_count}, 3),0) attribution_game_pay_7d_cost,
+          ifnull(round(sum(	attribution_active_pay_7d_per_count) /{day_count}, 3),0) attribution_active_pay_7d_per_count,
+          sum(game_pay_cost) game_pay_cost,
+          sum(game_pay_count) game_pay_count,
+          sum(phone) phone,
+          sum(form) form,
+          sum(map_search) map_search,
+          sum(button) button,
+          sum(`view`) `view`,
+          sum(download) download,
+          sum(qq) qq,
+          sum(lottery) lottery,
+          sum(vote) vote,
+          sum(message) message,
+          sum(redirect) redirect,
+          sum(shopping) shopping,
+          sum(consult) consult,
+          sum(wechat) wechat,
+          sum(phone_confirm) phone_confirm,
+          sum(phone_connect) phone_connect,
+          sum(consult_effective) consult_effective,
+          sum(coupon) coupon,
+          sum(coupon_single_page) coupon_single_page,
+          sum(redirect_to_shop) redirect_to_shop,
+          sum(poi_collect) poi_collect,
+          sum(poi_address_click) poi_address_click,
+          sum(luban_order_cnt) luban_order_cnt,
+          sum(luban_order_stat_amount) luban_order_stat_amount,
+          sum(luban_order_roi) luban_order_roi,
+          sum(luban_live_enter_cnt) luban_live_enter_cnt,
+          sum(live_watch_one_minute_count) live_watch_one_minute_count,
+          sum(luban_live_follow_cnt) luban_live_follow_cnt,
+          sum(live_fans_club_join_cnt) live_fans_club_join_cnt,
+          sum(luban_live_comment_cnt) luban_live_comment_cnt,
+          sum(luban_live_share_cnt) luban_live_share_cnt,
+          sum(luban_live_gift_cnt) luban_live_gift_cnt,
+          sum(luban_live_gift_amount) luban_live_gift_amount,
+          sum(luban_live_slidecart_click_cnt) luban_live_slidecart_click_cnt,
+          sum(luban_live_click_product_cnt) luban_live_click_product_cnt,
+          sum(luban_live_pay_order_count) luban_live_pay_order_count,
+          sum(luban_live_pay_order_stat_cost) luban_live_pay_order_stat_cost,
+          sum(wechat_login_count) wechat_login_count,
+          sum(attribution_wechat_login_30d_count) attribution_wechat_login_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_login_count),3),0) wechat_login_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_login_30d_count),3),0) attribution_wechat_login_30d_cost,
+          sum(wechat_first_pay_count) wechat_first_pay_count,
+          sum(attribution_wechat_first_pay_30d_count) attribution_wechat_first_pay_30d_count,
+          ifnull(round(sum(cost) / sum(wechat_first_pay_count),3),0) wechat_first_pay_cost,
+          ifnull(round(sum(cost) / sum(	attribution_wechat_first_pay_30d_count),3),0) attribution_wechat_first_pay_30d_cost,
+          ifnull(round(sum(wechat_first_pay_count) / sum(wechat_login_count) * 100,3),0) wechat_first_pay_rate,
+          ifnull(round(sum(	attribution_wechat_first_pay_30d_count) / sum(	attribution_wechat_login_30d_count) * 100,3),0) attribution_wechat_first_pay_30d_rate,
+          sum(wechat_pay_amount) wechat_pay_amount,
+          sum(attribution_wechat_pay_30d_amount) attribution_wechat_pay_30d_amount,
+          sum(attribution_wechat_pay_30d_roi) attribution_wechat_pay_30d_roi,
+          sum(phone_effective) phone_effective,
+          sum(total_play) total_play,
+          sum(valid_play) valid_play,
+          ifnull(round(sum(cost) / sum(valid_play),3),0) valid_play_cost,
+          ifnull(round(sum(valid_play) / sum(`show`) * 100,3),0) valid_play_rate,
+          sum(play_25_feed_break) play_25_feed_break,
+          sum(play_50_feed_break) play_50_feed_break,
+          sum(play_75_feed_break) play_75_feed_break,
+          sum(play_100_feed_break) play_100_feed_break,
+          ifnull(round(sum(average_play_time_per_play) /{day_count}, 3),0) average_play_time_per_play,
+          ifnull(round(sum(play_over_rate) /{day_count} * 100,3),0) play_over_rate,
+          ifnull(round(sum(wifi_play) / sum(total_play) * 100,3),0) wifi_play_rate,
+          sum(wifi_play) wifi_play,
+          ifnull(round(sum(play_duration_sum) /{day_count}, 3),0) play_duration_sum,
+          sum(advanced_creative_phone_click) advanced_creative_phone_click,
+          sum(advanced_creative_counsel_click) advanced_creative_counsel_click,
+          sum(advanced_creative_form_click) advanced_creative_form_click,
+          sum(advanced_creative_coupon_addition) advanced_creative_coupon_addition,
+          sum(advanced_creative_form_submit) advanced_creative_form_submit,
+          sum(card_show) card_show,
+          sum(`share`) `share`,
+          sum(`comment`) `comment`,
+          sum(`like`) `like`,
+          sum(follow) follow,
+          sum(home_visited) home_visited,
+          sum(ies_challenge_click) ies_challenge_click,
+          sum(ies_music_click) ies_music_click,
+          sum(location_click) location_click,
+          sum(message_action) message_action,
+          sum(click_landing_page) click_landing_page,
+          sum(click_shopwindow) click_shopwindow,
+          sum(click_website) click_website,
+          sum(click_download) click_download,
+          sum(click_call_dy) click_call_dy,
+          sum(cpc) cpc,
+          sum(cpm) cpm,
+          sum(cpa) cpa,
+          ifnull(round(sum(interact_per_cost) /{day_count}, 3),0) interact_per_cost,
+          ifnull(round(sum(convert_show_rate) /{day_count}, 3),0) convert_show_rate,
+          ifnull(round(sum(play_duration_10s) / sum(total_play) * 100,3),0) play_duration_10s_rate,
+          ifnull(round(sum(play_50_feed_break) / sum(total_play) * 100,3),0) play_50_feed_break_rate,
+          ifnull(round(sum(play_25_feed_break) / sum(total_play) * 100,3),0) play_25_feed_break_rate,
+          ifnull(round(sum(play_duration_5s_rate) /{day_count}, 3),0) play_duration_5s_rate,
+          ifnull(round(sum(play_duration_2s) / sum(total_play) * 100,3),0) play_duration_2s_rate,
+          ifnull(round(sum(average_play_progress) /{day_count}, 3),0) average_play_progress,
+          ifnull(round(sum(play_100_feed_break) / sum(total_play) * 100,3),0) play_100_feed_break_rate,
+          ifnull(round(sum(play_duration_3s) / sum(total_play) * 100,3),0) play_duration_3s_rate,
+          ifnull(round(sum(play_75_feed_break) / sum(total_play) * 100,3),0) play_75_feed_break_rate,
+          sum(play_duration) play_duration,
+          sum(play_duration_3s) play_duration_3s,
+          sum(play_duration_2s) play_duration_2s,
+          sum(play_duration_10s) play_duration_10s,
+          ifnull(round(sum(average_video_play) /{day_count}, 3),0) average_video_play,
+          sum(avg_rank) avg_rank,
+          sum(submit_certification_count) submit_certification_count,
+          sum(approval_count) approval_count,
+          sum(first_order_count) first_rental_order_count,
+          sum(first_rental_order_count) first_order_count,
+          sum(commute_first_pay_count) commute_first_pay_count
+         FROM media_api.bytedance_material_report_daily zhubiao
+         WHERE image_mode IN (
+                              'CREATIVE_IMAGE_MODE_VIDEO',
+                              'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL',
+                              'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO'
+             )
+           AND stat_datetime >= date_format(
+                 date_sub({date_time}, INTERVAL {day_count}  DAY),
+                 '%Y%m%d'
+             )
+             and advertiser_id={account_id}
+         GROUP BY advertiser_id,
+                  material_id,
+                  image_mode,
+                  inventory
+     ) report
+         LEFT JOIN (select advertiser_id, signature, material_id, filename, video_url
+                    from `media_api`.bytedance_file_video_get where advertiser_id={account_id}
+                    group by advertiser_id, signature, material_id, filename, video_url) video
+                   ON report.account_id = video.advertiser_id AND report.material_id = video.material_id
+         LEFT JOIN (select  code,material_name,cover_url from `jeecg-boot`.ctop_material_info where status=1) cmi 
+          ON video.signature = cmi.code
+         LEFT JOIN (select material_id, clip_id, shot_id, plan_id, leader_id
+                    from (
+                             select material_id,
+                                    clip_id,
+                                    shot_id,
+                                    plan_id,
+                                    leader_id,
+                                    row_number() over (partition by material_id order by create_time) rn
+                             from `jeecg-boot`.ctop_material_ascription) m
+                    where rn = 1) cma
+                   ON video.signature = cma.material_id
+         LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.account_id = cua.account_id
+         LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON cua.advertiser_id = ca.id
+         LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+         LEFT JOIN `jeecg-boot`.user_company uc ON cua.user_id = uc.user_id
+         LEFT JOIN `jeecg-boot`.ctop_product product ON cp.product_id = product.id
+         LEFT JOIN `jeecg-boot`.sys_user su ON cua.user_id = su.id
+         LEFT JOIN `jeecg-boot`.sys_user clip ON clip.id = cma.clip_id
+         LEFT JOIN `jeecg-boot`.sys_user shot ON shot.id = cma.shot_id
+         LEFT JOIN `jeecg-boot`.sys_user plan ON plan.id = cma.plan_id
+         LEFT JOIN `jeecg-boot`.sys_user leader ON leader.id = cma.leader_id
+         LEFT JOIN `jeecg-boot`.user_company up ON up.user_id = cma.clip_id
+         where cua.account_id = {account_id};""".format(date_time=self.date_time,
+                                                        day_count=self.day_count,
+                                                        account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceVideoReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 8 - 0
Apietl/bytedance_etl/__init__.py

@@ -0,0 +1,8 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/10/21 5:35 下午
+# @Site    : 
+# @File    : __init__.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553

+ 302 - 0
Apietl/bytedance_etl/report/AdDailyReport.py

@@ -0,0 +1,302 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : AdDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_ad_report_daily where 
+            stat_datetime between {start_time} and {end_time}""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+        replace into application.bytedance_ad_report_daily_dw(account_id, campaign_id, campaign_name, ad_id, ad_name,
+                                                      stat_datetime, project_id, product_id, advertiser_id, company_id,
+                                                      account_name, project_name, advertiser_name, company_name,
+                                                      sale_id, user_id, org_code, cost, `show`, avg_show_cost, click,
+                                                      avg_click_cost, ctr, `convert`, convert_cost, convert_rate,
+                                                      deep_convert, deep_convert_cost, deep_convert_rate,
+                                                      attribution_convert, attribution_convert_cost,
+                                                      attribution_deep_convert, attribution_deep_convert_cost,
+                                                      download_start, download_start_cost, download_start_rate,
+                                                      download_finish, download_finish_cost, download_finish_rate,
+                                                      click_install, install_finish, install_finish_cost,
+                                                      install_finish_rate, active, active_cost, active_rate, register,
+                                                      active_register_cost, active_register_rate, next_day_open,
+                                                      next_day_open_cost, next_day_open_rate,
+                                                      attribution_next_day_open_cnt, attribution_next_day_open_cost,
+                                                      attribution_next_day_open_rate, game_addiction,
+                                                      game_addiction_cost, game_addiction_rate, pay_count,
+                                                      active_pay_cost, active_pay_rate, loan_completion,
+                                                      loan_completion_cost, loan_completion_rate, pre_loan_credit,
+                                                      pre_loan_credit_cost, loan_credit, loan_credit_cost,
+                                                      loan_credit_rate, in_app_uv, in_app_detail_uv, in_app_cart,
+                                                      in_app_pay, in_app_order, attribution_game_pay_7d_count,
+                                                      attribution_game_pay_7d_cost, attribution_active_pay_7d_per_count,
+                                                      game_pay_cost, game_pay_count, phone, form, map_search, button,
+                                                      view, download, qq, lottery, vote, message, redirect, shopping,
+                                                      consult, wechat, phone_confirm, phone_connect, consult_effective,
+                                                      coupon, coupon_single_page, redirect_to_shop, poi_collect,
+                                                      poi_address_click, luban_order_cnt, luban_order_stat_amount,
+                                                      luban_order_roi, luban_live_enter_cnt,
+                                                      live_watch_one_minute_count, luban_live_follow_cnt,
+                                                      live_fans_club_join_cnt, luban_live_comment_cnt,
+                                                      luban_live_share_cnt, luban_live_gift_cnt, luban_live_gift_amount,
+                                                      luban_live_slidecart_click_cnt, luban_live_click_product_cnt,
+                                                      luban_live_pay_order_count, luban_live_pay_order_stat_cost,
+                                                      live_component_click_count, live_component_click_cost,
+                                                      live_component_click_rate, wechat_login_count,
+                                                      attribution_wechat_login_30d_count, wechat_login_cost,
+                                                      attribution_wechat_login_30d_cost, wechat_first_pay_count,
+                                                      attribution_wechat_first_pay_30d_count, wechat_first_pay_cost,
+                                                      attribution_wechat_first_pay_30d_cost, wechat_first_pay_rate,
+                                                      attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                      attribution_wechat_pay_30d_amount, attribution_wechat_pay_30d_roi,
+                                                      phone_effective, total_play, valid_play, valid_play_cost,
+                                                      valid_play_rate, play_25_feed_break, play_50_feed_break,
+                                                      play_75_feed_break, play_100_feed_break,
+                                                      average_play_time_per_play, play_over_rate, wifi_play_rate,
+                                                      wifi_play, play_duration_sum, advanced_creative_phone_click,
+                                                      advanced_creative_counsel_click, advanced_creative_form_click,
+                                                      advanced_creative_coupon_addition, advanced_creative_form_submit,
+                                                      card_show, share, comment, `like`, follow, home_visited,
+                                                      ies_challenge_click, ies_music_click, location_click,
+                                                      message_action, click_landing_page, click_shopwindow,
+                                                      click_website, click_download, click_call_dy,
+                                                      submit_certification_count, approval_count,
+                                                      first_rental_order_count, first_order_count,
+                                                      commute_first_pay_count)
+select a.advertiser_id account_id,
+       a.campaign_id,
+       a.campaign_name,
+       a.ad_id,
+       a.ad_name,
+       a.stat_datetime,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count
+from `media_api`.bytedance_ad_report_daily a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+order by a.advertiser_id, a.stat_datetime,a.ad_id
+limit {num},20000;
+        """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AdDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 300 - 0
Apietl/bytedance_etl/report/AdHourlyReport.py

@@ -0,0 +1,300 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : AdHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdHourlyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def Hourly_report(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_ad_report_hourly where 
+            stat_datetime between {start_time} and {end_time} group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn,sql)
+            for account_id in account_list:
+                sql = """
+replace into application.bytedance_ad_report_hourly_dw(account_id, campaign_id, campaign_name, ad_id, ad_name,
+                                                      stat_datetime,hour, project_id, product_id, advertiser_id, company_id,
+                                                      account_name, project_name, advertiser_name, company_name,
+                                                      sale_id, user_id, org_code, cost, `show`, avg_show_cost, click,
+                                                      avg_click_cost, ctr, `convert`, convert_cost, convert_rate,
+                                                      deep_convert, deep_convert_cost, deep_convert_rate,
+                                                      attribution_convert, attribution_convert_cost,
+                                                      attribution_deep_convert, attribution_deep_convert_cost,
+                                                      download_start, download_start_cost, download_start_rate,
+                                                      download_finish, download_finish_cost, download_finish_rate,
+                                                      click_install, install_finish, install_finish_cost,
+                                                      install_finish_rate, active, active_cost, active_rate, register,
+                                                      active_register_cost, active_register_rate, next_day_open,
+                                                      next_day_open_cost, next_day_open_rate,
+                                                      attribution_next_day_open_cnt, attribution_next_day_open_cost,
+                                                      attribution_next_day_open_rate, game_addiction,
+                                                      game_addiction_cost, game_addiction_rate, pay_count,
+                                                      active_pay_cost, active_pay_rate, loan_completion,
+                                                      loan_completion_cost, loan_completion_rate, pre_loan_credit,
+                                                      pre_loan_credit_cost, loan_credit, loan_credit_cost,
+                                                      loan_credit_rate, in_app_uv, in_app_detail_uv, in_app_cart,
+                                                      in_app_pay, in_app_order, attribution_game_pay_7d_count,
+                                                      attribution_game_pay_7d_cost, attribution_active_pay_7d_per_count,
+                                                      game_pay_cost, game_pay_count, phone, form, map_search, button,
+                                                      view, download, qq, lottery, vote, message, redirect, shopping,
+                                                      consult, wechat, phone_confirm, phone_connect, consult_effective,
+                                                      coupon, coupon_single_page, redirect_to_shop, poi_collect,
+                                                      poi_address_click, luban_order_cnt, luban_order_stat_amount,
+                                                      luban_order_roi, luban_live_enter_cnt,
+                                                      live_watch_one_minute_count, luban_live_follow_cnt,
+                                                      live_fans_club_join_cnt, luban_live_comment_cnt,
+                                                      luban_live_share_cnt, luban_live_gift_cnt, luban_live_gift_amount,
+                                                      luban_live_slidecart_click_cnt, luban_live_click_product_cnt,
+                                                      luban_live_pay_order_count, luban_live_pay_order_stat_cost,
+                                                      live_component_click_count, live_component_click_cost,
+                                                      live_component_click_rate, wechat_login_count,
+                                                      attribution_wechat_login_30d_count, wechat_login_cost,
+                                                      attribution_wechat_login_30d_cost, wechat_first_pay_count,
+                                                      attribution_wechat_first_pay_30d_count, wechat_first_pay_cost,
+                                                      attribution_wechat_first_pay_30d_cost, wechat_first_pay_rate,
+                                                      attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                      attribution_wechat_pay_30d_amount, attribution_wechat_pay_30d_roi,
+                                                      phone_effective, total_play, valid_play, valid_play_cost,
+                                                      valid_play_rate, play_25_feed_break, play_50_feed_break,
+                                                      play_75_feed_break, play_100_feed_break,
+                                                      average_play_time_per_play, play_over_rate, wifi_play_rate,
+                                                      wifi_play, play_duration_sum, advanced_creative_phone_click,
+                                                      advanced_creative_counsel_click, advanced_creative_form_click,
+                                                      advanced_creative_coupon_addition, advanced_creative_form_submit,
+                                                      card_show, share, comment, `like`, follow, home_visited,
+                                                      ies_challenge_click, ies_music_click, location_click,
+                                                      message_action, click_landing_page, click_shopwindow,
+                                                      click_website, click_download, click_call_dy,
+                                                      submit_certification_count, approval_count,
+                                                      first_rental_order_count, first_order_count,
+                                                      commute_first_pay_count)
+select a.advertiser_id account_id,
+       a.campaign_id,
+       a.campaign_name,
+       a.ad_id,
+       a.ad_name,
+       a.stat_datetime,
+       a.hour,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count
+from `media_api`.bytedance_ad_report_hourly a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+and a.advertiser_id={account_id}
+        """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AdHourlyReport().Hourly_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 324 - 0
Apietl/bytedance_etl/report/AdvertiserDailyInventoryReport.py

@@ -0,0 +1,324 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : AdvertiserDailyInventoryReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdvertiserDailyInventoryReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_advertiser_inventory_report_daily where 
+            stat_datetime between {start_time} and {end_time}""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+            replace into application.bytedance_advertiser_inventory_report_daily_dw(account_id, stat_datetime, project_id, product_id,
+                                                              advertiser_id, company_id, account_name, project_name,
+                                                              advertiser_name, company_name, sale_id, user_id, org_code,
+                                                              cost, `show`, avg_show_cost, click, avg_click_cost, ctr,
+                                                              `convert`, convert_cost, convert_rate, deep_convert,
+                                                              deep_convert_cost, deep_convert_rate, attribution_convert,
+                                                              attribution_convert_cost, attribution_deep_convert,
+                                                              attribution_deep_convert_cost, download_start,
+                                                              download_start_cost, download_start_rate, download_finish,
+                                                              download_finish_cost, download_finish_rate, click_install,
+                                                              install_finish, install_finish_cost, install_finish_rate,
+                                                              active, active_cost, active_rate, register,
+                                                              active_register_cost, active_register_rate, next_day_open,
+                                                              next_day_open_cost, next_day_open_rate,
+                                                              attribution_next_day_open_cnt,
+                                                              attribution_next_day_open_cost,
+                                                              attribution_next_day_open_rate, game_addiction,
+                                                              game_addiction_cost, game_addiction_rate, pay_count,
+                                                              active_pay_cost, active_pay_rate, loan_completion,
+                                                              loan_completion_cost, loan_completion_rate,
+                                                              pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                              loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                              in_app_detail_uv, in_app_cart, in_app_pay, in_app_order,
+                                                              attribution_game_pay_7d_count,
+                                                              attribution_game_pay_7d_cost,
+                                                              attribution_active_pay_7d_per_count, game_pay_cost,
+                                                              game_pay_count, phone, form, map_search, button, view,
+                                                              download, qq, lottery, vote, message, redirect, shopping,
+                                                              consult, wechat, phone_confirm, phone_connect,
+                                                              consult_effective, coupon, coupon_single_page,
+                                                              redirect_to_shop, poi_collect, poi_address_click,
+                                                              luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                              luban_live_enter_cnt, live_watch_one_minute_count,
+                                                              luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                              luban_live_comment_cnt, luban_live_share_cnt,
+                                                              luban_live_gift_cnt, luban_live_gift_amount,
+                                                              luban_live_slidecart_click_cnt,
+                                                              luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                              luban_live_pay_order_stat_cost,
+                                                              live_component_click_count, live_component_click_cost,
+                                                              live_component_click_rate, wechat_login_count,
+                                                              attribution_wechat_login_30d_count, wechat_login_cost,
+                                                              attribution_wechat_login_30d_cost, wechat_first_pay_count,
+                                                              attribution_wechat_first_pay_30d_count,
+                                                              wechat_first_pay_cost,
+                                                              attribution_wechat_first_pay_30d_cost,
+                                                              wechat_first_pay_rate,
+                                                              attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                              attribution_wechat_pay_30d_amount,
+                                                              attribution_wechat_pay_30d_roi, phone_effective,
+                                                              total_play, valid_play, valid_play_cost, valid_play_rate,
+                                                              play_25_feed_break, play_50_feed_break,
+                                                              play_75_feed_break, play_100_feed_break,
+                                                              average_play_time_per_play, play_over_rate,
+                                                              wifi_play_rate, wifi_play, play_duration_sum,
+                                                              advanced_creative_phone_click,
+                                                              advanced_creative_counsel_click,
+                                                              advanced_creative_form_click,
+                                                              advanced_creative_coupon_addition,
+                                                              advanced_creative_form_submit, card_show, share, comment,
+                                                              `like`, follow, home_visited, ies_challenge_click,
+                                                              ies_music_click, location_click, message_action,
+                                                              click_landing_page, click_shopwindow, click_website,
+                                                              click_download, click_call_dy, submit_certification_count,
+                                                              approval_count, first_rental_order_count,
+                                                              first_order_count, commute_first_pay_count,
+                                                              attribution_active_pay_intra_one_day_roi
+,attribution_active_pay_intra_one_day_amount
+,attribution_active_pay_intra_one_day_rate
+,attribution_active_pay_intra_one_day_cost
+,attribution_active_pay_intra_one_day_count,stat_pay_amount,
+inventory,
+inventory_name)
+select a.advertiser_id account_id,
+       a.stat_datetime,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count,
+              a.attribution_active_pay_intra_one_day_roi,
+a.attribution_active_pay_intra_one_day_amount,
+a.attribution_active_pay_intra_one_day_rate,
+a.attribution_active_pay_intra_one_day_cost,
+attribution_active_pay_intra_one_day_count,
+a.stat_pay_amount,
+inventory,
+inventory_name
+from `media_api`.bytedance_advertiser_inventory_report_daily a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+order by account_id limit {num},20000;
+        """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AdvertiserDailyInventoryReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 320 - 0
Apietl/bytedance_etl/report/AdvertiserDailyReport.py

@@ -0,0 +1,320 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : AdvertiserDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdvertiserDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_advertiser_report_daily where 
+            stat_datetime between {start_time} and {end_time}""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+            replace into application.bytedance_advertiser_report_daily_dw(account_id, stat_datetime, project_id, product_id,
+                                                              advertiser_id, company_id, account_name, project_name,
+                                                              advertiser_name, company_name, sale_id, user_id, org_code,
+                                                              cost, `show`, avg_show_cost, click, avg_click_cost, ctr,
+                                                              `convert`, convert_cost, convert_rate, deep_convert,
+                                                              deep_convert_cost, deep_convert_rate, attribution_convert,
+                                                              attribution_convert_cost, attribution_deep_convert,
+                                                              attribution_deep_convert_cost, download_start,
+                                                              download_start_cost, download_start_rate, download_finish,
+                                                              download_finish_cost, download_finish_rate, click_install,
+                                                              install_finish, install_finish_cost, install_finish_rate,
+                                                              active, active_cost, active_rate, register,
+                                                              active_register_cost, active_register_rate, next_day_open,
+                                                              next_day_open_cost, next_day_open_rate,
+                                                              attribution_next_day_open_cnt,
+                                                              attribution_next_day_open_cost,
+                                                              attribution_next_day_open_rate, game_addiction,
+                                                              game_addiction_cost, game_addiction_rate, pay_count,
+                                                              active_pay_cost, active_pay_rate, loan_completion,
+                                                              loan_completion_cost, loan_completion_rate,
+                                                              pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                              loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                              in_app_detail_uv, in_app_cart, in_app_pay, in_app_order,
+                                                              attribution_game_pay_7d_count,
+                                                              attribution_game_pay_7d_cost,
+                                                              attribution_active_pay_7d_per_count, game_pay_cost,
+                                                              game_pay_count, phone, form, map_search, button, view,
+                                                              download, qq, lottery, vote, message, redirect, shopping,
+                                                              consult, wechat, phone_confirm, phone_connect,
+                                                              consult_effective, coupon, coupon_single_page,
+                                                              redirect_to_shop, poi_collect, poi_address_click,
+                                                              luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                              luban_live_enter_cnt, live_watch_one_minute_count,
+                                                              luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                              luban_live_comment_cnt, luban_live_share_cnt,
+                                                              luban_live_gift_cnt, luban_live_gift_amount,
+                                                              luban_live_slidecart_click_cnt,
+                                                              luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                              luban_live_pay_order_stat_cost,
+                                                              live_component_click_count, live_component_click_cost,
+                                                              live_component_click_rate, wechat_login_count,
+                                                              attribution_wechat_login_30d_count, wechat_login_cost,
+                                                              attribution_wechat_login_30d_cost, wechat_first_pay_count,
+                                                              attribution_wechat_first_pay_30d_count,
+                                                              wechat_first_pay_cost,
+                                                              attribution_wechat_first_pay_30d_cost,
+                                                              wechat_first_pay_rate,
+                                                              attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                              attribution_wechat_pay_30d_amount,
+                                                              attribution_wechat_pay_30d_roi, phone_effective,
+                                                              total_play, valid_play, valid_play_cost, valid_play_rate,
+                                                              play_25_feed_break, play_50_feed_break,
+                                                              play_75_feed_break, play_100_feed_break,
+                                                              average_play_time_per_play, play_over_rate,
+                                                              wifi_play_rate, wifi_play, play_duration_sum,
+                                                              advanced_creative_phone_click,
+                                                              advanced_creative_counsel_click,
+                                                              advanced_creative_form_click,
+                                                              advanced_creative_coupon_addition,
+                                                              advanced_creative_form_submit, card_show, share, comment,
+                                                              `like`, follow, home_visited, ies_challenge_click,
+                                                              ies_music_click, location_click, message_action,
+                                                              click_landing_page, click_shopwindow, click_website,
+                                                              click_download, click_call_dy, submit_certification_count,
+                                                              approval_count, first_rental_order_count,
+                                                              first_order_count, commute_first_pay_count,
+                                                              attribution_active_pay_intra_one_day_roi
+,attribution_active_pay_intra_one_day_amount
+,attribution_active_pay_intra_one_day_rate
+,attribution_active_pay_intra_one_day_cost
+,attribution_active_pay_intra_one_day_count,stat_pay_amount)
+select a.advertiser_id account_id,
+       a.stat_datetime,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count,
+              a.attribution_active_pay_intra_one_day_roi,
+a.attribution_active_pay_intra_one_day_amount,
+a.attribution_active_pay_intra_one_day_rate,
+a.attribution_active_pay_intra_one_day_cost,
+attribution_active_pay_intra_one_day_count,
+a.stat_pay_amount
+from `media_api`.bytedance_advertiser_report_daily a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+order by account_id limit {num},20000;
+        """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AdvertiserDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 321 - 0
Apietl/bytedance_etl/report/AdvertiserHourlyReport.py

@@ -0,0 +1,321 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : AdvertiserHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdvertiserHourlyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def hourly_report(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_advertiser_report_hourly where 
+            stat_datetime between {start_time} and {end_time}""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+            replace into application.bytedance_advertiser_report_hourly_dw(account_id, stat_datetime,hour, project_id, product_id,
+                                                              advertiser_id, company_id, account_name, project_name,
+                                                              advertiser_name, company_name, sale_id, user_id, org_code,
+                                                              cost, `show`, avg_show_cost, click, avg_click_cost, ctr,
+                                                              `convert`, convert_cost, convert_rate, deep_convert,
+                                                              deep_convert_cost, deep_convert_rate, attribution_convert,
+                                                              attribution_convert_cost, attribution_deep_convert,
+                                                              attribution_deep_convert_cost, download_start,
+                                                              download_start_cost, download_start_rate, download_finish,
+                                                              download_finish_cost, download_finish_rate, click_install,
+                                                              install_finish, install_finish_cost, install_finish_rate,
+                                                              active, active_cost, active_rate, register,
+                                                              active_register_cost, active_register_rate, next_day_open,
+                                                              next_day_open_cost, next_day_open_rate,
+                                                              attribution_next_day_open_cnt,
+                                                              attribution_next_day_open_cost,
+                                                              attribution_next_day_open_rate, game_addiction,
+                                                              game_addiction_cost, game_addiction_rate, pay_count,
+                                                              active_pay_cost, active_pay_rate, loan_completion,
+                                                              loan_completion_cost, loan_completion_rate,
+                                                              pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                              loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                              in_app_detail_uv, in_app_cart, in_app_pay, in_app_order,
+                                                              attribution_game_pay_7d_count,
+                                                              attribution_game_pay_7d_cost,
+                                                              attribution_active_pay_7d_per_count, game_pay_cost,
+                                                              game_pay_count, phone, form, map_search, button, view,
+                                                              download, qq, lottery, vote, message, redirect, shopping,
+                                                              consult, wechat, phone_confirm, phone_connect,
+                                                              consult_effective, coupon, coupon_single_page,
+                                                              redirect_to_shop, poi_collect, poi_address_click,
+                                                              luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                              luban_live_enter_cnt, live_watch_one_minute_count,
+                                                              luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                              luban_live_comment_cnt, luban_live_share_cnt,
+                                                              luban_live_gift_cnt, luban_live_gift_amount,
+                                                              luban_live_slidecart_click_cnt,
+                                                              luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                              luban_live_pay_order_stat_cost,
+                                                              live_component_click_count, live_component_click_cost,
+                                                              live_component_click_rate, wechat_login_count,
+                                                              attribution_wechat_login_30d_count, wechat_login_cost,
+                                                              attribution_wechat_login_30d_cost, wechat_first_pay_count,
+                                                              attribution_wechat_first_pay_30d_count,
+                                                              wechat_first_pay_cost,
+                                                              attribution_wechat_first_pay_30d_cost,
+                                                              wechat_first_pay_rate,
+                                                              attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                              attribution_wechat_pay_30d_amount,
+                                                              attribution_wechat_pay_30d_roi, phone_effective,
+                                                              total_play, valid_play, valid_play_cost, valid_play_rate,
+                                                              play_25_feed_break, play_50_feed_break,
+                                                              play_75_feed_break, play_100_feed_break,
+                                                              average_play_time_per_play, play_over_rate,
+                                                              wifi_play_rate, wifi_play, play_duration_sum,
+                                                              advanced_creative_phone_click,
+                                                              advanced_creative_counsel_click,
+                                                              advanced_creative_form_click,
+                                                              advanced_creative_coupon_addition,
+                                                              advanced_creative_form_submit, card_show, share, comment,
+                                                              `like`, follow, home_visited, ies_challenge_click,
+                                                              ies_music_click, location_click, message_action,
+                                                              click_landing_page, click_shopwindow, click_website,
+                                                              click_download, click_call_dy, submit_certification_count,
+                                                              approval_count, first_rental_order_count,
+                                                              first_order_count, commute_first_pay_count,
+                                                              attribution_active_pay_intra_one_day_count,
+attribution_active_pay_intra_one_day_cost,
+attribution_active_pay_intra_one_day_rate,
+attribution_active_pay_intra_one_day_amount,
+attribution_active_pay_intra_one_day_roi,stat_pay_amount)
+select a.advertiser_id account_id,
+       a.stat_datetime,
+       a.hour,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count,
+       a.attribution_active_pay_intra_one_day_count,
+ a.attribution_active_pay_intra_one_day_cost,
+ a.attribution_active_pay_intra_one_day_rate,
+ a.attribution_active_pay_intra_one_day_amount,
+ a.attribution_active_pay_intra_one_day_roi,
+ a.stat_pay_amount
+from `media_api`.bytedance_advertiser_report_hourly a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+order by a.advertiser_id,a.stat_datetime,a.hour limit {num},20000;
+        """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AdvertiserHourlyReport().hourly_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 587 - 0
Apietl/bytedance_etl/report/AdvertiserReport.py

@@ -0,0 +1,587 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : AdvertiserReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdvertiserReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+
+    def daily_report(self, account_id, start_date, end_date):
+        try:
+            sql = """
+            replace into application.bytedance_advertiser_report_daily_dw(account_id, stat_datetime, project_id, product_id,
+                                                              advertiser_id, company_id, account_name, project_name,
+                                                              advertiser_name, company_name, sale_id, user_id, org_code,
+                                                              cost, `show`, avg_show_cost, click, avg_click_cost, ctr,
+                                                              `convert`, convert_cost, convert_rate, deep_convert,
+                                                              deep_convert_cost, deep_convert_rate, attribution_convert,
+                                                              attribution_convert_cost, attribution_deep_convert,
+                                                              attribution_deep_convert_cost, download_start,
+                                                              download_start_cost, download_start_rate, download_finish,
+                                                              download_finish_cost, download_finish_rate, click_install,
+                                                              install_finish, install_finish_cost, install_finish_rate,
+                                                              active, active_cost, active_rate, register,
+                                                              active_register_cost, active_register_rate, next_day_open,
+                                                              next_day_open_cost, next_day_open_rate,
+                                                              attribution_next_day_open_cnt,
+                                                              attribution_next_day_open_cost,
+                                                              attribution_next_day_open_rate, game_addiction,
+                                                              game_addiction_cost, game_addiction_rate, pay_count,
+                                                              active_pay_cost, active_pay_rate, loan_completion,
+                                                              loan_completion_cost, loan_completion_rate,
+                                                              pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                              loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                              in_app_detail_uv, in_app_cart, in_app_pay, in_app_order,
+                                                              attribution_game_pay_7d_count,
+                                                              attribution_game_pay_7d_cost,
+                                                              attribution_active_pay_7d_per_count, game_pay_cost,
+                                                              game_pay_count, phone, form, map_search, button, view,
+                                                              download, qq, lottery, vote, message, redirect, shopping,
+                                                              consult, wechat, phone_confirm, phone_connect,
+                                                              consult_effective, coupon, coupon_single_page,
+                                                              redirect_to_shop, poi_collect, poi_address_click,
+                                                              luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                              luban_live_enter_cnt, live_watch_one_minute_count,
+                                                              luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                              luban_live_comment_cnt, luban_live_share_cnt,
+                                                              luban_live_gift_cnt, luban_live_gift_amount,
+                                                              luban_live_slidecart_click_cnt,
+                                                              luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                              luban_live_pay_order_stat_cost,
+                                                              live_component_click_count, live_component_click_cost,
+                                                              live_component_click_rate, wechat_login_count,
+                                                              attribution_wechat_login_30d_count, wechat_login_cost,
+                                                              attribution_wechat_login_30d_cost, wechat_first_pay_count,
+                                                              attribution_wechat_first_pay_30d_count,
+                                                              wechat_first_pay_cost,
+                                                              attribution_wechat_first_pay_30d_cost,
+                                                              wechat_first_pay_rate,
+                                                              attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                              attribution_wechat_pay_30d_amount,
+                                                              attribution_wechat_pay_30d_roi, phone_effective,
+                                                              total_play, valid_play, valid_play_cost, valid_play_rate,
+                                                              play_25_feed_break, play_50_feed_break,
+                                                              play_75_feed_break, play_100_feed_break,
+                                                              average_play_time_per_play, play_over_rate,
+                                                              wifi_play_rate, wifi_play, play_duration_sum,
+                                                              advanced_creative_phone_click,
+                                                              advanced_creative_counsel_click,
+                                                              advanced_creative_form_click,
+                                                              advanced_creative_coupon_addition,
+                                                              advanced_creative_form_submit, card_show, share, comment,
+                                                              `like`, follow, home_visited, ies_challenge_click,
+                                                              ies_music_click, location_click, message_action,
+                                                              click_landing_page, click_shopwindow, click_website,
+                                                              click_download, click_call_dy, submit_certification_count,
+                                                              approval_count, first_rental_order_count,
+                                                              first_order_count, commute_first_pay_count,
+                                                              attribution_active_pay_intra_one_day_roi
+,attribution_active_pay_intra_one_day_amount
+,attribution_active_pay_intra_one_day_rate
+,attribution_active_pay_intra_one_day_cost
+,attribution_active_pay_intra_one_day_count,
+stat_pay_amount)
+select a.advertiser_id account_id,
+       a.stat_datetime,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count,
+       a.attribution_active_pay_intra_one_day_roi,
+a.attribution_active_pay_intra_one_day_amount,
+a.attribution_active_pay_intra_one_day_rate,
+a.attribution_active_pay_intra_one_day_cost,
+attribution_active_pay_intra_one_day_count,
+a.stat_pay_amount
+from `media_api`.bytedance_advertiser_report_daily a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_date} and {end_date}
+  and a.advertiser_id = {account_id}
+            """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            print(account_id, '头条账户日报执行完成')
+        except Exception as e:
+            logging.error(account_id, '执行失败')
+
+    def hourly_report(self, account_id, start_date, end_date):
+        try:
+            sql = """
+            replace into application.bytedance_advertiser_report_hourly_dw(account_id, stat_datetime, hour, project_id, product_id,
+                                                               advertiser_id, company_id, account_name, project_name,
+                                                               advertiser_name, company_name, sale_id, user_id,
+                                                               org_code, cost, `show`, avg_show_cost, click,
+                                                               avg_click_cost, ctr, `convert`, convert_cost,
+                                                               convert_rate, deep_convert, deep_convert_cost,
+                                                               deep_convert_rate, attribution_convert,
+                                                               attribution_convert_cost, attribution_deep_convert,
+                                                               attribution_deep_convert_cost, download_start,
+                                                               download_start_cost, download_start_rate,
+                                                               download_finish, download_finish_cost,
+                                                               download_finish_rate, click_install, install_finish,
+                                                               install_finish_cost, install_finish_rate, active,
+                                                               active_cost, active_rate, register, active_register_cost,
+                                                               active_register_rate, next_day_open, next_day_open_cost,
+                                                               next_day_open_rate, attribution_next_day_open_cnt,
+                                                               attribution_next_day_open_cost,
+                                                               attribution_next_day_open_rate, game_addiction,
+                                                               game_addiction_cost, game_addiction_rate, pay_count,
+                                                               active_pay_cost, active_pay_rate, loan_completion,
+                                                               loan_completion_cost, loan_completion_rate,
+                                                               pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                               loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                               in_app_detail_uv, in_app_cart, in_app_pay, in_app_order,
+                                                               attribution_game_pay_7d_count,
+                                                               attribution_game_pay_7d_cost,
+                                                               attribution_active_pay_7d_per_count, game_pay_cost,
+                                                               game_pay_count, phone, form, map_search, button, view,
+                                                               download, qq, lottery, vote, message, redirect, shopping,
+                                                               consult, wechat, phone_confirm, phone_connect,
+                                                               consult_effective, coupon, coupon_single_page,
+                                                               redirect_to_shop, poi_collect, poi_address_click,
+                                                               luban_order_cnt, luban_order_stat_amount,
+                                                               luban_order_roi, luban_live_enter_cnt,
+                                                               live_watch_one_minute_count, luban_live_follow_cnt,
+                                                               live_fans_club_join_cnt, luban_live_comment_cnt,
+                                                               luban_live_share_cnt, luban_live_gift_cnt,
+                                                               luban_live_gift_amount, luban_live_slidecart_click_cnt,
+                                                               luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                               luban_live_pay_order_stat_cost,
+                                                               live_component_click_count, live_component_click_cost,
+                                                               live_component_click_rate, wechat_login_count,
+                                                               attribution_wechat_login_30d_count, wechat_login_cost,
+                                                               attribution_wechat_login_30d_cost,
+                                                               wechat_first_pay_count,
+                                                               attribution_wechat_first_pay_30d_count,
+                                                               wechat_first_pay_cost,
+                                                               attribution_wechat_first_pay_30d_cost,
+                                                               wechat_first_pay_rate,
+                                                               attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                               attribution_wechat_pay_30d_amount,
+                                                               attribution_wechat_pay_30d_roi, phone_effective,
+                                                               total_play, valid_play, valid_play_cost, valid_play_rate,
+                                                               play_25_feed_break, play_50_feed_break,
+                                                               play_75_feed_break, play_100_feed_break,
+                                                               average_play_time_per_play, play_over_rate,
+                                                               wifi_play_rate, wifi_play, play_duration_sum,
+                                                               advanced_creative_phone_click,
+                                                               advanced_creative_counsel_click,
+                                                               advanced_creative_form_click,
+                                                               advanced_creative_coupon_addition,
+                                                               advanced_creative_form_submit, card_show, share, comment,
+                                                               `like`, follow, home_visited, ies_challenge_click,
+                                                               ies_music_click, location_click, message_action,
+                                                               click_landing_page, click_shopwindow, click_website,
+                                                               click_download, click_call_dy,
+                                                               submit_certification_count, approval_count,
+                                                               first_rental_order_count, first_order_count,
+                                                               commute_first_pay_count,
+                                                               attribution_active_pay_intra_one_day_count,
+attribution_active_pay_intra_one_day_cost,
+attribution_active_pay_intra_one_day_rate,
+attribution_active_pay_intra_one_day_amount,
+attribution_active_pay_intra_one_day_roi,stat_pay_amount)
+select a.advertiser_id account_id,
+       a.stat_datetime,
+       a.`hour`,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count,
+       a.attribution_active_pay_intra_one_day_count,
+a.attribution_active_pay_intra_one_day_cost,
+a.attribution_active_pay_intra_one_day_rate,
+a.attribution_active_pay_intra_one_day_amount,
+a.attribution_active_pay_intra_one_day_roi,
+a.stat_pay_amount
+from `media_api`.bytedance_advertiser_report_hourly a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_date} and {end_date}
+  and a.advertiser_id = {account_id}
+
+     """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            print(account_id, '头条账户时报执行完成')
+        except Exception as e:
+            logging.error(account_id, '执行失败')
+
+    def handler(self, date_type, account_id, start_date, end_date):
+        if date_type == "daily":
+            self.daily_report(account_id, start_date, end_date)
+        else:
+            self.hourly_report(account_id, start_date, end_date)

+ 322 - 0
Apietl/bytedance_etl/report/AdvertiserSumDailyReport.py

@@ -0,0 +1,322 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : AdvertiserDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+import logging
+from math import ceil
+
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AdvertiserSumDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = (datetime.date.today() + datetime.timedelta(0)).strftime("%Y%m%d")
+
+
+    def daily_report(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_advertiser_report_hourly where 
+            stat_datetime = {start_time} """.format(
+                start_time=self.start_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+            replace into application.bytedance_advertiser_report_daily_dw(account_id, stat_datetime, project_id, product_id,
+                                                              advertiser_id, company_id, account_name, project_name,
+                                                              advertiser_name, company_name, sale_id, user_id, org_code,
+                                                              cost, `show`, avg_show_cost, click, avg_click_cost, ctr,
+                                                              `convert`, convert_cost, convert_rate, deep_convert,
+                                                              deep_convert_cost, deep_convert_rate, attribution_convert,
+                                                              attribution_convert_cost, attribution_deep_convert,
+                                                              attribution_deep_convert_cost, download_start,
+                                                              download_start_cost, download_start_rate, download_finish,
+                                                              download_finish_cost, download_finish_rate, click_install,
+                                                              install_finish, install_finish_cost, install_finish_rate,
+                                                              active, active_cost, active_rate, register,
+                                                              active_register_cost, active_register_rate, next_day_open,
+                                                              next_day_open_cost, next_day_open_rate,
+                                                              attribution_next_day_open_cnt,
+                                                              attribution_next_day_open_cost,
+                                                              attribution_next_day_open_rate, game_addiction,
+                                                              game_addiction_cost, game_addiction_rate, pay_count,
+                                                              active_pay_cost, active_pay_rate, loan_completion,
+                                                              loan_completion_cost, loan_completion_rate,
+                                                              pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                              loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                              in_app_detail_uv, in_app_cart, in_app_pay, in_app_order,
+                                                              attribution_game_pay_7d_count,
+                                                              attribution_game_pay_7d_cost,
+                                                              attribution_active_pay_7d_per_count, game_pay_cost,
+                                                              game_pay_count, phone, form, map_search, button, view,
+                                                              download, qq, lottery, vote, message, redirect, shopping,
+                                                              consult, wechat, phone_confirm, phone_connect,
+                                                              consult_effective, coupon, coupon_single_page,
+                                                              redirect_to_shop, poi_collect, poi_address_click,
+                                                              luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                              luban_live_enter_cnt, live_watch_one_minute_count,
+                                                              luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                              luban_live_comment_cnt, luban_live_share_cnt,
+                                                              luban_live_gift_cnt, luban_live_gift_amount,
+                                                              luban_live_slidecart_click_cnt,
+                                                              luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                              luban_live_pay_order_stat_cost,
+                                                              live_component_click_count, live_component_click_cost,
+                                                              live_component_click_rate, wechat_login_count,
+                                                              attribution_wechat_login_30d_count, wechat_login_cost,
+                                                              attribution_wechat_login_30d_cost, wechat_first_pay_count,
+                                                              attribution_wechat_first_pay_30d_count,
+                                                              wechat_first_pay_cost,
+                                                              attribution_wechat_first_pay_30d_cost,
+                                                              wechat_first_pay_rate,
+                                                              attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                              attribution_wechat_pay_30d_amount,
+                                                              attribution_wechat_pay_30d_roi, phone_effective,
+                                                              total_play, valid_play, valid_play_cost, valid_play_rate,
+                                                              play_25_feed_break, play_50_feed_break,
+                                                              play_75_feed_break, play_100_feed_break,
+                                                              average_play_time_per_play, play_over_rate,
+                                                              wifi_play_rate, wifi_play, play_duration_sum,
+                                                              advanced_creative_phone_click,
+                                                              advanced_creative_counsel_click,
+                                                              advanced_creative_form_click,
+                                                              advanced_creative_coupon_addition,
+                                                              advanced_creative_form_submit, card_show, share, comment,
+                                                              `like`, follow, home_visited, ies_challenge_click,
+                                                              ies_music_click, location_click, message_action,
+                                                              click_landing_page, click_shopwindow, click_website,
+                                                              click_download, click_call_dy, submit_certification_count,
+                                                              approval_count, first_rental_order_count,
+                                                              first_order_count, commute_first_pay_count,
+                                                              attribution_active_pay_intra_one_day_roi
+,attribution_active_pay_intra_one_day_amount
+,attribution_active_pay_intra_one_day_rate
+,attribution_active_pay_intra_one_day_cost
+,attribution_active_pay_intra_one_day_count,stat_pay_amount)
+select a.advertiser_id account_id,
+       a.stat_datetime,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+         sum(a.cost) as  cost,
+       sum(a.show) as shows,
+       sum(a.avg_show_cost) as avg_show_cost,
+       sum(a.click) as click,
+       sum(a.avg_click_cost) as avg_click_cost,
+       sum(a.ctr) as ctr,
+       sum(a.convert) as converts,
+       sum(a.convert_cost) as convert_cost,
+       sum(a.convert_rate) as convert_rate,
+       sum(a.deep_convert) as deep_convert,
+       sum(a.deep_convert_cost) as deep_convert_cost,
+       sum(a.deep_convert_rate) as deep_convert_rate,
+       sum(a.attribution_convert)as attribution_convert,
+       sum(a.attribution_convert_cost) as attribution_convert_cost,
+       sum(a.attribution_deep_convert) as attribution_deep_convert_cost,
+       sum(a.attribution_deep_convert_cost) as attribution_deep_convert_cost,
+       sum(a.download_start) as download_start,
+       sum(a.download_start_cost) as download_start_cost,
+       sum(a.download_start_rate) as download_start_rate,
+       sum(a.download_finish) as download_finish,
+       sum(a.download_finish_cost) as download_finish_cost,
+       sum(a.download_finish_rate) as download_finish_rate,
+       sum(a.click_install) as click_install,
+       sum(a.install_finish) as install_finish,
+       sum(a.install_finish_cost) as install_finish_cost,
+       sum(a.install_finish_rate) as install_finish_rate,
+       sum(a.active) as active,
+       sum(a.active_cost) as active_cost,
+       sum(a.active_rate) as active_rate,
+       sum(a.register) as register,
+       sum(a.active_register_cost) as active_register_cost,
+       sum(a.active_register_rate)as active_register_rate,
+       sum(a.next_day_open) as next_day_open,
+       sum(a.next_day_open_cost)as next_day_open_cost,
+       sum(a.next_day_open_rate) as next_day_open_rate,
+       sum(a.attribution_next_day_open_cnt)as attribution_next_day_open_cost,
+       sum(a.attribution_next_day_open_cost)as attribution_next_day_open_cost  ,
+       sum(a.attribution_next_day_open_rate)as attribution_next_day_open_cost    ,
+       sum(a.game_addiction) as game_addiction,
+       sum(a.game_addiction_cost) as game_addiction_cost,
+       sum(a.game_addiction_rate) as game_addiction_rate,
+       sum(a.pay_count)as pay_count,
+       sum(a.active_pay_cost) as active_pay_cost,
+       sum(a.active_pay_rate) as active_pay_rate,
+       sum(a.loan_completion)as loan_completion,
+       sum(a.loan_completion_cost) as loan_completion_cost,
+       sum(a.loan_completion_rate) as loan_completion_rate,
+       sum(a.pre_loan_credit)as pre_loan_credit,
+       sum(a.pre_loan_credit_cost) as pre_loan_credit_cost,
+       sum(a.loan_credit)as loan_credit,
+       sum(a.loan_credit_cost)as loan_credit_cost,
+       sum(a.loan_credit_rate) as loan_credit_rate,
+       sum(a.in_app_uv)as in_app_uv,
+       sum(a.in_app_detail_uv)as in_app_detail_uv,
+       sum(a.in_app_cart)as in_app_cart,
+       sum(a.in_app_pay)as in_app_pay,
+       sum(a.in_app_order)as in_app_order,
+       sum(a.attribution_game_pay_7d_count)as attribution_game_pay_7d_count,
+       sum(a.attribution_game_pay_7d_cost)as attribution_game_pay_7d_cost,
+       sum(a.attribution_active_pay_7d_per_count)as attribution_active_pay_7d_count,
+       sum(a.game_pay_cost)as game_pay_cost,
+       sum(a.game_pay_count)as game_pay_count,
+       sum(a.phone)as phone,
+       sum(a.form)as form,
+       sum(a.map_search)as map_search,
+       sum(a.button)as button,
+       sum(a.view)as view,
+       sum(a.download)as download,
+       sum(a.qq)as qq,
+       sum(a.lottery)as lottery,
+       sum(a.vote)as vote,
+       sum(a.message)as message ,
+       sum(a.redirect)as redirect,
+       sum(a.shopping) as shopping,
+       sum(a.consult)as consult,
+       sum(a.wechat)as wechat,
+       sum(a.phone_confirm)as phone_confirm,
+       sum(a.phone_connect)as phone_connect,
+       sum(a.consult_effective)as consult_effective,
+       sum(a.coupon)as coupon,
+       sum(a.coupon_single_page)as coupon_single_page,
+       sum(a.redirect_to_shop)as redirect_to_shop,
+       sum(a.poi_collect)as poi_collect,
+       sum(a.poi_address_click)as poi_address_click,
+       sum(a.luban_order_cnt)as luban_order_cnt,
+       sum(a.luban_order_stat_amount)as luban_order_stat_amount,
+       sum(a.luban_order_roi)as luban_order_roi,
+       sum(a.luban_live_enter_cnt)as luban_live_enter_cnt,
+       sum(a.live_watch_one_minute_count)as live_watch_one_minute_count,
+       sum(a.luban_live_follow_cnt)as luban_live_follow_cnt,
+       sum(a.live_fans_club_join_cnt)as live_fans_club_join_cnt,
+       sum(a.luban_live_comment_cnt)as luban_live_comment_cnt,
+       sum(a.luban_live_share_cnt)as luban_live_share_cnt,
+       sum(a.luban_live_gift_cnt)as luban_live_gift_cnt,
+       sum(a.luban_live_gift_amount)as luban_live_gift_amount,
+       sum(a.luban_live_slidecart_click_cnt)as luban_live_slidecart_click_cnt,
+       sum(a.luban_live_click_product_cnt)as luban_live_click_product_cnt,
+       sum(a.luban_live_pay_order_count)as luban_live_pay_order_count,
+       sum(a.luban_live_pay_order_stat_cost)as luban_live_pay_order_stat_cost,
+       sum(a.live_component_click_count)as live_component_click_count,
+       sum(a.live_component_click_cost)as live_component_click_cost,
+       sum(a.live_component_click_rate)as live_component_click_rate,
+       sum(a.wechat_login_count)as wechat_login_count,
+       sum(a.attribution_wechat_login_30d_count)as attribution_wechat_login_30d_count,
+       sum(a.wechat_login_cost)as wechat_login_cost,
+       sum(a.attribution_wechat_login_30d_cost)as attribution_wechat_login_30d_cost,
+       sum(a.wechat_first_pay_count)as wechat_first_pay_count,
+       sum(a.attribution_wechat_first_pay_30d_count)as attribution_wechat_first_pay_30d_count,
+       sum(a.wechat_first_pay_cost)as wechat_first_pay_cost,
+       sum(a.attribution_wechat_first_pay_30d_cost)as attribution_wechat_first_pay_30d_cost,
+       sum(a.wechat_first_pay_rate)as wechat_first_pay_rate,
+       sum(a.attribution_wechat_first_pay_30d_rate)as attribution_wechat_first_pay_30d_rate,
+       sum(a.wechat_pay_amount)as wechat_pay_amount,
+       sum(a.attribution_wechat_pay_30d_amount)as attribution_wechat_pay_30d_amount,
+       sum(a.attribution_wechat_pay_30d_roi)as attribution_wechat_pay_30d_roi,
+       sum(a.phone_effective)as phone_effective,
+       sum(a.total_play)as total_play,
+       sum(a.valid_play)as valid_play,
+       sum(a.valid_play_cost)as valid_play_cost,
+       sum(a.valid_play_rate)as valid_play_rate,
+       sum(a.play_25_feed_break)as play_25_feed_break,
+       sum(a.play_50_feed_break)as play_50_feed_break,
+       sum(a.play_75_feed_break)as play_75_feed_break,
+       sum(a.play_100_feed_break)as play_100_feed_break,
+       sum(a.average_play_time_per_play)as average_play_time_per_play,
+       sum(a.play_over_rate)as play_over_rate,
+       sum(a.wifi_play_rate)as wifi_play_rate,
+       sum(a.wifi_play)as wifi_play,
+       sum(a.play_duration_sum)as play_duration_sum,
+       sum(a.advanced_creative_phone_click)as advanced_creative_phone_click,
+       sum(a.advanced_creative_counsel_click)as advanced_creative_counsel_click,
+       sum(a.advanced_creative_form_click)as advanced_creative_form_click,
+       sum(a.advanced_creative_coupon_addition)as advanced_creative_coupon_addition,
+       sum(a.advanced_creative_form_submit)as advanced_creative_form_submit,
+       sum(a.card_show)as card_show,
+       sum(a.share)as share,
+       sum(a.comment)as comment,
+       sum(a.like)as likes,
+       sum(a.follow)as follow,
+       sum(a.home_visited)as home_visited,
+       sum(a.ies_challenge_click)as ies_challenge_click,
+       sum(a.ies_music_click)as ies_music_click,
+       sum(a.location_click)as location_click,
+       sum(a.message_action)as message_action,
+       sum(a.click_landing_page)as click_landing_page,
+       sum(a.click_shopwindow)as click_shopwindow,
+       sum(a.click_website)as click_website,
+       sum(a.click_download)as click_download,
+       sum(a.click_call_dy)as click_call_dy,
+       sum(a.submit_certification_count)as submit_certification_count,
+       sum(a.approval_count)as approval_count,
+       sum(a.first_rental_order_count)as first_rental_order_count,
+       sum(a.first_order_count)as first_order_count,
+       sum(a.commute_first_pay_count)as commute_first_pay_count,
+              sum(a.attribution_active_pay_intra_one_day_roi)as attribution_active_pay_intra_one_day_roi,
+sum(a.attribution_active_pay_intra_one_day_amount)as attribution_active_pay_intra_one_day_amount,
+sum(a.attribution_active_pay_intra_one_day_rate)as attribution_active_pay_intra_one_day_rate,
+sum(a.attribution_active_pay_intra_one_day_cost)as attribution_active_pay_intra_one_day_cost,
+sum(attribution_active_pay_intra_one_day_count)as attribution_active_pay_intra_one_day_count,
+sum(a.stat_pay_amount)as stat_pay_amount
+from `media_api`.bytedance_advertiser_report_hourly a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime = {start_time} 
+group by a.advertiser_id
+order by account_id limit {num},20000;
+        """.format(start_time=self.start_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AdvertiserSumDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 127 - 0
Apietl/bytedance_etl/report/AppBytedanceAgentReport.py

@@ -0,0 +1,127 @@
+# Author zhaohailiang
+# coding=utf-8
+# @Time    : 2021/12/20 2:22 下午
+# @Site    :
+# @File    : AppBytedanceAgentReport.py
+# @Software: PyCharm
+# @contact: zhaohailiang@c-top.com.cn
+# @Tel 18201631162
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceAgentReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+
+    def taskHandler(self):
+        try:
+            sql = """
+    SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+replace into application.app_bytedance_agent_report_d
+(account_id, stat_datetime, project_id, product_id, advertiser_id, company_id, account_name, project_name,
+ advertiser_name, company_name, sale_id, user_id, org_code, agent_id, agent_name, second_industry, first_industry,
+ account_source, audit_pass_time, account_status, account_company_id, account_company_name, register_time, cost,
+ cash_cost, bid_cost, brand_cost, grants_cost, cpc_cost, cpm_cost, ocpc_cost, cpa_cost, ocpm_cost, cpv_cost, cpt_cost,
+ gd_cost, `show`, cpm, click, ctr, cpc, transfer_count, today_cost, transfer_amount, history_cost, register_days,
+ first_cost_time)
+select report.advertiser_id as account_id,
+       report.stat_datetime,
+       project_id,
+       product_id,
+       bei.advertiser_id,
+       bei.company_id,
+       account_name,
+       project_name,
+       bei.advertiser_name,
+       bei.company_name,
+       sale_id,
+       user_id,
+       org_code,
+       agent_id,
+       agent_name,
+       second_industry,
+       first_industry,
+       account_source,
+       audit_pass_time,
+       account_status,
+       report.company_id    as account_company_id,
+       report.company_name  as account_company_name,
+       register_time,
+       cost,
+       cash_cost,
+       bid_cost,
+       brand_cost,
+       grants_cost,
+       cpc_cost,
+       cpm_cost,
+       ocpc_cost,
+       cpa_cost,
+       ocpm_cost,
+       cpv_cost,
+       cpt_cost,
+       gd_cost,
+       `show`,
+       cpm,
+       click,
+       ctr,
+       cpc,
+       transfer_count,
+       today_cost,
+       transfer_amount,
+       history_cost,
+       register_days,
+       first_cost_time
+from media_api.bytedance_agent_report_daily report
+         LEFT JOIN (
+    SELECT cua.account_id  id,
+           cua.account_id  account_id,
+           cp.id           project_id,
+           product.id      product_id,
+           ca.id           advertiser_id,
+           uc.company_id   company_id,
+           cua.auth_name   account_name,
+           cp.project_name project_name,
+           ca.`name`       advertiser_name,
+           uc.company_name company_name,
+           cp.sale_id      sale_id,
+           sale.realname   sale_name,
+           su.id           user_id,
+           su.realname     user_name,
+           su.leader_id,
+           lea.realname    leader_name,
+           su.org_code     org_code
+    FROM `jeecg-boot`.ctop_user_allocation cua
+             LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+             LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+             LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+             LEFT JOIN `jeecg-boot`.sys_user sale ON sale.id = cp.sale_id
+             LEFT JOIN `jeecg-boot`.sys_user lea ON lea.id = su.leader_id
+             LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+             LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+    WHERE cua.account_id IS NOT NULL) bei
+                   on report.advertiser_id = bei.account_id
+where report.stat_datetime={date_time}
+
+ """.format(date_time=self.date_time)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceAgentReport().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")

+ 309 - 0
Apietl/bytedance_etl/report/CampaignDailyReport.py

@@ -0,0 +1,309 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : CampaignDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class CampaignDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_campaign_report_daily where 
+            stat_datetime between {start_time} and {end_time}""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+          replace into application.bytedance_campaign_report_daily_dw(account_id, campaign_id, campaign_name, stat_datetime,
+                                                            project_id, product_id, advertiser_id, company_id,
+                                                            account_name, project_name, advertiser_name, company_name,
+                                                            sale_id, user_id, org_code, cost, `show`, avg_show_cost,
+                                                            click, avg_click_cost, ctr, `convert`, convert_cost,
+                                                            convert_rate, deep_convert, deep_convert_cost,
+                                                            deep_convert_rate, attribution_convert,
+                                                            attribution_convert_cost, attribution_deep_convert,
+                                                            attribution_deep_convert_cost, download_start,
+                                                            download_start_cost, download_start_rate, download_finish,
+                                                            download_finish_cost, download_finish_rate, click_install,
+                                                            install_finish, install_finish_cost, install_finish_rate,
+                                                            active, active_cost, active_rate, register,
+                                                            active_register_cost, active_register_rate, next_day_open,
+                                                            next_day_open_cost, next_day_open_rate,
+                                                            attribution_next_day_open_cnt,
+                                                            attribution_next_day_open_cost,
+                                                            attribution_next_day_open_rate, game_addiction,
+                                                            game_addiction_cost, game_addiction_rate, pay_count,
+                                                            active_pay_cost, active_pay_rate, loan_completion,
+                                                            loan_completion_cost, loan_completion_rate, pre_loan_credit,
+                                                            pre_loan_credit_cost, loan_credit, loan_credit_cost,
+                                                            loan_credit_rate, in_app_uv, in_app_detail_uv, in_app_cart,
+                                                            in_app_pay, in_app_order, attribution_game_pay_7d_count,
+                                                            attribution_game_pay_7d_cost,
+                                                            attribution_active_pay_7d_per_count, game_pay_cost,
+                                                            game_pay_count, phone, form, map_search, button, view,
+                                                            download, qq, lottery, vote, message, redirect, shopping,
+                                                            consult, wechat, phone_confirm, phone_connect,
+                                                            consult_effective, coupon, coupon_single_page,
+                                                            redirect_to_shop, poi_collect, poi_address_click,
+                                                            luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                            luban_live_enter_cnt, live_watch_one_minute_count,
+                                                            luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                            luban_live_comment_cnt, luban_live_share_cnt,
+                                                            luban_live_gift_cnt, luban_live_gift_amount,
+                                                            luban_live_slidecart_click_cnt,
+                                                            luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                            luban_live_pay_order_stat_cost, live_component_click_count,
+                                                            live_component_click_cost, live_component_click_rate,
+                                                            wechat_login_count, attribution_wechat_login_30d_count,
+                                                            wechat_login_cost, attribution_wechat_login_30d_cost,
+                                                            wechat_first_pay_count,
+                                                            attribution_wechat_first_pay_30d_count,
+                                                            wechat_first_pay_cost,
+                                                            attribution_wechat_first_pay_30d_cost,
+                                                            wechat_first_pay_rate,
+                                                            attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                            attribution_wechat_pay_30d_amount,
+                                                            attribution_wechat_pay_30d_roi, phone_effective, total_play,
+                                                            valid_play, valid_play_cost, valid_play_rate,
+                                                            play_25_feed_break, play_50_feed_break, play_75_feed_break,
+                                                            play_100_feed_break, average_play_time_per_play,
+                                                            play_over_rate, wifi_play_rate, wifi_play,
+                                                            play_duration_sum, advanced_creative_phone_click,
+                                                            advanced_creative_counsel_click,
+                                                            advanced_creative_form_click,
+                                                            advanced_creative_coupon_addition,
+                                                            advanced_creative_form_submit, card_show, share, comment,
+                                                            `like`, follow, home_visited, ies_challenge_click,
+                                                            ies_music_click, location_click, message_action,
+                                                            click_landing_page, click_shopwindow, click_website,
+                                                            click_download, click_call_dy, submit_certification_count,
+                                                            approval_count, first_rental_order_count, first_order_count,
+                                                            commute_first_pay_count)
+select a.advertiser_id account_id,
+       a.campaign_id,
+       a.campaign_name,
+       a.stat_datetime,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count
+from `media_api`.bytedance_campaign_report_daily a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+order by a.advertiser_id,a.campaign_id,a.stat_datetime
+limit {num},20000;
+        """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = CampaignDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 311 - 0
Apietl/bytedance_etl/report/CampaignHourlyReport.py

@@ -0,0 +1,311 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : CampaignHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class CampaignHourlyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def hourly_report(self):
+        try:
+            sql = """select count(1) from media_api.bytedance_campaign_report_hourly where 
+            stat_datetime between {start_time} and {end_time}""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+          replace into application.bytedance_campaign_report_hourly_dw(account_id, campaign_id, campaign_name,
+                                                            stat_datetime,hour,
+                                                            project_id, product_id, advertiser_id, company_id,
+                                                            account_name, project_name, advertiser_name, company_name,
+                                                            sale_id, user_id, org_code, cost, `show`, avg_show_cost,
+                                                            click, avg_click_cost, ctr, `convert`, convert_cost,
+                                                            convert_rate, deep_convert, deep_convert_cost,
+                                                            deep_convert_rate, attribution_convert,
+                                                            attribution_convert_cost, attribution_deep_convert,
+                                                            attribution_deep_convert_cost, download_start,
+                                                            download_start_cost, download_start_rate, download_finish,
+                                                            download_finish_cost, download_finish_rate, click_install,
+                                                            install_finish, install_finish_cost, install_finish_rate,
+                                                            active, active_cost, active_rate, register,
+                                                            active_register_cost, active_register_rate, next_day_open,
+                                                            next_day_open_cost, next_day_open_rate,
+                                                            attribution_next_day_open_cnt,
+                                                            attribution_next_day_open_cost,
+                                                            attribution_next_day_open_rate, game_addiction,
+                                                            game_addiction_cost, game_addiction_rate, pay_count,
+                                                            active_pay_cost, active_pay_rate, loan_completion,
+                                                            loan_completion_cost, loan_completion_rate, pre_loan_credit,
+                                                            pre_loan_credit_cost, loan_credit, loan_credit_cost,
+                                                            loan_credit_rate, in_app_uv, in_app_detail_uv, in_app_cart,
+                                                            in_app_pay, in_app_order, attribution_game_pay_7d_count,
+                                                            attribution_game_pay_7d_cost,
+                                                            attribution_active_pay_7d_per_count, game_pay_cost,
+                                                            game_pay_count, phone, form, map_search, button, view,
+                                                            download, qq, lottery, vote, message, redirect, shopping,
+                                                            consult, wechat, phone_confirm, phone_connect,
+                                                            consult_effective, coupon, coupon_single_page,
+                                                            redirect_to_shop, poi_collect, poi_address_click,
+                                                            luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                            luban_live_enter_cnt, live_watch_one_minute_count,
+                                                            luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                            luban_live_comment_cnt, luban_live_share_cnt,
+                                                            luban_live_gift_cnt, luban_live_gift_amount,
+                                                            luban_live_slidecart_click_cnt,
+                                                            luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                            luban_live_pay_order_stat_cost, live_component_click_count,
+                                                            live_component_click_cost, live_component_click_rate,
+                                                            wechat_login_count, attribution_wechat_login_30d_count,
+                                                            wechat_login_cost, attribution_wechat_login_30d_cost,
+                                                            wechat_first_pay_count,
+                                                            attribution_wechat_first_pay_30d_count,
+                                                            wechat_first_pay_cost,
+                                                            attribution_wechat_first_pay_30d_cost,
+                                                            wechat_first_pay_rate,
+                                                            attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                            attribution_wechat_pay_30d_amount,
+                                                            attribution_wechat_pay_30d_roi, phone_effective, total_play,
+                                                            valid_play, valid_play_cost, valid_play_rate,
+                                                            play_25_feed_break, play_50_feed_break, play_75_feed_break,
+                                                            play_100_feed_break, average_play_time_per_play,
+                                                            play_over_rate, wifi_play_rate, wifi_play,
+                                                            play_duration_sum, advanced_creative_phone_click,
+                                                            advanced_creative_counsel_click,
+                                                            advanced_creative_form_click,
+                                                            advanced_creative_coupon_addition,
+                                                            advanced_creative_form_submit, card_show, share, comment,
+                                                            `like`, follow, home_visited, ies_challenge_click,
+                                                            ies_music_click, location_click, message_action,
+                                                            click_landing_page, click_shopwindow, click_website,
+                                                            click_download, click_call_dy, submit_certification_count,
+                                                            approval_count, first_rental_order_count, first_order_count,
+                                                            commute_first_pay_count)
+select a.advertiser_id account_id,
+       a.campaign_id,
+       a.campaign_name,
+       a.stat_datetime,
+       a.hour,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count
+from `media_api`.bytedance_campaign_report_hourly a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+order by a.advertiser_id,a.campaign_id,a.stat_datetime,a.hour
+limit {num},20000;
+        """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = CampaignHourlyReport().hourly_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 309 - 0
Apietl/bytedance_etl/report/CreativeDailyReport.py

@@ -0,0 +1,309 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    :
+# @File    : CreativeDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class CreativeDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_creative_report_daily where 
+            stat_datetime between {start_time} and {end_time} group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """
+replace into application.bytedance_creative_report_daily_dw(account_id, campaign_id, campaign_name, ad_id, ad_name,
+                                                            creative_id, stat_datetime, project_id, product_id,
+                                                            advertiser_id, company_id, account_name, project_name,
+                                                            advertiser_name, company_name, sale_id, user_id, org_code,
+                                                            cost, `show`, avg_show_cost, click, avg_click_cost, ctr,
+                                                            `convert`, convert_cost, convert_rate, deep_convert,
+                                                            deep_convert_cost, deep_convert_rate, attribution_convert,
+                                                            attribution_convert_cost, attribution_deep_convert,
+                                                            attribution_deep_convert_cost, download_start,
+                                                            download_start_cost, download_start_rate, download_finish,
+                                                            download_finish_cost, download_finish_rate, click_install,
+                                                            install_finish, install_finish_cost, install_finish_rate,
+                                                            active, active_cost, active_rate, register,
+                                                            active_register_cost, active_register_rate, next_day_open,
+                                                            next_day_open_cost, next_day_open_rate,
+                                                            attribution_next_day_open_cnt,
+                                                            attribution_next_day_open_cost,
+                                                            attribution_next_day_open_rate, game_addiction,
+                                                            game_addiction_cost, game_addiction_rate, pay_count,
+                                                            active_pay_cost, active_pay_rate, loan_completion,
+                                                            loan_completion_cost, loan_completion_rate, pre_loan_credit,
+                                                            pre_loan_credit_cost, loan_credit, loan_credit_cost,
+                                                            loan_credit_rate, in_app_uv, in_app_detail_uv, in_app_cart,
+                                                            in_app_pay, in_app_order, attribution_game_pay_7d_count,
+                                                            attribution_game_pay_7d_cost,
+                                                            attribution_active_pay_7d_per_count, game_pay_cost,
+                                                            game_pay_count, phone, form, map_search, button, view,
+                                                            download, qq, lottery, vote, message, redirect, shopping,
+                                                            consult, wechat, phone_confirm, phone_connect,
+                                                            consult_effective, coupon, coupon_single_page,
+                                                            redirect_to_shop, poi_collect, poi_address_click,
+                                                            luban_order_cnt, luban_order_stat_amount, luban_order_roi,
+                                                            luban_live_enter_cnt, live_watch_one_minute_count,
+                                                            luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                            luban_live_comment_cnt, luban_live_share_cnt,
+                                                            luban_live_gift_cnt, luban_live_gift_amount,
+                                                            luban_live_slidecart_click_cnt,
+                                                            luban_live_click_product_cnt, luban_live_pay_order_count,
+                                                            luban_live_pay_order_stat_cost, live_component_click_count,
+                                                            live_component_click_cost, live_component_click_rate,
+                                                            wechat_login_count, attribution_wechat_login_30d_count,
+                                                            wechat_login_cost, attribution_wechat_login_30d_cost,
+                                                            wechat_first_pay_count,
+                                                            attribution_wechat_first_pay_30d_count,
+                                                            wechat_first_pay_cost,
+                                                            attribution_wechat_first_pay_30d_cost,
+                                                            wechat_first_pay_rate,
+                                                            attribution_wechat_first_pay_30d_rate, wechat_pay_amount,
+                                                            attribution_wechat_pay_30d_amount,
+                                                            attribution_wechat_pay_30d_roi, phone_effective, total_play,
+                                                            valid_play, valid_play_cost, valid_play_rate,
+                                                            play_25_feed_break, play_50_feed_break, play_75_feed_break,
+                                                            play_100_feed_break, average_play_time_per_play,
+                                                            play_over_rate, wifi_play_rate, wifi_play,
+                                                            play_duration_sum, advanced_creative_phone_click,
+                                                            advanced_creative_counsel_click,
+                                                            advanced_creative_form_click,
+                                                            advanced_creative_coupon_addition,
+                                                            advanced_creative_form_submit, card_show, share, comment,
+                                                            `like`, follow, home_visited, ies_challenge_click,
+                                                            ies_music_click, location_click, message_action,
+                                                            click_landing_page, click_shopwindow, click_website,
+                                                            click_download, click_call_dy, submit_certification_count,
+                                                            approval_count, first_rental_order_count, first_order_count,
+                                                            commute_first_pay_count)
+select a.advertiser_id account_id,
+       a.campaign_id,
+       a.campaign_name,
+       a.ad_id,
+       a.ad_name,
+       a.creative_id,
+       a.stat_datetime,
+       b.project_id,
+       b.product_id,
+       b.advertiser_id,
+       b.company_id,
+       b.account_name,
+       b.project_name,
+       b.advertiser_name,
+       b.company_name,
+       b.sale_id,
+       b.user_id,
+       b.org_code,
+       a.cost,
+       a.show,
+       a.avg_show_cost,
+       a.click,
+       a.avg_click_cost,
+       a.ctr,
+       a.convert,
+       a.convert_cost,
+       a.convert_rate,
+       a.deep_convert,
+       a.deep_convert_cost,
+       a.deep_convert_rate,
+       a.attribution_convert,
+       a.attribution_convert_cost,
+       a.attribution_deep_convert,
+       a.attribution_deep_convert_cost,
+       a.download_start,
+       a.download_start_cost,
+       a.download_start_rate,
+       a.download_finish,
+       a.download_finish_cost,
+       a.download_finish_rate,
+       a.click_install,
+       a.install_finish,
+       a.install_finish_cost,
+       a.install_finish_rate,
+       a.active,
+       a.active_cost,
+       a.active_rate,
+       a.register,
+       a.active_register_cost,
+       a.active_register_rate,
+       a.next_day_open,
+       a.next_day_open_cost,
+       a.next_day_open_rate,
+       a.attribution_next_day_open_cnt,
+       a.attribution_next_day_open_cost,
+       a.attribution_next_day_open_rate,
+       a.game_addiction,
+       a.game_addiction_cost,
+       a.game_addiction_rate,
+       a.pay_count,
+       a.active_pay_cost,
+       a.active_pay_rate,
+       a.loan_completion,
+       a.loan_completion_cost,
+       a.loan_completion_rate,
+       a.pre_loan_credit,
+       a.pre_loan_credit_cost,
+       a.loan_credit,
+       a.loan_credit_cost,
+       a.loan_credit_rate,
+       a.in_app_uv,
+       a.in_app_detail_uv,
+       a.in_app_cart,
+       a.in_app_pay,
+       a.in_app_order,
+       a.attribution_game_pay_7d_count,
+       a.attribution_game_pay_7d_cost,
+       a.attribution_active_pay_7d_per_count,
+       a.game_pay_cost,
+       a.game_pay_count,
+       a.phone,
+       a.form,
+       a.map_search,
+       a.button,
+       a.view,
+       a.download,
+       a.qq,
+       a.lottery,
+       a.vote,
+       a.message,
+       a.redirect,
+       a.shopping,
+       a.consult,
+       a.wechat,
+       a.phone_confirm,
+       a.phone_connect,
+       a.consult_effective,
+       a.coupon,
+       a.coupon_single_page,
+       a.redirect_to_shop,
+       a.poi_collect,
+       a.poi_address_click,
+       a.luban_order_cnt,
+       a.luban_order_stat_amount,
+       a.luban_order_roi,
+       a.luban_live_enter_cnt,
+       a.live_watch_one_minute_count,
+       a.luban_live_follow_cnt,
+       a.live_fans_club_join_cnt,
+       a.luban_live_comment_cnt,
+       a.luban_live_share_cnt,
+       a.luban_live_gift_cnt,
+       a.luban_live_gift_amount,
+       a.luban_live_slidecart_click_cnt,
+       a.luban_live_click_product_cnt,
+       a.luban_live_pay_order_count,
+       a.luban_live_pay_order_stat_cost,
+       a.live_component_click_count,
+       a.live_component_click_cost,
+       a.live_component_click_rate,
+       a.wechat_login_count,
+       a.attribution_wechat_login_30d_count,
+       a.wechat_login_cost,
+       a.attribution_wechat_login_30d_cost,
+       a.wechat_first_pay_count,
+       a.attribution_wechat_first_pay_30d_count,
+       a.wechat_first_pay_cost,
+       a.attribution_wechat_first_pay_30d_cost,
+       a.wechat_first_pay_rate,
+       a.attribution_wechat_first_pay_30d_rate,
+       a.wechat_pay_amount,
+       a.attribution_wechat_pay_30d_amount,
+       a.attribution_wechat_pay_30d_roi,
+       a.phone_effective,
+       a.total_play,
+       a.valid_play,
+       a.valid_play_cost,
+       a.valid_play_rate,
+       a.play_25_feed_break,
+       a.play_50_feed_break,
+       a.play_75_feed_break,
+       a.play_100_feed_break,
+       a.average_play_time_per_play,
+       a.play_over_rate,
+       a.wifi_play_rate,
+       a.wifi_play,
+       a.play_duration_sum,
+       a.advanced_creative_phone_click,
+       a.advanced_creative_counsel_click,
+       a.advanced_creative_form_click,
+       a.advanced_creative_coupon_addition,
+       a.advanced_creative_form_submit,
+       a.card_show,
+       a.share,
+       a.comment,
+       a.like,
+       a.follow,
+       a.home_visited,
+       a.ies_challenge_click,
+       a.ies_music_click,
+       a.location_click,
+       a.message_action,
+       a.click_landing_page,
+       a.click_shopwindow,
+       a.click_website,
+       a.click_download,
+       a.click_call_dy,
+       a.submit_certification_count,
+       a.approval_count,
+       a.first_rental_order_count,
+       a.first_order_count,
+       a.commute_first_pay_count
+from `media_api`.bytedance_creative_report_daily a
+         left join
+     (select cua.account_id  id,
+             cua.account_id  account_id,
+             cp.id           project_id,
+             product.id      product_id,
+             ca.id           advertiser_id,
+             uc.company_id   company_id,
+             cua.auth_name   account_name,
+             cp.project_name project_name,
+             ca.name         advertiser_name,
+             uc.company_name company_name,
+             cp.sale_id      sale_id,
+             su.id           user_id,
+             su.org_code     org_code
+      from `jeecg-boot`.ctop_user_allocation cua
+               left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+               left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+               left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+               left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+               left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+      where cua.account_id is not null) b
+     on a.advertiser_id = b.account_id
+where a.stat_datetime between {start_time} and {end_time}
+and a.advertiser_id={account_id}
+        """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = CreativeDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 8 - 0
Apietl/bytedance_etl/report/CreativeHourlyReport.py

@@ -0,0 +1,8 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/10/21 5:53 下午
+# @Site    : 
+# @File    : CreativeHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553

+ 349 - 0
Apietl/bytedance_etl/report/MaterialImageDailyReport.py

@@ -0,0 +1,349 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialImageDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class MateirialImageDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_material_report_daily where
+             stat_datetime between {start_time} and {end_time} and image_mode in
+      ('CREATIVE_IMAGE_MODE_SMALL', 'CREATIVE_IMAGE_MODE_LARGE', 'CREATIVE_IMAGE_MODE_LARGE_VERTICAL',
+       'CREATIVE_IMAGE_MODE_GROUP', 'CREATIVE_IMAGE_MODE_GIF', 'TOUTIAO_SEARCH_AD_IMAGE', 'SEARCH_AD_SMALL_IMAGE',
+       'CREATIVE_IMAGE_MODE_UNION_SPLASH') 
+           group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """
+replace into application.bytedance_material_image_report_daily_dw(account_id, material_id, image_mode, inventory,
+                                                                  stat_datetime, project_id, product_id, advertiser_id,
+                                                                  company_id, account_name, project_name,
+                                                                  advertiser_name, company_name, sale_id, user_id,
+                                                                  image_name,
+                                                                  cost, `show`, avg_show_cost, click,
+                                                                  avg_click_cost, ctr, `convert`, convert_cost,
+                                                                  convert_rate, deep_convert, deep_convert_cost,
+                                                                  deep_convert_rate, attribution_convert,
+                                                                  attribution_convert_cost, attribution_deep_convert,
+                                                                  attribution_deep_convert_cost, download_start,
+                                                                  download_start_cost, download_start_rate,
+                                                                  download_finish, download_finish_cost,
+                                                                  download_finish_rate, click_install, install_finish,
+                                                                  install_finish_cost, install_finish_rate, active,
+                                                                  active_cost, active_rate, register,
+                                                                  active_register_cost, active_register_rate,
+                                                                  active_pay_amount, next_day_open, next_day_open_cost,
+                                                                  next_day_open_rate, attribution_next_day_open_cnt,
+                                                                  attribution_next_day_open_cost,
+                                                                  attribution_next_day_open_rate, game_addiction,
+                                                                  game_addiction_cost, game_addiction_rate, pay_count,
+                                                                  active_pay_cost, active_pay_rate, loan_completion,
+                                                                  loan_completion_cost, loan_completion_rate,
+                                                                  pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                                  loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                                  in_app_detail_uv, in_app_cart, in_app_pay,
+                                                                  in_app_order, attribution_game_pay_7d_count,
+                                                                  attribution_game_pay_7d_cost,
+                                                                  attribution_active_pay_7d_per_count, game_pay_cost,
+                                                                  game_pay_count, phone, form, map_search, button, view,
+                                                                  download, qq, lottery, vote, message, redirect,
+                                                                  shopping, consult, wechat, phone_confirm,
+                                                                  phone_connect, consult_effective, coupon,
+                                                                  coupon_single_page, redirect_to_shop, poi_collect,
+                                                                  poi_address_click, luban_order_cnt,
+                                                                  luban_order_stat_amount, luban_order_roi,
+                                                                  luban_live_enter_cnt, live_watch_one_minute_count,
+                                                                  luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                                  luban_live_comment_cnt, luban_live_share_cnt,
+                                                                  luban_live_gift_cnt, luban_live_gift_amount,
+                                                                  luban_live_slidecart_click_cnt,
+                                                                  luban_live_click_product_cnt,
+                                                                  luban_live_pay_order_count,
+                                                                  luban_live_pay_order_stat_cost, wechat_login_count,
+                                                                  attribution_wechat_login_30d_count, wechat_login_cost,
+                                                                  attribution_wechat_login_30d_cost,
+                                                                  wechat_first_pay_count,
+                                                                  attribution_wechat_first_pay_30d_count,
+                                                                  wechat_first_pay_cost,
+                                                                  attribution_wechat_first_pay_30d_cost,
+                                                                  wechat_first_pay_rate,
+                                                                  attribution_wechat_first_pay_30d_rate,
+                                                                  wechat_pay_amount, attribution_wechat_pay_30d_amount,
+                                                                  attribution_wechat_pay_30d_roi, phone_effective,
+                                                                  total_play, valid_play, valid_play_cost,
+                                                                  valid_play_rate, play_25_feed_break,
+                                                                  play_50_feed_break, play_75_feed_break,
+                                                                  play_100_feed_break, average_play_time_per_play,
+                                                                  play_over_rate, wifi_play_rate, wifi_play,
+                                                                  play_duration_sum, advanced_creative_phone_click,
+                                                                  advanced_creative_counsel_click,
+                                                                  advanced_creative_form_click,
+                                                                  advanced_creative_coupon_addition,
+                                                                  advanced_creative_form_submit, card_show, share,
+                                                                  comment, `like`, follow, home_visited,
+                                                                  ies_challenge_click, ies_music_click, location_click,
+                                                                  message_action, click_landing_page, click_shopwindow,
+                                                                  click_website, click_download, click_call_dy, cpc,
+                                                                  cpm, cpa, interact_per_cost, convert_show_rate,
+                                                                  play_duration_10s_rate, play_50_feed_break_rate,
+                                                                  play_25_feed_break_rate, play_duration_5s_rate,
+                                                                  play_duration_2s_rate, average_play_progress,
+                                                                  play_100_feed_break_rate, play_duration_3s_rate,
+                                                                  play_75_feed_break_rate, play_duration,
+                                                                  play_duration_3s, play_duration_2s, play_duration_10s,
+                                                                  average_video_play, avg_rank,
+                                                                  submit_certification_count, approval_count,
+                                                                  first_order_count, first_rental_order_count,
+                                                                  commute_first_pay_count,image_url,signature)
+
+select report.advertiser_id account_id,      #'账户ID'
+       report.material_id,                   #头条素材id
+       report.image_mode,                    #素材类型
+       report.inventory,                     #广告位
+       report.stat_datetime,                 #数据起始时间
+       cua.project_id,                       #项目id
+       product.id           product_id,      #项目中产品id
+       cua.advertiser_id,                    #广告主id
+       uc.company_id,                        #运营人的公司id
+       cua.auth_name        account_name,    #账户名称
+       cp.project_name,                      #项目名称
+       ca.name              advertiser_name, #广告主名称
+       uc.company_name,                      #运营人的公司名称
+       cp.sale_id,                           #项目里的销售id
+       cua.user_id,                          #运营id
+       image.filename,
+       report.`cost`,
+       report.`show`,
+       report.`avg_show_cost`,
+       report.`click`,
+       report.`avg_click_cost`,
+       report.`ctr`,
+       report.`convert`,
+       report.`convert_cost`,
+       report.`convert_rate`,
+       report.`deep_convert`,
+       report.`deep_convert_cost`,
+       report.`deep_convert_rate`,
+       report.`attribution_convert`,
+       report.`attribution_convert_cost`,
+       report.`attribution_deep_convert`,
+       report.`attribution_deep_convert_cost`,
+       report.`download_start`,
+       report.`download_start_cost`,
+       report.`download_start_rate`,
+       report.`download_finish`,
+       report.`download_finish_cost`,
+       report.`download_finish_rate`,
+       report.`click_install`,
+       report.`install_finish`,
+       report.`install_finish_cost`,
+       report.`install_finish_rate`,
+       report.`active`,
+       report.`active_cost`,
+       report.`active_rate`,
+       report.`register`,
+       report.`active_register_cost`,
+       report.`active_register_rate`,
+       report.`active_pay_amount`,
+       report.`next_day_open`,
+       report.`next_day_open_cost`,
+       report.`next_day_open_rate`,
+       report.`attribution_next_day_open_cnt`,
+       report.`attribution_next_day_open_cost`,
+       report.`attribution_next_day_open_rate`,
+       report.`game_addiction`,
+       report.`game_addiction_cost`,
+       report.`game_addiction_rate`,
+       report.`pay_count`,
+       report.`active_pay_cost`,
+       report.`active_pay_rate`,
+       report.`loan_completion`,
+       report.`loan_completion_cost`,
+       report.`loan_completion_rate`,
+       report.`pre_loan_credit`,
+       report.`pre_loan_credit_cost`,
+       report.`loan_credit`,
+       report.`loan_credit_cost`,
+       report.`loan_credit_rate`,
+       report.`in_app_uv`,
+       report.`in_app_detail_uv`,
+       report.`in_app_cart`,
+       report.`in_app_pay`,
+       report.`in_app_order`,
+       report.`attribution_game_pay_7d_count`,
+       report.`attribution_game_pay_7d_cost`,
+       report.`attribution_active_pay_7d_per_count`,
+       report.`game_pay_cost`,
+       report.`game_pay_count`,
+       report.`phone`,
+       report.`form`,
+       report.`map_search`,
+       report.`button`,
+       report.`view`,
+       report.`download`,
+       report.`qq`,
+       report.`lottery`,
+       report.`vote`,
+       report.`message`,
+       report.`redirect`,
+       report.`shopping`,
+       report.`consult`,
+       report.`wechat`,
+       report.`phone_confirm`,
+       report.`phone_connect`,
+       report.`consult_effective`,
+       report.`coupon`,
+       report.`coupon_single_page`,
+       report.`redirect_to_shop`,
+       report.`poi_collect`,
+       report.`poi_address_click`,
+       report.`luban_order_cnt`,
+       report.`luban_order_stat_amount`,
+       report.`luban_order_roi`,
+       report.`luban_live_enter_cnt`,
+       report.`live_watch_one_minute_count`,
+       report.`luban_live_follow_cnt`,
+       report.`live_fans_club_join_cnt`,
+       report.`luban_live_comment_cnt`,
+       report.`luban_live_share_cnt`,
+       report.`luban_live_gift_cnt`,
+       report.`luban_live_gift_amount`,
+       report.`luban_live_slidecart_click_cnt`,
+       report.`luban_live_click_product_cnt`,
+       report.`luban_live_pay_order_count`,
+       report.`luban_live_pay_order_stat_cost`,
+       report.`wechat_login_count`,
+       report.`attribution_wechat_login_30d_count`,
+       report.`wechat_login_cost`,
+       report.`attribution_wechat_login_30d_cost`,
+       report.`wechat_first_pay_count`,
+       report.`attribution_wechat_first_pay_30d_count`,
+       report.`wechat_first_pay_cost`,
+       report.`attribution_wechat_first_pay_30d_cost`,
+       report.`wechat_first_pay_rate`,
+       report.`attribution_wechat_first_pay_30d_rate`,
+       report.`wechat_pay_amount`,
+       report.`attribution_wechat_pay_30d_amount`,
+       report.`attribution_wechat_pay_30d_roi`,
+       report.`phone_effective`,
+       report.`total_play`,
+       report.`valid_play`,
+       report.`valid_play_cost`,
+       report.`valid_play_rate`,
+       report.`play_25_feed_break`,
+       report.`play_50_feed_break`,
+       report.`play_75_feed_break`,
+       report.`play_100_feed_break`,
+       report.`average_play_time_per_play`,
+       report.`play_over_rate`,
+       report.`wifi_play_rate`,
+       report.`wifi_play`,
+       report.`play_duration_sum`,
+       report.`advanced_creative_phone_click`,
+       report.`advanced_creative_counsel_click`,
+       report.`advanced_creative_form_click`,
+       report.`advanced_creative_coupon_addition`,
+       report.`advanced_creative_form_submit`,
+       report.`card_show`,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.`follow`,
+       report.`home_visited`,
+       report.`ies_challenge_click`,
+       report.`ies_music_click`,
+       report.`location_click`,
+       report.`message_action`,
+       report.`click_landing_page`,
+       report.`click_shopwindow`,
+       report.`click_website`,
+       report.`click_download`,
+       report.`click_call_dy`,
+       report.`cpc`,
+       report.`cpm`,
+       report.`cpa`,
+       report.`interact_per_cost`,
+       report.`convert_show_rate`,
+       report.`play_duration_10s_rate`,
+       report.`play_50_feed_break_rate`,
+       report.`play_25_feed_break_rate`,
+       report.`play_duration_5s_rate`,
+       report.`play_duration_2s_rate`,
+       report.`average_play_progress`,
+       report.`play_100_feed_break_rate`,
+       report.`play_duration_3s_rate`,
+       report.`play_75_feed_break_rate`,
+       report.`play_duration`,
+       report.`play_duration_3s`,
+       report.`play_duration_2s`,
+       report.`play_duration_10s`,
+       report.`average_video_play`,
+       report.`avg_rank`,
+       report.`submit_certification_count`,
+       report.`approval_count`,
+       report.`first_order_count`,
+       report.`first_rental_order_count`,
+       report.`commute_first_pay_count`,
+       image.image_url,
+       image.signature
+from media_api.bytedance_material_report_daily report
+         left join
+     `media_api`.bytedance_file_image_get image
+     on report.material_id = image.material_id
+         and report.advertiser_id = image.advertiser_id
+         left join `jeecg-boot`.ctop_user_allocation cua
+                   on report.advertiser_id = cua.account_id
+         left join `jeecg-boot`.ctop_advertiser ca
+                   on cua.advertiser_id = ca.id
+         left join `jeecg-boot`.ctop_project cp
+                   on cua.project_id = cp.id
+         left join `jeecg-boot`.user_company uc
+                   on cua.user_id = uc.user_id
+         left join `jeecg-boot`.ctop_product product
+                   on cp.product_id = product.id
+         left join `jeecg-boot`.sys_user su
+                   on cua.user_id = su.id
+where report.image_mode in
+      ('CREATIVE_IMAGE_MODE_SMALL', 'CREATIVE_IMAGE_MODE_LARGE', 'CREATIVE_IMAGE_MODE_LARGE_VERTICAL',
+       'CREATIVE_IMAGE_MODE_GROUP', 'CREATIVE_IMAGE_MODE_GIF', 'TOUTIAO_SEARCH_AD_IMAGE', 'SEARCH_AD_SMALL_IMAGE',
+       'CREATIVE_IMAGE_MODE_UNION_SPLASH')
+  and report.stat_datetime between {start_time} and {end_time}
+  and report.advertiser_id = {account_id}
+ """.format( start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = MateirialImageDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 368 - 0
Apietl/bytedance_etl/report/MaterialVideoDailyReport.py

@@ -0,0 +1,368 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialVideoDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class MateirialVideoDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_material_report_daily where 
+         stat_datetime between {start_time} and {end_time} 
+         and image_mode in ('CREATIVE_IMAGE_MODE_VIDEO', 'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL', 'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+         group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+
+                try:
+                    sql = """
+                                    replace into application.bytedance_material_video_report_daily_dw(account_id, material_id, image_mode, inventory,
+                                                                                      stat_datetime, project_id, product_id, advertiser_id,
+                                                                                      company_id, account_name, project_name,
+                                                                                      advertiser_name, company_name, sale_id, user_id,
+                                                                                      clip_id, shot_id, plan_id, leader_id, org_code,
+                                                                                      signature, channel_type, clip_name, shot_name,
+                                                                                      plan_name, leader_name, video_name, channel_name,
+                                                                                      clip_company_id, cost, `show`, avg_show_cost, click,
+                                                                                      avg_click_cost, ctr, `convert`, convert_cost,
+                                                                                      convert_rate, deep_convert, deep_convert_cost,
+                                                                                      deep_convert_rate, attribution_convert,
+                                                                                      attribution_convert_cost, attribution_deep_convert,
+                                                                                      attribution_deep_convert_cost, download_start,
+                                                                                      download_start_cost, download_start_rate,
+                                                                                      download_finish, download_finish_cost,
+                                                                                      download_finish_rate, click_install, install_finish,
+                                                                                      install_finish_cost, install_finish_rate, active,
+                                                                                      active_cost, active_rate, register,
+                                                                                      active_register_cost, active_register_rate,
+                                                                                      active_pay_amount, next_day_open, next_day_open_cost,
+                                                                                      next_day_open_rate, attribution_next_day_open_cnt,
+                                                                                      attribution_next_day_open_cost,
+                                                                                      attribution_next_day_open_rate, game_addiction,
+                                                                                      game_addiction_cost, game_addiction_rate, pay_count,
+                                                                                      active_pay_cost, active_pay_rate, loan_completion,
+                                                                                      loan_completion_cost, loan_completion_rate,
+                                                                                      pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                                                      loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                                                      in_app_detail_uv, in_app_cart, in_app_pay,
+                                                                                      in_app_order, attribution_game_pay_7d_count,
+                                                                                      attribution_game_pay_7d_cost,
+                                                                                      attribution_active_pay_7d_per_count, game_pay_cost,
+                                                                                      game_pay_count, phone, form, map_search, button, view,
+                                                                                      download, qq, lottery, vote, message, redirect,
+                                                                                      shopping, consult, wechat, phone_confirm,
+                                                                                      phone_connect, consult_effective, coupon,
+                                                                                      coupon_single_page, redirect_to_shop, poi_collect,
+                                                                                      poi_address_click, luban_order_cnt,
+                                                                                      luban_order_stat_amount, luban_order_roi,
+                                                                                      luban_live_enter_cnt, live_watch_one_minute_count,
+                                                                                      luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                                                      luban_live_comment_cnt, luban_live_share_cnt,
+                                                                                      luban_live_gift_cnt, luban_live_gift_amount,
+                                                                                      luban_live_slidecart_click_cnt,
+                                                                                      luban_live_click_product_cnt,
+                                                                                      luban_live_pay_order_count,
+                                                                                      luban_live_pay_order_stat_cost, wechat_login_count,
+                                                                                      attribution_wechat_login_30d_count, wechat_login_cost,
+                                                                                      attribution_wechat_login_30d_cost,
+                                                                                      wechat_first_pay_count,
+                                                                                      attribution_wechat_first_pay_30d_count,
+                                                                                      wechat_first_pay_cost,
+                                                                                      attribution_wechat_first_pay_30d_cost,
+                                                                                      wechat_first_pay_rate,
+                                                                                      attribution_wechat_first_pay_30d_rate,
+                                                                                      wechat_pay_amount, attribution_wechat_pay_30d_amount,
+                                                                                      attribution_wechat_pay_30d_roi, phone_effective,
+                                                                                      total_play, valid_play, valid_play_cost,
+                                                                                      valid_play_rate, play_25_feed_break,
+                                                                                      play_50_feed_break, play_75_feed_break,
+                                                                                      play_100_feed_break, average_play_time_per_play,
+                                                                                      play_over_rate, wifi_play_rate, wifi_play,
+                                                                                      play_duration_sum, advanced_creative_phone_click,
+                                                                                      advanced_creative_counsel_click,
+                                                                                      advanced_creative_form_click,
+                                                                                      advanced_creative_coupon_addition,
+                                                                                      advanced_creative_form_submit, card_show, share,
+                                                                                      comment, `like`, follow, home_visited,
+                                                                                      ies_challenge_click, ies_music_click, location_click,
+                                                                                      message_action, click_landing_page, click_shopwindow,
+                                                                                      click_website, click_download, click_call_dy, cpc,
+                                                                                      cpm, cpa, interact_per_cost, convert_show_rate,
+                                                                                      play_duration_10s_rate, play_50_feed_break_rate,
+                                                                                      play_25_feed_break_rate, play_duration_5s_rate,
+                                                                                      play_duration_2s_rate, average_play_progress,
+                                                                                      play_100_feed_break_rate, play_duration_3s_rate,
+                                                                                      play_75_feed_break_rate, play_duration,
+                                                                                      play_duration_3s, play_duration_2s, play_duration_10s,
+                                                                                      average_video_play, avg_rank,
+                                                                                      submit_certification_count, approval_count,
+                                                                                      first_order_count, first_rental_order_count,
+                                                                                      commute_first_pay_count,video_url, cover_url,
+                                                                                     shot_leader_id, shot_leader_name)
+                            select
+                           report.advertiser_id account_id,   #'账户ID'
+                           report.material_id,                #头条素材id
+                           report.image_mode,                 #素材类型
+                           report.inventory,                  #广告位
+                           report.stat_datetime,              #数据起始时间
+                           cua.project_id,                    #项目id
+                           product.id product_id,             #项目中产品id
+                           cua.advertiser_id,                 #广告主id
+                           uc.company_id,                     #运营人的公司id
+                           cua.auth_name account_name,        #账户名称
+                           cp.project_name,                   #项目名称
+                           ca.name advertiser_name,           #广告主名称
+                           uc.company_name,                   #运营人的公司名称
+                           cp.sale_id,                        #项目里的销售id
+                           cua.user_id,                       #运营id
+                           cma.clip_id,                       #剪辑id
+                           cma.shot_id,                       #拍摄id
+                           cma.plan_id,                       #编导id
+                           cma.leader_id,                     #负责人id
+                           leader.org_code,                       #机构编码
+                           video.signature,                #素材id
+                           null chl_type,                       #渠道类型
+                           clip.realname clip_name,           #剪辑名称
+                           shot.realname shot_name,           #拍摄名称
+                           plan.realname plan_name,           #编导名称
+                           leader.realname lead_name,         #负责人名称
+                           case when cmi.material_name is not null then cmi.material_name else video.filename end as video_name, ##视频名称
+                           '' chl_name,                       #渠道名称
+                           up.company_id clip_company_id,     #剪辑公司
+                           report.`cost`,
+                           report.`show`,
+                           report.`avg_show_cost`,
+                           report.`click`,
+                           report.`avg_click_cost`,
+                           report.`ctr`,
+                           report.`convert`,
+                           report.`convert_cost`,
+                           report.`convert_rate`,
+                           report.`deep_convert`,
+                           report.`deep_convert_cost`,
+                           report.`deep_convert_rate`,
+                           report.`attribution_convert`,
+                           report.`attribution_convert_cost`,
+                           report.`attribution_deep_convert`,
+                           report.`attribution_deep_convert_cost`,
+                           report.`download_start`,
+                           report.`download_start_cost`,
+                           report.`download_start_rate`,
+                           report.`download_finish`,
+                           report.`download_finish_cost`,
+                           report.`download_finish_rate`,
+                           report.`click_install`,
+                           report.`install_finish`,
+                           report.`install_finish_cost`,
+                           report.`install_finish_rate`,
+                           report.`active`,
+                           report.`active_cost`,
+                           report.`active_rate`,
+                           report.`register`,
+                           report.`active_register_cost`,
+                           report.`active_register_rate`,
+                           report.`active_pay_amount`,
+                           report.`next_day_open`,
+                           report.`next_day_open_cost`,
+                           report.`next_day_open_rate`,
+                           report.`attribution_next_day_open_cnt`,
+                           report.`attribution_next_day_open_cost`,
+                           report.`attribution_next_day_open_rate`,
+                           report.`game_addiction`,
+                           report.`game_addiction_cost`,
+                           report.`game_addiction_rate`,
+                           report.`pay_count`,
+                           report.`active_pay_cost`,
+                           report.`active_pay_rate`,
+                           report.`loan_completion`,
+                           report.`loan_completion_cost`,
+                           report.`loan_completion_rate`,
+                           report.`pre_loan_credit`,
+                           report.`pre_loan_credit_cost`,
+                           report.`loan_credit`,
+                           report.`loan_credit_cost`,
+                           report.`loan_credit_rate`,
+                           report.`in_app_uv`,
+                           report.`in_app_detail_uv`,
+                           report.`in_app_cart`,
+                           report.`in_app_pay`,
+                           report.`in_app_order`,
+                           report.`attribution_game_pay_7d_count`,
+                           report.`attribution_game_pay_7d_cost`,
+                           report.`attribution_active_pay_7d_per_count`,
+                           report.`game_pay_cost`,
+                           report.`game_pay_count`,
+                           report.`phone`,
+                           report.`form`,
+                           report.`map_search`,
+                           report.`button`,
+                           report.`view`,
+                           report.`download`,
+                           report.`qq`,
+                           report.`lottery`,
+                           report.`vote`,
+                           report.`message`,
+                           report.`redirect`,
+                           report.`shopping`,
+                           report.`consult`,
+                           report.`wechat`,
+                           report.`phone_confirm`,
+                           report.`phone_connect`,
+                           report.`consult_effective`,
+                           report.`coupon`,
+                           report.`coupon_single_page`,
+                           report.`redirect_to_shop`,
+                           report.`poi_collect`,
+                           report.`poi_address_click`,
+                           report.`luban_order_cnt`,
+                           report.`luban_order_stat_amount`,
+                           report.`luban_order_roi`,
+                           report.`luban_live_enter_cnt`,
+                           report.`live_watch_one_minute_count`,
+                           report.`luban_live_follow_cnt`,
+                           report.`live_fans_club_join_cnt`,
+                           report.`luban_live_comment_cnt`,
+                           report.`luban_live_share_cnt`,
+                           report.`luban_live_gift_cnt`,
+                           report.`luban_live_gift_amount`,
+                           report.`luban_live_slidecart_click_cnt`,
+                           report.`luban_live_click_product_cnt`,
+                           report.`luban_live_pay_order_count`,
+                           report.`luban_live_pay_order_stat_cost`,
+                           report.`wechat_login_count`,
+                           report.`attribution_wechat_login_30d_count`,
+                           report.`wechat_login_cost`,
+                           report.`attribution_wechat_login_30d_cost`,
+                           report.`wechat_first_pay_count`,
+                           report.`attribution_wechat_first_pay_30d_count`,
+                           report.`wechat_first_pay_cost`,
+                           report.`attribution_wechat_first_pay_30d_cost`,
+                           report.`wechat_first_pay_rate`,
+                           report.`attribution_wechat_first_pay_30d_rate`,
+                           report.`wechat_pay_amount`,
+                           report.`attribution_wechat_pay_30d_amount`,
+                           report.`attribution_wechat_pay_30d_roi`,
+                           report.`phone_effective`,
+                           report.`total_play`,
+                           report.`valid_play`,
+                           report.`valid_play_cost`,
+                           report.`valid_play_rate`,
+                           report.`play_25_feed_break`,
+                           report.`play_50_feed_break`,
+                           report.`play_75_feed_break`,
+                           report.`play_100_feed_break`,
+                           report.`average_play_time_per_play`,
+                           report.`play_over_rate`,
+                           report.`wifi_play_rate`,
+                           report.`wifi_play`,
+                           report.`play_duration_sum`,
+                           report.`advanced_creative_phone_click`,
+                           report.`advanced_creative_counsel_click`,
+                           report.`advanced_creative_form_click`,
+                           report.`advanced_creative_coupon_addition`,
+                           report.`advanced_creative_form_submit`,
+                           report.`card_show`,
+                           report.`share`,
+                           report.`comment`,
+                           report.`like`,
+                           report.`follow`,
+                           report.`home_visited`,
+                           report.`ies_challenge_click`,
+                           report.`ies_music_click`,
+                           report.`location_click`,
+                           report.`message_action`,
+                           report.`click_landing_page`,
+                           report.`click_shopwindow`,
+                           report.`click_website`,
+                           report.`click_download`,
+                           report.`click_call_dy`,
+                           report.`cpc`,
+                           report.`cpm`,
+                           report.`cpa`,
+                           report.`interact_per_cost`,
+                           report.`convert_show_rate`,
+                           report.`play_duration_10s_rate`,
+                           report.`play_50_feed_break_rate`,
+                           report.`play_25_feed_break_rate`,
+                           report.`play_duration_5s_rate`,
+                           report.`play_duration_2s_rate`,
+                           report.`average_play_progress`,
+                           report.`play_100_feed_break_rate`,
+                           report.`play_duration_3s_rate`,
+                           report.`play_75_feed_break_rate`,
+                           report.`play_duration`,
+                           report.`play_duration_3s`,
+                           report.`play_duration_2s`,
+                           report.`play_duration_10s`,
+                           report.`average_video_play`,
+                           report.`avg_rank`,
+                           report.`submit_certification_count`,
+                           report.`approval_count`,
+                           report.`first_order_count`,
+                           report.`first_rental_order_count`,
+                           report.`commute_first_pay_count`,
+                           ifnull(concat('http://i.snssdk.com/video/code/1/toutiao/',video.id),'-') as video_url, cmi.cover_url,
+                           cma.shot_leader_id,
+                           cma.shot_leader_name
+                    from media_api.bytedance_material_report_daily report
+                    left join
+                    `media_api`.bytedance_file_video_get video
+                    on report.advertiser_id=video.advertiser_id and report.material_id=video.material_id
+                    left join (select  code,material_name,cover_url from `jeecg-boot`.ctop_material_info where status=1) cmi
+                    on video.signature=cmi.code
+                    left join `jeecg-boot`.ctop_material_ascription cma
+                    on video.signature=cma.material_id
+                    left join `jeecg-boot`.ctop_user_allocation cua
+                    on report.advertiser_id=cua.account_id
+                    left join `jeecg-boot`.ctop_advertiser ca
+                    on cua.advertiser_id = ca.id
+                    left join `jeecg-boot`.ctop_project cp
+                    on cua.project_id = cp.id
+                    left join `jeecg-boot`.user_company uc
+                    on cua.user_id=uc.user_id
+                    left join `jeecg-boot`.ctop_product product
+                    on cp.product_id=product.id
+                    left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+                    left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+                    left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+                    left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+                    left join `jeecg-boot`.user_company up on up.user_id=cma.clip_id where
+                    report.image_mode in ('CREATIVE_IMAGE_MODE_VIDEO', 'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL', 'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+                    and report.stat_datetime between {start_time} and {end_time}  and report.advertiser_id={account_id}
+                                    """.format(start_time=self.start_time, end_time=self.end_time,
+                                               account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = MateirialVideoDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 329 - 0
Apietl/bytedance_etl/report/NewMaterialImageDailyReport.py

@@ -0,0 +1,329 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : NewMaterialimageDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class NewMaterialimageDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_live_args()
+        self.start_time = self.parm
+
+    def daily_report(self):
+        try:
+            sql = """
+replace into application.bytedance_material_image_report_daily_dw(account_id, material_id, image_mode, inventory,
+                                                                  stat_datetime, project_id, product_id, advertiser_id,
+                                                                  company_id, account_name, project_name,
+                                                                  advertiser_name, company_name, sale_id, user_id,
+                                                                  image_name,
+                                                                  cost, `show`, avg_show_cost, click,
+                                                                  avg_click_cost, ctr, `convert`, convert_cost,
+                                                                  convert_rate, deep_convert, deep_convert_cost,
+                                                                  deep_convert_rate, attribution_convert,
+                                                                  attribution_convert_cost, attribution_deep_convert,
+                                                                  attribution_deep_convert_cost, download_start,
+                                                                  download_start_cost, download_start_rate,
+                                                                  download_finish, download_finish_cost,
+                                                                  download_finish_rate, click_install, install_finish,
+                                                                  install_finish_cost, install_finish_rate, active,
+                                                                  active_cost, active_rate, register,
+                                                                  active_register_cost, active_register_rate,
+                                                                  active_pay_amount, next_day_open, next_day_open_cost,
+                                                                  next_day_open_rate, attribution_next_day_open_cnt,
+                                                                  attribution_next_day_open_cost,
+                                                                  attribution_next_day_open_rate, game_addiction,
+                                                                  game_addiction_cost, game_addiction_rate, pay_count,
+                                                                  active_pay_cost, active_pay_rate, loan_completion,
+                                                                  loan_completion_cost, loan_completion_rate,
+                                                                  pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                                  loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                                  in_app_detail_uv, in_app_cart, in_app_pay,
+                                                                  in_app_order, attribution_game_pay_7d_count,
+                                                                  attribution_game_pay_7d_cost,
+                                                                  attribution_active_pay_7d_per_count, game_pay_cost,
+                                                                  game_pay_count, phone, form, map_search, button, view,
+                                                                  download, qq, lottery, vote, message, redirect,
+                                                                  shopping, consult, wechat, phone_confirm,
+                                                                  phone_connect, consult_effective, coupon,
+                                                                  coupon_single_page, redirect_to_shop, poi_collect,
+                                                                  poi_address_click, luban_order_cnt,
+                                                                  luban_order_stat_amount, luban_order_roi,
+                                                                  luban_live_enter_cnt, live_watch_one_minute_count,
+                                                                  luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                                  luban_live_comment_cnt, luban_live_share_cnt,
+                                                                  luban_live_gift_cnt, luban_live_gift_amount,
+                                                                  luban_live_slidecart_click_cnt,
+                                                                  luban_live_click_product_cnt,
+                                                                  luban_live_pay_order_count,
+                                                                  luban_live_pay_order_stat_cost, wechat_login_count,
+                                                                  attribution_wechat_login_30d_count, wechat_login_cost,
+                                                                  attribution_wechat_login_30d_cost,
+                                                                  wechat_first_pay_count,
+                                                                  attribution_wechat_first_pay_30d_count,
+                                                                  wechat_first_pay_cost,
+                                                                  attribution_wechat_first_pay_30d_cost,
+                                                                  wechat_first_pay_rate,
+                                                                  attribution_wechat_first_pay_30d_rate,
+                                                                  wechat_pay_amount, attribution_wechat_pay_30d_amount,
+                                                                  attribution_wechat_pay_30d_roi, phone_effective,
+                                                                  total_play, valid_play, valid_play_cost,
+                                                                  valid_play_rate, play_25_feed_break,
+                                                                  play_50_feed_break, play_75_feed_break,
+                                                                  play_100_feed_break, average_play_time_per_play,
+                                                                  play_over_rate, wifi_play_rate, wifi_play,
+                                                                  play_duration_sum, advanced_creative_phone_click,
+                                                                  advanced_creative_counsel_click,
+                                                                  advanced_creative_form_click,
+                                                                  advanced_creative_coupon_addition,
+                                                                  advanced_creative_form_submit, card_show, share,
+                                                                  comment, `like`, follow, home_visited,
+                                                                  ies_challenge_click, ies_music_click, location_click,
+                                                                  message_action, click_landing_page, click_shopwindow,
+                                                                  click_website, click_download, click_call_dy, cpc,
+                                                                  cpm, cpa, interact_per_cost, convert_show_rate,
+                                                                  play_duration_10s_rate, play_50_feed_break_rate,
+                                                                  play_25_feed_break_rate, play_duration_5s_rate,
+                                                                  play_duration_2s_rate, average_play_progress,
+                                                                  play_100_feed_break_rate, play_duration_3s_rate,
+                                                                  play_75_feed_break_rate, play_duration,
+                                                                  play_duration_3s, play_duration_2s, play_duration_10s,
+                                                                  average_video_play, avg_rank,
+                                                                  submit_certification_count, approval_count,
+                                                                  first_order_count, first_rental_order_count,
+                                                                  commute_first_pay_count, image_url, signature)
+select report.advertiser_id account_id,      #'账户ID'
+       report.material_id,                   #头条素材id
+       '' as                image_mode,      #素材类型
+       '' as                inventory,       #广告位
+       report.stat_datetime,                 #数据起始时间
+       cua.project_id,                       #项目id
+       product.id           product_id,      #项目中产品id
+       cua.advertiser_id,                    #广告主id
+       uc.company_id,                        #运营人的公司id
+       cua.auth_name        account_name,    #账户名称
+       cp.project_name,                      #项目名称
+       ca.name              advertiser_name, #广告主名称
+       uc.company_name,                      #运营人的公司名称
+       cp.sale_id,                           #项目里的销售id
+       cua.user_id,                          #运营id
+       image.filename,
+       report.stat_cost,
+       report.show_cnt,
+       0 as                show_cost,
+       report.`convert_cnt`,
+       report.`cpc_platform`,
+       report.`ctr`,
+       report.`convert_cnt`,
+       report.`conversion_cost`,
+       report.`conversion_rate`,
+       report.`deep_convert_cnt`,
+       report.`deep_convert_cost`,
+       report.`deep_convert_rate`,
+       0 as                `attribution_convert`,
+       0 as                `attribution_convert_cost`,
+       0 as                `attribution_deep_convert`,
+       0 as                `attribution_deep_convert_cost`,
+       report.`click_start_cnt`,
+       report.`click_start_cost`,
+       report.`click_start_rate`,
+       report.`download_finish_cnt`,
+       report.`download_finish_cost`,
+       report.`download_finish_rate`,
+       0 as                `click_install`,
+       report.`install_finish_cnt`,
+       report.`install_finish_cost`,
+       report.`install_finish_rate`,
+       report.`active`,
+       report.`active_cost`,
+       report.`active_rate`,
+       report.`active_register`,
+       report.`active_register_cost`,
+       report.`active_register_rate`,
+       0 as                `active_pay_amount`,
+       report.`next_day_open`,
+       0 as                `next_day_open_cost`,
+       0 as                `next_day_open_rate`,
+       report.`attribution_next_day_open_cnt`,
+       report.`attribution_next_day_open_cost`,
+       report.`attribution_next_day_open_rate`,
+       report.`game_addiction`,
+       report.`game_addiction_cost`,
+       report.`game_addiction_rate`,
+       0 as                `cnt`,
+       report.`active_pay_cost`,
+       report.`active_pay_rate`,
+       0 as                `loan_completion`,
+       0 as                `loan_completion_cost`,
+       0 as                `loan_completion_rate`,
+       0 as                `pre_loan_credit`,
+       0 as                `pre_loan_credit_cost`,
+       0 as                `loan_credit`,
+       0 as                `loan_credit_cost`,
+       0 as                `loan_credit_rate`,
+       report.`in_app_uv`,
+       report.`in_app_detail_uv`,
+       report.`in_app_cart`,
+       report.`in_app_pay`,
+       report.`in_app_order`,
+       report.`attribution_game_pay_7d_count`,
+       report.`attribution_game_pay_7d_cost`,
+       0 as                `attribution_active_pay_7d_per_count`,
+       report.`game_pay_cost`,
+       report.`game_pay_count`,
+       0 as                `phone`,
+       0 as                `form`,
+       0 as                `map_search`,
+       0 as                `button`,
+       0 as                `view`,
+       0 as                `download`,
+       0 as                `qq`,
+       0 as                `lottery`,
+       0 as                `vote`,
+       0 as                `message`,
+       0 as                `redirect`,
+       0 as                `shopping`,
+       0 as                `consult`,
+       0 as                `wechat`,
+       0 as                `phone_confirm`,
+       0 as                `phone_connect`,
+       0 as                `consult_effective`,
+       0 as                `coupon`,
+       0 as                `coupon_single_page`,
+       0 as                `redirect_to_shop`,
+       0 as                `poi_collect`,
+       0 as                `poi_address_click`,
+       0 as                `luban_order_cnt`,
+       0 as                `luban_order_stat_amount`,
+       0 as                `luban_order_roi`,
+       0 as                `luban_live_enter_cnt`,
+       0 as                `live_watch_one_minute_count`,
+       0 as                `luban_live_follow_cnt`,
+       0 as                `live_fans_club_join_cnt`,
+       0 as                `luban_live_comment_cnt`,
+       0 as                `luban_live_share_cnt`,
+       0 as                `luban_live_gift_cnt`,
+       0 as                `luban_live_gift_amount`,
+       0 as                `luban_live_slidecart_click_cnt`,
+       0 as                `luban_live_click_product_cnt`,
+       0 as                `luban_live_pay_order_count`,
+       0 as                `luban_live_pay_order_stat_cost`,
+       0 as                `wechat_login_count`,
+       0 as                `attribution_wechat_login_30d_count`,
+       0 as                `wechat_login_cost`,
+       0 as                `attribution_wechat_login_30d_cost`,
+       0 as                `wechat_first_pay_count`,
+       0 as                `attribution_wechat_first_pay_30d_count`,
+       0 as                `wechat_first_pay_cost`,
+       0 as                `attribution_wechat_first_pay_30d_cost`,
+       0 as                `wechat_first_pay_rate`,
+       0 as                `attribution_wechat_first_pay_30d_rate`,
+       0 as                `wechat_pay_amount`,
+       0 as                `attribution_wechat_pay_30d_amount`,
+       0 as                `attribution_wechat_pay_30d_roi`,
+       0 as                `phone_effective`,
+       0 as                `total_play`,
+       0 as                `valid_play`,
+       0 as                `valid_play_cost`,
+       0 as                `valid_play_rate`,
+       0 as                `play_25_feed_break`,
+       0 as                `play_50_feed_break`,
+       0 as                `play_75_feed_break`,
+       0 as                `play_100_feed_break`,
+       0 as                `average_play_time_per_play`,
+       0 as                `play_over_rate`,
+       0 as                `wifi_play_rate`,
+       0 as                `wifi_play`,
+       0 as                `play_duration_sum`,
+       0 as                `advanced_creative_phone_click`,
+       0 as                `advanced_creative_counsel_click`,
+       0 as                `advanced_creative_form_click`,
+       0 as                `advanced_creative_coupon_addition`,
+       0 as                `advanced_creative_form_submit`,
+       0 as                `card_show`,
+       0 as                `share`,
+       0 as                `comment`,
+       0 as                `like`,
+       0 as                `follow`,
+       0 as                `home_visited`,
+       0 as                `ies_challenge_click`,
+       0 as                `ies_music_click`,
+       0 as                `location_click`,
+       0 as                `message_action`,
+       0 as                `click_landing_page`,
+       0 as                `click_shopwindow`,
+       0 as                `click_website`,
+       0 as                `click_download`,
+       0 as                `click_call_dy`,
+       0 as                `cpc`,
+       0 as                `cpm`,
+       0 as                `cpa`,
+       0 as                `interact_per_cost`,
+       0 as                `convert_show_rate`,
+       0 as                `play_duration_10s_rate`,
+       0 as                `play_50_feed_break_rate`,
+       0 as                `play_25_feed_break_rate`,
+       0 as                `play_duration_5s_rate`,
+       0 as                `play_duration_2s_rate`,
+       0 as                `average_play_progress`,
+       0 as                `play_100_feed_break_rate`,
+       0 as                `play_duration_3s_rate`,
+       0 as                `play_75_feed_break_rate`,
+       0 as                `play_duration`,
+       0 as                `play_duration_3s`,
+       0 as                `play_duration_2s`,
+       0 as                `play_duration_10s`,
+       0 as                `average_video_play`,
+       0 as                `avg_rank`,
+       0 as                `submit_certification_count`,
+       0 as                `approval_count`,
+       0 as                `first_order_count`,
+       0 as                `first_rental_order_count`,
+       0 as                `commute_first_pay_count`,
+       image.image_url,
+       image.signature
+from media_api.bytedance_material_image_daily report
+         left join
+     `media_api`.bytedance_file_image_get image
+     on report.material_id = image.material_id
+         and report.advertiser_id = image.advertiser_id
+         left join `jeecg-boot`.ctop_user_allocation cua
+                   on report.advertiser_id = cua.account_id
+         left join `jeecg-boot`.ctop_advertiser ca
+                   on cua.advertiser_id = ca.id
+         left join `jeecg-boot`.ctop_project cp
+                   on cua.project_id = cp.id
+         left join `jeecg-boot`.user_company uc
+                   on cua.user_id = uc.user_id
+         left join `jeecg-boot`.ctop_product product
+                   on cp.product_id = product.id
+         left join `jeecg-boot`.sys_user su
+                   on cua.user_id = su.id
+where report.stat_datetime = {start_time}
+                                    """.format(start_time=self.start_time)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = NewMaterialimageDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 205 - 0
Apietl/bytedance_etl/report/NewMaterialVideoDailyReport.py

@@ -0,0 +1,205 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : NewMaterialVideoDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class NewMaterialVideoDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_live_args()
+        self.start_time = self.parm
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_material_video_daily where 
+         stat_datetime = {start_time}  
+         group by advertiser_id""".format(
+                start_time=self.start_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+
+                try:
+                    sql = """
+  replace into application.bytedance_material_video_report_daily_dw(account_id, material_id, image_mode, inventory,
+                                                                  stat_datetime, project_id, product_id, advertiser_id,
+                                                                  company_id, account_name, project_name,
+                                                                  advertiser_name, company_name, sale_id, user_id,
+                                                                  clip_id, shot_id, plan_id, leader_id, org_code,
+                                                                  signature, channel_type, clip_name, shot_name,
+                                                                  plan_name, leader_name, video_name, channel_name,
+                                                                  clip_company_id,
+                                                                  cost,
+                                                                  `show`,
+                                                                  click,
+                                                                  ctr,
+                                                                  `convert`,
+                                                                  convert_cost,
+                                                                  convert_rate,
+                                                                  deep_convert,
+                                                                  deep_convert_cost,
+                                                                  deep_convert_rate,
+                                                                  download_start,
+                                                                  download_start_cost,
+                                                                  download_start_rate,
+                                                                  download_finish,
+                                                                  download_finish_cost,
+                                                                  download_finish_rate,
+                                                                  install_finish,
+                                                                  install_finish_cost,
+                                                                  install_finish_rate,
+                                                                  active,
+                                                                  active_cost,
+                                                                  active_rate,
+                                                                  register,
+                                                                  active_register_rate,
+                                                                  game_addiction,
+                                                                  game_addiction_cost,
+                                                                  game_addiction_rate,
+                                                                  attribution_next_day_open_cnt,
+                                                                  attribution_next_day_open_cost,
+                                                                  attribution_next_day_open_rate,
+                                                                  next_day_open,
+                                                                  pay_count,
+                                                                  active_pay_cost,
+                                                                  active_pay_rate,
+                                                                  game_pay_count,
+                                                                  game_pay_cost,
+                                                                  attribution_game_pay_7d_count,
+                                                                  attribution_game_pay_7d_cost,
+                                                                  in_app_uv,
+                                                                  in_app_detail_uv,
+                                                                  in_app_cart,
+                                                                  in_app_pay,
+                                                                  in_app_order, video_url, cover_url,shot_leader_id,shot_leader_name)
+select report.advertiser_id                                                                      account_id,      #'账户ID'
+       report.material_id,                                                                                        #头条素材id
+       ''                                                                                     as image_mode,      #素材类型
+       ''                                                                                     as inventory,       #广告位
+       stat_datetime,                                                                                             #数据起始时间
+       cua.project_id,                                                                                            #项目id
+       product.id                                                                                product_id,      #项目中产品id
+       cua.advertiser_id,                                                                                         #广告主id
+       uc.company_id,                                                                                             #运营人的公司id
+       cua.auth_name                                                                             account_name,    #账户名称
+       cp.project_name,                                                                                           #项目名称
+       ca.name                                                                                   advertiser_name, #广告主名称
+       uc.company_name,                                                                                           #运营人的公司名称
+       cp.sale_id,                                                                                                #项目里的销售id
+       cua.user_id,                                                                                               #运营id
+       cma.clip_id,                                                                                               #剪辑id
+       cma.shot_id,                                                                                               #拍摄id
+       cma.plan_id,                                                                                               #编导id
+       cma.leader_id,                                                                                             #负责人id
+       leader.org_code,                                                                                           #机构编码
+       video.signature,                                                                                           #素材id
+       null                                                                                      chl_type,        #渠道类型
+       clip.realname                                                                             clip_name,       #剪辑名称
+       shot.realname                                                                             shot_name,       #拍摄名称
+       plan.realname                                                                             plan_name,       #编导名称
+       leader.realname                                                                           lead_name,       #负责人名称
+       case when cmi.material_name is not null then cmi.material_name else video.filename end as video_name,      ##视频名称
+       ''                                                                                        chl_name,        #渠道名称
+       up.company_id                                                                             clip_company_id, #剪辑公司
+       stat_cost                                                                              as cost,
+       show_cnt                                                                               as `show`,
+       click_cnt                                                                              as click,
+       ctr,
+       convert_cnt                                                                            as `convert`,
+       conversion_cost                                                                        as convert_cost,
+       conversion_rate                                                                        as convert_rate,
+       deep_convert_cnt                                                                       as deep_convert,
+       deep_convert_cost,
+       deep_convert_rate,
+       click_start_cnt                                                                        as download_start,
+       click_start_cost                                                                       as download_start_cost,
+       click_start_rate                                                                       as download_start_rate,
+       download_finish_cnt                                                                    as download_finish,
+       download_finish_cost,
+       download_finish_rate,
+       install_finish_cnt                                                                     as install_finish,
+       install_finish_cost,
+       install_finish_rate,
+       active,
+       active_cost,
+       active_rate,
+       active_register                                                                   as register,
+       active_register_rate,
+       game_addiction,
+       game_addiction_cost,
+       game_addiction_rate,
+       attribution_next_day_open_cnt,
+       attribution_next_day_open_cost,
+       attribution_next_day_open_rate,
+       next_day_open,
+       active_pay                                                                             as pay_count,
+       active_pay_cost,
+       active_pay_rate,
+       game_pay_count,
+       game_pay_cost,
+       attribution_game_pay_7d_count,
+       attribution_game_pay_7d_cost,
+       in_app_uv,
+       in_app_detail_uv,
+       in_app_cart,
+       in_app_pay,
+       in_app_order,
+       ifnull(concat('http://i.snssdk.com/video/code/1/toutiao/',video.id),'-') as video_url ,
+       cmi.cover_url,
+       cma.shot_leader_id,
+       cma.shot_leader_name
+from media_api.bytedance_material_video_daily report
+         left join
+     `media_api`.bytedance_file_video_get video
+     on report.advertiser_id = video.advertiser_id and report.material_id = video.material_id
+         left join (select code, material_name, cover_url from `jeecg-boot`.ctop_material_info where status = 1) cmi
+                   on video.signature = cmi.code
+         left join `jeecg-boot`.ctop_material_ascription cma
+                   on video.signature = cma.material_id
+         left join `jeecg-boot`.ctop_user_allocation cua
+                   on report.advertiser_id = cua.account_id
+         left join `jeecg-boot`.ctop_advertiser ca
+                   on cua.advertiser_id = ca.id
+         left join `jeecg-boot`.ctop_project cp
+                   on cua.project_id = cp.id
+         left join `jeecg-boot`.user_company uc
+                   on cua.user_id = uc.user_id
+         left join `jeecg-boot`.ctop_product product
+                   on cp.product_id = product.id
+         left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+         left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+         left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+         left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+         left join `jeecg-boot`.user_company up on up.user_id = cma.clip_id
+where report.stat_datetime ={start_time} 
+  and report.advertiser_id = {account_id}
+                                    """.format(start_time=self.start_time,
+                                               account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = NewMaterialVideoDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 8 - 0
Apietl/bytedance_etl/report/__init__.py

@@ -0,0 +1,8 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/10/21 5:35 下午
+# @Site    : 
+# @File    : __init__.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553

+ 29 - 0
Apietl/config.ini

@@ -0,0 +1,29 @@
+[xxl_job]
+xxl_url = 192.168.1.187
+
+[tidb]
+tidb_host = 192.168.0.249
+tidb_port = 3390
+tidb_user = hcst
+tidb_password = hcst@2024
+
+[tidb_pro]
+tidb_host = 139.186.172.149
+tidb_port = 3390
+tidb_user = hcst
+tidb_password = hcst@2025
+
+[webhook]
+url = 192.168.0.85
+[livewebhook]
+liveurl=192.168.0.203
+
+[log]
+Log_home=/data/etl_logs
+[cdc]
+cdc_url = 192.168.0.198
+
+
+;[Log]
+;log_dir = /opt/log/
+;log_level =INFO

+ 118 - 0
Apietl/conn/TidbConn.py

@@ -0,0 +1,118 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 11:06 上午
+# @Site    : 
+# @File    : TidbConn.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+import logging
+import time
+from sqlite3 import OperationalError
+
+import pymysql
+from Apietl.constant.ConfConstant import ConfConstant
+
+
+class TidbConn:
+    """
+    Tidb连接管理
+    """
+
+    @staticmethod
+    def get_connection():
+        """
+        获取Tidb连接
+        :return: 连接
+        """
+        return pymysql.connect(host=ConfConstant.TIDB_HOST, user=ConfConstant.TIDB_USER,
+                               passwd=ConfConstant.TIDB_PASSWORD,
+                               port=ConfConstant.TIDB_PORT, charset="utf8")
+
+    @staticmethod
+    def get_pro_connection():
+        """
+        获取Tidb连接
+        :return: 连接
+        """
+        return pymysql.connect(host=ConfConstant.TIDB_PRO_HOST, user=ConfConstant.TIDB_PRO_USER,
+                               passwd=ConfConstant.TIDB_PRO_PASSWORD,
+                               port=ConfConstant.TIDB_PRO_PORT, charset="utf8")
+
+    @classmethod
+    def _reConn(cls, num=28800, stime=5):  # 重试连接总次数为1天,这里根据实际情况自己设置,如果服务器宕机1天都没发现就......
+        _number = 0
+        _status = True
+        while _status and _number <= num:
+            try:
+                TidbConn.get_connection().ping()  # cping 校验连接是否异常
+                _status = False
+            except:
+                if TidbConn.get_connection() == True:  # 重新连接,成功退出
+                    _status = False
+                    break
+                _number += 1
+                time.sleep(stime)  # 连接不成功,休眠3秒钟,继续循环,知道成功或重试次数结束
+
+    @classmethod
+    def quary(cls, conn, sql):
+        # mysql_conn = TidbConn.get_connection()
+        TidbConn._reCon(conn)
+        cur = conn.cursor()
+        cur.execute(sql)
+        result = cur.fetchall()
+        return result
+
+    @staticmethod
+    def conn_close(conn: pymysql.connections.Connection):
+        """
+        关闭资源
+        """
+        if conn:
+            try:
+                conn.close()
+            except pymysql.err.Error:
+                pass
+
+    @staticmethod
+    def exec_sql(conn, sql_buff):
+        sql_list = sql_buff.split(";")
+        cur = conn.cursor()
+        for sql in sql_list:
+            if sql.strip():
+                cur.execute(sql)
+        return cur
+
+    @staticmethod
+    def upsert(conn, sql_buff):
+        while True:
+            try:
+                conn.ping()
+                break
+            except OperationalError:
+                conn.ping(True)
+        try:
+            TidbConn.exec_sql(conn, sql_buff)
+        except Exception as e:
+            logging.error('失败详细信息:', repr(e))
+        conn.commit()
+
+    @staticmethod
+    def _reCon(conn):
+        """ MySQLdb.OperationalError异常"""
+        # self.con.close()
+        while True:
+            try:
+                conn.ping()
+                break
+            except OperationalError:
+                conn.ping(True)
+
+
+if __name__ == '__main__':
+    db = TidbConn.get_connection()
+    cursor = db.cursor()
+    # 执行sql语句
+    sql = "show databases"
+    cursor.execute(sql)
+    result = cursor.fetchall()
+    print("result====", result)

+ 102 - 0
Apietl/conn/TidbProConn.py

@@ -0,0 +1,102 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 11:06 上午
+# @Site    :
+# @File    : TidbProConn.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+import logging
+import time
+from sqlite3 import OperationalError
+
+import pymysql
+from Apietl.constant.ConfConstant import ConfConstant
+
+
+class TidbProConn:
+    """
+    Tidb连接管理
+    """
+
+    @staticmethod
+    def get_connection():
+        """
+        获取Tidb连接
+        :return: 连接
+        """
+        return pymysql.connect(host=ConfConstant.TIDB_PRO_HOST, user=ConfConstant.TIDB_PRO_USER,
+                               passwd=ConfConstant.TIDB_PRO_PASSWORD,
+                               port=ConfConstant.TIDB_PRO_PORT, charset="utf8")
+
+    @classmethod
+    def _reConn(cls, num=28800, stime=5):  # 重试连接总次数为1天,这里根据实际情况自己设置,如果服务器宕机1天都没发现就......
+        _number = 0
+        _status = True
+        while _status and _number <= num:
+            try:
+                TidbProConn.get_connection().ping()  # cping 校验连接是否异常
+                _status = False
+            except:
+                if TidbProConn.get_connection() == True:  # 重新连接,成功退出
+                    _status = False
+                    break
+                _number += 1
+                time.sleep(stime)  # 连接不成功,休眠3秒钟,继续循环,知道成功或重试次数结束
+
+    @classmethod
+    def quary(cls, conn, sql):
+        TidbProConn._reCon(conn)
+        cur = conn.cursor()
+        cur.execute(sql)
+        result = cur.fetchall()
+        return result
+
+    @staticmethod
+    def conn_close(conn: pymysql.connections.Connection):
+        """
+        关闭资源
+        """
+        if conn:
+            try:
+                conn.close()
+            except pymysql.err.Error:
+                pass
+
+    @staticmethod
+    def exec_sql(conn, sql_buff):
+        sql_list = sql_buff.split(";")
+        cur = conn.cursor()
+        for sql in sql_list:
+            if sql.strip():
+                cur.execute(sql)
+        return cur
+
+    @staticmethod
+    def upsert(conn, sql_buff):
+        TidbProConn._reCon(conn)
+        try:
+            TidbProConn.exec_sql(conn, sql_buff)
+        except Exception as e:
+            logging.error('失败详细信息:', repr(e))
+        TidbProConn.get_connection().commit()
+
+    @staticmethod
+    def _reCon(conn):
+        """ MySQLdb.OperationalError异常"""
+        # self.con.close()
+        while True:
+            try:
+                conn.ping()
+                break
+            except OperationalError:
+                conn.ping(True)
+
+
+if __name__ == '__main__':
+    db = TidbProConn.get_connection()
+    cursor = db.cursor()
+    # 执行sql语句
+    sql = "show databases"
+    cursor.execute(sql)
+    result = cursor.fetchall()
+    print("result====", result)

+ 51 - 0
Apietl/constant/ConfConstant.py

@@ -0,0 +1,51 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 11:14 上午
+# @Site    : 
+# @File    : ConfConstant.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import os
+from configparser import ConfigParser
+
+dir_name = os.path.dirname(os.path.dirname(__file__))
+
+config_file = dir_name + '/config.ini'
+_cfp = ConfigParser()
+_cfp.read(config_file, encoding='utf-8')
+
+
+class ConfConstant:
+    """
+    配置文件常量类
+    """
+    # tidb 配置信息
+    TIDB_HOST = _cfp.get('tidb', 'tidb_host')
+    TIDB_PORT = _cfp.getint('tidb', 'tidb_port')
+    TIDB_USER = _cfp.get('tidb', 'tidb_user')
+    TIDB_PASSWORD = _cfp.get('tidb', 'tidb_password')
+
+    # tidb 配置信息
+    TIDB_PRO_HOST = _cfp.get('tidb_pro', 'tidb_host')
+    TIDB_PRO_PORT = _cfp.getint('tidb_pro', 'tidb_port')
+    TIDB_PRO_USER = _cfp.get('tidb_pro', 'tidb_user')
+    TIDB_PRO_PASSWORD = _cfp.get('tidb_pro', 'tidb_password')
+
+    # xxl-job 配置
+    xxl_url = _cfp.get('xxl_job', 'xxl_url')
+
+    # ### log
+    # # 日志存放路径
+    Log_home = _cfp.get('log', 'Log_home')
+    # # 日志等级
+    # LOG_LEVEL = _cfp.get('log', 'log_level')
+
+    # webhook配置
+    URL = _cfp.get('webhook', 'url')
+
+    #livewebhook'
+    LiveURL = _cfp.get('livewebhook', 'liveurl')
+
+    # cdc url
+    CDC_URL = _cfp.get('cdc', 'cdc_url')

+ 97 - 0
Apietl/email/BytedanceHotMaterial.py

@@ -0,0 +1,97 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : BytedanceHotMaterial.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class BytedanceHotMaterial:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select t1.company_name               AS '公司',
+       round(t2.hot_cost, 3)         AS '上线两周消耗',
+       ifnull(t1.clip_name, '-')     AS '剪辑',
+       ifnull(t1.shot_name, '-')     AS '拍摄',
+       ifnull(t1.plan_name, '-')     AS '编导',
+       ifnull(t1.leader_name, '-')   as '负责人',
+       case
+           when t1.material_innovate = 1 then '非创新素材'
+           when t1.material_innovate = 2 then '创新素材'
+           else '其它' end             AS '是否创新素材',
+       ifnull(t1.video_url, '-')     AS '素材链接',
+       ifnull(pro.project_name, '-') as '项目名称'
+from (select material_id,
+             company_id,
+             company_name,
+             clip_name,
+             shot_name,
+             plan_name,
+             leader_name,
+             video_url,
+             material_innovate
+      from application.app_bytedance_material_info
+      where faddish_date >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d') AND faddish_date<={stat_date}
+        and company_id is not null) t1
+         left join
+         (select * from application.hot_material_two_week_cost where media_id = 1) t2
+         on t1.material_id = t2.signature
+         left join `jeecg-boot`.ctop_material_info info
+                   on t1.material_id = info.code
+         left join `jeecg-boot`.ctop_project pro
+                   on info.project_id = pro.id
+order by hot_cost desc;
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/bytedance_hot_material_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn',
+                      'shiyuke@s-hcst.com',
+                      'lihuan@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条-近一周爆款素材全明细"
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="bytedance_hot_material.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    BytedanceHotMaterial().handler()

+ 90 - 0
Apietl/email/KuaishouClipMaterialToQingQMonth.py

@@ -0,0 +1,90 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQMonth:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       photo_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             photo_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(charge) as charge
+      from application.kuaishou_material_video_report_daily_dw
+WHERE stat_date >= date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m01')
+  AND stat_date <=  date_format(date_sub(DATE_FORMAT(NOW() , '%%Y-%%m-01'),interval 1 day),'%%Y%%m%%d')
+      group by signature, project_id) a
+         left join application.app_kuaishou_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60156
+
+        """
+        print(sql)
+        path = """/data/excel/快手剪辑素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手剪辑素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="kuaishou_month.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQMonth().handler()

+ 89 - 0
Apietl/email/KuaishouClipMaterialToQingQWeek.py

@@ -0,0 +1,89 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQWeek.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQWeek:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       photo_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             photo_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(charge) as charge
+      from application.kuaishou_material_video_report_daily_dw
+WHERE stat_date >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d') 
+and stat_date<={stat_date}
+      group by signature, project_id) a
+         left join application.app_kuaishou_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60156
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/快手剪辑素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手剪辑素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="kuaishou_Week.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQWeek().handler()

+ 97 - 0
Apietl/email/KuaishouHotMaterial.py

@@ -0,0 +1,97 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : BytedanceHotMaterial.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class KuaishouHotMaterial:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select t1.company_name             AS '公司',
+       round(t2.hot_cost, 3)       AS '上线两周消耗',
+       ifnull(t1.clip_name, '-')   AS '剪辑',
+       ifnull(t1.shot_name, '-')   AS '拍摄',
+       ifnull(t1.plan_name, '-')   AS '编导',
+       ifnull(t1.leader_name, '-') as '负责人',
+       case
+           when t1.material_innovate = 1 then '非创新素材'
+           when t1.material_innovate = 2 then '创新素材'
+           else '其它' end           AS '是否创新素材',
+       ifnull(t1.video_url, '-')   AS '素材链接',
+       ifnull(pro.project_name,'-') as '项目名称'
+from (select material_id,
+             company_id,
+             company_name,
+             clip_name,
+             shot_name,
+             plan_name,
+             leader_name,
+             video_url,
+             material_innovate
+      from application.app_kuaishou_material_info
+      where faddish_date >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d') AND faddish_date<={stat_date}
+        and company_id is not null) t1
+         left join
+         (select * from application.hot_material_two_week_cost where media_id = 2) t2
+         on t1.material_id = t2.signature
+left join `jeecg-boot`.ctop_material_info info
+on t1.material_id=info.code
+left join `jeecg-boot`.ctop_project pro
+on info.project_id=pro.id
+order by hot_cost desc;
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/kuaishou_hot_material_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn',
+                      'shiyuke@s-hcst.com',
+                      'lihuan@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手-近一周爆款素材全明细"
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="kuaishou_hot_material.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    KuaishouHotMaterial().handler()

+ 90 - 0
Apietl/email/KuaishouMaterialToQingQMonth.py

@@ -0,0 +1,90 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQMonth:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       photo_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             photo_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(charge) as charge
+      from application.kuaishou_material_video_report_daily_dw
+WHERE stat_date >= date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m01')
+  AND stat_date <=  date_format(date_sub(DATE_FORMAT(NOW() , '%%Y-%%m-01'),interval 1 day),'%%Y%%m%%d')
+      group by signature, project_id) a
+         left join application.app_kuaishou_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60155
+
+        """
+        print(sql)
+        path = """/data/excel/快手真人素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手真人素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="kuaishou_month.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQMonth().handler()

+ 89 - 0
Apietl/email/KuaishouMaterialToQingQWeek.py

@@ -0,0 +1,89 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQWeek.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQWeek:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       photo_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             photo_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(charge) as charge
+      from application.kuaishou_material_video_report_daily_dw
+WHERE stat_date >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d') 
+and stat_date<={stat_date}
+      group by signature, project_id) a
+         left join application.app_kuaishou_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60155
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/快手真人素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手真人素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="kuaishou_Week.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQWeek().handler()

+ 85 - 0
Apietl/email/NewBytedanceAllMaterialFourD.py

@@ -0,0 +1,85 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : NewKuaiShouAllMaterialFourD.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class NewBytedanceAllMaterialFourD:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+        SELECT
+        ( CASE WHEN t1.company_name IS NOT NULL THEN t1.company_name ELSE '-' END ) '公司',
+        ifnull(t2.project_name,'-') AS '项目',
+        ifnull(t2.signature,'-') AS '素材标识',
+        ( CASE WHEN t1.material_innovate = 1 THEN '非创新素材' WHEN t1.material_innovate = 2 THEN '创新素材' ELSE '-' END ) AS '是否创新素材',
+        ifnull(t1.media_time,'-') AS '上线时间',
+        ifnull(t2.channel_name,'-') AS '渠道',
+        sum( t2.cost ) AS '消耗',
+        ifnull(t2.clip_name,'-') AS '剪辑',
+        ifnull(t2.shot_name,'-') AS '拍摄',
+        ifnull(t2.plan_name,'-') AS '编导',
+        ifnull(t2.leader_name,'-') AS '负责人',
+        ifnull(t2.video_name,'-') AS '素材名称',
+        ifnull(t2.video_url,'-') AS '素材'
+from (select company_name,material_id,media_time,material_innovate 
+from application.app_bytedance_material_info where media_time >=  date_format(date_sub({stat_date}, INTERVAL 13 DAY),'%%Y%%m%%d')  and media_time<={stat_date}) t1
+left join 
+(select * from application.bytedance_material_video_report_daily_dw where stat_datetime>=date_format(date_sub({stat_date}, INTERVAL 13 DAY),'%%Y%%m%%d')) t2
+on t1.material_id=t2.signature where t2.signature is not null
+group by t1.material_id;
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/NewBytedanceAllMaterialFourD_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn',
+                      'shiyuke@s-hcst.com',
+                      'lihuan@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条--近14天上线素材消耗全明细"
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="NewBytedanceAllMaterialFourD.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    NewBytedanceAllMaterialFourD().handler()

+ 91 - 0
Apietl/email/NewKuaiShouAllMaterialFourD.py

@@ -0,0 +1,91 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : NewKuaiShouAllMaterialFourD.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class NewKuaiShouAllMaterialFourD:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """SELECT (CASE WHEN t2.company_name IS NOT NULL THEN t2.company_name ELSE '-' END)              AS '公司',
+       ifnull(t2.project_name, '-')                                                                             AS '项目',
+       ifnull(t2.signature, '-')                                                                                AS '素材标识',
+       (CASE
+            WHEN t3.material_innovate = 1 THEN '非创新素材'
+            WHEN t3.material_innovate = 2 THEN '创新素材'
+            ELSE '-' END)                                                                                       AS '是否创新素材',
+       ifnull(t1.frast_cost_time, '-')                                                                               AS '上线时间',
+       ifnull(t2.channel_name, '-')                                                                             AS '渠道',
+       sum(t2.charge)                                                                                           AS '消耗',
+       ifnull(t2.clip_name, '-')                                                                                AS '剪辑',
+       ifnull(t2.shot_name, '-')                                                                                AS '拍摄',
+       ifnull(t2.plan_name, '-')                                                                                AS '编导',
+       ifnull(t2.leader_name, '-')                                                                              AS '负责人',
+       ifnull(t2.video_name, '-')                                                                               AS '素材名称',
+       ifnull(t2.photo_url, '-')                                                                                AS '素材'
+from (select signature,frast_cost_time
+      from application.material_farst_cost_time
+      where frast_cost_time >= date_format(date_sub({stat_date}, INTERVAL 13 DAY), '%%Y%%m%%d') and frast_cost_time<={stat_date}) t1
+         left join
+     (select *
+      from application.kuaishou_material_video_report_daily_dw
+      where stat_date >= date_format(date_sub({stat_date}, INTERVAL 13 DAY), '%%Y%%m%%d')) t2
+     on t1.signature = t2.signature
+left join `jeecg-boot`.ctop_material_info t3
+on t1.signature=t3.code
+where t2.signature is not null
+group by t1.signature;""".format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/NewKuaiShouAllMaterialFourD_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn',
+                      'shiyuke@s-hcst.com',
+                      'lihuan@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手-近14天上线素材消耗全明细"
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="NewKuaiShouAllMaterialFourD.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    NewKuaiShouAllMaterialFourD().handler()

+ 72 - 0
Apietl/email/OpenWindowMaterial.py

@@ -0,0 +1,72 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : OpenWindowMaterial.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class OpenWindowMaterial:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select project_name as '项目',
+       signature    as '唯一标识',
+       video_name   as '素材名称',
+       sum(charge)  as '消耗',
+       photo_url    as '素材链接'
+from application.kuaishou_material_video_report_daily_dw
+where stat_date >= date_format(date_sub({date},interval 6 day),'%%Y%%m%%d')
+  and stat_date <= {date}
+  and video_name like '%%开屏%%'
+group by signature """.format(date=self.stat_date)
+        print(sql)
+        path = """/data/excel/开屏素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn', 'shiyuke@s-hcst.com', 'videomanager@c-top.com.cn', 'jiajingbo@s-hcst.com',
+                      'renhaozhe@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "开屏素材数据_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="OpenWindowMaterial.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    OpenWindowMaterial().handler()

+ 101 - 0
Apietl/email/TidbKuaiShouAllMaterialEffeW.py

@@ -0,0 +1,101 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : NewKuaiShouAllMaterialFourD.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TidbKuaiShouAllMaterialEffeW:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select t1.company_name               AS '公司',
+       round(t2.single_week_cost, 3) AS '上线一周消耗',
+       ifnull(t1.clip_name, '-')     AS '剪辑',
+       ifnull(t1.shot_name, '-')     AS '拍摄',
+       ifnull(t1.plan_name, '-')     AS '编导',
+       ifnull(t1.leader_name, '-')   as '负责人',
+       case
+           when t1.material_innovate = 1 then '非创新素材'
+           when t1.material_innovate = 2 then '创新素材'
+           else '其它' end             AS '是否创新素材',
+       ifnull(t1.video_url, '-')     AS '素材链接'
+from (select material_id,
+             company_id,
+             company_name,
+             clip_name,
+             shot_name,
+             plan_name,
+             leader_name,
+             video_url,
+             material_innovate
+      from application.app_kuaishou_material_info
+      where effect_date >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d') AND effect_date<={stat_date}
+        and company_id is not null) t1
+         left join
+     (select a.signature, sum(c.charge) single_week_cost
+      from application.effect_material a
+               inner join application.material_farst_cost_time b
+                          on a.signature = b.signature
+               inner join
+           application.kuaishou_material_video_report_daily_dw c
+           on a.signature = c.signature
+      where c.stat_date >= b.frast_cost_time and c.stat_date<=date_format(date_add(b.frast_cost_time, interval 6 day), '%%Y%%m%%d')
+        and a.media_id = 2
+      group by a.signature) t2
+     on t1.material_id = t2.signature
+order by single_week_cost desc;
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/TidbKuaiShouAllMaterialEffeW_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn',
+                      'shiyuke@s-hcst.com',
+                      'lihuan@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手-近一周有效素材全明细"
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="TidbKuaiShouAllMaterialEffeW.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TidbKuaiShouAllMaterialEffeW().handler()

+ 90 - 0
Apietl/email/TouTiaoClipMaterialToQingQMonth.py

@@ -0,0 +1,90 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQMonth:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       a.video_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             video_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(cost) as charge
+      from application.bytedance_material_video_report_daily_dw
+WHERE stat_datetime >= date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m01')
+  AND stat_datetime <=  date_format(date_sub(DATE_FORMAT(NOW() , '%%Y-%%m-01'),interval 1 day),'%%Y%%m%%d')
+      group by signature, project_id) a
+         left join application.app_bytedance_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60156
+
+        """
+        print(sql)
+        path = """/data/excel/头条剪辑素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条剪辑素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="bytedance_month.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQMonth().handler()

+ 91 - 0
Apietl/email/TouTiaoClipMaterialToQingQWeek.py

@@ -0,0 +1,91 @@
+
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQWeek.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQWeek:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       a.video_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             video_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(cost) as charge
+      from application.bytedance_material_video_report_daily_dw
+WHERE stat_datetime >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d')
+        and stat_datetime <= {stat_date}
+      group by signature, project_id) a
+         left join application.app_bytedance_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60156
+
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/头条剪辑素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条剪辑素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="bytedance_Week.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQWeek().handler()

+ 90 - 0
Apietl/email/TouTiaoMaterialToQingQMonth.py

@@ -0,0 +1,90 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQMonth:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       a.video_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             video_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(cost) as charge
+      from application.bytedance_material_video_report_daily_dw
+WHERE stat_datetime >= date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m01')
+  AND stat_datetime <=  date_format(date_sub(DATE_FORMAT(NOW() , '%%Y-%%m-01'),interval 1 day),'%%Y%%m%%d')
+      group by signature, project_id) a
+         left join application.app_bytedance_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60155
+
+        """
+        print(sql)
+        path = """/data/excel/头条真人素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条真人素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="bytedance_month.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQMonth().handler()

+ 90 - 0
Apietl/email/TouTiaoMaterialToQingQWeek.py

@@ -0,0 +1,90 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : TouTiaoMaterialToQingQWeek.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class TouTiaoMaterialToQingQWeek:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select a.project_name as `项目名称`,
+       signature      as `md5`,
+       video_name     as `视频名称`,
+       a.video_url      as `视频链接`,
+       a.leader_name  as `设计负责人`,
+       a.shot_name    as `拍摄`,
+       a.clip_name    as `剪辑`,
+       a.plan_name    as `编导`,
+       charge         as `消耗`
+from (select project_name,
+             project_id,
+             signature,
+             video_name,
+             video_url,
+             leader_name,
+             shot_name,
+             clip_name,
+             plan_name,
+             sum(cost) as charge
+      from application.bytedance_material_video_report_daily_dw
+WHERE stat_datetime >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d')
+        and stat_datetime <= {stat_date}
+      group by signature, project_id) a
+         left join application.app_bytedance_material_info b
+                   on a.signature = b.material_id
+where b.tag_id = 60155
+
+        """.format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/头条真人素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['keqingqing@c-top.com.cn', 'shiyuke@s-hcst.com','zhangwei@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条真人素材_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="bytedance_Week.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    TouTiaoMaterialToQingQWeek().handler()

+ 8 - 0
Apietl/email/__init__.py

@@ -0,0 +1,8 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2024/5/21 14:28
+# @Site    : 
+# @File    : __init__.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553

+ 82 - 0
Apietl/email/kwai_caiwu.py

@@ -0,0 +1,82 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : send_material_report_weekly.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class kwai_caiwu:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+        SELECT
+      date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m') as '年月',
+      t1.company_name            as '公司',
+       t1.project_name            as '项目',
+       ifnull(t6.realname ,'-')            as '上级',
+       ifnull(t5.realname,'-')                as '运营',
+       CONCAT(t1.account_id, '_') as '账户id',
+       t1.account_name            as '账户名称',
+       sum(t1.charge)               as '消耗'
+from application.kuaishou_account_report_daily_dw t1
+         LEFT JOIN `jeecg-boot`.sys_user t5
+                   ON t1.user_id = t5.id
+       left join `jeecg-boot`.sys_user t6
+on t5.leader_id=t6.id
+WHERE t1.stat_date >= date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m01')
+  AND t1.stat_date <=  date_format(date_sub(DATE_FORMAT(NOW() , '%%Y-%%m-01'),interval 1 day),'%%Y%%m%%d')
+  AND t1.charge > 0
+GROUP BY t1.account_id;
+
+        """
+        print(sql)
+        path = """/data/excel/快手数据_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['caiwu@c-top.com.cn', 'yumengmeng@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手数据_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="kuaishou.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    kwai_caiwu().handler()

+ 115 - 0
Apietl/email/send_bytedance_material_report_weekly.py

@@ -0,0 +1,115 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    : 
+# @File    : send_material_report_weekly.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class send_material_report_weekly:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """select (CASE
+            WHEN a.clip_company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+                '北京汇创思拓数字科技有限公司'
+            WHEN a.clip_company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+                '广州汇创思拓数字科技有限公司'
+            WHEN a.clip_company_id = '4b10089775c040119e139087517aed88' THEN
+                '上海汇创思拓数字科技有限公司'
+            WHEN a.clip_company_id = '5e46b5dea3d740eeb8ff1a2895015a4b' THEN
+                '北京骄阳数字科技有限公司'
+            WHEN a.clip_company_id = '1648c7ceb4d449a9a13fbed7a8d9976c' THEN
+                '杭州妙构思数字科技有限公司'
+            WHEN a.clip_company_id = '3321d5893bcd430cab1238a99bc7db11' THEN
+                '北京次元像素'
+            WHEN a.clip_company_id = 'ace886fa51a244818e918b5d6d1bcc62' then
+                '北京聚意创量文化传媒有限公司'
+            when a.clip_company_id = '3f362f527a804e8a9db6552ea12f7574'
+                then '石家庄汇创思拓数字科技有限公司'
+            else '-'
+    END)                   as '公司',
+       a.project_name      AS '项目',
+       a.signature         AS '素材标识',
+       b.frast_cost_time   AS '上线时间',
+       '-'                    渠道,
+       a.charge            AS '消耗',
+       trim(a.clip_name)   AS '剪辑',
+       trim(a.shot_name)   AS '拍摄',
+       trim(a.plan_name)   AS '编导',
+       trim(a.leader_name) AS '负责人',
+       shot_leader_name    as `拍摄负责人`,
+       a.video_name        AS '素材名称',
+       a.video_url         AS '链接'
+from (select clip_company_id,
+             shot_leader_name,
+             company_name,
+             project_name,
+             signature,
+             channel_name,
+             sum(cost) charge,
+             clip_name,
+             shot_name,
+             plan_name,
+             a.leader_name,
+             video_name,
+             video_url
+      from application.bytedance_material_video_report_daily_dw a
+      where stat_datetime >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d')
+        and stat_datetime <= {stat_date}
+      group by signature) a
+         left join
+     application.bytedance_material_farst_cost_time b
+     on a.signature = b.signature""".format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/toutiao_material_week_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn',
+                      'shiyuke@s-hcst.com',
+                      'lihuan@c-top.com.cn',
+                      'kashipin@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条-近一周素材消耗全明细"
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="bytedance_material_report_weekly.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    send_material_report_weekly().handler()

+ 109 - 0
Apietl/email/send_material_report_weekly.py

@@ -0,0 +1,109 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    : 
+# @File    : send_material_report_weekly.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class send_material_report_weekly:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """select (CASE
+            WHEN t1.clip_company_id = 'd57fecdcf7a94d009736d9c850731582' THEN '北京汇创思拓数字科技有限公司'
+            WHEN t1.clip_company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+                '广州汇创思拓数字科技有限公司'
+            WHEN t1.clip_company_id = '4b10089775c040119e139087517aed88' THEN
+                '上海汇创思拓数字科技有限公司'
+            WHEN t1.clip_company_id = '5e46b5dea3d740eeb8ff1a2895015a4b' THEN
+                '北京骄阳数字科技有限公司'
+            WHEN t1.clip_company_id = '1648c7ceb4d449a9a13fbed7a8d9976c' THEN
+                '杭州妙构思数字科技有限公司'
+            WHEN t1.clip_company_id = '3321d5893bcd430cab1238a99bc7db11' THEN
+                '北京次元像素'
+            WHEN t1.clip_company_id = 'ace886fa51a244818e918b5d6d1bcc62' then
+                '北京聚意创量文化传媒有限公司'
+            when t1.clip_company_id='3f362f527a804e8a9db6552ea12f7574' then '石家庄汇创思拓数字科技有限公司' 
+            when t1.clip_company_id is null and pro.agent_code = 1 then '北京汇创思拓数字科技有限公司'
+            when t1.clip_company_id is null and pro.agent_code = 2 then '北京骄阳数字科技有限公司'
+            when t1.clip_company_id is null and pro.agent_code = 3 then '上海汇创思拓数字科技有限公司'
+            else '-'
+
+    END)                            as '公司',
+       ifnull(t1.project_name, '-') AS '项目',
+       ifnull(t1.signature, '-')    AS '素材标识',
+       ifnull(t3.media_time, '-')   AS '上线时间',
+       ifnull(t1.channel_name, '-') AS '渠道',
+       round(sum(t1.charge), 3)     AS '消耗',
+       ifnull(t1.clip_name, '-')    AS '剪辑',
+       ifnull(t1.shot_name, '-')    AS '拍摄',
+       ifnull(t1.plan_name, '-')    AS '编导',
+       ifnull(t1.leader_name, '-')  AS '负责人',
+       case
+           when t3.material_innovate = 1 then '非创新素材'
+           when t3.material_innovate = 2 then '创新素材'
+           else '其它' end            AS '是否创新素材',
+       ifnull(t1.video_name, '-')   AS '素材名称',
+       ifnull(t1.photo_url, '-')    AS '链接'
+from application.kuaishou_material_video_report_daily_dw t1
+         left join (select material_id, media_time, company_name, material_innovate
+                    from application.app_kuaishou_material_info) t3
+                   on t1.signature = t3.material_id
+         left join `jeecg-boot`.user_company t2
+                   on t1.user_id = t2.user_id
+         left join `jeecg-boot`.ctop_project pro
+                   on t1.project_id = pro.id
+where t1.stat_date >= date_format(date_sub({stat_date}, INTERVAL 6 DAY), '%%Y%%m%%d') and t1.stat_date<={stat_date}
+group by t1.signature;""".format(stat_date=self.stat_date)
+        print(sql)
+        path = """/data/excel/kuaishou_material_week_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn',
+                      'shiyuke@s-hcst.com','fengxue@c-top.com.cn',
+                      'lihuan@c-top.com.cn','kashipin@s-hcst.com']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "快手-近一周素材消耗全明细"
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="kuaishou_material_report_weekly.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    send_material_report_weekly().handler()

+ 75 - 0
Apietl/email/suzaomaterial.py

@@ -0,0 +1,75 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : send_material_report_weekly.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class suzaomaterial:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+select project_name,a.signature,frast_cost_time,video_name,photo_url from
+  (select signature, frast_cost_time
+      from application.material_farst_cost_time
+      where frast_cost_time = date_format(date_sub(now(), interval 1 day), '%%Y%%m%%d')) a
+inner join(
+    select signature,video_name,photo_url,project_name from application.kuaishou_material_video_report_daily_dw where stat_date= date_format(date_sub(now(), interval 1 day), '%%Y%%m%%d')
+    and channel_type=1
+    group by signature
+      )b
+on a.signature=b.signature
+
+
+""".format(date=self.stat_date)
+        print(sql)
+        path = """/data/excel/塑造素材_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['wanglei@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "塑造素材数据_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="SuZaoMaterial.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    suzaomaterial().handler()

+ 81 - 0
Apietl/email/toutiao_caiwu.py

@@ -0,0 +1,81 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/12/20 11:18 上午
+# @Site    :
+# @File    : send_material_report_weekly.py.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+from urllib import parse
+import pandas as pd
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from sqlalchemy import create_engine
+
+
+class kwai_caiwu:
+    def __init__(self):
+        self.OuterStr = "mysql+pymysql://%s:%s@%s:%d/%s" % (
+            "hcst", parse.quote_plus("hcst@2024"), "192.168.0.249", 3390, "application")
+        self.OuterConn = create_engine(self.OuterStr,
+                                       pool_size=10,
+                                       max_overflow=5,
+                                       pool_timeout=10,
+                                       pool_recycle=3600)
+        self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
+
+    def handler(self):
+        sql = """
+SELECT       date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m') as '年月',
+       t1.company_name            as '公司',
+       t1.project_name            as '项目',
+       ifnull(t6.realname ,'-')            as '上级',
+       ifnull(t5.realname,'-')                as '运营',
+       CONCAT(t1.account_id, '_') as '账户id',
+       t1.account_name            as '账户名称',
+       sum(t1.cost)               as '消耗'
+from application.bytedance_advertiser_report_daily_dw t1
+         LEFT JOIN `jeecg-boot`.sys_user t5
+                   ON t1.user_id = t5.id
+       left join `jeecg-boot`.sys_user t6
+on t5.leader_id=t6.id
+WHERE t1.stat_datetime >= date_format(date_sub(NOW() , INTERVAL 1 MONTH), '%%Y%%m01')
+  AND t1.stat_datetime <=  date_format(date_sub(DATE_FORMAT(NOW() , '%%Y-%%m-01'),interval 1 day),'%%Y%%m%%d')
+  AND t1.cost > 0
+GROUP BY t1.account_id;
+
+        """
+        print(sql)
+        path = """/data/excel/头条数据_{stat_date}.xls""".format(stat_date=self.stat_date)
+        print(path)
+        pd.read_sql(sql, self.OuterConn).to_excel(path, index=False, engine='openpyxl')
+        my_sender = 'huichuangsituo@c-top.com.cn'  # 发件人邮箱账号
+        my_pass = 'rfZV5qRfendzM3Ph'  # 发件人邮箱授权码 / 腾讯企业邮箱请使用登陆密码
+        recipients = ['caiwu@c-top.com.cn', 'yumengmeng@c-top.com.cn']  # 收件人邮箱账号
+        # content = "Please check the attachment. "
+        msg = MIMEMultipart()
+        # [发件人的邮箱昵称、发件人邮箱账号], 昵称随便
+        msg['From'] = my_sender
+        # [收件人邮箱昵称、收件人邮箱账号], 昵称随便
+        msg['To'] = ",".join(recipients)
+        # 邮件的主题,也就是邮件的标题
+        msg['Subject'] = "头条数据_{stat_date}".format(stat_date=self.stat_date)
+
+        att1 = MIMEText(
+            open(path, 'rb').read(), 'base64', 'utf-8')
+        att1["Content-Type"] = 'application/octet-stream'
+        att1["Content-Disposition"] = 'attachment; filename="toutiao.xls"'
+        msg.attach(att1)
+        server = smtplib.SMTP_SSL("smtp.exmail.qq.com", port=465)
+        server.login(my_sender, my_pass)
+        try:
+            server.sendmail(my_sender, recipients, msg.as_string())
+            print("发送成功")
+        except Exception as e:
+            print("发送失败", e)
+
+
+if __name__ == '__main__':
+    kwai_caiwu().handler()

+ 348 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouAccountReportBusi.py

@@ -0,0 +1,348 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouAccountReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouAccountReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+            SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_account_report_busi_{day_count}d 
+SELECT
+concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+report.account_id,
+bei.project_id,
+bei.product_id,
+bei.advertiser_id,
+bei.company_id,
+bei.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.user_id,
+bei.leader_id,
+bei.org_code,
+report.charge,
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((report.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+report.`show`,
+report.photo_click,
+report.aclick,
+report.bclick,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio,
+report.impression_1k_cost,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.cancel_follow,
+report.report,
+report.block,
+report.negative,
+report.submit,
+report.download_started,
+report.download_completed,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost,
+report.event_next_day_stay,
+report.event_next_day_stay_cost,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost,
+report.form_action_ratio,
+report.event_jin_jian_landing_page,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans,
+report.event_order_amount_roi,
+report.event_goods_view_cost,
+report.merchant_reco_fans_cost,
+report.event_button_click,
+report.event_button_click_cost,
+report.event_button_click_ratio,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi,
+report.event_appoint_form,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio,
+report.event_appoint_jump_click,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad,
+report.event_multi_conversion_cost,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from 
+ media_api.kuaishou_account_report_daily a 
+  where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count} -1  DAY),'%Y%m%d') and {date_time}
+ group by advertiser_id) report
+LEFT JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL) bei 
+ON report.account_id = bei.account_id
+LEFT JOIN  (select advertiser_id account_id,round(sum(charge),3) charge from `media_api`.kuaishou_account_report_daily a  
+where stat_date between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d') and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') 
+group by advertiser_id) ratio
+ON report.account_id = ratio.account_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouAccountReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 343 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouAccountReportBusiMonth.py

@@ -0,0 +1,343 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouAccountReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouAccountReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+            SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_account_report_busi_m
+SELECT
+date_format({date_time},'%Y%m') stat_mon,
+report.account_id,
+bei.project_id,
+bei.product_id,
+bei.advertiser_id,
+bei.company_id,
+bei.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.user_id,
+bei.leader_id,
+bei.org_code,
+report.charge,
+report.`show`,
+report.photo_click,
+report.aclick,
+report.bclick,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio,
+report.impression_1k_cost,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.cancel_follow,
+report.report,
+report.block,
+report.negative,
+report.submit,
+report.download_started,
+report.download_completed,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost,
+report.event_next_day_stay,
+report.event_next_day_stay_cost,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost,
+report.form_action_ratio,
+report.event_jin_jian_landing_page,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans,
+report.event_order_amount_roi,
+report.event_goods_view_cost,
+report.merchant_reco_fans_cost,
+report.event_button_click,
+report.event_button_click_cost,
+report.event_button_click_ratio,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi,
+report.event_appoint_form,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio,
+report.event_appoint_jump_click,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad,
+report.event_multi_conversion_cost,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from 
+ media_api.kuaishou_account_report_daily a  
+ where stat_date between  date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') and {date_time}
+group by advertiser_id) report
+LEFT JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL) bei 
+ON report.account_id = bei.account_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouAccountReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 354 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouCampaignReportBusi.py

@@ -0,0 +1,354 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouCampaignReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouCampaignReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """#插入数据
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_campaign_report_busi_{day_count}d 
+SELECT
+concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+report.account_id,
+bei.advertiser_id,
+report.campaign_id,
+report.campaign_name,
+bei.project_id,
+bei.product_id,
+bei.company_id,
+bei.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.user_id,
+bei.leader_id,
+bei.org_code,
+report.charge,
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((report.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+report.`show`,
+report.photo_click,
+report.aclick,
+report.bclick,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio,
+report.impression_1k_cost,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.cancel_follow,
+report.report,
+report.block,
+report.negative,
+report.submit,
+report.download_started,
+report.download_completed,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost,
+report.event_next_day_stay,
+report.event_next_day_stay_cost,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost,
+report.form_action_ratio,
+report.event_jin_jian_landing_page,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans,
+report.event_order_amount_roi,
+report.event_goods_view_cost,
+report.merchant_reco_fans_cost,
+report.event_button_click,
+report.event_button_click_cost,
+report.event_button_click_ratio,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi,
+report.event_appoint_form,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio,
+report.event_appoint_jump_click,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad,
+report.event_multi_conversion_cost,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+campaign_id,
+campaign_name,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from 
+ media_api.kuaishou_campaign_report_daily  a  
+ where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count} -1  DAY),'%Y%m%d') and {date_time}
+ group by advertiser_id,campaign_id,campaign_name) report
+LEFT JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL) bei 
+ON report.account_id = bei.account_id
+LEFT JOIN
+(select advertiser_id account_id,campaign_id,round(sum(charge),3) charge from `media_api`.kuaishou_campaign_report_daily a  
+where stat_date between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d') and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id,campaign_id) ratio
+on report.account_id = ratio.account_id and report.campaign_id=ratio.campaign_id;
+""".format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouCampaignReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 347 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouCampaignReportBusiMonth.py

@@ -0,0 +1,347 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouCampaignReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouCampaignReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """#插入数据
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_campaign_report_busi_m
+SELECT
+date_format({date_time},'%Y%m') as stat_mon,
+report.account_id,
+bei.advertiser_id,
+report.campaign_id,
+report.campaign_name,
+bei.project_id,
+bei.product_id,
+bei.company_id,
+bei.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.user_id,
+bei.leader_id,
+bei.org_code,
+report.charge,
+report.`show`,
+report.photo_click,
+report.aclick,
+report.bclick,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio,
+report.impression_1k_cost,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.cancel_follow,
+report.report,
+report.block,
+report.negative,
+report.submit,
+report.download_started,
+report.download_completed,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost,
+report.event_next_day_stay,
+report.event_next_day_stay_cost,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost,
+report.form_action_ratio,
+report.event_jin_jian_landing_page,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans,
+report.event_order_amount_roi,
+report.event_goods_view_cost,
+report.merchant_reco_fans_cost,
+report.event_button_click,
+report.event_button_click_cost,
+report.event_button_click_ratio,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi,
+report.event_appoint_form,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio,
+report.event_appoint_jump_click,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad,
+report.event_multi_conversion_cost,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+campaign_id,
+campaign_name,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from 
+ media_api.kuaishou_campaign_report_daily  a  
+ where stat_date between  date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') and {date_time}
+ group by advertiser_id,campaign_id,campaign_name) report
+LEFT JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL) bei 
+ON report.account_id = bei.account_id;
+""".format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouCampaignReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 519 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouCompanyDesignReportBusi.py

@@ -0,0 +1,519 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouCompanyDesignReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouCompanyDesignReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_company_design_report_busi_{day_count}d 
+select
+new.start_end_time,
+ifnull(new.company_id,'-') company_id,
+new.company_name,
+new.org_code,
+new.org_type,
+new.dep_id,
+new.dep_name,
+ifnull(new.leader_id,'-') leader_id,
+new.leader_name,
+new.material_count,
+new.innovate_num,
+new.charge,
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((new.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+new.show ,
+new.photo_click,
+new.aclick,
+new.bclick,
+new.photo_click_ratio,
+new.play_3s_ratio ,
+new.action_ratio,
+new.impression_1k_cost,
+new.photo_click_cost ,
+new.action_cost,
+new.share,
+new.comment ,
+new.like ,
+new.follow,
+new.cancel_follow ,
+new.report,
+new.block,
+new.negative,
+new.submit,
+new.download_started ,
+new.download_completed,
+new.activation ,
+new.event_pay_first_day ,
+new.event_pay_purchase_amount_first_day,
+new.event_pay_first_day_roi,
+new.event_pay,
+new.event_pay_purchase_amount ,
+new.event_pay_roi ,
+new.event_register,
+new.event_register_cost ,
+new.event_register_ratio,
+new.event_jin_jian_app,
+new.event_jin_jian_app_cost,
+new.event_credit_grant_app ,
+new.event_credit_grant_app_cost,
+new.event_credit_grant_app_ratio ,
+new.event_order_paid ,
+new.event_order_paid_purchase_amount,
+new.event_order_paid_cost,
+new.event_next_day_stay ,
+new.event_next_day_stay_cost,
+new.event_next_day_stay_ratio ,
+new.form_count ,
+new.form_cost,
+new.form_action_ratio,
+new.event_jin_jian_landing_page,
+new.event_jin_jian_landing_page_cost,
+new.event_credit_grant_landing_page ,
+new.event_credit_grant_landing_page_cost,
+new.event_credit_grant_landing_page_ratio ,
+new.event_valid_clues,
+new.event_valid_clues_cost ,
+new.event_add_wechat ,
+new.event_add_wechat_cost,
+new.event_add_wechat_ratio ,
+new.event_get_through,
+new.event_get_through_cost ,
+new.event_get_through_ratio,
+new.event_app_invoked,
+new.event_app_invoked_cost ,
+new.event_app_invoked_ratio,
+new.event_credit_grant_landing_ratio,
+new.play_5s_ratio ,
+new.play_end_ratio,
+new.event_new_user_pay,
+new.event_new_user_pay_cost,
+new.event_new_user_pay_ratio,
+new.click_1k_cost ,
+new.ad_product_cnt,
+new.event_goods_view ,
+new.merchant_reco_fans,
+new.event_order_amount_roi ,
+new.event_goods_view_cost,
+new.merchant_reco_fans_cost,
+new.event_button_click,
+new.event_button_click_cost,
+new.event_button_click_ratio,
+new.event_order_paid_roi,
+new.event_new_user_jinjian_app,
+new.event_new_user_jinjian_app_cost ,
+new.event_new_user_jinjian_app_roi,
+new.event_new_user_credit_grant_app ,
+new.event_new_user_credit_grant_app_cost,
+new.event_new_user_credit_grant_app_roi,
+new.event_new_user_jinjian_page,
+new.event_new_user_jinjian_page_cost,
+new.event_new_user_jinjian_page_roi ,
+new.event_new_user_credit_grant_page,
+new.event_new_user_credit_grant_page_cost ,
+new.event_new_user_credit_grant_page_roi,
+new.event_appoint_form,
+new.event_appoint_form_cost,
+new.event_appoint_form_ratio,
+new.event_appoint_jump_click,
+new.event_appoint_jump_click_cost,
+new.event_appoint_jump_click_ratio,
+new.union_event_pay_purchase_amount_7d ,
+new.union_event_pay_purchase_amount_7d_roi,
+new.event_order_successed,
+new.ad_photo_played_10s_ratio ,
+new.key_action_cost,
+new.event_add_shopping_cart_cost ,
+new.event_ad_watch_times_ratio,
+new.event_outbound_call_cost,
+new.event_outbound_call ,
+new.action_new_ratio ,
+new.event_watch_app_ad,
+new.event_multi_conversion_cost,
+new.event_phone_card_activate ,
+new.ad_photo_played_75percent_ratio ,
+new.key_action_ratio ,
+new.event_ad_watch_times_cost ,
+new.event_add_shopping_cart,
+new.key_action ,
+new.event_multi_conversion ,
+new.event_wechat_connected ,
+new.event_dsp_gift_form ,
+new.event_intention_confirmed ,
+new.event_phone_get_through,
+new.event_multi_conversion_ratio ,
+new.event_measurement_house,
+new.ad_photo_played_2s_ratio,
+new.event_outbound_call_ratio ,
+new.event_ad_watch_times
+from
+(select
+  concat(date_format(date_sub({date_time}, INTERVAL {day_count}-1 DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+  company_id,
+  company_name,
+  org_code,
+  org_type,
+  id dep_id,
+  depart_name dep_name,
+  leader_id,
+  leader_name,
+  count(DISTINCT signature) material_count,
+  count(distinct innovate_code) innovate_num,
+  sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+round(sum(photo_click)/sum(`show`),3) photo_click_ratio,
+round(sum(play_3s_ratio)/{day_count}*100,3)  play_3s_ratio,
+round(sum(bclick)/sum(aclick)*100,3) action_ratio,
+round(sum(impression_1k_cost)/{day_count},3) impression_1k_cost,
+round(sum(charge)/sum(photo_click),3) photo_click_cost,
+round(sum(charge)/sum(bclick),3) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+round(sum(event_pay_purchase_amount)/sum(charge)*100,3) event_pay_roi,
+sum(event_register) event_register,
+round(sum(charge)/sum(event_register),3) event_register_cost,
+round(sum(event_register)/sum(activation)*100,3) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+round(sum(charge)/sum(event_jin_jian_app),3) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+round(sum(charge)/sum(event_credit_grant_app),3) event_credit_grant_app_cost ,
+round(sum(event_credit_grant_app_ratio)/{day_count}*100,3) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+round(sum(charge)/sum(event_next_day_stay),3) event_next_day_stay_cost,
+round(sum(event_next_day_stay)/sum(activation)*100,3) event_next_day_stay_ratio,
+sum(form_count) form_count,
+round(sum(charge)/sum(form_count),3) form_cost,
+round(sum(form_count)/sum(`bclick`)*100,3) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+round(sum(charge)/sum(event_jin_jian_landing_page),3) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+round(sum(charge)/sum(event_credit_grant_landing_page),3) event_credit_grant_landing_page_cost,
+round(sum(event_credit_grant_landing_page)/{day_count}*100,3) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+round(sum(charge)/sum(event_valid_clues),3) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+round(sum(charge)/sum(event_add_wechat),3) event_add_wechat_cost,
+round(sum(event_add_wechat)/sum(bclick)*100,3) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+round(sum(charge)/sum(event_get_through),3) event_get_through_cost,
+round(sum(event_get_through)/sum(bclick)*100,3) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+round(sum(charge)/sum(event_app_invoked),3) event_app_invoked_cost ,
+round(sum(event_app_invoked)/sum(bclick)*100,3) event_app_invoked_ratio,
+round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3) event_credit_grant_landing_ratio,
+round(sum(play_5s_ratio)/{day_count}*100,3) play_5s_ratio,
+round(sum(play_end_ratio)/{day_count}*100,3) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+round(sum(charge)/sum(event_new_user_pay),3) event_new_user_pay_cost,
+round(sum(event_new_user_pay)/sum(activation)*100,3) event_new_user_pay_ratio,
+round(sum(click_1k_cost)/{day_count}*100) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+round(sum(event_order_amount_roi)/{day_count}*100,3) event_order_amount_roi,
+round(sum(charge)/sum(event_goods_view) ,3) event_goods_view_cost,
+round(sum(charge)/sum(merchant_reco_fans),3) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+round(sum(charge)/sum(event_button_click),3) event_button_click_cost,
+round(sum(event_button_click)/sum(bclick)*100,3) event_button_click_ratio ,
+round(sum(event_order_paid_roi)/{day_count}*100,3) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+round(sum(charge)/sum(event_new_user_jinjian_app),3) event_new_user_jinjian_app_cost,
+round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+round(sum(charge)/sum(event_new_user_credit_grant_app),3)  event_new_user_credit_grant_app_cost,
+round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+round(sum(charge)/sum(event_new_user_jinjian_page),3) event_new_user_jinjian_page_cost,
+round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+round(sum(charge)/sum(event_new_user_credit_grant_page) ,3) event_new_user_credit_grant_page_cost,
+round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+round(sum(charge)/sum(event_appoint_form),3) event_appoint_form_cost,
+round(sum(event_appoint_form)/sum(form_count)*100,3) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+round(sum(charge)/sum(event_appoint_jump_click),3)  event_appoint_jump_click_cost,
+round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3) ad_photo_played_10s_ratio,
+round(sum(charge)/sum(key_action),3) key_action_cost,
+round(sum(charge)/sum(event_add_shopping_cart) ,3) event_add_shopping_cart_cost ,
+round(sum(event_ad_watch_times)/sum(bclick)*100,3) event_ad_watch_times_ratio,
+round(sum(charge)/sum(event_outbound_call),3) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+round(sum(action_new_ratio)/{day_count}*100,3) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+round(sum(charge)/sum(event_multi_conversion) ,3) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3) ad_photo_played_75percent_ratio,
+round(sum(key_action)/sum(activation)*100,3) key_action_ratio ,
+round(sum(charge)/sum(event_ad_watch_times),3) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+round(sum(event_multi_conversion)/sum(activation)*100,3)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3) ad_photo_played_2s_ratio,
+round(sum(event_outbound_call_ratio)/sum(activation)*100,3) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from
+(SELECT
+video.signature,
+uc.company_id,
+CASE WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN '北京汇创思拓数字科技有限公司'
+     WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN '广州汇创思拓数字科技有限公司'
+     WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN '上海汇创思拓数字科技有限公司'
+ELSE
+uc.company_name
+END company_name,
+cma.leader_id,
+us.realname leader_name,
+ur.org_type,
+ur.org_code,
+ur.id,
+ur.depart_name,
+case when cmi.material_innovate=2 then video.signature else '' end  innovate_code,
+report.charge ,
+report.`show`,
+    report.photo_click,
+    report.aclick ,
+    report.bclick ,
+    report.photo_click_ratio,
+    report.play_3s_ratio,
+    report.action_ratio ,
+    report.impression_1k_cost ,
+    report.photo_click_cost,
+    report.action_cost,
+    report.`share`,
+    report.`comment`,
+    report.`like`,
+    report.follow ,
+    report.cancel_follow,
+    report.report ,
+    report.block,
+    report.negative,
+    report.submit ,
+    report.download_started,
+    report.download_completed ,
+    report.activation,
+    report.event_pay_first_day,
+    report.event_pay_purchase_amount_first_day,
+    report.event_pay_first_day_roi,
+    report.event_pay ,
+    report.event_pay_purchase_amount,
+    report.event_pay_roi,
+    report.event_register,
+    report.event_register_cost,
+    report.event_register_ratio,
+    report.event_jin_jian_app ,
+    report.event_jin_jian_app_cost,
+    report.event_credit_grant_app,
+    report.event_credit_grant_app_cost ,
+    report.event_credit_grant_app_ratio,
+    report.event_order_paid,
+    report.event_order_paid_purchase_amount,
+    report.event_order_paid_cost ,
+    report.event_next_day_stay,
+    report.event_next_day_stay_cost ,
+    report.event_next_day_stay_ratio,
+    report.form_count,
+    report.form_cost ,
+    report.form_action_ratio,
+    report.event_jin_jian_landing_page ,
+    report.event_jin_jian_landing_page_cost,
+    report.event_credit_grant_landing_page,
+    report.event_credit_grant_landing_page_cost ,
+    report.event_credit_grant_landing_page_ratio,
+    report.event_valid_clues,
+    report.event_valid_clues_cost,
+    report.event_add_wechat,
+    report.event_add_wechat_cost ,
+    report.event_add_wechat_ratio,
+    report.event_get_through,
+    report.event_get_through_cost,
+    report.event_get_through_ratio,
+    report.event_app_invoked,
+    report.event_app_invoked_cost,
+    report.event_app_invoked_ratio,
+    report.event_credit_grant_landing_ratio,
+    report.play_5s_ratio,
+    report.play_end_ratio,
+    report.event_new_user_pay ,
+    report.event_new_user_pay_cost,
+    report.event_new_user_pay_ratio ,
+    report.click_1k_cost,
+    report.ad_product_cnt,
+    report.event_goods_view,
+    report.merchant_reco_fans ,
+    report.event_order_amount_roi,
+    report.event_goods_view_cost ,
+    report.merchant_reco_fans_cost,
+    report.event_button_click ,
+    report.event_button_click_cost,
+    report.event_button_click_ratio ,
+    report.event_order_paid_roi,
+    report.event_new_user_jinjian_app,
+    report.event_new_user_jinjian_app_cost,
+    report.event_new_user_jinjian_app_roi ,
+    report.event_new_user_credit_grant_app,
+    report.event_new_user_credit_grant_app_cost ,
+    report.event_new_user_credit_grant_app_roi,
+    report.event_new_user_jinjian_page ,
+    report.event_new_user_jinjian_page_cost,
+    report.event_new_user_jinjian_page_roi,
+    report.event_new_user_credit_grant_page,
+    report.event_new_user_credit_grant_page_cost,
+    report.event_new_user_credit_grant_page_roi ,
+    report.event_appoint_form ,
+    report.event_appoint_form_cost,
+    report.event_appoint_form_ratio ,
+    report.event_appoint_jump_click ,
+    report.event_appoint_jump_click_cost,
+    report.event_appoint_jump_click_ratio ,
+    report.union_event_pay_purchase_amount_7d,
+    report.union_event_pay_purchase_amount_7d_roi,
+    report.placement_type,
+    report.ad_scene,
+    report.event_order_successed ,
+    report.ad_photo_played_10s_ratio,
+    report.key_action_cost ,
+    report.event_add_shopping_cart_cost,
+    report.event_ad_watch_times_ratio,
+    report.event_outbound_call_cost ,
+    report.event_outbound_call,
+    report.action_new_ratio,
+    report.event_watch_app_ad ,
+    report.event_multi_conversion_cost ,
+    report.event_phone_card_activate,
+    report.ad_photo_played_75percent_ratio,
+    report.key_action_ratio,
+    report.event_ad_watch_times_cost,
+    report.event_add_shopping_cart,
+    report.key_action,
+    report.event_multi_conversion,
+    report.event_wechat_connected,
+    report.event_dsp_gift_form,
+    report.event_intention_confirmed,
+    report.event_phone_get_through,
+    report.event_multi_conversion_ratio,
+    report.event_measurement_house,
+    report.ad_photo_played_2s_ratio ,
+    report.event_outbound_call_ratio,
+    report.event_ad_watch_times
+    FROM
+     (select * from media_api.kuaishou_material_video_report_daily 
+     where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count}-1 DAY),'%Y%m%d') and {date_time}) report
+     LEFT JOIN
+(select account_id,signature,photo_id from media_api.kuaishou_video_list  group by account_id,signature,photo_id) video
+ON report.advertiser_id = video.account_id AND report.photo_id = video.photo_id
+LEFT JOIN (
+SELECT
+CODE,
+material_innovate,
+cover_url
+FROM
+`jeecg-boot`.ctop_material_info
+GROUP BY
+CODE,
+material_innovate,
+cover_url
+) cmi ON video.signature = cmi. CODE
+LEFT JOIN (
+SELECT
+material_id,
+leader_id
+FROM
+(
+SELECT
+material_id,
+leader_id,
+row_number () over (PARTITION BY material_id ORDER BY create_time) rn FROM `jeecg-boot`.ctop_material_ascription) m
+WHERE rn = 1) cma ON video.signature = cma.material_id
+left join `jeecg-boot`.sys_user us on us.id=cma.leader_id
+LEFT JOIN `jeecg-boot`.user_company uc ON cma.leader_id = uc.user_id
+LEFT JOIN `jeecg-boot`.sys_user_depart de ON cma.leader_id = de.user_id
+LEFT JOIN `jeecg-boot`.sys_depart ur ON de.dep_id = ur.id) def
+group by
+company_id,
+company_name,
+org_code,
+org_type,
+id,
+depart_name,
+leader_id,
+leader_name) new
+left join
+(select  ifnull(cma.leader_id,'-') leader_id,sum(report.charge) charge from
+(select advertiser_id account_id,photo_id,sum(charge) charge from media_api.kuaishou_material_video_report_daily
+where stat_date between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count} -1 DAY),'%Y%m%d')
+and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') group by advertiser_id,photo_id) report
+LEFT JOIN
+(select account_id,signature,photo_id from media_api.kuaishou_video_list  group by account_id,signature,photo_id) video
+ON report.account_id = video.account_id AND report.photo_id=video.photo_id
+LEFT JOIN (SELECT material_id,leader_id FROM (SELECT material_id,leader_id,row_number () over (PARTITION BY material_id ORDER BY create_time ) rn
+FROM  `jeecg-boot`.ctop_material_ascription) m WHERE rn = 1 ) cma ON video.signature = cma.material_id
+group by cma.leader_id) ratio
+on new.leader_id=ratio.leader_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouCompanyDesignReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 375 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouCompanyDesignReportBusiMonth.py

@@ -0,0 +1,375 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouCompanyDesignReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouCompanyDesignReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+delete from application.app_kuaishou_company_design_report_busi_m  where stat_mon=date_format({date_time},'%Y%m%') ;
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_company_design_report_busi_m 
+select
+date_format({date_time},'%Y%m') stat_mon,
+ifnull(company_id,'-') company_id,
+company_name,
+org_code,
+org_type,
+id,
+depart_name,
+ifnull(leader_id,'-') leader_id,
+leader_name,
+count(DISTINCT signature) num,
+count(distinct innovate_code) innovate_num,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from 
+(SELECT
+video.signature,
+uc.company_id,
+CASE
+WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+uc.company_name
+END company_name,
+cma.leader_id,
+    us.realname leader_name,
+ur.org_type,
+ur.org_code,
+ur.id,
+ur.depart_name,
+case when cmi.material_innovate=2 then video.signature else '' end  innovate_code,
+report.charge ,
+    report.`show`,
+    report.photo_click,
+    report.aclick ,
+    report.bclick ,
+    report.photo_click_ratio,
+    report.play_3s_ratio,
+    report.action_ratio ,
+    report.impression_1k_cost ,
+    report.photo_click_cost,
+    report.action_cost,
+    report.`share`,
+    report.`comment`,
+    report.`like`,
+    report.follow ,
+    report.cancel_follow,
+    report.report ,
+    report.block,
+    report.negative,
+    report.submit ,
+    report.download_started,
+    report.download_completed ,
+    report.activation,
+    report.event_pay_first_day,
+    report.event_pay_purchase_amount_first_day,
+    report.event_pay_first_day_roi,
+    report.event_pay ,
+    report.event_pay_purchase_amount,
+    report.event_pay_roi,
+    report.event_register,
+    report.event_register_cost,
+    report.event_register_ratio,
+    report.event_jin_jian_app ,
+    report.event_jin_jian_app_cost,
+    report.event_credit_grant_app,
+    report.event_credit_grant_app_cost ,
+    report.event_credit_grant_app_ratio,
+    report.event_order_paid,
+    report.event_order_paid_purchase_amount,
+    report.event_order_paid_cost ,
+    report.event_next_day_stay,
+    report.event_next_day_stay_cost ,
+    report.event_next_day_stay_ratio,
+    report.form_count,
+    report.form_cost ,
+    report.form_action_ratio,
+    report.event_jin_jian_landing_page ,
+    report.event_jin_jian_landing_page_cost,
+    report.event_credit_grant_landing_page,
+    report.event_credit_grant_landing_page_cost ,
+    report.event_credit_grant_landing_page_ratio,
+    report.event_valid_clues,
+    report.event_valid_clues_cost,
+    report.event_add_wechat,
+    report.event_add_wechat_cost ,
+    report.event_add_wechat_ratio,
+    report.event_get_through,
+    report.event_get_through_cost,
+    report.event_get_through_ratio,
+    report.event_app_invoked,
+    report.event_app_invoked_cost,
+    report.event_app_invoked_ratio,
+    report.event_credit_grant_landing_ratio,
+    report.play_5s_ratio,
+    report.play_end_ratio,
+    report.event_new_user_pay ,
+    report.event_new_user_pay_cost,
+    report.event_new_user_pay_ratio ,
+    report.click_1k_cost,
+    report.ad_product_cnt,
+    report.event_goods_view,
+    report.merchant_reco_fans ,
+    report.event_order_amount_roi,
+    report.event_goods_view_cost ,
+    report.merchant_reco_fans_cost,
+    report.event_button_click ,
+    report.event_button_click_cost,
+    report.event_button_click_ratio ,
+    report.event_order_paid_roi,
+    report.event_new_user_jinjian_app,
+    report.event_new_user_jinjian_app_cost,
+    report.event_new_user_jinjian_app_roi ,
+    report.event_new_user_credit_grant_app,
+    report.event_new_user_credit_grant_app_cost ,
+    report.event_new_user_credit_grant_app_roi,
+    report.event_new_user_jinjian_page ,
+    report.event_new_user_jinjian_page_cost,
+    report.event_new_user_jinjian_page_roi,
+    report.event_new_user_credit_grant_page,
+    report.event_new_user_credit_grant_page_cost,
+    report.event_new_user_credit_grant_page_roi ,
+    report.event_appoint_form ,
+    report.event_appoint_form_cost,
+    report.event_appoint_form_ratio ,
+    report.event_appoint_jump_click ,
+    report.event_appoint_jump_click_cost,
+    report.event_appoint_jump_click_ratio ,
+    report.union_event_pay_purchase_amount_7d,
+    report.union_event_pay_purchase_amount_7d_roi,
+    report.placement_type,
+    report.ad_scene,
+    report.event_order_successed ,
+    report.ad_photo_played_10s_ratio,
+    report.key_action_cost ,
+    report.event_add_shopping_cart_cost,
+    report.event_ad_watch_times_ratio,
+    report.event_outbound_call_cost ,
+    report.event_outbound_call,
+    report.action_new_ratio,
+    report.event_watch_app_ad ,
+    report.event_multi_conversion_cost ,
+    report.event_phone_card_activate,
+    report.ad_photo_played_75percent_ratio,
+    report.key_action_ratio,
+    report.event_ad_watch_times_cost,
+    report.event_add_shopping_cart,
+    report.key_action,
+    report.event_multi_conversion,
+    report.event_wechat_connected,
+    report.event_dsp_gift_form,
+    report.event_intention_confirmed,
+    report.event_phone_get_through,
+    report.event_multi_conversion_ratio,
+    report.event_measurement_house,
+    report.ad_photo_played_2s_ratio ,
+    report.event_outbound_call_ratio,
+    report.event_ad_watch_times
+FROM
+(select * from media_api.kuaishou_material_video_report_daily 
+where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') and {date_time}
+) report
+LEFT JOIN 
+(select account_id,signature,photo_id from media_api.kuaishou_video_list  group by account_id,signature,photo_id) video 
+ ON report.advertiser_id = video.account_id AND report.photo_id = video.photo_id 
+LEFT JOIN (
+SELECT
+CODE,
+material_innovate,
+cover_url
+FROM
+`jeecg-boot`.ctop_material_info
+GROUP BY
+CODE,
+material_innovate,
+cover_url
+) cmi ON video.signature = cmi. CODE
+LEFT JOIN (
+SELECT
+material_id,
+leader_id
+FROM
+(
+SELECT
+material_id,
+leader_id,
+row_number () over (
+PARTITION BY material_id
+ORDER BY
+create_time
+) rn
+FROM
+`jeecg-boot`.ctop_material_ascription
+) m
+WHERE
+rn = 1
+) cma ON video.signature = cma.material_id
+ left join `jeecg-boot`.sys_user us on us.id=cma.leader_id
+LEFT JOIN `jeecg-boot`.user_company uc ON cma.leader_id = uc.user_id
+LEFT JOIN `jeecg-boot`.sys_user_depart de ON cma.leader_id = de.user_id
+LEFT JOIN `jeecg-boot`.sys_depart ur ON de.dep_id = ur.id) def
+group by
+  company_id,
+  leader_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouCompanyDesignReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 515 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouCompanyUserReportBusi.py

@@ -0,0 +1,515 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouCompanyUserReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouCompanyUserReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_company_user_report_busi_{day_count}d
+select
+new.start_end_time,
+new.company_id,
+new.company_name,
+new.org_code,
+new.org_type,
+new.dep_id,
+new.dep_name,
+new.leader_id,
+new.leader_name,
+new.account_num,
+new.charge,
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((new.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+new.show,
+new.photo_click,
+new.aclick,
+new.bclick,
+new.photo_click_ratio,
+new.play_3s_ratio,
+new.action_ratio,
+new.impression_1k_cost,
+new.photo_click_cost,
+new.action_cost,
+new.share,
+new.comment,
+new.like,
+new.follow,
+new.cancel_follow,
+new.report,
+new.block,
+new.negative,
+new.submit,
+new.download_started,
+new.download_completed,
+new.activation,
+new.event_pay_first_day,
+new.event_pay_purchase_amount_first_day,
+new.event_pay_first_day_roi,
+new.event_pay,
+new.event_pay_purchase_amount,
+new.event_pay_roi,
+new.event_register,
+new.event_register_cost,
+new.event_register_ratio,
+new.event_jin_jian_app,
+new.event_jin_jian_app_cost,
+new.event_credit_grant_app,
+new.event_credit_grant_app_cost,
+new.event_credit_grant_app_ratio,
+new.event_order_paid,
+new.event_order_paid_purchase_amount,
+new.event_order_paid_cost,
+new.event_next_day_stay,
+new.event_next_day_stay_cost,
+new.event_next_day_stay_ratio,
+new.form_count,
+new.form_cost,
+new.form_action_ratio,
+new.event_jin_jian_landing_page,
+new.event_jin_jian_landing_page_cost,
+new.event_credit_grant_landing_page,
+new.event_credit_grant_landing_page_cost,
+new.event_credit_grant_landing_page_ratio,
+new.event_valid_clues,
+new.event_valid_clues_cost,
+new.event_add_wechat,
+new.event_add_wechat_cost,
+new.event_add_wechat_ratio,
+new.event_get_through,
+new.event_get_through_cost,
+new.event_get_through_ratio,
+new.event_app_invoked,
+new.event_app_invoked_cost,
+new.event_app_invoked_ratio,
+new.event_credit_grant_landing_ratio,
+new.play_5s_ratio,
+new.play_end_ratio,
+new.event_new_user_pay,
+new.event_new_user_pay_cost,
+new.event_new_user_pay_ratio,
+new.click_1k_cost,
+new.ad_product_cnt,
+new.event_goods_view,
+new.merchant_reco_fans,
+new.event_order_amount_roi,
+new.event_goods_view_cost,
+new.merchant_reco_fans_cost,
+new.event_button_click,
+new.event_button_click_cost,
+new.event_button_click_ratio,
+new.event_order_paid_roi,
+new.event_new_user_jinjian_app,
+new.event_new_user_jinjian_app_cost,
+new.event_new_user_jinjian_app_roi,
+new.event_new_user_credit_grant_app,
+new.event_new_user_credit_grant_app_cost,
+new.event_new_user_credit_grant_app_roi,
+new.event_new_user_jinjian_page,
+new.event_new_user_jinjian_page_cost,
+new.event_new_user_jinjian_page_roi,
+new.event_new_user_credit_grant_page,
+new.event_new_user_credit_grant_page_cost,
+new.event_new_user_credit_grant_page_roi,
+new.event_appoint_form,
+new.event_appoint_form_cost,
+new.event_appoint_form_ratio,
+new.event_appoint_jump_click,
+new.event_appoint_jump_click_cost,
+new.event_appoint_jump_click_ratio,
+new.union_event_pay_purchase_amount_7d,
+new.union_event_pay_purchase_amount_7d_roi,
+new.event_order_successed,
+new.ad_photo_played_10s_ratio,
+new.key_action_cost,
+new.event_add_shopping_cart_cost,
+new.event_ad_watch_times_ratio,
+new.event_outbound_call_cost,
+new.event_outbound_call,
+new.action_new_ratio,
+new.event_watch_app_ad,
+new.event_multi_conversion_cost,
+new.event_phone_card_activate,
+new.ad_photo_played_75percent_ratio,
+new.key_action_ratio,
+new.event_ad_watch_times_cost,
+new.event_add_shopping_cart,
+new.key_action,
+new.event_multi_conversion,
+new.event_wechat_connected,
+new.event_dsp_gift_form,
+new.event_intention_confirmed,
+new.event_phone_get_through,
+new.event_multi_conversion_ratio,
+new.event_measurement_house,
+new.ad_photo_played_2s_ratio,
+new.event_outbound_call_ratio,
+new.event_ad_watch_times
+from
+(SELECT
+concat(date_format(date_sub({date_time}, INTERVAL {day_count}-1 DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+company_id,
+company_name,
+org_code,
+org_type,
+dep_id,
+dep_name,
+leader_id,
+trim(leader_name) leader_name,
+count(DISTINCT account_id) account_num,
+sum(charge) charge,
+sum(`show`) `show`,
+  sum(photo_click)photo_click,
+  sum(aclick)  aclick,
+  sum(bclick) bclick,
+  round(sum(photo_click)/sum(`show`),3) photo_click_ratio,
+  round(sum(play_3s_ratio)/{day_count}*100,3)  play_3s_ratio,
+  round(sum(bclick)/sum(aclick)*100,3) action_ratio,
+  round(sum(impression_1k_cost)/{day_count},3) impression_1k_cost,
+  round(sum(charge)/sum(photo_click),3) photo_click_cost,
+  round(sum(charge)/sum(bclick),3) action_cost,
+  sum(`share`) `share`,
+  sum(`comment`) `comment`,
+  sum(`like`)  `like`,
+  sum(follow) follow,
+  sum(cancel_follow)  cancel_follow,
+  sum(report) report,
+  sum(block) block,
+  sum(negative) negative,
+  sum(submit) submit,
+  sum(download_started) download_started,
+  sum(download_completed) download_completed,
+  sum(activation) activation,
+  sum(event_pay_first_day) event_pay_first_day,
+  sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+  round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3) event_pay_first_day_roi ,
+  sum(event_pay) event_pay,
+  sum(event_pay_purchase_amount) event_pay_purchase_amount,
+  round(sum(event_pay_purchase_amount)/sum(charge)*100,3) event_pay_roi,
+  sum(event_register) event_register,
+  round(sum(charge)/sum(event_register),3) event_register_cost,
+  round(sum(event_register)/sum(activation)*100,3) event_register_ratio,
+  sum(event_jin_jian_app) event_jin_jian_app,
+  round(sum(charge)/sum(event_jin_jian_app),3) event_jin_jian_app_cost,
+  sum(event_credit_grant_app) event_credit_grant_app,
+  round(sum(charge)/sum(event_credit_grant_app),3) event_credit_grant_app_cost ,
+  round(sum(event_credit_grant_app_ratio)/{day_count}*100,3) event_credit_grant_app_ratio,
+  sum(event_order_paid) event_order_paid,
+  sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+  round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3) event_order_paid_cost,
+  sum(event_next_day_stay) event_next_day_stay,
+  round(sum(charge)/sum(event_next_day_stay),3) event_next_day_stay_cost,
+  round(sum(event_next_day_stay)/sum(activation)*100,3) event_next_day_stay_ratio,
+  sum(form_count) form_count,
+  round(sum(charge)/sum(form_count),3) form_cost,
+  round(sum(form_count)/sum(`bclick`)*100,3) form_action_ratio,
+  sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+  round(sum(charge)/sum(event_jin_jian_landing_page),3) event_jin_jian_landing_page_cost,
+  sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+  round(sum(charge)/sum(event_credit_grant_landing_page),3) event_credit_grant_landing_page_cost,
+  round(sum(event_credit_grant_landing_page)/{day_count}*100,3) event_credit_grant_landing_page_ratio,
+  sum(event_valid_clues) event_valid_clues,
+  round(sum(charge)/sum(event_valid_clues),3) event_valid_clues_cost,
+  sum(event_add_wechat) event_add_wechat,
+  round(sum(charge)/sum(event_add_wechat),3) event_add_wechat_cost,
+  round(sum(event_add_wechat)/sum(bclick)*100,3) event_add_wechat_ratio,
+  sum(event_get_through) event_get_through,
+  round(sum(charge)/sum(event_get_through),3) event_get_through_cost,
+  round(sum(event_get_through)/sum(bclick)*100,3) event_get_through_ratio,
+  sum(event_app_invoked) event_app_invoked,
+  round(sum(charge)/sum(event_app_invoked),3) event_app_invoked_cost ,
+  round(sum(event_app_invoked)/sum(bclick)*100,3) event_app_invoked_ratio,
+  round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3) event_credit_grant_landing_ratio,
+  round(sum(play_5s_ratio)/{day_count}*100,3) play_5s_ratio,
+  round(sum(play_end_ratio)/{day_count}*100,3) play_end_ratio,
+  sum(event_new_user_pay) event_new_user_pay,
+  round(sum(charge)/sum(event_new_user_pay),3) event_new_user_pay_cost,
+  round(sum(event_new_user_pay)/sum(activation)*100,3) event_new_user_pay_ratio,
+  round(sum(click_1k_cost)/{day_count}*100) click_1k_cost,
+  sum(ad_product_cnt) ad_product_cnt,
+  sum(event_goods_view) event_goods_view,
+  sum(merchant_reco_fans) merchant_reco_fans,
+  round(sum(event_order_amount_roi)/{day_count}*100,3) event_order_amount_roi,
+  round(sum(charge)/sum(event_goods_view) ,3) event_goods_view_cost,
+  round(sum(charge)/sum(merchant_reco_fans),3) merchant_reco_fans_cost,
+  sum(event_button_click) event_button_click,
+  round(sum(charge)/sum(event_button_click),3) event_button_click_cost,
+  round(sum(event_button_click)/sum(bclick)*100,3) event_button_click_ratio ,
+  round(sum(event_order_paid_roi)/{day_count}*100,3) event_order_paid_roi,
+  sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+  round(sum(charge)/sum(event_new_user_jinjian_app),3) event_new_user_jinjian_app_cost,
+  round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3) event_new_user_jinjian_app_roi ,
+  sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+  round(sum(charge)/sum(event_new_user_credit_grant_app),3)  event_new_user_credit_grant_app_cost,
+  round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3) event_new_user_credit_grant_app_roi,
+  sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+  round(sum(charge)/sum(event_new_user_jinjian_page),3) event_new_user_jinjian_page_cost,
+  round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3) event_new_user_jinjian_page_roi ,
+  sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+  round(sum(charge)/sum(event_new_user_credit_grant_page) ,3) event_new_user_credit_grant_page_cost,
+  round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3) event_new_user_credit_grant_page_roi,
+  sum(event_appoint_form)  event_appoint_form,
+  round(sum(charge)/sum(event_appoint_form),3) event_appoint_form_cost,
+  round(sum(event_appoint_form)/sum(form_count)*100,3) event_appoint_form_ratio,
+  sum(event_appoint_jump_click) event_appoint_jump_click,
+  round(sum(charge)/sum(event_appoint_jump_click),3)  event_appoint_jump_click_cost,
+  round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3) event_appoint_jump_click_ratio,
+  sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+  round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3) union_event_pay_purchase_amount_7d_roi,
+  sum(event_order_successed) event_order_successed,
+  round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3) ad_photo_played_10s_ratio,
+  round(sum(charge)/sum(key_action),3) key_action_cost,
+  round(sum(charge)/sum(event_add_shopping_cart) ,3) event_add_shopping_cart_cost ,
+  round(sum(event_ad_watch_times)/sum(bclick)*100,3) event_ad_watch_times_ratio,
+  round(sum(charge)/sum(event_outbound_call),3) event_outbound_call_cost,
+  sum(event_outbound_call) event_outbound_call,
+  round(sum(action_new_ratio)/{day_count}*100,3) action_new_ratio,
+  sum(event_watch_app_ad) event_watch_app_ad,
+  round(sum(charge)/sum(event_multi_conversion) ,3) event_multi_conversion_cost,
+  sum(event_phone_card_activate) event_phone_card_activate,
+  round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3) ad_photo_played_75percent_ratio,
+  round(sum(key_action)/sum(activation)*100,3) key_action_ratio ,
+  round(sum(charge)/sum(event_ad_watch_times),3) event_ad_watch_times_cost,
+  sum(event_add_shopping_cart) event_add_shopping_cart,
+  sum(key_action) key_action,
+  sum(event_multi_conversion) event_multi_conversion,
+  sum(event_wechat_connected) event_wechat_connected,
+  sum(event_dsp_gift_form) event_dsp_gift_form,
+  sum(event_intention_confirmed) event_intention_confirmed,
+  sum(event_phone_get_through) event_phone_get_through,
+  round(sum(event_multi_conversion)/sum(activation)*100,3)  event_multi_conversion_ratio,
+  sum(event_measurement_house) event_measurement_house,
+  round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3) ad_photo_played_2s_ratio,
+  round(sum(event_outbound_call_ratio)/sum(activation)*100,3) event_outbound_call_ratio,
+  sum(event_ad_watch_times) event_ad_watch_times
+from (
+SELECT
+report.advertiser_id account_id,
+dep.company_id,
+CASE
+WHEN dep.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN dep.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN dep.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+dep.company_name
+END company_name,
+dep.org_code,
+dep.org_type,
+dep.dep_id,
+dep.dep_name,
+dep.user_id,
+dep.realname,
+dep.leader_id,
+dep.leader_name,
+report.charge ,
+report.`show`,
+    report.photo_click,
+    report.aclick ,
+    report.bclick ,
+    report.photo_click_ratio,
+    report.play_3s_ratio,
+    report.action_ratio ,
+    report.impression_1k_cost ,
+    report.photo_click_cost,
+    report.action_cost,
+    report.`share`,
+    report.`comment`,
+    report.`like`,
+    report.follow ,
+    report.cancel_follow,
+    report.report ,
+    report.block,
+    report.negative,
+    report.submit ,
+    report.download_started,
+    report.download_completed ,
+    report.activation,
+    report.event_pay_first_day,
+    report.event_pay_purchase_amount_first_day,
+    report.event_pay_first_day_roi,
+    report.event_pay ,
+    report.event_pay_purchase_amount,
+    report.event_pay_roi,
+    report.event_register,
+    report.event_register_cost,
+    report.event_register_ratio,
+    report.event_jin_jian_app ,
+    report.event_jin_jian_app_cost,
+    report.event_credit_grant_app,
+    report.event_credit_grant_app_cost ,
+    report.event_credit_grant_app_ratio,
+    report.event_order_paid,
+    report.event_order_paid_purchase_amount,
+    report.event_order_paid_cost ,
+    report.event_next_day_stay,
+    report.event_next_day_stay_cost ,
+    report.event_next_day_stay_ratio,
+    report.form_count,
+    report.form_cost ,
+    report.form_action_ratio,
+    report.event_jin_jian_landing_page ,
+    report.event_jin_jian_landing_page_cost,
+    report.event_credit_grant_landing_page,
+    report.event_credit_grant_landing_page_cost ,
+    report.event_credit_grant_landing_page_ratio,
+    report.event_valid_clues,
+    report.event_valid_clues_cost,
+    report.event_add_wechat,
+    report.event_add_wechat_cost ,
+    report.event_add_wechat_ratio,
+    report.event_get_through,
+    report.event_get_through_cost,
+    report.event_get_through_ratio,
+    report.event_app_invoked,
+    report.event_app_invoked_cost,
+    report.event_app_invoked_ratio,
+    report.event_credit_grant_landing_ratio,
+    report.play_5s_ratio,
+    report.play_end_ratio,
+    report.event_new_user_pay ,
+    report.event_new_user_pay_cost,
+    report.event_new_user_pay_ratio ,
+    report.click_1k_cost,
+    report.ad_product_cnt,
+    report.event_goods_view,
+    report.merchant_reco_fans ,
+    report.event_order_amount_roi,
+    report.event_goods_view_cost ,
+    report.merchant_reco_fans_cost,
+    report.event_button_click ,
+    report.event_button_click_cost,
+    report.event_button_click_ratio ,
+    report.event_order_paid_roi,
+    report.event_new_user_jinjian_app,
+    report.event_new_user_jinjian_app_cost,
+    report.event_new_user_jinjian_app_roi ,
+    report.event_new_user_credit_grant_app,
+    report.event_new_user_credit_grant_app_cost ,
+    report.event_new_user_credit_grant_app_roi,
+    report.event_new_user_jinjian_page ,
+    report.event_new_user_jinjian_page_cost,
+    report.event_new_user_jinjian_page_roi,
+    report.event_new_user_credit_grant_page,
+    report.event_new_user_credit_grant_page_cost,
+    report.event_new_user_credit_grant_page_roi ,
+    report.event_appoint_form ,
+    report.event_appoint_form_cost,
+    report.event_appoint_form_ratio ,
+    report.event_appoint_jump_click ,
+    report.event_appoint_jump_click_cost,
+    report.event_appoint_jump_click_ratio ,
+    report.union_event_pay_purchase_amount_7d,
+    report.union_event_pay_purchase_amount_7d_roi,
+    report.placement_type,
+    report.ad_scene,
+    report.event_order_successed ,
+    report.ad_photo_played_10s_ratio,
+    report.key_action_cost ,
+    report.event_add_shopping_cart_cost,
+    report.event_ad_watch_times_ratio,
+    report.event_outbound_call_cost ,
+    report.event_outbound_call,
+    report.action_new_ratio,
+    report.event_watch_app_ad ,
+    report.event_multi_conversion_cost ,
+    report.event_phone_card_activate,
+    report.ad_photo_played_75percent_ratio,
+    report.key_action_ratio,
+    report.event_ad_watch_times_cost,
+    report.event_add_shopping_cart,
+    report.key_action,
+    report.event_multi_conversion,
+    report.event_wechat_connected,
+    report.event_dsp_gift_form,
+    report.event_intention_confirmed,
+    report.event_phone_get_through,
+    report.event_multi_conversion_ratio,
+    report.event_measurement_house,
+    report.ad_photo_played_2s_ratio ,
+    report.event_outbound_call_ratio,
+    report.event_ad_watch_times
+   from
+   (select * from  media_api.kuaishou_account_report_daily  
+   where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count}-1 DAY),'%Y%m%d') and {date_time}) report
+   left join
+   (
+SELECT
+cua.account_id,
+uc.company_id,
+uc.company_name,
+pa.org_code,
+pa.org_type,
+pa.id dep_id,
+pa.depart_name dep_name,
+cua.user_id,
+su.realname,
+cp.sale_id,
+su.leader_id,
+se.realname leader_name
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user se ON su.leader_id=se.id
+LEFT JOIN `jeecg-boot`.sys_user_depart de ON su.leader_id = de.user_id
+LEFT JOIN `jeecg-boot`.sys_depart pa ON pa.id = de.dep_id) dep
+ON report.advertiser_id = dep.account_id) com
+group by
+company_id,
+org_code,
+org_type,
+dep_id,
+leader_id)new
+left join
+(select  ifnull(b.leader_id,'-') leader_id ,sum(a.charge) charge from
+(select advertiser_id account_id,sum(charge)charge from  media_api.kuaishou_account_report_daily
+where stat_date between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d')
+and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id) a
+left join
+(SELECT cua.account_id,su.leader_id
+FROM  `jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id) b
+on  a.account_id=b.account_id
+group by b.leader_id) ratio
+on new.leader_id=ratio.leader_id;
+""".format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouCompanyUserReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 367 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouCompanyUserReportBusiMonth.py

@@ -0,0 +1,367 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouCompanyUserReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouCompanyUserReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_company_user_report_busi_m
+SELECT
+date_format({date_time},'%Y%m') start_end_time,
+ifnull(company_id,'-') company_id,
+company_name,
+org_code,
+org_type,
+dep_id,
+depart_name,
+ifnull(leader_id,'-') leader_id,
+ leader_name,
+count(DISTINCT account_id) account_num,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times  
+from (
+SELECT
+report.advertiser_id account_id,
+dep.company_id,
+CASE
+WHEN dep.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN
+'北京汇创思拓数字科技有限公司'
+WHEN dep.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN
+'广州汇创思拓数字科技有限公司'
+WHEN dep.company_id = '4b10089775c040119e139087517aed88' THEN
+'上海汇创思拓数字科技有限公司'
+ELSE
+dep.company_name
+END company_name,
+dep.org_code,
+dep.org_type,
+dep.dep_id,
+dep.depart_name,
+dep.user_id,
+dep.realname,
+dep.leader_id,
+dep.leader_name,
+report.charge ,
+    report.`show`,
+    report.photo_click,
+    report.aclick ,
+    report.bclick ,
+    report.photo_click_ratio,
+    report.play_3s_ratio,
+    report.action_ratio ,
+    report.impression_1k_cost ,
+    report.photo_click_cost,
+    report.action_cost,
+    report.`share`,
+    report.`comment`,
+    report.`like`,
+    report.follow ,
+    report.cancel_follow,
+    report.report ,
+    report.block,
+    report.negative,
+    report.submit ,
+    report.download_started,
+    report.download_completed ,
+    report.activation,
+    report.event_pay_first_day,
+    report.event_pay_purchase_amount_first_day,
+    report.event_pay_first_day_roi,
+    report.event_pay ,
+    report.event_pay_purchase_amount,
+    report.event_pay_roi,
+    report.event_register,
+    report.event_register_cost,
+    report.event_register_ratio,
+    report.event_jin_jian_app ,
+    report.event_jin_jian_app_cost,
+    report.event_credit_grant_app,
+    report.event_credit_grant_app_cost ,
+    report.event_credit_grant_app_ratio,
+    report.event_order_paid,
+    report.event_order_paid_purchase_amount,
+    report.event_order_paid_cost ,
+    report.event_next_day_stay,
+    report.event_next_day_stay_cost ,
+    report.event_next_day_stay_ratio,
+    report.form_count,
+    report.form_cost ,
+    report.form_action_ratio,
+    report.event_jin_jian_landing_page ,
+    report.event_jin_jian_landing_page_cost,
+    report.event_credit_grant_landing_page,
+    report.event_credit_grant_landing_page_cost ,
+    report.event_credit_grant_landing_page_ratio,
+    report.event_valid_clues,
+    report.event_valid_clues_cost,
+    report.event_add_wechat,
+    report.event_add_wechat_cost ,
+    report.event_add_wechat_ratio,
+    report.event_get_through,
+    report.event_get_through_cost,
+    report.event_get_through_ratio,
+    report.event_app_invoked,
+    report.event_app_invoked_cost,
+    report.event_app_invoked_ratio,
+    report.event_credit_grant_landing_ratio,
+    report.play_5s_ratio,
+    report.play_end_ratio,
+    report.event_new_user_pay ,
+    report.event_new_user_pay_cost,
+    report.event_new_user_pay_ratio ,
+    report.click_1k_cost,
+    report.ad_product_cnt,
+    report.event_goods_view,
+    report.merchant_reco_fans ,
+    report.event_order_amount_roi,
+    report.event_goods_view_cost ,
+    report.merchant_reco_fans_cost,
+    report.event_button_click ,
+    report.event_button_click_cost,
+    report.event_button_click_ratio ,
+    report.event_order_paid_roi,
+    report.event_new_user_jinjian_app,
+    report.event_new_user_jinjian_app_cost,
+    report.event_new_user_jinjian_app_roi ,
+    report.event_new_user_credit_grant_app,
+    report.event_new_user_credit_grant_app_cost ,
+    report.event_new_user_credit_grant_app_roi,
+    report.event_new_user_jinjian_page ,
+    report.event_new_user_jinjian_page_cost,
+    report.event_new_user_jinjian_page_roi,
+    report.event_new_user_credit_grant_page,
+    report.event_new_user_credit_grant_page_cost,
+    report.event_new_user_credit_grant_page_roi ,
+    report.event_appoint_form ,
+    report.event_appoint_form_cost,
+    report.event_appoint_form_ratio ,
+    report.event_appoint_jump_click ,
+    report.event_appoint_jump_click_cost,
+    report.event_appoint_jump_click_ratio ,
+    report.union_event_pay_purchase_amount_7d,
+    report.union_event_pay_purchase_amount_7d_roi,
+    report.placement_type,
+    report.ad_scene,
+    report.event_order_successed ,
+    report.ad_photo_played_10s_ratio,
+    report.key_action_cost ,
+    report.event_add_shopping_cart_cost,
+    report.event_ad_watch_times_ratio,
+    report.event_outbound_call_cost ,
+    report.event_outbound_call,
+    report.action_new_ratio,
+    report.event_watch_app_ad ,
+    report.event_multi_conversion_cost ,
+    report.event_phone_card_activate,
+    report.ad_photo_played_75percent_ratio,
+    report.key_action_ratio,
+    report.event_ad_watch_times_cost,
+    report.event_add_shopping_cart,
+    report.key_action,
+    report.event_multi_conversion,
+    report.event_wechat_connected,
+    report.event_dsp_gift_form,
+    report.event_intention_confirmed,
+    report.event_phone_get_through,
+    report.event_multi_conversion_ratio,
+    report.event_measurement_house,
+    report.ad_photo_played_2s_ratio ,
+    report.event_outbound_call_ratio,
+    report.event_ad_watch_times
+   from  
+   (select * from  media_api.kuaishou_account_report_daily 
+    where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') and {date_time}
+    ) report
+   left join 
+   (
+SELECT
+cua.account_id,
+uc.company_id,
+uc.company_name,
+pa.org_code,
+pa.org_type,
+pa.id dep_id,
+pa.depart_name,
+cua.user_id,
+su.realname,
+cp.sale_id,
+su.leader_id,
+se.realname leader_name
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user se ON su.leader_id=se.id
+LEFT JOIN `jeecg-boot`.sys_user_depart de ON su.leader_id = de.user_id
+LEFT JOIN `jeecg-boot`.sys_depart pa ON pa.id = de.dep_id) dep 
+ON report.advertiser_id = dep.account_id) com
+where com.company_id is not null
+group by 
+company_id,
+company_name,
+org_code,
+org_type,
+dep_id,
+depart_name,
+leader_id,
+leader_name;
+
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouCompanyUserReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 337 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouProductReportBusi.py

@@ -0,0 +1,337 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouProductReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouProductReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_product_report_busi_{day_count}d 
+select
+new.start_end_time,
+new.company_id,
+new.company_name,
+new.product_id,
+new.product_name,
+new.account_num,
+new.charge,
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((new.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+new.show,
+new.photo_click,
+new.aclick,
+new.bclick,
+new.photo_click_ratio,
+new.play_3s_ratio,
+new.action_ratio,
+new.impression_1k_cost,
+new.photo_click_cost,
+new.action_cost,
+new.share,
+new.comment,
+new.like,
+new.follow,
+new.cancel_follow,
+new.report,
+new.block,
+new.negative,
+new.submit,
+new.download_started,
+new.download_completed,
+new.activation,
+new.event_pay_first_day,
+new.event_pay_purchase_amount_first_day,
+new.event_pay_first_day_roi,
+new.event_pay,
+new.event_pay_purchase_amount,
+new.event_pay_roi,
+new.event_register,
+new.event_register_cost,
+new.event_register_ratio,
+new.event_jin_jian_app,
+new.event_jin_jian_app_cost,
+new.event_credit_grant_app,
+new.event_credit_grant_app_cost,
+new.event_credit_grant_app_ratio,
+new.event_order_paid,
+new.event_order_paid_purchase_amount,
+new.event_order_paid_cost,
+new.event_next_day_stay,
+new.event_next_day_stay_cost,
+new.event_next_day_stay_ratio,
+new.form_count,
+new.form_cost,
+new.form_action_ratio,
+new.event_jin_jian_landing_page,
+new.event_jin_jian_landing_page_cost,
+new.event_credit_grant_landing_page,
+new.event_credit_grant_landing_page_cost,
+new.event_credit_grant_landing_page_ratio,
+new.event_valid_clues,
+new.event_valid_clues_cost,
+new.event_add_wechat,
+new.event_add_wechat_cost,
+new.event_add_wechat_ratio,
+new.event_get_through,
+new.event_get_through_cost,
+new.event_get_through_ratio,
+new.event_app_invoked,
+new.event_app_invoked_cost,
+new.event_app_invoked_ratio,
+new.event_credit_grant_landing_ratio,
+new.play_5s_ratio,
+new.play_end_ratio,
+new.event_new_user_pay,
+new.event_new_user_pay_cost,
+new.event_new_user_pay_ratio,
+new.click_1k_cost,
+new.ad_product_cnt,
+new.event_goods_view,
+new.merchant_reco_fans,
+new.event_order_amount_roi,
+new.event_goods_view_cost,
+new.merchant_reco_fans_cost,
+new.event_button_click,
+new.event_button_click_cost,
+new.event_button_click_ratio,
+new.event_order_paid_roi,
+new.event_new_user_jinjian_app,
+new.event_new_user_jinjian_app_cost,
+new.event_new_user_jinjian_app_roi,
+new.event_new_user_credit_grant_app,
+new.event_new_user_credit_grant_app_cost,
+new.event_new_user_credit_grant_app_roi,
+new.event_new_user_jinjian_page,
+new.event_new_user_jinjian_page_cost,
+new.event_new_user_jinjian_page_roi,
+new.event_new_user_credit_grant_page,
+new.event_new_user_credit_grant_page_cost,
+new.event_new_user_credit_grant_page_roi,
+new.event_appoint_form,
+new.event_appoint_form_cost,
+new.event_appoint_form_ratio,
+new.event_appoint_jump_click,
+new.event_appoint_jump_click_cost,
+new.event_appoint_jump_click_ratio,
+new.union_event_pay_purchase_amount_7d,
+new.union_event_pay_purchase_amount_7d_roi,
+new.event_order_successed,
+new.ad_photo_played_10s_ratio,
+new.key_action_cost,
+new.event_add_shopping_cart_cost,
+new.event_ad_watch_times_ratio,
+new.event_outbound_call_cost,
+new.event_outbound_call,
+new.action_new_ratio,
+new.event_watch_app_ad,
+new.event_multi_conversion_cost,
+new.event_phone_card_activate,
+new.ad_photo_played_75percent_ratio,
+new.key_action_ratio,
+new.event_ad_watch_times_cost,
+new.event_add_shopping_cart,
+new.key_action,
+new.event_multi_conversion,
+new.event_wechat_connected,
+new.event_dsp_gift_form,
+new.event_intention_confirmed,
+new.event_phone_get_through,
+new.event_multi_conversion_ratio,
+new.event_measurement_house,
+new.ad_photo_played_2s_ratio,
+new.event_outbound_call_ratio,
+new.event_ad_watch_times
+from
+(SELECT
+concat(date_format(date_sub({date_time},INTERVAL {day_count}-1 DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+uc.company_id,
+CASE WHEN uc.company_id='d57fecdcf7a94d009736d9c850731582' THEN '北京汇创思拓数字科技有限公司'
+WHEN uc.company_id='6608ed13c0dd42de93c790dbdf124234' THEN '广州汇创思拓数字科技有限公司'
+WHEN uc.company_id='4b10089775c040119e139087517aed88' THEN '上海汇创思拓数字科技有限公司'
+ELSE uc.company_name END company_name,
+cp.product_id,
+cn.product_name,
+count(report.advertiser_id) account_num,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click) photo_click,
+sum(aclick) aclick,
+sum(bclick) bclick,
+round(sum(photo_click)/sum(`show`),3) photo_click_ratio,
+round(sum(play_3s_ratio)/{day_count}*100,3) play_3s_ratio,
+round(sum(bclick)/sum(aclick)*100,3) action_ratio,
+round(sum(impression_1k_cost)/{day_count},3) impression_1k_cost,
+round(sum(charge)/sum(photo_click),3) photo_click_cost,
+round(sum(charge)/sum(bclick),3) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`) `like`,
+sum(follow) follow,
+sum(cancel_follow) cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3) event_pay_first_day_roi,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+round(sum(event_pay_purchase_amount)/sum(charge)*100,3) event_pay_roi,
+sum(event_register) event_register,
+round(sum(charge)/sum(event_register),3) event_register_cost,
+round(sum(event_register)/sum(activation)*100,3) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+round(sum(charge)/sum(event_jin_jian_app),3) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+round(sum(charge)/sum(event_credit_grant_app),3) event_credit_grant_app_cost,
+round(sum(event_credit_grant_app_ratio)/{day_count}*100,3) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+round(sum(event_order_paid_purchase_amount)/sum(event_order_paid),3) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+round(sum(charge)/sum(event_next_day_stay),3) event_next_day_stay_cost,
+round(sum(event_next_day_stay)/sum(activation)*100,3) event_next_day_stay_ratio,
+sum(form_count) form_count,
+round(sum(charge)/sum(form_count),3) form_cost,
+round(sum(form_count)/sum(`bclick`)*100,3) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+round(sum(charge)/sum(event_jin_jian_landing_page),3) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+round(sum(charge)/sum(event_credit_grant_landing_page),3) event_credit_grant_landing_page_cost,
+round(sum(event_credit_grant_landing_page)/{day_count}*100,3) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+round(sum(charge)/sum(event_valid_clues),3) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+round(sum(charge)/sum(event_add_wechat),3) event_add_wechat_cost,
+round(sum(event_add_wechat)/sum(bclick)*100,3) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+round(sum(charge)/sum(event_get_through),3) event_get_through_cost,
+round(sum(event_get_through)/sum(bclick)*100,3) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+round(sum(charge)/sum(event_app_invoked),3) event_app_invoked_cost,
+round(sum(event_app_invoked)/sum(bclick)*100,3) event_app_invoked_ratio,
+round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3) event_credit_grant_landing_ratio,
+round(sum(play_5s_ratio)/{day_count}*100,3) play_5s_ratio,
+round(sum(play_end_ratio)/{day_count}*100,3) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+round(sum(charge)/sum(event_new_user_pay),3) event_new_user_pay_cost,
+round(sum(event_new_user_pay)/sum(activation)*100,3) event_new_user_pay_ratio,
+round(sum(click_1k_cost)/{day_count}*100) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+round(sum(event_order_amount_roi)/{day_count}*100,3) event_order_amount_roi,
+round(sum(charge)/sum(event_goods_view),3) event_goods_view_cost,
+round(sum(charge)/sum(merchant_reco_fans),3) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+round(sum(charge)/sum(event_button_click),3) event_button_click_cost,
+round(sum(event_button_click)/sum(bclick)*100,3) event_button_click_ratio,
+round(sum(event_order_paid_roi)/{day_count}*100,3) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+round(sum(charge)/sum(event_new_user_jinjian_app),3) event_new_user_jinjian_app_cost,
+round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3) event_new_user_jinjian_app_roi,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+round(sum(charge)/sum(event_new_user_credit_grant_app),3) event_new_user_credit_grant_app_cost,
+round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+round(sum(charge)/sum(event_new_user_jinjian_page),3) event_new_user_jinjian_page_cost,
+round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3) event_new_user_jinjian_page_roi,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+round(sum(charge)/sum(event_new_user_credit_grant_page),3) event_new_user_credit_grant_page_cost,
+round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form) event_appoint_form,
+round(sum(charge)/sum(event_appoint_form),3) event_appoint_form_cost,
+round(sum(event_appoint_form)/sum(form_count)*100,3) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+round(sum(charge)/sum(event_appoint_jump_click),3) event_appoint_jump_click_cost,
+round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3) ad_photo_played_10s_ratio,
+round(sum(charge)/sum(key_action),3) key_action_cost,
+round(sum(charge)/sum(event_add_shopping_cart),3) event_add_shopping_cart_cost,
+round(sum(event_ad_watch_times)/sum(bclick)*100,3) event_ad_watch_times_ratio,
+round(sum(charge)/sum(event_outbound_call),3) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+round(sum(action_new_ratio)/{day_count}*100,3) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+round(sum(charge)/sum(event_multi_conversion),3) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3) ad_photo_played_75percent_ratio,
+round(sum(key_action)/sum(activation)*100,3) key_action_ratio,
+round(sum(charge)/sum(event_ad_watch_times),3) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+round(sum(event_multi_conversion)/sum(activation)*100,3) event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3) ad_photo_played_2s_ratio,
+round(sum(event_outbound_call_ratio)/sum(activation)*100,3) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from
+(select * from media_api.kuaishou_account_report_daily 
+where stat_date between date_format(date_sub({date_time},INTERVAL {day_count}-1 DAY),'%Y%m%d') and {date_time}) report
+LEFT JOIN`jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id=cua.account_id
+LEFT JOIN`jeecg-boot`.ctop_project cp ON cua.project_id=cp.id
+LEFT JOIN`jeecg-boot`.ctop_product cn ON cp.product_id=cn.id
+LEFT JOIN`jeecg-boot`.user_company uc ON uc.user_id=cua.user_id
+GROUP BY
+uc.company_id,
+cp.product_id) new
+LEFT JOIN
+(select ifnull(b.product_id,'-') product_id,sum(a.charge) charge from
+(select advertiser_id account_id,sum(charge) charge from media_api.kuaishou_account_report_daily
+where stat_date between date_format(date_sub(date_format(date_sub({date_time},INTERVAL {day_count} DAY),'%Y%m%d'),INTERVAL {day_count}-1 DAY),'%Y%m%d')
+and date_format(date_sub({date_time},INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id) a
+left join
+(SELECT cua.account_id,cp.product_id,cp.id project_id
+FROM`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN`jeecg-boot`.ctop_project cp ON cua.project_id=cp.id) b
+on a.account_id=b.account_id
+group by b.product_id) ratio
+on new.product_id=ratio.product_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouProductReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 187 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouProductReportBusiMonth.py

@@ -0,0 +1,187 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouProductReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouProductReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_product_report_busi_m
+SELECT
+date_format({date_time},'%Y%m') as  stat_mon,
+ifnull(uc.company_id,'-') company_id,
+CASE WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN '北京汇创思拓数字科技有限公司'
+       WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN '广州汇创思拓数字科技有限公司'
+       WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN '上海汇创思拓数字科技有限公司' ELSE uc.company_name END 
+       company_name,
+ ifnull(cp.product_id,'-') product_id,
+ cn.product_name,
+ count(report.advertiser_id) account_num,
+ sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+ from  
+ (select * from  media_api.kuaishou_account_report_daily  
+ where stat_date between  date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') and {date_time}) report
+LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_product cn ON cp.product_id = cn.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+GROUP BY
+cp.product_id;
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouProductReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 341 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouProjectReportBusi.py

@@ -0,0 +1,341 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouProjectReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouProjectReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_project_report_busi_{day_count}d
+select
+new.start_end_time,
+new.company_id,
+new.company_name,
+new.product_id,
+new.product_name,
+new.project_id,
+new.project_name,
+new.account_num,
+new.charge,
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((new.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+new.show,
+new.photo_click,
+new.aclick,
+new.bclick,
+new.photo_click_ratio,
+new.play_3s_ratio,
+new.action_ratio,
+new.impression_1k_cost,
+new.photo_click_cost,
+new.action_cost,
+new.share,
+new.comment,
+new.like,
+new.follow,
+new.cancel_follow,
+new.report,
+new.block,
+new.negative,
+new.submit,
+new.download_started,
+new.download_completed,
+new.activation,
+new.event_pay_first_day,
+new.event_pay_purchase_amount_first_day,
+new.event_pay_first_day_roi,
+new.event_pay,
+new.event_pay_purchase_amount,
+new.event_pay_roi,
+new.event_register,
+new.event_register_cost,
+new.event_register_ratio,
+new.event_jin_jian_app,
+new.event_jin_jian_app_cost,
+new.event_credit_grant_app,
+new.event_credit_grant_app_cost,
+new.event_credit_grant_app_ratio,
+new.event_order_paid,
+new.event_order_paid_purchase_amount,
+new.event_order_paid_cost,
+new.event_next_day_stay,
+new.event_next_day_stay_cost,
+new.event_next_day_stay_ratio,
+new.form_count,
+new.form_cost,
+new.form_action_ratio,
+new.event_jin_jian_landing_page,
+new.event_jin_jian_landing_page_cost,
+new.event_credit_grant_landing_page,
+new.event_credit_grant_landing_page_cost,
+new.event_credit_grant_landing_page_ratio,
+new.event_valid_clues,
+new.event_valid_clues_cost,
+new.event_add_wechat,
+new.event_add_wechat_cost,
+new.event_add_wechat_ratio,
+new.event_get_through,
+new.event_get_through_cost,
+new.event_get_through_ratio,
+new.event_app_invoked,
+new.event_app_invoked_cost,
+new.event_app_invoked_ratio,
+new.event_credit_grant_landing_ratio,
+new.play_5s_ratio,
+new.play_end_ratio,
+new.event_new_user_pay,
+new.event_new_user_pay_cost,
+new.event_new_user_pay_ratio,
+new.click_1k_cost,
+new.ad_product_cnt,
+new.event_goods_view,
+new.merchant_reco_fans,
+new.event_order_amount_roi,
+new.event_goods_view_cost,
+new.merchant_reco_fans_cost,
+new.event_button_click,
+new.event_button_click_cost,
+new.event_button_click_ratio,
+new.event_order_paid_roi,
+new.event_new_user_jinjian_app,
+new.event_new_user_jinjian_app_cost,
+new.event_new_user_jinjian_app_roi,
+new.event_new_user_credit_grant_app,
+new.event_new_user_credit_grant_app_cost,
+new.event_new_user_credit_grant_app_roi,
+new.event_new_user_jinjian_page,
+new.event_new_user_jinjian_page_cost,
+new.event_new_user_jinjian_page_roi,
+new.event_new_user_credit_grant_page,
+new.event_new_user_credit_grant_page_cost,
+new.event_new_user_credit_grant_page_roi,
+new.event_appoint_form,
+new.event_appoint_form_cost,
+new.event_appoint_form_ratio,
+new.event_appoint_jump_click,
+new.event_appoint_jump_click_cost,
+new.event_appoint_jump_click_ratio,
+new.union_event_pay_purchase_amount_7d,
+new.union_event_pay_purchase_amount_7d_roi,
+new.event_order_successed,
+new.ad_photo_played_10s_ratio,
+new.key_action_cost,
+new.event_add_shopping_cart_cost,
+new.event_ad_watch_times_ratio,
+new.event_outbound_call_cost,
+new.event_outbound_call,
+new.action_new_ratio,
+new.event_watch_app_ad,
+new.event_multi_conversion_cost,
+new.event_phone_card_activate,
+new.ad_photo_played_75percent_ratio,
+new.key_action_ratio,
+new.event_ad_watch_times_cost,
+new.event_add_shopping_cart,
+new.key_action,
+new.event_multi_conversion,
+new.event_wechat_connected,
+new.event_dsp_gift_form,
+new.event_intention_confirmed,
+new.event_phone_get_through,
+new.event_multi_conversion_ratio,
+new.event_measurement_house,
+new.ad_photo_played_2s_ratio,
+new.event_outbound_call_ratio,
+new.event_ad_watch_times
+from
+(SELECT
+concat(date_format(date_sub({date_time},INTERVAL {day_count}-1 DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+uc.company_id,
+CASE WHEN uc.company_id='d57fecdcf7a94d009736d9c850731582' THEN '北京汇创思拓数字科技有限公司'
+WHEN uc.company_id='6608ed13c0dd42de93c790dbdf124234' THEN '广州汇创思拓数字科技有限公司'
+WHEN uc.company_id='4b10089775c040119e139087517aed88' THEN '上海汇创思拓数字科技有限公司'
+ELSE uc.company_name END company_name,
+cp.product_id,
+cn.product_name,
+cua.project_id,
+cp.project_name,
+count(report.advertiser_id) account_num,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click) photo_click,
+sum(aclick) aclick,
+sum(bclick) bclick,
+round(sum(photo_click)/sum(`show`),3) photo_click_ratio,
+round(sum(play_3s_ratio)/{day_count}*100,3) play_3s_ratio,
+round(sum(bclick)/sum(aclick)*100,3) action_ratio,
+round(sum(impression_1k_cost)/{day_count},3) impression_1k_cost,
+round(sum(charge)/sum(photo_click),3) photo_click_cost,
+round(sum(charge)/sum(bclick),3) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`) `like`,
+sum(follow) follow,
+sum(cancel_follow) cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3) event_pay_first_day_roi,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+round(sum(event_pay_purchase_amount)/sum(charge)*100,3) event_pay_roi,
+sum(event_register) event_register,
+round(sum(charge)/sum(event_register),3) event_register_cost,
+round(sum(event_register)/sum(activation)*100,3) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+round(sum(charge)/sum(event_jin_jian_app),3) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+round(sum(charge)/sum(event_credit_grant_app),3) event_credit_grant_app_cost,
+round(sum(event_credit_grant_app_ratio)/{day_count}*100,3) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+round(sum(event_order_paid_purchase_amount)/sum(event_order_paid),3) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+round(sum(charge)/sum(event_next_day_stay),3) event_next_day_stay_cost,
+round(sum(event_next_day_stay)/sum(activation)*100,3) event_next_day_stay_ratio,
+sum(form_count) form_count,
+round(sum(charge)/sum(form_count),3) form_cost,
+round(sum(form_count)/sum(`bclick`)*100,3) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+round(sum(charge)/sum(event_jin_jian_landing_page),3) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+round(sum(charge)/sum(event_credit_grant_landing_page),3) event_credit_grant_landing_page_cost,
+round(sum(event_credit_grant_landing_page)/{day_count}*100,3) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+round(sum(charge)/sum(event_valid_clues),3) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+round(sum(charge)/sum(event_add_wechat),3) event_add_wechat_cost,
+round(sum(event_add_wechat)/sum(bclick)*100,3) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+round(sum(charge)/sum(event_get_through),3) event_get_through_cost,
+round(sum(event_get_through)/sum(bclick)*100,3) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+round(sum(charge)/sum(event_app_invoked),3) event_app_invoked_cost,
+round(sum(event_app_invoked)/sum(bclick)*100,3) event_app_invoked_ratio,
+round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3) event_credit_grant_landing_ratio,
+round(sum(play_5s_ratio)/{day_count}*100,3) play_5s_ratio,
+round(sum(play_end_ratio)/{day_count}*100,3) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+round(sum(charge)/sum(event_new_user_pay),3) event_new_user_pay_cost,
+round(sum(event_new_user_pay)/sum(activation)*100,3) event_new_user_pay_ratio,
+round(sum(click_1k_cost)/{day_count}*100) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+round(sum(event_order_amount_roi)/{day_count}*100,3) event_order_amount_roi,
+round(sum(charge)/sum(event_goods_view),3) event_goods_view_cost,
+round(sum(charge)/sum(merchant_reco_fans),3) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+round(sum(charge)/sum(event_button_click),3) event_button_click_cost,
+round(sum(event_button_click)/sum(bclick)*100,3) event_button_click_ratio,
+round(sum(event_order_paid_roi)/{day_count}*100,3) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+round(sum(charge)/sum(event_new_user_jinjian_app),3) event_new_user_jinjian_app_cost,
+round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3) event_new_user_jinjian_app_roi,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+round(sum(charge)/sum(event_new_user_credit_grant_app),3) event_new_user_credit_grant_app_cost,
+round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+round(sum(charge)/sum(event_new_user_jinjian_page),3) event_new_user_jinjian_page_cost,
+round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3) event_new_user_jinjian_page_roi,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+round(sum(charge)/sum(event_new_user_credit_grant_page),3) event_new_user_credit_grant_page_cost,
+round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form) event_appoint_form,
+round(sum(charge)/sum(event_appoint_form),3) event_appoint_form_cost,
+round(sum(event_appoint_form)/sum(form_count)*100,3) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+round(sum(charge)/sum(event_appoint_jump_click),3) event_appoint_jump_click_cost,
+round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3) ad_photo_played_10s_ratio,
+round(sum(charge)/sum(key_action),3) key_action_cost,
+round(sum(charge)/sum(event_add_shopping_cart),3) event_add_shopping_cart_cost,
+round(sum(event_ad_watch_times)/sum(bclick)*100,3) event_ad_watch_times_ratio,
+round(sum(charge)/sum(event_outbound_call),3) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+round(sum(action_new_ratio)/{day_count}*100,3) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+round(sum(charge)/sum(event_multi_conversion),3) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3) ad_photo_played_75percent_ratio,
+round(sum(key_action)/sum(activation)*100,3) key_action_ratio,
+round(sum(charge)/sum(event_ad_watch_times),3) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+round(sum(event_multi_conversion)/sum(activation)*100,3) event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3) ad_photo_played_2s_ratio,
+round(sum(event_outbound_call_ratio)/sum(activation)*100,3) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from
+(select * from media_api.kuaishou_account_report_daily 
+where stat_date between date_format(date_sub({date_time},INTERVAL {day_count}-1 DAY),'%Y%m%d') and {date_time}) report
+LEFT JOIN`jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id=cua.account_id
+LEFT JOIN`jeecg-boot`.ctop_project cp ON cua.project_id=cp.id
+LEFT JOIN`jeecg-boot`.ctop_product cn ON cp.product_id=cn.id
+LEFT JOIN`jeecg-boot`.user_company uc ON uc.user_id=cua.user_id
+GROUP BY
+uc.company_id,
+cp.product_id,
+cua.project_id) new
+LEFT JOIN
+(select ifnull(b.project_id,'-') project_id,ifnull(b.product_id,'-') product_id,sum(a.charge) charge from
+(select advertiser_id account_id,sum(charge) charge from media_api.kuaishou_account_report_daily
+where stat_date between date_format(date_sub(date_format(date_sub({date_time},INTERVAL {day_count} DAY),'%Y%m%d'),INTERVAL {day_count}-1 DAY),'%Y%m%d')
+and date_format(date_sub({date_time},INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id) a
+left join
+(SELECT cua.account_id,cp.product_id,cp.id project_id
+FROM`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN`jeecg-boot`.ctop_project cp ON cua.project_id=cp.id) b
+on a.account_id=b.account_id
+group by b.project_id) ratio
+on new.project_id=ratio.project_id;
+        """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouProjectReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 191 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouProjectReportBusiMonth.py

@@ -0,0 +1,191 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouProjectReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouProjectReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql = """
+SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_project_report_busi_m
+SELECT
+date_format({date_time},'%Y%m') as  stat_mon,
+ifnull(uc.company_id,'-') company_id,
+CASE WHEN uc.company_id = 'd57fecdcf7a94d009736d9c850731582' THEN '北京汇创思拓数字科技有限公司'
+       WHEN uc.company_id = '6608ed13c0dd42de93c790dbdf124234' THEN '广州汇创思拓数字科技有限公司'
+       WHEN uc.company_id = '4b10089775c040119e139087517aed88' THEN '上海汇创思拓数字科技有限公司' ELSE uc.company_name END 
+       company_name,
+ ifnull(cp.product_id,'-') product_id,
+ cn.product_name,
+ ifnull(cua.project_id,'-') project_id,
+ cp.project_name,
+ count(report.advertiser_id) account_num,
+ sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+ from  
+ (select * from  media_api.kuaishou_account_report_daily  
+ where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') and {date_time}
+ ) report
+LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.advertiser_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_product cn ON cp.product_id = cn.id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+GROUP BY
+cua.project_id;
+ 
+            """.format(date_time=self.date_time, day_count=self.day_count)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouProjectReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 368 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouUnitReportBusi.py

@@ -0,0 +1,368 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouUnitReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouUnitReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select account_id from `jeecg-boot`.ctop_user_allocation where media_id=2
+group by account_id """
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_unit_report_busi_{day_count}d 
+SELECT
+concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+report.account_id,
+bei.advertiser_id,
+report.campaign_id,
+report.campaign_name,
+report.unit_id,
+report.unit_name,
+bei.project_id,
+bei.product_id,
+bei.company_id,
+bei.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.user_id,
+bei.leader_id,
+bei.org_code,
+report.charge,
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((report.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+report.`show`,
+report.photo_click,
+report.aclick,
+report.bclick,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio,
+report.impression_1k_cost,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.cancel_follow,
+report.report,
+report.block,
+report.negative,
+report.submit,
+report.download_started,
+report.download_completed,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost,
+report.event_next_day_stay,
+report.event_next_day_stay_cost,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost,
+report.form_action_ratio,
+report.event_jin_jian_landing_page,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans,
+report.event_order_amount_roi,
+report.event_goods_view_cost,
+report.merchant_reco_fans_cost,
+report.event_button_click,
+report.event_button_click_cost,
+report.event_button_click_ratio,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi,
+report.event_appoint_form,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio,
+report.event_appoint_jump_click,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad,
+report.event_multi_conversion_cost,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+campaign_id,
+campaign_name,
+unit_id,
+unit_name,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from 
+ media_api.kuaishou_unit_report_daily  a 
+where stat_date between date_format(date_sub({date_time}, INTERVAL ({day_count} -1 ) DAY),'%Y%m%d') and {date_time}
+and advertiser_id={account_id}
+ group by advertiser_id,campaign_id,campaign_name,unit_id,unit_name) report
+LEFT JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+and cua.account_id ={account_id}) bei 
+ON report.account_id = bei.account_id
+LEFT JOIN
+(select advertiser_id account_id,unit_id,round(sum(charge),3) charge from `media_api`.kuaishou_unit_report_daily a  
+where stat_date between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d') 
+and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d')
+group by advertiser_id,unit_id) ratio
+on report.account_id = ratio.account_id and report.unit_id=ratio.unit_id
+;""".format(date_time=self.date_time, day_count=self.day_count, account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouUnitReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 360 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouUnitReportBusiMonth.py

@@ -0,0 +1,360 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouUnitReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouUnitReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select account_id from `jeecg-boot`.ctop_user_allocation where media_id=2
+group by account_id """
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_unit_report_busi_m
+SELECT
+date_format({date_time},'%Y%m')stat_mon,
+report.account_id,
+bei.advertiser_id,
+report.campaign_id,
+report.campaign_name,
+report.unit_id,
+report.unit_name,
+bei.project_id,
+bei.product_id,
+bei.company_id,
+bei.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.user_id,
+bei.leader_id,
+bei.org_code,
+report.charge,
+report.`show`,
+report.photo_click,
+report.aclick,
+report.bclick,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio,
+report.impression_1k_cost,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.cancel_follow,
+report.report,
+report.block,
+report.negative,
+report.submit,
+report.download_started,
+report.download_completed,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost,
+report.event_next_day_stay,
+report.event_next_day_stay_cost,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost,
+report.form_action_ratio,
+report.event_jin_jian_landing_page,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans,
+report.event_order_amount_roi,
+report.event_goods_view_cost,
+report.merchant_reco_fans_cost,
+report.event_button_click,
+report.event_button_click_cost,
+report.event_button_click_ratio,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi,
+report.event_appoint_form,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio,
+report.event_appoint_jump_click,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad,
+report.event_multi_conversion_cost,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+campaign_id,
+campaign_name,
+unit_id,
+unit_name,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from 
+ media_api.kuaishou_unit_report_daily  a 
+where stat_date between date_format(date_sub({date_time}, INTERVAL {day_count}  DAY),'%Y%m%d') and {date_time}
+and advertiser_id={account_id}
+ group by advertiser_id,campaign_id,campaign_name,unit_id,unit_name) report
+LEFT JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+and cua.account_id ={account_id}) bei 
+ON report.account_id = bei.account_id
+;""".format(date_time=self.date_time, day_count=self.day_count, account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouUnitReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 258 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouVideoList.py

@@ -0,0 +1,258 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/12 10:20 上午
+# @Site    :
+# @File    : AppKuaishouVideoList.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+import logging
+from math import ceil
+
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AppKuaishouVideoList:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.date_time = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y-%m-%d")
+
+    def execSql(self):
+        try:
+            sql = """select account_id from `jeecg-boot`.ctop_user_allocation where  account_status=0 and media_id=2""".format(date_time=self.date_time)
+            cur = self.conn.cursor()
+            cur.execute(sql)
+            result = cur.fetchall()
+            for account_id in result:
+                try:
+                    sql = """
+            replace into application.app_kuaishou_video_list
+            select account_id,
+                   status,
+                   photo_id,
+                   width,
+                   height,
+                   url,
+                   cover_url,
+                   material_type,
+                   channel_type,
+                   v.stat_date,
+                   case when v.signature is null then '' else v.signature end as signature,
+                   photo_name,
+                   case when tag_category_id is null then '' else tag_category_id end as tag_category_id,
+                   tag_category_name,
+                   case when parent_id is null then '' else parent_id end as parent_id,
+                   parent_name,
+                   case when tag_code is null then '' else tag_code end as tag_code,
+                   tag_name,
+                   clip_id,
+                   clip_name,
+                   shot_id,
+                   shot_name,
+                   plan_id,
+                   plan_name,
+                   leader_id,
+                   leader_name,
+                   material_innovate,
+                   hot.stat_date,
+                   effect.stat_date
+            from (
+                     select `account_id`,
+                            `status`,
+                            `photo_id`,
+                            `width`,
+                            `height`,
+                            `url`,
+                            `cover_url`,
+                            `material_type`,
+                            `channel_type`,
+                            `stat_date`,
+                            `signature`,
+                            case when t.material_name is null then a.photo_name else t.material_name end as photo_name,
+                            clip_id,
+                            clip_name,
+                            shot_id,
+                            shot_name,
+                            plan_id,
+                            plan_name,
+                            leader_id,
+                            leader_name,
+                            material_innovate
+
+                     from (select `account_id`,
+                                  `status`,
+                                  `photo_id`,
+                                  `width`,
+                                  `height`,
+                                  `url`,
+                                  `cover_url`,
+                                  `material_type`,
+                                  `channel_type`,
+                                  `stat_date`,
+                                  `signature`,
+                                  `photo_name`
+
+                           from media_api.kuaishou_video_list 
+                           where date_format(stat_date, '%Y-%m-%d') >= date_sub('{date_time}', INTERVAL 13 DAY)
+                           and account_id ={account_id}
+                          ) a
+                              left join
+                          (select material_id,
+                                  clip_id,
+                                  clip_name,
+                                  shot_id,
+                                  shot_name,
+                                  plan_id,
+                                  plan_name,
+                                  leader_id,
+                                  leader_name,
+                                  material_name,
+                                  material_innovate
+                           from (select cma.material_id,
+                                        clip_id,
+                                        clip.realname   as clip_name,
+                                        shot_id,
+                                        shot.realname   as shot_name,
+                                        plan_id,
+                                        plan.realname   as plan_name,
+                                        cma.leader_id,
+                                        leader.realname as leader_name,
+                                        material_name,
+                                        material_innovate
+                                 from `jeecg-boot`.ctop_material_ascription cma
+                                          left join
+                                      `jeecg-boot`.ctop_material_info b
+                                      on cma.material_id = b.id
+                                          left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+                                          left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+                                          left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+                                          left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+                                ) v) t
+                          on a.signature = t.material_id) v
+                     left join
+                 (select code,
+                         tag_category_id,
+                         tag_category_name,
+                         parent_id,
+                         parent_name,
+                         group_concat(tag_code order by tag_code separator '-') tag_code,
+                         group_concat(tag_name order by tag_code separator '-') tag_name
+                  from (select a.code, b.tag_category_id, b.tag_category_name, b.parent_id, b.parent_name, b.tag_code, b.tag_name
+                        from `jeecg-boot`.ctop_material_tag_info a
+                                 left join
+                             (select tag_category_id,
+                                     tag_category_name,
+                                     case
+                                         when parent_id is not null then parent_id
+                                         when tag_category_id = 2 then null
+                                         else tag_code end parent_id,
+                                     case
+                                         when parent_id = 60155 then '真人'
+                                         when parent_id = 60156 then '制作'
+                                         when parent_id = 60160 then '外拍场景'
+                                         when parent_id = 60161 then '公司内景'
+                                         when tag_category_id = 2 then null
+                                         else '其他' end     parent_name,
+                                     tag_code,
+                                     tag_name
+                              from `jeecg-boot`.ctop_tag_info) b
+                             on a.tag_code = b.tag_code) c
+                  group by code, tag_category_id, tag_category_name, parent_id, parent_name) m
+                 on v.signature = m.code
+                     left join (
+                select stat_date, signature, sum
+                from (
+                         select stat_date, signature, sum, row_number() over (partition by signature order by sum) as rn
+                         from (
+                                  select stat_date,
+                                         signature,
+                                         sum(cost) over (partition by signature order by stat_date) as sum
+                                  from (
+                                           select rep.stat_date, video.signature, cost
+                                           from (select photo_id, signature, stat_date
+                                                 from (select photo_id,
+                                                              stat_date,
+                                                              signature,
+                                                              row_number() over (partition by photo_id order by stat_date) rn
+                                                       from media_api.kuaishou_video_list
+                                                       where photo_id is not null
+                                                         and photo_id != 0 and account_id ={account_id}
+                                                         and date_format(stat_date, '%Y-%m-%d') >=
+                                                             date_sub('{date_time}', INTERVAL 6 DAY)
+                                                       group by photo_id, signature) s
+                                                 where rn = 1) video
+                                                    left join
+                                                (
+                                                    select stat_date, photo_id, sum(charge) as cost
+                                                    from media_api.kuaishou_material_video_report_daily
+                                                    where photo_id is not null
+                                                      and photo_id != 0 and advertiser_id ={account_id}
+                                                    group by stat_date, photo_id
+                                                ) rep
+                                                on video.photo_id = rep.photo_id
+                                           where rep.stat_date between date_format(video.stat_date, '%Y%m%d') and date_format(date_add(video.stat_date, INTERVAL 6 DAY), '%Y%m%d')) t) t
+                         where sum >= 5000
+                     ) t
+                where rn = 1
+            ) effect
+                               on effect.signature = v.signature
+                     left join (
+                select stat_date, signature, sum
+                from (
+                         select stat_date, signature, sum, row_number() over (partition by signature order by sum) as rn
+                         from (
+                                  select stat_date,
+                                         signature,
+                                         sum(cost) over (partition by signature order by stat_date) as sum
+                                  from (
+                                           select rep.stat_date, video.signature, cost
+                                           from (select photo_id, signature, stat_date
+                                                 from (select photo_id,
+                                                              stat_date,
+                                                              signature,
+                                                              row_number() over (partition by photo_id order by stat_date) rn
+                                                       from media_api.kuaishou_video_list
+                                                       where photo_id is not null 
+                                                         and photo_id != 0 and account_id ={account_id}
+                                                         and date_format(stat_date, '%Y-%m-%d') >=
+                                                             date_sub('{date_time}', INTERVAL 13 DAY)
+                                                       group by photo_id, signature) s
+                                                 where rn = 1) video
+                                                    left join
+                                                (
+                                                    select stat_date, photo_id, sum(charge) as cost
+                                                    from media_api.kuaishou_material_video_report_daily
+                                                    where photo_id is not null and advertiser_id ={account_id}
+                                                      and photo_id != 0
+                                                    group by stat_date, photo_id
+                                                ) rep
+                                                on video.photo_id = rep.photo_id
+                                           where rep.stat_date between date_format(video.stat_date, '%Y%m%d') and date_format(date_add(video.stat_date, INTERVAL 13 DAY), '%Y%m%d')) t) t
+                         where sum >= 50000
+                     ) t
+                where rn = 1
+            ) hot
+                               on hot.signature = v.signature
+            group by account_id, photo_id, v.stat_date, v.signature, tag_category_id, parent_id, tag_code
+                                """.format(date_time=self.date_time, account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouVideoList().execSql()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 186 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouVideoListAll.py

@@ -0,0 +1,186 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/12 10:20 上午
+# @Site    :
+# @File    : AppKuaishouVideoListAll.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+import logging
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AppKuaishouVideoListAll:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+
+    def execSql(self):
+        try:
+            sql = """select account_id from `jeecg-boot`.ctop_user_allocation where  account_status=0 and media_id=2"""
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                try:
+                    sql = """
+replace into application.app_kuaishou_video_list
+select account_id,
+       status,
+       photo_id,
+       width,
+       height,
+       url,
+       cover_url,
+       material_type,
+       channel_type,
+       v.stat_date,
+       case when v.signature is null then '' else v.signature end as signature,
+       photo_name,
+       case when tag_category_id is null then '' else tag_category_id end as tag_category_id,
+       tag_category_name,
+       case when parent_id is null then '' else parent_id end as parent_id,
+       parent_name,
+       case when tag_code is null then '' else tag_code end as tag_code,
+       tag_name,
+       clip_id,
+       clip_name,
+       shot_id,
+       shot_name,
+       plan_id,
+       plan_name,
+       leader_id,
+       leader_name,
+       material_innovate,
+       hot.stat_date,
+       effect.stat_date
+from (
+         select `account_id`,
+                `status`,
+                `photo_id`,
+                `width`,
+                `height`,
+                `url`,
+                `cover_url`,
+                `material_type`,
+                `channel_type`,
+                `stat_date`,
+                `signature`,
+                case when t.material_name is null then a.photo_name else t.material_name end as photo_name,
+                clip_id,
+                clip_name,
+                shot_id,
+                shot_name,
+                plan_id,
+                plan_name,
+                leader_id,
+                leader_name,
+                material_innovate
+
+         from (select `account_id`,
+                      `status`,
+                      `photo_id`,
+                      `width`,
+                      `height`,
+                      `url`,
+                      `cover_url`,
+                      `material_type`,
+                      `channel_type`,
+                      `stat_date`,
+                      `signature`,
+                      `photo_name`
+
+               from media_api.kuaishou_video_list 
+               where  account_id ={account_id}
+              ) a
+                  left join
+              (select material_id,
+                      clip_id,
+                      clip_name,
+                      shot_id,
+                      shot_name,
+                      plan_id,
+                      plan_name,
+                      leader_id,
+                      leader_name,
+                      material_name,
+                      material_innovate
+               from (select cma.material_id,
+                            clip_id,
+                            clip.realname   as clip_name,
+                            shot_id,
+                            shot.realname   as shot_name,
+                            plan_id,
+                            plan.realname   as plan_name,
+                            cma.leader_id,
+                            leader.realname as leader_name,
+                            material_name,
+                            material_innovate
+                     from `jeecg-boot`.ctop_material_ascription cma
+                              left join
+                          `jeecg-boot`.ctop_material_info b
+                          on cma.material_id = b.id
+                              left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+                              left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+                              left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+                              left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+                    ) v) t
+              on a.signature = t.material_id) v
+         left join
+     (select code,
+             tag_category_id,
+             tag_category_name,
+             parent_id,
+             parent_name,
+             group_concat(tag_code order by tag_code separator '-') tag_code,
+             group_concat(tag_name order by tag_code separator '-') tag_name
+      from (select a.code, b.tag_category_id, b.tag_category_name, b.parent_id, b.parent_name, b.tag_code, b.tag_name
+            from `jeecg-boot`.ctop_material_tag_info a
+                     left join
+                 (select tag_category_id,
+                         tag_category_name,
+                         case
+                             when parent_id is not null then parent_id
+                             when tag_category_id = 2 then null
+                             else tag_code end parent_id,
+                         case
+                             when parent_id = 60155 then '真人'
+                             when parent_id = 60156 then '制作'
+                             when parent_id = 60160 then '外拍场景'
+                             when parent_id = 60161 then '公司内景'
+                             when tag_category_id = 2 then null
+                             else '其他' end     parent_name,
+                         tag_code,
+                         tag_name
+                  from `jeecg-boot`.ctop_tag_info) b
+                 on a.tag_code = b.tag_code) c
+      group by code, tag_category_id, tag_category_name, parent_id, parent_name) m
+     on v.signature = m.code
+         left join (
+                select signature,effect_date as stat_date from application.effect_material where media_id =2
+) effect
+                   on effect.signature = v.signature
+         left join (
+     select signature,hot_date as stat_date from application.hot_material where media_id =2
+) hot
+                   on hot.signature = v.signature
+group by account_id, photo_id, v.stat_date, v.signature, tag_category_id, parent_id, tag_code
+                    """.format(account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouVideoListAll().execSql()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 377 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouVideoReportBusi.py

@@ -0,0 +1,377 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouVideoReportBusi.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouVideoReportBusi:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = self.parm[1]
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select advertiser_id from media_api.kuaishou_material_video_report_daily 
+where  stat_date between date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d')  and {date_time}
+group by advertiser_id """.format(date_time=self.date_time,day_count=self.day_count)
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_video_report_busi_{day_count}d 
+SELECT
+concat(date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d'),'-',{date_time}) start_end_time,
+report.account_id,
+report.photo_id,
+report.image_token,
+report.photo_caption,
+report.photo_url,
+report.cover_url,
+cua.project_id, #项目id  
+product.id product_id, #项目中产品id
+cua.advertiser_id,#广告主id
+uc.company_id,#运营人的公司id                      
+cua.auth_name account_name,#账户名称
+cp.project_name,#项目名称  
+ca.`name` advertiser_name,#广告主名称
+uc.company_name, #运营人的公司名称
+cp.sale_id, #项目里的销售id
+cua.user_id, #运营id                                                                                                          
+cma.clip_id, #剪辑id
+cma.shot_id, #拍摄id
+cma.plan_id, #编导id
+cma.leader_id, #负责人id
+su.org_code, #机构编码
+video.signature, #md5
+video.channel_type chl_type, #渠道类型
+clip.realname clip_name, #剪辑名称
+shot.realname shot_name, #拍摄名称
+plan.realname plan_name, #编导名称
+leader.realname lead_name,#负责人名称
+CASE WHEN cmi.material_name IS NOT NULL THEN cmi.material_name 
+     ELSE video.photo_name END AS video_name,##视频名称
+case when video.channel_type = 1 then '素造' else '内部' end as  chl_name, #渠道名称
+up.company_id clip_company_id, #剪辑公司
+ifnull(ratio.charge,0) last_charge,
+ifnull(round((report.charge-ratio.charge)/ratio.charge*100,3),0) charge_ratio,
+report.charge ,
+report.`show`,
+report.photo_click,
+report.aclick ,
+report.bclick ,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio ,
+report.impression_1k_cost ,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow ,
+report.cancel_follow,
+report.report ,
+report.block,
+report.negative,
+report.submit ,
+report.download_started,
+report.download_completed ,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay ,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app ,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost ,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost ,
+report.event_next_day_stay,
+report.event_next_day_stay_cost ,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost ,
+report.form_action_ratio,
+report.event_jin_jian_landing_page ,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost ,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost ,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay ,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio ,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans ,
+report.event_order_amount_roi,
+report.event_goods_view_cost ,
+report.merchant_reco_fans_cost,
+report.event_button_click ,
+report.event_button_click_cost,
+report.event_button_click_ratio ,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi ,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost ,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page ,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi ,
+report.event_appoint_form ,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio ,
+report.event_appoint_jump_click ,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio ,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed ,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost ,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost ,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad ,
+report.event_multi_conversion_cost ,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio ,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+photo_id,
+image_token,
+photo_caption,
+photo_url,
+cover_url,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from  media_api.kuaishou_material_video_report_daily 
+where  stat_date between date_format(date_sub({date_time}, INTERVAL ({day_count} -1 )  DAY),'%Y%m%d') and {date_time}
+and advertiser_id={account_id}
+group by 
+advertiser_id,
+photo_id) report
+left join media_api.kuaishou_video_list video
+on report.account_id = video.account_id and report.photo_id = video.photo_id 
+LEFT JOIN (select  code,material_name from `jeecg-boot`.ctop_material_info where status=1) cmi ON video.signature = cmi.CODE
+LEFT JOIN (select material_id,clip_id,shot_id,plan_id,leader_id from (
+select  material_id,clip_id,shot_id,plan_id,leader_id,row_number()over(partition by material_id order by create_time)rn
+ from `jeecg-boot`.ctop_material_ascription )m where  rn=1) cma 
+ON video.signature = cma.material_id
+LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.account_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON cua.advertiser_id = ca.id
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.user_company uc ON cua.user_id = uc.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON cp.product_id = product.id
+LEFT JOIN `jeecg-boot`.sys_user su ON cua.user_id = su.id
+LEFT JOIN `jeecg-boot`.sys_user clip ON clip.id = cma.clip_id
+LEFT JOIN `jeecg-boot`.sys_user shot ON shot.id = cma.shot_id
+LEFT JOIN `jeecg-boot`.sys_user plan ON plan.id = cma.plan_id
+LEFT JOIN `jeecg-boot`.sys_user leader ON leader.id = cma.leader_id
+LEFT JOIN `jeecg-boot`.user_company up ON up.user_id = cma.clip_id 
+LEFT JOIN 
+(select advertiser_id account_id,photo_id,round(sum(charge),3) charge from `media_api`.kuaishou_material_video_report_daily a  
+where a.advertiser_id={account_id} and  stat_date between date_format(date_sub(date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d'), INTERVAL {day_count}-1 DAY),'%Y%m%d') 
+and date_format(date_sub({date_time}, INTERVAL {day_count} DAY),'%Y%m%d') 
+group by advertiser_id,photo_id) ratio
+on report.account_id=ratio.account_id and report.photo_id=ratio.photo_id
+where cua.account_id={account_id};
+""".format(date_time=self.date_time, day_count=self.day_count, account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouVideoReportBusi().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 369 - 0
Apietl/kwai_etl/Q4Task/AppKuaishouVideoReportBusiMonth.py

@@ -0,0 +1,369 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/11/10 3:37 下午
+# @Site    :
+# @File    : AppKuaishouVideoReportBusiMonth.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouVideoReportBusiMonth:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+        self.day_count = Utils.get_days()
+
+    def taskHandler(self):
+        try:
+            sql_porject = """select advertiser_id from media_api.kuaishou_material_video_report_daily 
+            where  stat_date between date_format(date_sub({date_time}, INTERVAL ({day_count})  DAY),'%Y%m%d') and {date_time}
+            group by advertiser_id """.format(date_time=self.date_time,day_count=self.day_count)
+            account_list = TidbConn.quary(self.conn, sql_porject)
+            for account_id in account_list:
+                try:
+                    sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+REPLACE INTO application.app_kuaishou_video_report_busi_m
+SELECT
+date_format({date_time},'%Y%m') stat_mon,
+report.account_id,
+report.photo_id,
+report.image_token,
+report.photo_caption,
+report.photo_url,
+report.cover_url,
+cua.project_id, #项目id  
+product.id product_id, #项目中产品id
+cua.advertiser_id,#广告主id
+uc.company_id,#运营人的公司id                      
+cua.auth_name account_name,#账户名称
+cp.project_name,#项目名称  
+ca.`name` advertiser_name,#广告主名称
+uc.company_name, #运营人的公司名称
+cp.sale_id, #项目里的销售id
+cua.user_id, #运营id                                                                                                          
+cma.clip_id, #剪辑id
+cma.shot_id, #拍摄id
+cma.plan_id, #编导id
+cma.leader_id, #负责人id
+su.org_code, #机构编码
+video.signature, #md5
+video.channel_type chl_type, #渠道类型
+clip.realname clip_name, #剪辑名称
+shot.realname shot_name, #拍摄名称
+plan.realname plan_name, #编导名称
+leader.realname lead_name,#负责人名称
+CASE WHEN cmi.material_name IS NOT NULL THEN cmi.material_name 
+     ELSE video.photo_name END AS video_name,##视频名称
+case when video.channel_type = 1 then '素造' else '内部' end as  chl_name, #渠道名称
+up.company_id clip_company_id, #剪辑公司
+report.charge ,
+report.`show`,
+report.photo_click,
+report.aclick ,
+report.bclick ,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio ,
+report.impression_1k_cost ,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow ,
+report.cancel_follow,
+report.report ,
+report.block,
+report.negative,
+report.submit ,
+report.download_started,
+report.download_completed ,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay ,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app ,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost ,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost ,
+report.event_next_day_stay,
+report.event_next_day_stay_cost ,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost ,
+report.form_action_ratio,
+report.event_jin_jian_landing_page ,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost ,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost ,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay ,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio ,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans ,
+report.event_order_amount_roi,
+report.event_goods_view_cost ,
+report.merchant_reco_fans_cost,
+report.event_button_click ,
+report.event_button_click_cost,
+report.event_button_click_ratio ,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi ,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost ,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page ,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi ,
+report.event_appoint_form ,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio ,
+report.event_appoint_jump_click ,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio ,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+report.event_order_successed ,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost ,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost ,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad ,
+report.event_multi_conversion_cost ,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio ,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times
+from 
+(select 
+advertiser_id account_id,
+photo_id,
+image_token,
+photo_caption,
+photo_url,
+cover_url,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(play_3s_ratio)/{day_count}*100,3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost)/{day_count},3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)/{day_count}*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)/{day_count}*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)/{day_count}*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)/{day_count}*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)/{day_count}*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)/{day_count}*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)/{day_count}*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)/{day_count}*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)/{day_count}*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)/{day_count}*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)/{day_count}*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)/{day_count}*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)/{day_count}*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times
+from  media_api.kuaishou_material_video_report_daily 
+where  stat_date between  date_format(date_sub({date_time}, INTERVAL ({day_count})  DAY),'%Y%m%d') and {date_time}
+and advertiser_id={account_id}
+group by 
+advertiser_id,
+photo_id)report
+left join media_api.kuaishou_video_list video
+on report.account_id = video.account_id and report.photo_id = video.photo_id 
+LEFT JOIN (select  code,material_name from `jeecg-boot`.ctop_material_info where status=1) cmi ON video.signature = cmi.CODE
+LEFT JOIN (select material_id,clip_id,shot_id,plan_id,leader_id from (
+select  material_id,clip_id,shot_id,plan_id,leader_id,row_number()over(partition by material_id order by create_time)rn
+ from `jeecg-boot`.ctop_material_ascription )m where  rn=1) cma 
+ON video.signature = cma.material_id
+LEFT JOIN `jeecg-boot`.ctop_user_allocation cua ON report.account_id = cua.account_id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON cua.advertiser_id = ca.id
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.user_company uc ON cua.user_id = uc.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON cp.product_id = product.id
+LEFT JOIN `jeecg-boot`.sys_user su ON cua.user_id = su.id
+LEFT JOIN `jeecg-boot`.sys_user clip ON clip.id = cma.clip_id
+LEFT JOIN `jeecg-boot`.sys_user shot ON shot.id = cma.shot_id
+LEFT JOIN `jeecg-boot`.sys_user plan ON plan.id = cma.plan_id
+LEFT JOIN `jeecg-boot`.sys_user leader ON leader.id = cma.leader_id
+LEFT JOIN `jeecg-boot`.user_company up ON up.user_id = cma.clip_id
+where cua.account_id={account_id};
+""".format(date_time=self.date_time, day_count=self.day_count, account_id=account_id[0])
+                    print(sql)
+                    TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                    self.conn.commit()
+                except Exception as e:
+                    logging.error(e)
+                    return -1
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouVideoReportBusiMonth().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 158 - 0
Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceAgeReportDaily.py

@@ -0,0 +1,158 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2022/2/17 4:13 下午
+# @Site    : 
+# @File    : KuaishouUnitAudienceAgeReportDaily.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class KuaishouUnitAudienceAgeReportDaily:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+
+    def ageReport(self):
+        try:
+            sql = """
+            replace into application.kuaishou_unit_audience_age_report_daily_d
+select age.account_id,
+       campaign_id,
+       age.unit_id,
+       event_register_cost,
+       age_segment,
+       photo_click,
+       key_action_cost,
+       event_next_day_stay,
+       photo_click_ratio,
+       photo_click_cost,
+       charge,
+       event_pay,
+       action_cost,
+       conversion_cost,
+       bclick,
+       key_action_ratio,
+       play_3s_ratio,
+       play_5s_ratio,
+       activation,
+       event_app_invoked_ratio,
+       `show`,
+       event_new_user_pay_cost,
+       aclick,
+       deep_conversion_num,
+       action_ratio,
+       conversion_ratio,
+       event_pay_purchase_amount_first_day,
+       event_pay_first_day,
+       conversion_num_cost,
+       event_new_user_pay_ratio,
+       event_app_invoked,
+       event_next_day_stay_ratio,
+       event_register,
+       event_new_user_pay,
+       event_next_day_stay_cost,
+       deep_conversion_ratio,
+       deep_conversion_cost,
+       `like`,
+       event_app_invoked_cost,
+       key_action,
+       ad_photo_played_10s,
+       form_cost,
+       event_pay_purchase_amount,
+       click_1k_cost,
+       impression_1k_cost,
+       conversion_num,
+       stat_date,
+       ad_photo_played_2s_ratio,
+       follow,
+       signature,
+       project_id,
+       project_name
+from (select account_id,
+             campaign_id,
+             unit_id,
+             event_register_cost,
+             age_segment,
+             photo_click,
+             key_action_cost,
+             event_next_day_stay,
+             photo_click_ratio,
+             photo_click_cost,
+             charge,
+             event_pay,
+             action_cost,
+             conversion_cost,
+             bclick,
+             key_action_ratio,
+             play_3s_ratio,
+             play_5s_ratio,
+             activation,
+             event_app_invoked_ratio,
+             `show`,
+             event_new_user_pay_cost,
+             aclick,
+             deep_conversion_num,
+             action_ratio,
+             conversion_ratio,
+             event_pay_purchase_amount_first_day,
+             event_pay_first_day,
+             conversion_num_cost,
+             event_new_user_pay_ratio,
+             event_app_invoked,
+             event_next_day_stay_ratio,
+             event_register,
+             event_new_user_pay,
+             event_next_day_stay_cost,
+             deep_conversion_ratio,
+             deep_conversion_cost,
+             `like`,
+             event_app_invoked_cost,
+             key_action,
+             ad_photo_played_10s,
+             form_cost,
+             event_pay_purchase_amount,
+             click_1k_cost,
+             impression_1k_cost,
+             conversion_num,
+             stat_date,
+             ad_photo_played_2s_ratio,
+             follow
+      from media_api.kuaishou_unit_audience_age_report_daily
+      where stat_date = {stat_date}) age
+         inner join
+     (select account_id, unit_id, photo_id
+      from `jeecg-boot`.ctop_ai_kuaishou_creative_level_operation_record
+      group by account_id, unit_id, photo_id) operation
+     on age.account_id = operation.account_id and age.unit_id = operation.unit_id
+         inner join
+     (select account_id, signature, photo_id
+      from media_api.kuaishou_video_list
+      group by account_id, signature, photo_id) video
+     on operation.account_id = video.account_id and operation.photo_id = video.photo_id
+         left join (
+    select account_id, project_id
+    from `jeecg-boot`.ctop_user_allocation
+) acc_info
+                   on age.account_id = acc_info.account_id
+         left join (
+    select id, project_name
+    from `jeecg-boot`.ctop_project
+) pro
+                   on acc_info.project_id = pro.id
+            """.format(stat_date=self.start_time)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    KuaishouUnitAudienceAgeReportDaily().ageReport()

+ 168 - 0
Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceCityReportDaily.py

@@ -0,0 +1,168 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2022/2/17 4:13 下午
+# @Site    : 
+# @File    : KuaishouUnitAudienceCityReportDaily.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class KuaishouUnitAudienceCityReportDaily:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+
+    def cityReport(self):
+        try:
+            sql = """select account_id from media_api.kuaishou_unit_audience_city_report_daily where 
+            stat_date = {start_time} group by account_id
+            """.format(start_time=self.start_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """
+replace into application.kuaishou_unit_audience_city_report_daily_d
+select city.account_id,
+       campaign_id,
+       city.unit_id,
+       event_register_cost,
+       province,
+       city,
+       photo_click,
+       key_action_cost,
+       event_next_day_stay,
+       photo_click_ratio,
+       photo_click_cost,
+       charge,
+       event_pay,
+       action_cost,
+       conversion_cost,
+       bclick,
+       key_action_ratio,
+       play_3s_ratio,
+       play_5s_ratio,
+       activation,
+       event_app_invoked_ratio,
+       `show`,
+       event_new_user_pay_cost,
+       aclick,
+       deep_conversion_num,
+       action_ratio,
+       conversion_ratio,
+       event_pay_purchase_amount_first_day,
+       event_pay_first_day,
+       conversion_num_cost,
+       event_new_user_pay_ratio,
+       event_app_invoked,
+       event_next_day_stay_ratio,
+       event_register,
+       event_new_user_pay,
+       event_next_day_stay_cost,
+       deep_conversion_ratio,
+       deep_conversion_cost,
+       `like`,
+       event_app_invoked_cost,
+       key_action,
+       ad_photo_played_10s,
+       form_cost,
+       event_pay_purchase_amount,
+       click_1k_cost,
+       impression_1k_cost,
+       conversion_num,
+       stat_date,
+       ad_photo_played_2s_ratio,
+       follow,
+       signature,
+       project_id,
+       project_name
+from (select account_id,
+             campaign_id,
+             unit_id,
+             event_register_cost,
+             province,
+             city,
+             photo_click,
+             key_action_cost,
+             event_next_day_stay,
+             photo_click_ratio,
+             photo_click_cost,
+             charge,
+             event_pay,
+             action_cost,
+             conversion_cost,
+             bclick,
+             key_action_ratio,
+             play_3s_ratio,
+             play_5s_ratio,
+             activation,
+             event_app_invoked_ratio,
+             `show`,
+             event_new_user_pay_cost,
+             aclick,
+             deep_conversion_num,
+             action_ratio,
+             conversion_ratio,
+             event_pay_purchase_amount_first_day,
+             event_pay_first_day,
+             conversion_num_cost,
+             event_new_user_pay_ratio,
+             event_app_invoked,
+             event_next_day_stay_ratio,
+             event_register,
+             event_new_user_pay,
+             event_next_day_stay_cost,
+             deep_conversion_ratio,
+             deep_conversion_cost,
+             `like`,
+             event_app_invoked_cost,
+             key_action,
+             ad_photo_played_10s,
+             form_cost,
+             event_pay_purchase_amount,
+             click_1k_cost,
+             impression_1k_cost,
+             conversion_num,
+             stat_date,
+             ad_photo_played_2s_ratio,
+             follow
+      from media_api.kuaishou_unit_audience_city_report_daily
+      where stat_date = 20210705 and account_id={account_id}) city
+         inner join
+     (select account_id, unit_id, photo_id
+      from `jeecg-boot`.ctop_ai_kuaishou_creative_level_operation_record where account_id={account_id}
+      group by account_id, unit_id, photo_id) operation
+     on city.account_id = operation.account_id and city.unit_id = operation.unit_id
+         inner join
+     (select account_id, signature, photo_id
+      from media_api.kuaishou_video_list where account_id={account_id}
+      group by account_id, signature, photo_id) video
+     on operation.account_id = video.account_id and operation.photo_id = video.photo_id
+         left join (
+    select account_id, project_id
+    from `jeecg-boot`.ctop_user_allocation where account_id={account_id}
+) acc_info
+                   on city.account_id = acc_info.account_id
+         left join (
+    select id, project_name
+    from `jeecg-boot`.ctop_project
+) pro
+                   on acc_info.project_id = pro.id       
+                    """.format(stat_date=self.start_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+                TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    KuaishouUnitAudienceCityReportDaily().cityReport()

+ 159 - 0
Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceGenderReportDaily.py

@@ -0,0 +1,159 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2022/2/17 4:13 下午
+# @Site    : 
+# @File    : KuaishouUnitAudienceGenderReportDaily.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class KuaishouUnitAudienceGenderReportDaily:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+
+    def genderReport(self):
+        try:
+            sql = """
+            replace into application.kuaishou_unit_audience_gender_report_daily_d
+select gender.account_id,
+       campaign_id,
+       gender.unit_id,
+       event_register_cost,
+       gender,
+       photo_click,
+       key_action_cost,
+       event_next_day_stay,
+       photo_click_ratio,
+       photo_click_cost,
+       charge,
+       event_pay,
+       action_cost,
+       conversion_cost,
+       bclick,
+       key_action_ratio,
+       play_3s_ratio,
+       play_5s_ratio,
+       activation,
+       event_app_invoked_ratio,
+       `show`,
+       event_new_user_pay_cost,
+       aclick,
+       deep_conversion_num,
+       action_ratio,
+       conversion_ratio,
+       event_pay_purchase_amount_first_day,
+       event_pay_first_day,
+       conversion_num_cost,
+       event_new_user_pay_ratio,
+       event_app_invoked,
+       event_next_day_stay_ratio,
+       event_register,
+       event_new_user_pay,
+       event_next_day_stay_cost,
+       deep_conversion_ratio,
+       deep_conversion_cost,
+       `like`,
+       event_app_invoked_cost,
+       key_action,
+       ad_photo_played_10s,
+       form_cost,
+       event_pay_purchase_amount,
+       click_1k_cost,
+       impression_1k_cost,
+       conversion_num,
+       stat_date,
+       ad_photo_played_2s_ratio,
+       follow,
+       signature,
+       project_id,
+       project_name
+from (select account_id,
+             campaign_id,
+             unit_id,
+             event_register_cost,
+             gender,
+             photo_click,
+             key_action_cost,
+             event_next_day_stay,
+             photo_click_ratio,
+             photo_click_cost,
+             charge,
+             event_pay,
+             action_cost,
+             conversion_cost,
+             bclick,
+             key_action_ratio,
+             play_3s_ratio,
+             play_5s_ratio,
+             activation,
+             event_app_invoked_ratio,
+             `show`,
+             event_new_user_pay_cost,
+             aclick,
+             deep_conversion_num,
+             action_ratio,
+             conversion_ratio,
+             event_pay_purchase_amount_first_day,
+             event_pay_first_day,
+             conversion_num_cost,
+             event_new_user_pay_ratio,
+             event_app_invoked,
+             event_next_day_stay_ratio,
+             event_register,
+             event_new_user_pay,
+             event_next_day_stay_cost,
+             deep_conversion_ratio,
+             deep_conversion_cost,
+             `like`,
+             event_app_invoked_cost,
+             key_action,
+             ad_photo_played_10s,
+             form_cost,
+             event_pay_purchase_amount,
+             click_1k_cost,
+             impression_1k_cost,
+             conversion_num,
+             stat_date,
+             ad_photo_played_2s_ratio,
+             follow
+      from media_api.kuaishou_unit_audience_gender_report_daily
+      where stat_date = {stat_date}) gender
+         inner join
+     (select account_id, unit_id, photo_id
+      from `jeecg-boot`.ctop_ai_kuaishou_creative_level_operation_record
+      group by account_id, unit_id, photo_id) operation
+     on gender.account_id = operation.account_id and gender.unit_id = operation.unit_id
+         inner join
+     (select account_id, signature, photo_id
+      from media_api.kuaishou_video_list
+      group by account_id, signature, photo_id) video
+     on operation.account_id = video.account_id and operation.photo_id = video.photo_id
+         left join (
+    select account_id, project_id
+    from `jeecg-boot`.ctop_user_allocation
+) acc_info
+                   on gender.account_id = acc_info.account_id
+         left join (
+    select id, project_name
+    from `jeecg-boot`.ctop_project
+) pro
+                   on acc_info.project_id = pro.id
+            """.format(stat_date=self.start_time)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    KuaishouUnitAudienceGenderReportDaily().genderReport()

+ 168 - 0
Apietl/kwai_etl/audienceReport/KuaishouUnitAudienceProvinceReportDaily.py

@@ -0,0 +1,168 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2022/2/17 4:13 下午
+# @Site    : 
+# @File    : KuaishouUnitAudienceProvinceReportDaily.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class KuaishouUnitAudienceProvinceReportDaily:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+
+    def provinceReport(self):
+        try:
+            sql = """select count(1) from media_api.kuaishou_unit_audience_province_report_daily where 
+            stat_date = {start_time}""".format(
+                start_time=self.start_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 40000)
+            for i in range(pageCount):
+                sql = """
+                    replace into application.kuaishou_unit_audience_province_report_daily_d
+select province.account_id,
+       campaign_id,
+       province.unit_id,
+       event_register_cost,
+       province,
+       photo_click,
+       key_action_cost,
+       event_next_day_stay,
+       photo_click_ratio,
+       photo_click_cost,
+       charge,
+       event_pay,
+       action_cost,
+       conversion_cost,
+       bclick,
+       key_action_ratio,
+       play_3s_ratio,
+       play_5s_ratio,
+       activation,
+       event_app_invoked_ratio,
+       `show`,
+       event_new_user_pay_cost,
+       aclick,
+       deep_conversion_num,
+       action_ratio,
+       conversion_ratio,
+       event_pay_purchase_amount_first_day,
+       event_pay_first_day,
+       conversion_num_cost,
+       event_new_user_pay_ratio,
+       event_app_invoked,
+       event_next_day_stay_ratio,
+       event_register,
+       event_new_user_pay,
+       event_next_day_stay_cost,
+       deep_conversion_ratio,
+       deep_conversion_cost,
+       `like`,
+       event_app_invoked_cost,
+       key_action,
+       ad_photo_played_10s,
+       form_cost,
+       event_pay_purchase_amount,
+       click_1k_cost,
+       impression_1k_cost,
+       conversion_num,
+       stat_date,
+       ad_photo_played_2s_ratio,
+       follow,
+       signature,
+       project_id,
+       project_name
+from (select account_id,
+             campaign_id,
+             unit_id,
+             event_register_cost,
+             province,
+             photo_click,
+             key_action_cost,
+             event_next_day_stay,
+             photo_click_ratio,
+             photo_click_cost,
+             charge,
+             event_pay,
+             action_cost,
+             conversion_cost,
+             bclick,
+             key_action_ratio,
+             play_3s_ratio,
+             play_5s_ratio,
+             activation,
+             event_app_invoked_ratio,
+             `show`,
+             event_new_user_pay_cost,
+             aclick,
+             deep_conversion_num,
+             action_ratio,
+             conversion_ratio,
+             event_pay_purchase_amount_first_day,
+             event_pay_first_day,
+             conversion_num_cost,
+             event_new_user_pay_ratio,
+             event_app_invoked,
+             event_next_day_stay_ratio,
+             event_register,
+             event_new_user_pay,
+             event_next_day_stay_cost,
+             deep_conversion_ratio,
+             deep_conversion_cost,
+             `like`,
+             event_app_invoked_cost,
+             key_action,
+             ad_photo_played_10s,
+             form_cost,
+             event_pay_purchase_amount,
+             click_1k_cost,
+             impression_1k_cost,
+             conversion_num,
+             stat_date,
+             ad_photo_played_2s_ratio,
+             follow
+      from media_api.kuaishou_unit_audience_province_report_daily
+      where stat_date = {stat_date}) province
+         inner join
+     (select account_id, unit_id, photo_id
+      from `jeecg-boot`.ctop_ai_kuaishou_creative_level_operation_record
+      group by account_id, unit_id, photo_id) operation
+     on province.account_id = operation.account_id and province.unit_id = operation.unit_id
+         inner join
+     (select account_id, signature, photo_id
+      from media_api.kuaishou_video_list
+      group by account_id, signature, photo_id) video
+     on operation.account_id = video.account_id and operation.photo_id = video.photo_id
+         left join (
+    select account_id, project_id
+    from `jeecg-boot`.ctop_user_allocation
+) acc_info
+                   on province.account_id = acc_info.account_id
+         left join (
+    select id, project_name
+    from `jeecg-boot`.ctop_project
+) pro
+                   on acc_info.project_id = pro.id
+ order by unit_id limit {num},20000        
+                    """.format(stat_date=self.start_time, num=i * 40000)
+
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+                TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    KuaishouUnitAudienceProvinceReportDaily().provinceReport()

A különbségek nem kerülnek megjelenítésre, a fájl túl nagy
+ 386 - 0
Apietl/kwai_etl/report/AccountDailyReport.py


+ 480 - 0
Apietl/kwai_etl/report/AccountHourlyReport.py

@@ -0,0 +1,480 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/27 10:50 上午
+# @Site    : 
+# @File    : AccountHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AccountHourlyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.kuaishou_account_report_hourly where 
+            stat_date between {start_time} and {end_time}
+            group by advertiser_id
+            """.format(
+                start_time=self.start_time, end_time=self.end_time)
+            adv_list = TidbConn.quary(self.conn, sql)
+
+            for account_id in adv_list:
+                sql = """
+            replace into application.kuaishou_account_report_hourly_dw
+(account_id, stat_date, `hour`, project_id, product_id, advertiser_id, company_id, account_name, project_name, advertiser_name,
+ company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio,
+ play_3s_ratio, action_ratio, impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow,
+ cancel_follow, report, block, negative, submit, download_started, download_completed, activation, event_pay_first_day,
+ event_pay_purchase_amount_first_day, event_pay_first_day_roi, event_pay, event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost, event_register_ratio, event_jin_jian_app, event_jin_jian_app_cost,
+ event_credit_grant_app, event_credit_grant_app_cost, event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount, event_order_paid_cost, event_next_day_stay, event_next_day_stay_cost,
+ event_next_day_stay_ratio, form_count, form_cost, form_action_ratio, event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost, event_credit_grant_landing_page, event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio, event_valid_clues, event_valid_clues_cost, event_add_wechat,
+ event_add_wechat_cost, event_add_wechat_ratio, event_get_through, event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost, event_app_invoked_ratio, event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay, event_new_user_pay_cost, event_new_user_pay_ratio, click_1k_cost, ad_product_cnt,
+ event_goods_view, merchant_reco_fans, event_order_amount_roi, event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost, event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app, event_new_user_jinjian_app_cost, event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app, event_new_user_credit_grant_app_cost, event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page, event_new_user_jinjian_page_cost, event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page, event_new_user_credit_grant_page_cost, event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost, event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost, event_appoint_jump_click_ratio, union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type, ad_scene, event_order_successed, ad_photo_played_10s_ratio,
+ key_action_cost, event_add_shopping_cart_cost, event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio, event_watch_app_ad, event_multi_conversion_cost, event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio, event_ad_watch_times_cost, event_add_shopping_cart, key_action,
+ event_multi_conversion, event_wechat_connected, event_dsp_gift_form, event_intention_confirmed,
+ event_phone_get_through, event_multi_conversion_ratio, event_measurement_house, ad_photo_played_2s_ratio,
+ event_outbound_call_ratio, event_ad_watch_times,event_pay_purchase_amount_one_day_by_conversion, ad_photo_played_10s
+,ad_photo_played_2s
+,ad_photo_played_75percent
+,approx_pay_cost
+,approx_pay_count
+,approx_pay_ratio
+,cancel_like
+,click_conversion_ratio
+,conversion_cost
+,conversion_cost_by_impression_7d
+,conversion_num
+,conversion_num_by_impression_7d
+,conversion_num_cost
+,conversion_ratio
+,conversion_ratio_by_impression_7d
+,deep_conversion_cost
+,deep_conversion_cost_by_impression_7d
+,deep_conversion_num
+,deep_conversion_num_by_impression_7d
+,deep_conversion_ratio
+,deep_conversion_ratio_by_impression_7d
+,download_completed_cost
+,download_completed_ratio
+,download_conversion_ratio
+,download_installed
+,download_started_cost
+,download_started_ratio
+,event_24h_stay
+,event_24h_stay_by_conversion
+,event_24h_stay_by_conversion_cost
+,event_24h_stay_by_conversion_ratio
+,event_24h_stay_cost
+,event_24h_stay_ratio
+,event_ad_watch_10_times
+,event_ad_watch_10_times_cost
+,event_ad_watch_10_times_ratio
+,event_ad_watch_20_times
+,event_ad_watch_20_times_cost
+,event_ad_watch_20_times_ratio
+,event_ad_watch_5_times
+,event_ad_watch_5_times_cost
+,event_ad_watch_5_times_ratio
+,event_audition
+,event_consultation_valid_retained
+,event_consultation_valid_retained_cost
+,event_consultation_valid_retained_ratio
+,event_conversion_click_cost
+,event_conversion_click_ratio
+,event_credit_grant_first_day_app
+,event_credit_grant_first_day_app_cost
+,event_credit_grant_first_day_app_ratio
+,event_credit_grant_first_day_landing_page
+,event_credit_grant_first_day_landing_page_cost
+,event_credit_grant_first_day_landing_page_ratio
+,event_making_calls
+,event_making_calls_cost
+,event_making_calls_ratio
+,event_order_submit
+,event_pay_purchase_amount_one_day
+,event_pay_purchase_amount_one_day_by_conversion_roi
+,event_pay_purchase_amount_one_day_roi
+,event_pay_purchase_amount_three_day_by_conversion
+,event_pay_purchase_amount_three_day_by_conversion_roi
+,event_pay_purchase_amount_week_by_conversion
+,event_pay_purchase_amount_week_by_conversion_roi
+,event_pay_week_by_conversion
+,event_pay_week_by_conversion_cost
+,event_pay_weighted_purchase_amount
+,event_pay_weighted_purchase_amount_first_day
+,event_pre_component_consultation_valid_retained
+,event_three_day_stay_by_conversion
+,event_three_day_stay_by_conversion_cost
+,event_three_day_stay_by_conversion_ratio
+,event_wechat_qr_code_link_click
+,event_week_stay_by_conversion
+,event_week_stay_by_conversion_cost
+,event_week_stay_by_conversion_ratio
+,live_event_goods_view
+,live_played_3s
+,played_end
+,played_five_seconds
+,played_three_seconds
+,event_next_day_stay_new
+,event_next_day_stay_new_cost
+,event_next_day_stay_new_ratio
+,ad_show
+,event_credit_card_recheck
+,event_credit_card_recheck_first_day
+,event_no_intention
+,event_multi_pay_seven_day_by_conversion
+,event_multi_pay_seven_day_by_conversion_cost
+,mcb_left_time_perf_cpa_bid
+,mcb_left_time_perf_roi_ratio
+,mcb_all_day_perf_cpa_bid
+,mcb_all_day_perf_roi_ratio
+,live_room_avg_played_seconds
+,ad_live_share
+,ad_live_comment
+,live_played_started
+,live_played_started_cost
+,ad_live_follow
+,ad_live_follow_cost
+,simple_live_played_started
+,standard_live_played_started
+,conversion_component_impression
+,conversion_component_click
+,conversion_component_rate
+,ad_landing_page_impression
+,ad_app_download_half_impression
+,event_draw_credit_line
+,minigame_iaa_purchase_amount
+,minigame_iaa_purchase_roi
+)
+        select report.advertiser_id as account_id,
+               stat_date,
+                `hour`,
+               project_id,
+               product_id,
+               sys.advertiser_id,
+               company_id,
+               account_name,
+               project_name,
+               advertiser_name,
+               company_name,
+               sale_id,
+               user_id,
+               org_code,
+               charge,
+               `show`,
+               photo_click,
+               aclick,
+               bclick,
+               photo_click_ratio,
+               play_3s_ratio,
+               action_ratio,
+               impression_1k_cost,
+               photo_click_cost,
+               action_cost,
+               share,
+               comment,
+               `like`,
+               follow,
+               cancel_follow,
+               report,
+               block,
+               negative,
+               submit,
+               download_started,
+               download_completed,
+               activation,
+               event_pay_first_day,
+               event_pay_purchase_amount_first_day,
+               event_pay_first_day_roi,
+               event_pay,
+               event_pay_purchase_amount,
+               event_pay_roi,
+               event_register,
+               event_register_cost,
+               event_register_ratio,
+               event_jin_jian_app,
+               event_jin_jian_app_cost,
+               event_credit_grant_app,
+               event_credit_grant_app_cost,
+               event_credit_grant_app_ratio,
+               event_order_paid,
+               event_order_paid_purchase_amount,
+               event_order_paid_cost,
+               event_next_day_stay,
+               event_next_day_stay_cost,
+               event_next_day_stay_ratio,
+               form_count,
+               form_cost,
+               form_action_ratio,
+               event_jin_jian_landing_page,
+               event_jin_jian_landing_page_cost,
+               event_credit_grant_landing_page,
+               event_credit_grant_landing_page_cost,
+               event_credit_grant_landing_page_ratio,
+               event_valid_clues,
+               event_valid_clues_cost,
+               event_add_wechat,
+               event_add_wechat_cost,
+               event_add_wechat_ratio,
+               event_get_through,
+               event_get_through_cost,
+               event_get_through_ratio,
+               event_app_invoked,
+               event_app_invoked_cost,
+               event_app_invoked_ratio,
+               event_credit_grant_landing_ratio,
+               play_5s_ratio,
+               play_end_ratio,
+               event_new_user_pay,
+               event_new_user_pay_cost,
+               event_new_user_pay_ratio,
+               click_1k_cost,
+               ad_product_cnt,
+               event_goods_view,
+               merchant_reco_fans,
+               event_order_amount_roi,
+               event_goods_view_cost,
+               merchant_reco_fans_cost,
+               event_button_click,
+               event_button_click_cost,
+               event_button_click_ratio,
+               event_order_paid_roi,
+               event_new_user_jinjian_app,
+               event_new_user_jinjian_app_cost,
+               event_new_user_jinjian_app_roi,
+               event_new_user_credit_grant_app,
+               event_new_user_credit_grant_app_cost,
+               event_new_user_credit_grant_app_roi,
+               event_new_user_jinjian_page,
+               event_new_user_jinjian_page_cost,
+               event_new_user_jinjian_page_roi,
+               event_new_user_credit_grant_page,
+               event_new_user_credit_grant_page_cost,
+               event_new_user_credit_grant_page_roi,
+               event_appoint_form,
+               event_appoint_form_cost,
+               event_appoint_form_ratio,
+               event_appoint_jump_click,
+               event_appoint_jump_click_cost,
+               event_appoint_jump_click_ratio,
+               union_event_pay_purchase_amount_7d,
+               union_event_pay_purchase_amount_7d_roi,
+               placement_type,
+               ad_scene,
+               event_order_successed,
+               ad_photo_played_10s_ratio,
+               key_action_cost,
+               event_add_shopping_cart_cost,
+               event_ad_watch_times_ratio,
+               event_outbound_call_cost,
+               event_outbound_call,
+               action_new_ratio,
+               event_watch_app_ad,
+               event_multi_conversion_cost,
+               event_phone_card_activate,
+               ad_photo_played_75percent_ratio,
+               key_action_ratio,
+               event_ad_watch_times_cost,
+               event_add_shopping_cart,
+               key_action,
+               event_multi_conversion,
+               event_wechat_connected,
+               event_dsp_gift_form,
+               event_intention_confirmed,
+               event_phone_get_through,
+               event_multi_conversion_ratio,
+               event_measurement_house,
+               ad_photo_played_2s_ratio,
+               event_outbound_call_ratio,
+               event_ad_watch_times,
+               event_pay_purchase_amount_one_day_by_conversion,
+                ad_photo_played_10s
+,ad_photo_played_2s
+,ad_photo_played_75percent
+,approx_pay_cost
+,approx_pay_count
+,approx_pay_ratio
+,cancel_like
+,click_conversion_ratio
+,conversion_cost
+,conversion_cost_by_impression_7d
+,conversion_num
+,conversion_num_by_impression_7d
+,conversion_num_cost
+,conversion_ratio
+,conversion_ratio_by_impression_7d
+,deep_conversion_cost
+,deep_conversion_cost_by_impression_7d
+,deep_conversion_num
+,deep_conversion_num_by_impression_7d
+,deep_conversion_ratio
+,deep_conversion_ratio_by_impression_7d
+,download_completed_cost
+,download_completed_ratio
+,download_conversion_ratio
+,download_installed
+,download_started_cost
+,download_started_ratio
+,event_24h_stay
+,event_24h_stay_by_conversion
+,event_24h_stay_by_conversion_cost
+,event_24h_stay_by_conversion_ratio
+,event_24h_stay_cost
+,event_24h_stay_ratio
+,event_ad_watch_10_times
+,event_ad_watch_10_times_cost
+,event_ad_watch_10_times_ratio
+,event_ad_watch_20_times
+,event_ad_watch_20_times_cost
+,event_ad_watch_20_times_ratio
+,event_ad_watch_5_times
+,event_ad_watch_5_times_cost
+,event_ad_watch_5_times_ratio
+,event_audition
+,event_consultation_valid_retained
+,event_consultation_valid_retained_cost
+,event_consultation_valid_retained_ratio
+,event_conversion_click_cost
+,event_conversion_click_ratio
+,event_credit_grant_first_day_app
+,event_credit_grant_first_day_app_cost
+,event_credit_grant_first_day_app_ratio
+,event_credit_grant_first_day_landing_page
+,event_credit_grant_first_day_landing_page_cost
+,event_credit_grant_first_day_landing_page_ratio
+,event_making_calls
+,event_making_calls_cost
+,event_making_calls_ratio
+,event_order_submit
+,event_pay_purchase_amount_one_day
+,event_pay_purchase_amount_one_day_by_conversion_roi
+,event_pay_purchase_amount_one_day_roi
+,event_pay_purchase_amount_three_day_by_conversion
+,event_pay_purchase_amount_three_day_by_conversion_roi
+,event_pay_purchase_amount_week_by_conversion
+,event_pay_purchase_amount_week_by_conversion_roi
+,event_pay_week_by_conversion
+,event_pay_week_by_conversion_cost
+,event_pay_weighted_purchase_amount
+,event_pay_weighted_purchase_amount_first_day
+,event_pre_component_consultation_valid_retained
+,event_three_day_stay_by_conversion
+,event_three_day_stay_by_conversion_cost
+,event_three_day_stay_by_conversion_ratio
+,event_wechat_qr_code_link_click
+,event_week_stay_by_conversion
+,event_week_stay_by_conversion_cost
+,event_week_stay_by_conversion_ratio
+,live_event_goods_view
+,live_played_3s
+,played_end
+,played_five_seconds
+,played_three_seconds
+,event_next_day_stay_new
+,event_next_day_stay_new_cost
+,event_next_day_stay_new_ratio
+,ad_show
+,event_credit_card_recheck
+,event_credit_card_recheck_first_day
+,event_no_intention
+,event_multi_pay_seven_day_by_conversion
+,event_multi_pay_seven_day_by_conversion_cost
+,mcb_left_time_perf_cpa_bid
+,mcb_left_time_perf_roi_ratio
+,mcb_all_day_perf_cpa_bid
+,mcb_all_day_perf_roi_ratio
+,live_room_avg_played_seconds
+,ad_live_share
+,ad_live_comment
+,live_played_started
+,live_played_started_cost
+,ad_live_follow
+,ad_live_follow_cost
+,simple_live_played_started
+,standard_live_played_started
+,conversion_component_impression
+,conversion_component_click
+,conversion_component_rate
+,ad_landing_page_impression
+,ad_app_download_half_impression
+,event_draw_credit_line
+,minigame_iaa_purchase_amount
+,minigame_iaa_purchase_roi
+        from media_api.kuaishou_account_report_hourly report
+                 left join
+                          (SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+             ) sys
+             on report.advertiser_id = sys.account_id
+        where report.stat_date between {start_time} and {end_time}
+    and  report.advertiser_id = {account_id}
+        """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AccountHourlyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 569 - 0
Apietl/kwai_etl/report/AccountHourlySumToDayReport.py

@@ -0,0 +1,569 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2022/3/24 9:53 上午
+# @Site    : 
+# @File    : AccountHourlySumToDayReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import datetime
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+
+
+class AccountHourlySumToDayReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.stat_date = (datetime.date.today() + datetime.timedelta(0)).strftime("%Y%m%d")
+
+    def hourly_sum_report(self):
+        try:
+            sql = """
+            SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';  
+            replace into application.kuaishou_account_report_daily_dw
+SELECT
+report.account_id,
+report.stat_date  ,
+bei.project_id,
+bei.product_id,
+bei.advertiser_id,
+bei.company_id,
+bei.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.user_id,
+bei.org_code,
+report.charge,
+report.`show`,
+report.photo_click,
+report.aclick,
+report.bclick,
+report.photo_click_ratio,
+report.play_3s_ratio,
+report.action_ratio,
+report.impression_1k_cost,
+report.photo_click_cost,
+report.action_cost,
+report.`share`,
+report.`comment`,
+report.`like`,
+report.follow,
+report.cancel_follow,
+report.report,
+report.block,
+report.negative,
+report.submit,
+report.download_started,
+report.download_completed,
+report.activation,
+report.event_pay_first_day,
+report.event_pay_purchase_amount_first_day,
+report.event_pay_first_day_roi,
+report.event_pay,
+report.event_pay_purchase_amount,
+report.event_pay_roi,
+report.event_register,
+report.event_register_cost,
+report.event_register_ratio,
+report.event_jin_jian_app,
+report.event_jin_jian_app_cost,
+report.event_credit_grant_app,
+report.event_credit_grant_app_cost,
+report.event_credit_grant_app_ratio,
+report.event_order_paid,
+report.event_order_paid_purchase_amount,
+report.event_order_paid_cost,
+report.event_next_day_stay,
+report.event_next_day_stay_cost,
+report.event_next_day_stay_ratio,
+report.form_count,
+report.form_cost,
+report.form_action_ratio,
+report.event_jin_jian_landing_page,
+report.event_jin_jian_landing_page_cost,
+report.event_credit_grant_landing_page,
+report.event_credit_grant_landing_page_cost,
+report.event_credit_grant_landing_page_ratio,
+report.event_valid_clues,
+report.event_valid_clues_cost,
+report.event_add_wechat,
+report.event_add_wechat_cost,
+report.event_add_wechat_ratio,
+report.event_get_through,
+report.event_get_through_cost,
+report.event_get_through_ratio,
+report.event_app_invoked,
+report.event_app_invoked_cost,
+report.event_app_invoked_ratio,
+report.event_credit_grant_landing_ratio,
+report.play_5s_ratio,
+report.play_end_ratio,
+report.event_new_user_pay,
+report.event_new_user_pay_cost,
+report.event_new_user_pay_ratio,
+report.click_1k_cost,
+report.ad_product_cnt,
+report.event_goods_view,
+report.merchant_reco_fans,
+report.event_order_amount_roi,
+report.event_goods_view_cost,
+report.merchant_reco_fans_cost,
+report.event_button_click,
+report.event_button_click_cost,
+report.event_button_click_ratio,
+report.event_order_paid_roi,
+report.event_new_user_jinjian_app,
+report.event_new_user_jinjian_app_cost,
+report.event_new_user_jinjian_app_roi,
+report.event_new_user_credit_grant_app,
+report.event_new_user_credit_grant_app_cost,
+report.event_new_user_credit_grant_app_roi,
+report.event_new_user_jinjian_page,
+report.event_new_user_jinjian_page_cost,
+report.event_new_user_jinjian_page_roi,
+report.event_new_user_credit_grant_page,
+report.event_new_user_credit_grant_page_cost,
+report.event_new_user_credit_grant_page_roi,
+report.event_appoint_form,
+report.event_appoint_form_cost,
+report.event_appoint_form_ratio,
+report.event_appoint_jump_click,
+report.event_appoint_jump_click_cost,
+report.event_appoint_jump_click_ratio,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi,
+''  as placement_type,
+'' as  ad_scene,
+report.event_order_successed,
+report.ad_photo_played_10s_ratio,
+report.key_action_cost,
+report.event_add_shopping_cart_cost,
+report.event_ad_watch_times_ratio,
+report.event_outbound_call_cost,
+report.event_outbound_call,
+report.action_new_ratio,
+report.event_watch_app_ad,
+report.event_multi_conversion_cost,
+report.event_phone_card_activate,
+report.ad_photo_played_75percent_ratio,
+report.key_action_ratio,
+report.event_ad_watch_times_cost,
+report.event_add_shopping_cart,
+report.key_action,
+report.event_multi_conversion,
+report.event_wechat_connected,
+report.event_dsp_gift_form,
+report.event_intention_confirmed,
+report.event_phone_get_through,
+report.event_multi_conversion_ratio,
+report.event_measurement_house,
+report.ad_photo_played_2s_ratio,
+report.event_outbound_call_ratio,
+report.event_ad_watch_times,
+report.event_pay_purchase_amount_one_day_by_conversion,
+ report.ad_photo_played_10s
+,report.ad_photo_played_2s
+,report.ad_photo_played_75percent
+,report.approx_pay_cost
+,report.approx_pay_count
+,report.approx_pay_ratio
+,report.cancel_like
+,report.click_conversion_ratio
+,report.conversion_cost
+,report.conversion_cost_by_impression_7d
+,report.conversion_num
+,report.conversion_num_by_impression_7d
+,report.conversion_num_cost
+,report.conversion_ratio
+,report.conversion_ratio_by_impression_7d
+,report.deep_conversion_cost
+,report.deep_conversion_cost_by_impression_7d
+,report.deep_conversion_num
+,report.deep_conversion_num_by_impression_7d
+,report.deep_conversion_ratio
+,report.deep_conversion_ratio_by_impression_7d
+,report.download_completed_cost
+,report.download_completed_ratio
+,report.download_conversion_ratio
+,report.download_installed
+,report.download_started_cost
+,report.download_started_ratio
+,report.event_24h_stay
+,report.event_24h_stay_by_conversion
+,report.event_24h_stay_by_conversion_cost
+,report.event_24h_stay_by_conversion_ratio
+,report.event_24h_stay_cost
+,report.event_24h_stay_ratio
+,report.event_ad_watch_10_times
+,report.event_ad_watch_10_times_cost
+,report.event_ad_watch_10_times_ratio
+,report.event_ad_watch_20_times
+,report.event_ad_watch_20_times_cost
+,report.event_ad_watch_20_times_ratio
+,report.event_ad_watch_5_times
+,report.event_ad_watch_5_times_cost
+,report.event_ad_watch_5_times_ratio
+,report.event_audition
+,report.event_consultation_valid_retained
+,report.event_consultation_valid_retained_cost
+,report.event_consultation_valid_retained_ratio
+,report.event_conversion_click_cost
+,report.event_conversion_click_ratio
+,report.event_credit_grant_first_day_app
+,report.event_credit_grant_first_day_app_cost
+,report.event_credit_grant_first_day_app_ratio
+,report.event_credit_grant_first_day_landing_page
+,report.event_credit_grant_first_day_landing_page_cost
+,report.event_credit_grant_first_day_landing_page_ratio
+,report.event_making_calls
+,report.event_making_calls_cost
+,report.event_making_calls_ratio
+,report.event_order_submit
+,report.event_pay_purchase_amount_one_day
+,report.event_pay_purchase_amount_one_day_by_conversion_roi
+,report.event_pay_purchase_amount_one_day_roi
+,report.event_pay_purchase_amount_three_day_by_conversion
+,report.event_pay_purchase_amount_three_day_by_conversion_roi
+,report.event_pay_purchase_amount_week_by_conversion
+,report.event_pay_purchase_amount_week_by_conversion_roi
+,report.event_pay_week_by_conversion
+,report.event_pay_week_by_conversion_cost
+,report.event_pay_weighted_purchase_amount
+,report.event_pay_weighted_purchase_amount_first_day
+,report.event_pre_component_consultation_valid_retained
+,report.event_three_day_stay_by_conversion
+,report.event_three_day_stay_by_conversion_cost
+,report.event_three_day_stay_by_conversion_ratio
+,report.event_wechat_qr_code_link_click
+,report.event_week_stay_by_conversion
+,report.event_week_stay_by_conversion_cost
+,report.event_week_stay_by_conversion_ratio
+,report.live_event_goods_view
+,report.live_played_3s
+,report.played_end
+,report.played_five_seconds
+,report.played_three_seconds
+,report.event_next_day_stay_new
+,report.event_next_day_stay_new_cost
+,report.event_next_day_stay_new_ratio
+,report.ad_show
+,report.event_credit_card_recheck
+,report.event_credit_card_recheck_first_day
+,report.event_no_intention
+,report.event_multi_pay_seven_day_by_conversion
+,report.event_multi_pay_seven_day_by_conversion_cost
+,report.mcb_left_time_perf_cpa_bid
+,report.mcb_left_time_perf_roi_ratio
+,report.mcb_all_day_perf_cpa_bid
+,report.mcb_all_day_perf_roi_ratio
+,report.live_room_avg_played_seconds
+,report.ad_live_share
+,report.ad_live_comment
+,report.live_played_started
+,report.live_played_started_cost
+,report.ad_live_follow
+,report.ad_live_follow_cost
+,report.simple_live_played_started
+,report.standard_live_played_started
+,report.conversion_component_impression
+,report.conversion_component_click
+,report.conversion_component_rate
+,report.ad_landing_page_impression
+,report.ad_app_download_half_impression
+,report.event_draw_credit_line
+,minigame_iaa_purchase_roi
+,minigame_iaa_purchase_amount
+from 
+(select
+        stat_date,
+advertiser_id account_id,
+sum(charge) charge,
+sum(`show`) `show`,
+sum(photo_click)photo_click,
+sum(aclick)  aclick,
+sum(bclick) bclick,
+ifnull(round(sum(photo_click)/sum(`show`),3),0) photo_click_ratio,
+ifnull(round(sum(aclick*play_3s_ratio)/sum(aclick),3),0)  play_3s_ratio,
+ifnull(round(sum(bclick)/sum(aclick)*100,3),0) action_ratio,
+ifnull(round(sum(impression_1k_cost),3),0) impression_1k_cost,
+ifnull(round(sum(charge)/sum(photo_click),3),0) photo_click_cost,
+ifnull(round(sum(charge)/sum(bclick),3),0) action_cost,
+sum(`share`) `share`,
+sum(`comment`) `comment`,
+sum(`like`)  `like`,
+sum(follow) follow,
+sum(cancel_follow)  cancel_follow,
+sum(report) report,
+sum(block) block,
+sum(negative) negative,
+sum(submit) submit,
+sum(download_started) download_started,
+sum(download_completed) download_completed,
+sum(activation) activation,
+sum(event_pay_first_day) event_pay_first_day,
+sum(event_pay_purchase_amount_first_day) event_pay_purchase_amount_first_day,
+ifnull(round(sum(event_pay_purchase_amount_first_day)/sum(charge)*100,3),0) event_pay_first_day_roi ,
+sum(event_pay) event_pay,
+sum(event_pay_purchase_amount) event_pay_purchase_amount,
+ifnull(round(sum(event_pay_purchase_amount)/sum(charge)*100,3),0) event_pay_roi,
+sum(event_register) event_register,
+ifnull(round(sum(charge)/sum(event_register),3),0) event_register_cost,
+ifnull(round(sum(event_register)/sum(activation)*100,3),0) event_register_ratio,
+sum(event_jin_jian_app) event_jin_jian_app,
+ifnull(round(sum(charge)/sum(event_jin_jian_app),3),0) event_jin_jian_app_cost,
+sum(event_credit_grant_app) event_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_credit_grant_app),3),0) event_credit_grant_app_cost ,
+ifnull(round(sum(event_credit_grant_app_ratio)*100,3),0) event_credit_grant_app_ratio,
+sum(event_order_paid) event_order_paid,
+sum(event_order_paid_purchase_amount) event_order_paid_purchase_amount,
+ifnull(round(sum(event_order_paid_purchase_amount)/sum(event_order_paid) ,3),0) event_order_paid_cost,
+sum(event_next_day_stay) event_next_day_stay,
+ifnull(round(sum(charge)/sum(event_next_day_stay),3),0) event_next_day_stay_cost,
+ifnull(round(sum(event_next_day_stay)/sum(activation)*100,3),0) event_next_day_stay_ratio,
+sum(form_count) form_count,
+ifnull(round(sum(charge)/sum(form_count),3),0) form_cost,
+ifnull(round(sum(form_count)/sum(`bclick`)*100,3),0) form_action_ratio,
+sum(event_jin_jian_landing_page) event_jin_jian_landing_page,
+ifnull(round(sum(charge)/sum(event_jin_jian_landing_page),3),0) event_jin_jian_landing_page_cost,
+sum(event_credit_grant_landing_page) event_credit_grant_landing_page,
+ifnull(round(sum(charge)/sum(event_credit_grant_landing_page),3),0) event_credit_grant_landing_page_cost,
+ifnull(round(sum(event_credit_grant_landing_page)*100,3),0) event_credit_grant_landing_page_ratio,
+sum(event_valid_clues) event_valid_clues,
+ifnull(round(sum(charge)/sum(event_valid_clues),3),0) event_valid_clues_cost,
+sum(event_add_wechat) event_add_wechat,
+ifnull(round(sum(charge)/sum(event_add_wechat),3),0) event_add_wechat_cost,
+ifnull(round(sum(event_add_wechat)/sum(bclick)*100,3),0) event_add_wechat_ratio,
+sum(event_get_through) event_get_through,
+ifnull(round(sum(charge)/sum(event_get_through),3),0) event_get_through_cost,
+ifnull(round(sum(event_get_through)/sum(bclick)*100,3),0) event_get_through_ratio,
+sum(event_app_invoked) event_app_invoked,
+ifnull(round(sum(charge)/sum(event_app_invoked),3),0) event_app_invoked_cost ,
+ifnull(round(sum(event_app_invoked)/sum(bclick)*100,3),0) event_app_invoked_ratio,
+ifnull(round(sum(event_credit_grant_landing_ratio)*100,3),0) event_credit_grant_landing_ratio,
+ifnull(round(sum(play_5s_ratio)*100,3),0) play_5s_ratio,
+ifnull(round(sum(play_end_ratio)*100,3),0) play_end_ratio,
+sum(event_new_user_pay) event_new_user_pay,
+ifnull(round(sum(charge)/sum(event_new_user_pay),3),0) event_new_user_pay_cost,
+ifnull(round(sum(event_new_user_pay)/sum(activation)*100,3),0) event_new_user_pay_ratio,
+ifnull(round(sum(click_1k_cost)*100,3),0) click_1k_cost,
+sum(ad_product_cnt) ad_product_cnt,
+sum(event_goods_view) event_goods_view,
+sum(merchant_reco_fans) merchant_reco_fans,
+ifnull(round(sum(event_order_amount_roi)*100,3),0) event_order_amount_roi,
+ifnull(round(sum(charge)/sum(event_goods_view) ,3),0) event_goods_view_cost,
+ifnull(round(sum(charge)/sum(merchant_reco_fans),3),0) merchant_reco_fans_cost,
+sum(event_button_click) event_button_click,
+ifnull(round(sum(charge)/sum(event_button_click),3),0) event_button_click_cost,
+ifnull(round(sum(event_button_click)/sum(bclick)*100,3),0) event_button_click_ratio ,
+ifnull(round(sum(event_order_paid_roi)*100,3),0) event_order_paid_roi,
+sum(event_new_user_jinjian_app) event_new_user_jinjian_app,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_app),3),0) event_new_user_jinjian_app_cost,
+ifnull(round(sum(event_new_user_jinjian_app)/sum(form_count)*100,3),0) event_new_user_jinjian_app_roi ,
+sum(event_new_user_credit_grant_app) event_new_user_credit_grant_app,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_app),3),0)  event_new_user_credit_grant_app_cost,
+ifnull(round(sum(event_new_user_credit_grant_app)/sum(form_count)*100,3),0) event_new_user_credit_grant_app_roi,
+sum(event_new_user_jinjian_page) event_new_user_jinjian_page,
+ifnull(round(sum(charge)/sum(event_new_user_jinjian_page),3),0) event_new_user_jinjian_page_cost,
+ifnull(round(sum(event_new_user_jinjian_page)/sum(form_count)*100,3),0) event_new_user_jinjian_page_roi ,
+sum(event_new_user_credit_grant_page) event_new_user_credit_grant_page,
+ifnull(round(sum(charge)/sum(event_new_user_credit_grant_page) ,3),0) event_new_user_credit_grant_page_cost,
+ifnull(round(sum(event_new_user_credit_grant_page)/sum(form_count)*100,3),0) event_new_user_credit_grant_page_roi,
+sum(event_appoint_form)  event_appoint_form,
+ifnull(round(sum(charge)/sum(event_appoint_form),3),0) event_appoint_form_cost,
+ifnull(round(sum(event_appoint_form)/sum(form_count)*100,3),0) event_appoint_form_ratio,
+sum(event_appoint_jump_click) event_appoint_jump_click,
+ifnull(round(sum(charge)/sum(event_appoint_jump_click),3),0)  event_appoint_jump_click_cost,
+ifnull(round(sum(event_appoint_jump_click_cost)/sum(form_count)*100,3),0) event_appoint_jump_click_ratio,
+sum(union_event_pay_purchase_amount_7d) union_event_pay_purchase_amount_7d,
+ifnull(round(sum(union_event_pay_purchase_amount_7d_roi)*100,3),0) union_event_pay_purchase_amount_7d_roi,
+sum(event_order_successed) event_order_successed,
+ifnull(round(sum(ad_photo_played_10s_ratio)*100,3),0) ad_photo_played_10s_ratio,
+ifnull(round(sum(charge)/sum(key_action),3),0) key_action_cost,
+ifnull(round(sum(charge)/sum(event_add_shopping_cart) ,3),0) event_add_shopping_cart_cost ,
+ifnull(round(sum(event_ad_watch_times)/sum(bclick)*100,3),0) event_ad_watch_times_ratio,
+ifnull(round(sum(charge)/sum(event_outbound_call),3),0) event_outbound_call_cost,
+sum(event_outbound_call) event_outbound_call,
+ifnull(round(sum(action_new_ratio)*100,3),0) action_new_ratio,
+sum(event_watch_app_ad) event_watch_app_ad,
+ifnull(round(sum(charge)/sum(event_multi_conversion) ,3),0) event_multi_conversion_cost,
+sum(event_phone_card_activate) event_phone_card_activate,
+ifnull(round(sum(ad_photo_played_75percent_ratio)*100,3),0) ad_photo_played_75percent_ratio,
+ifnull(round(sum(key_action)/sum(activation)*100,3),0) key_action_ratio ,
+ifnull(round(sum(charge)/sum(event_ad_watch_times),3),0) event_ad_watch_times_cost,
+sum(event_add_shopping_cart) event_add_shopping_cart,
+sum(key_action) key_action,
+sum(event_multi_conversion) event_multi_conversion,
+sum(event_wechat_connected) event_wechat_connected,
+sum(event_dsp_gift_form) event_dsp_gift_form,
+sum(event_intention_confirmed) event_intention_confirmed,
+sum(event_phone_get_through) event_phone_get_through,
+ifnull(round(sum(event_multi_conversion)/sum(activation)*100,3),0)  event_multi_conversion_ratio,
+sum(event_measurement_house) event_measurement_house,
+ifnull(round(sum(ad_photo_played_2s_ratio)*100,3),0) ad_photo_played_2s_ratio,
+ifnull(round(sum(event_outbound_call_ratio)/sum(activation)*100,3),0) event_outbound_call_ratio,
+sum(event_ad_watch_times) event_ad_watch_times,
+sum(event_pay_purchase_amount_one_day_by_conversion) as event_pay_purchase_amount_one_day_by_conversion
+,sum(ad_photo_played_10s) as           ad_photo_played_10s
+,sum(ad_photo_played_2s) as           ad_photo_played_2s
+,sum(ad_photo_played_75percent) as           ad_photo_played_75percent
+,sum(approx_pay_cost) as           approx_pay_cost
+,sum(approx_pay_count) as           approx_pay_count
+,sum(approx_pay_ratio) as           approx_pay_ratio
+,sum(cancel_like) as           cancel_like
+,sum(click_conversion_ratio) as           click_conversion_ratio
+,sum(conversion_cost) as           conversion_cost
+,sum(conversion_cost_by_impression_7d) as           conversion_cost_by_impression_7d
+,sum(conversion_num) as           conversion_num
+,sum(conversion_num_by_impression_7d) as           conversion_num_by_impression_7d
+,sum(conversion_num_cost) as           conversion_num_cost
+,sum(conversion_ratio) as           conversion_ratio
+,sum(conversion_ratio_by_impression_7d) as           conversion_ratio_by_impression_7d
+,sum(deep_conversion_cost) as           deep_conversion_cost
+,sum(deep_conversion_cost_by_impression_7d) as           deep_conversion_cost_by_impression_7d
+,sum(deep_conversion_num) as           deep_conversion_num
+,sum(deep_conversion_num_by_impression_7d) as           deep_conversion_num_by_impression_7d
+,sum(deep_conversion_ratio) as           deep_conversion_ratio
+,sum(deep_conversion_ratio_by_impression_7d) as           deep_conversion_ratio_by_impression_7d
+,sum(download_completed_cost) as           download_completed_cost
+,sum(download_completed_ratio) as           download_completed_ratio
+,sum(download_conversion_ratio) as           download_conversion_ratio
+,sum(download_installed) as           download_installed
+,sum(download_started_cost) as           download_started_cost
+,sum(download_started_ratio) as           download_started_ratio
+,sum(event_24h_stay) as           event_24h_stay
+,sum(event_24h_stay_by_conversion) as           event_24h_stay_by_conversion
+,sum(event_24h_stay_by_conversion_cost) as           event_24h_stay_by_conversion_cost
+,sum(event_24h_stay_by_conversion_ratio) as           event_24h_stay_by_conversion_ratio
+,sum(event_24h_stay_cost) as           event_24h_stay_cost
+,sum(event_24h_stay_ratio) as           event_24h_stay_ratio
+,sum(event_ad_watch_10_times) as           event_ad_watch_10_times
+,sum(event_ad_watch_10_times_cost) as           event_ad_watch_10_times_cost
+,sum(event_ad_watch_10_times_ratio) as           event_ad_watch_10_times_ratio
+,sum(event_ad_watch_20_times) as           event_ad_watch_20_times
+,sum(event_ad_watch_20_times_cost) as           event_ad_watch_20_times_cost
+,sum(event_ad_watch_20_times_ratio) as           event_ad_watch_20_times_ratio
+,sum(event_ad_watch_5_times) as           event_ad_watch_5_times
+,sum(event_ad_watch_5_times_cost) as           event_ad_watch_5_times_cost
+,sum(event_ad_watch_5_times_ratio) as           event_ad_watch_5_times_ratio
+,sum(event_audition) as           event_audition
+,sum(event_consultation_valid_retained) as           event_consultation_valid_retained
+,sum(event_consultation_valid_retained_cost) as           event_consultation_valid_retained_cost
+,sum(event_consultation_valid_retained_ratio) as           event_consultation_valid_retained_ratio
+,sum(event_conversion_click_cost) as           event_conversion_click_cost
+,sum(event_conversion_click_ratio) as           event_conversion_click_ratio
+,sum(event_credit_grant_first_day_app) as           event_credit_grant_first_day_app
+,sum(event_credit_grant_first_day_app_cost) as           event_credit_grant_first_day_app_cost
+,sum(event_credit_grant_first_day_app_ratio) as           event_credit_grant_first_day_app_ratio
+,sum(event_credit_grant_first_day_landing_page) as           event_credit_grant_first_day_landing_page
+,sum(event_credit_grant_first_day_landing_page_cost) as           event_credit_grant_first_day_landing_page_cost
+,sum(event_credit_grant_first_day_landing_page_ratio) as           event_credit_grant_first_day_landing_page_ratio
+,sum(event_making_calls) as           event_making_calls
+,sum(event_making_calls_cost) as           event_making_calls_cost
+,sum(event_making_calls_ratio) as           event_making_calls_ratio
+,sum(event_order_submit) as           event_order_submit
+,sum(event_pay_purchase_amount_one_day) as           event_pay_purchase_amount_one_day
+,sum(event_pay_purchase_amount_one_day_by_conversion_roi) as           event_pay_purchase_amount_one_day_by_conversion_roi
+,sum(event_pay_purchase_amount_one_day_roi) as           event_pay_purchase_amount_one_day_roi
+,sum(event_pay_purchase_amount_three_day_by_conversion) as           event_pay_purchase_amount_three_day_by_conversion
+,sum(event_pay_purchase_amount_three_day_by_conversion_roi) as           event_pay_purchase_amount_three_day_by_conversion_roi
+,sum(event_pay_purchase_amount_week_by_conversion) as           event_pay_purchase_amount_week_by_conversion
+,sum(event_pay_purchase_amount_week_by_conversion_roi) as           event_pay_purchase_amount_week_by_conversion_roi
+,sum(event_pay_week_by_conversion) as           event_pay_week_by_conversion
+,sum(event_pay_week_by_conversion_cost) as           event_pay_week_by_conversion_cost
+,sum(event_pay_weighted_purchase_amount) as           event_pay_weighted_purchase_amount
+,sum(event_pay_weighted_purchase_amount_first_day) as           event_pay_weighted_purchase_amount_first_day
+,sum(event_pre_component_consultation_valid_retained) as           event_pre_component_consultation_valid_retained
+,sum(event_three_day_stay_by_conversion) as           event_three_day_stay_by_conversion
+,sum(event_three_day_stay_by_conversion_cost) as           event_three_day_stay_by_conversion_cost
+,sum(event_three_day_stay_by_conversion_ratio) as           event_three_day_stay_by_conversion_ratio
+,sum(event_wechat_qr_code_link_click) as           event_wechat_qr_code_link_click
+,sum(event_week_stay_by_conversion) as           event_week_stay_by_conversion
+,sum(event_week_stay_by_conversion_cost) as           event_week_stay_by_conversion_cost
+,sum(event_week_stay_by_conversion_ratio) as           event_week_stay_by_conversion_ratio
+,sum(live_event_goods_view) as           live_event_goods_view
+,sum(live_played_3s) as           live_played_3s
+,sum(played_end) as           played_end
+,sum(played_five_seconds) as           played_five_seconds
+,sum(played_three_seconds) as           played_three_seconds
+,sum(event_next_day_stay_new) as           event_next_day_stay_new
+,sum(event_next_day_stay_new_cost) as           event_next_day_stay_new_cost
+,sum(event_next_day_stay_new_ratio) as           event_next_day_stay_new_ratio
+,sum(ad_show) as           ad_show
+,sum(event_credit_card_recheck) as           event_credit_card_recheck
+,sum(event_credit_card_recheck_first_day) as           event_credit_card_recheck_first_day
+,sum(event_no_intention) as           event_no_intention
+,sum(event_multi_pay_seven_day_by_conversion) as           event_multi_pay_seven_day_by_conversion
+,sum(event_multi_pay_seven_day_by_conversion_cost) as           event_multi_pay_seven_day_by_conversion_cost
+,sum(mcb_left_time_perf_cpa_bid) as           mcb_left_time_perf_cpa_bid
+,sum(mcb_left_time_perf_roi_ratio) as           mcb_left_time_perf_roi_ratio
+,sum(mcb_all_day_perf_cpa_bid) as           mcb_all_day_perf_cpa_bid
+,sum(mcb_all_day_perf_roi_ratio) as           mcb_all_day_perf_roi_ratio
+,sum(live_room_avg_played_seconds) as           live_room_avg_played_seconds
+,sum(ad_live_share) as           ad_live_share
+,sum(ad_live_comment) as           ad_live_comment
+,sum(live_played_started) as           live_played_started
+,sum(live_played_started_cost) as           live_played_started_cost
+,sum(ad_live_follow) as           ad_live_follow
+,sum(ad_live_follow_cost) as           ad_live_follow_cost
+,sum(simple_live_played_started) as           simple_live_played_started
+,sum(standard_live_played_started) as           standard_live_played_started
+,sum(conversion_component_impression) as           conversion_component_impression
+,sum(conversion_component_click) as           conversion_component_click
+,sum(conversion_component_rate) as           conversion_component_rate
+,sum(ad_landing_page_impression) as           ad_landing_page_impression
+,sum(ad_app_download_half_impression) as           ad_app_download_half_impression
+,sum(event_draw_credit_line) as event_draw_credit_line
+  ,sum(minigame_iaa_purchase_roi) as minigame_iaa_purchase_roi
+,sum(minigame_iaa_purchase_amount) as minigame_iaa_purchase_amount
+from 
+ media_api.kuaishou_account_report_hourly a
+ where stat_date={stat_date}
+group by advertiser_id) report
+LEFT JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL) bei 
+ON report.account_id = bei.account_id;
+            """.format(stat_date=self.stat_date)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AccountHourlySumToDayReport().hourly_sum_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 899 - 0
Apietl/kwai_etl/report/AccountReport.py

@@ -0,0 +1,899 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : AccountReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Utils import Utils
+
+
+class AccountReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+
+    def daily_report(self, account_id, start_date, end_date):
+        try:
+            sql = """
+            replace into application.kuaishou_account_report_daily_dw
+(account_id, stat_date, project_id, product_id, advertiser_id, company_id, account_name, project_name, advertiser_name,
+ company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio,
+ play_3s_ratio, action_ratio, impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow,
+ cancel_follow, report, block, negative, submit, download_started, download_completed, activation, event_pay_first_day,
+ event_pay_purchase_amount_first_day, event_pay_first_day_roi, event_pay, event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost, event_register_ratio, event_jin_jian_app, event_jin_jian_app_cost,
+ event_credit_grant_app, event_credit_grant_app_cost, event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount, event_order_paid_cost, event_next_day_stay, event_next_day_stay_cost,
+ event_next_day_stay_ratio, form_count, form_cost, form_action_ratio, event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost, event_credit_grant_landing_page, event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio, event_valid_clues, event_valid_clues_cost, event_add_wechat,
+ event_add_wechat_cost, event_add_wechat_ratio, event_get_through, event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost, event_app_invoked_ratio, event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay, event_new_user_pay_cost, event_new_user_pay_ratio, click_1k_cost, ad_product_cnt,
+ event_goods_view, merchant_reco_fans, event_order_amount_roi, event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost, event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app, event_new_user_jinjian_app_cost, event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app, event_new_user_credit_grant_app_cost, event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page, event_new_user_jinjian_page_cost, event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page, event_new_user_credit_grant_page_cost, event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost, event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost, event_appoint_jump_click_ratio, union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type, ad_scene, event_order_successed, ad_photo_played_10s_ratio,
+ key_action_cost, event_add_shopping_cart_cost, event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio, event_watch_app_ad, event_multi_conversion_cost, event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio, event_ad_watch_times_cost, event_add_shopping_cart, key_action,
+ event_multi_conversion, event_wechat_connected, event_dsp_gift_form, event_intention_confirmed,
+ event_phone_get_through, event_multi_conversion_ratio, event_measurement_house, ad_photo_played_2s_ratio,
+ event_outbound_call_ratio, event_ad_watch_times,event_pay_purchase_amount_one_day_by_conversion,
+  ad_photo_played_10s
+,ad_photo_played_2s
+,ad_photo_played_75percent
+,approx_pay_cost
+,approx_pay_count
+,approx_pay_ratio
+,cancel_like
+,click_conversion_ratio
+,conversion_cost
+,conversion_cost_by_impression_7d
+,conversion_num
+,conversion_num_by_impression_7d
+,conversion_num_cost
+,conversion_ratio
+,conversion_ratio_by_impression_7d
+,deep_conversion_cost
+,deep_conversion_cost_by_impression_7d
+,deep_conversion_num
+,deep_conversion_num_by_impression_7d
+,deep_conversion_ratio
+,deep_conversion_ratio_by_impression_7d
+,download_completed_cost
+,download_completed_ratio
+,download_conversion_ratio
+,download_installed
+,download_started_cost
+,download_started_ratio
+,event_24h_stay
+,event_24h_stay_by_conversion
+,event_24h_stay_by_conversion_cost
+,event_24h_stay_by_conversion_ratio
+,event_24h_stay_cost
+,event_24h_stay_ratio
+,event_ad_watch_10_times
+,event_ad_watch_10_times_cost
+,event_ad_watch_10_times_ratio
+,event_ad_watch_20_times
+,event_ad_watch_20_times_cost
+,event_ad_watch_20_times_ratio
+,event_ad_watch_5_times
+,event_ad_watch_5_times_cost
+,event_ad_watch_5_times_ratio
+,event_audition
+,event_consultation_valid_retained
+,event_consultation_valid_retained_cost
+,event_consultation_valid_retained_ratio
+,event_conversion_click_cost
+,event_conversion_click_ratio
+,event_credit_grant_first_day_app
+,event_credit_grant_first_day_app_cost
+,event_credit_grant_first_day_app_ratio
+,event_credit_grant_first_day_landing_page
+,event_credit_grant_first_day_landing_page_cost
+,event_credit_grant_first_day_landing_page_ratio
+,event_making_calls
+,event_making_calls_cost
+,event_making_calls_ratio
+,event_order_submit
+,event_pay_purchase_amount_one_day
+,event_pay_purchase_amount_one_day_by_conversion_roi
+,event_pay_purchase_amount_one_day_roi
+,event_pay_purchase_amount_three_day_by_conversion
+,event_pay_purchase_amount_three_day_by_conversion_roi
+,event_pay_purchase_amount_week_by_conversion
+,event_pay_purchase_amount_week_by_conversion_roi
+,event_pay_week_by_conversion
+,event_pay_week_by_conversion_cost
+,event_pay_weighted_purchase_amount
+,event_pay_weighted_purchase_amount_first_day
+,event_pre_component_consultation_valid_retained
+,event_three_day_stay_by_conversion
+,event_three_day_stay_by_conversion_cost
+,event_three_day_stay_by_conversion_ratio
+,event_wechat_qr_code_link_click
+,event_week_stay_by_conversion
+,event_week_stay_by_conversion_cost
+,event_week_stay_by_conversion_ratio
+,live_event_goods_view
+,live_played_3s
+,played_end
+,played_five_seconds
+,played_three_seconds
+,event_next_day_stay_new
+,event_next_day_stay_new_cost
+,event_next_day_stay_new_ratio
+,ad_show
+,event_credit_card_recheck
+,event_credit_card_recheck_first_day
+,event_no_intention
+,event_multi_pay_seven_day_by_conversion
+,event_multi_pay_seven_day_by_conversion_cost
+,mcb_left_time_perf_cpa_bid
+,mcb_left_time_perf_roi_ratio
+,mcb_all_day_perf_cpa_bid
+,mcb_all_day_perf_roi_ratio
+,live_room_avg_played_seconds
+,ad_live_share
+,ad_live_comment
+,live_played_started
+,live_played_started_cost
+,ad_live_follow
+,ad_live_follow_cost
+,simple_live_played_started
+,standard_live_played_started
+,conversion_component_impression
+,conversion_component_click
+,conversion_component_rate
+,ad_landing_page_impression
+,ad_app_download_half_impression
+,event_draw_credit_line
+  ,minigame_iaa_purchase_roi
+,minigame_iaa_purchase_amount
+)
+        select report.advertiser_id as account_id,
+               stat_date,
+               project_id,
+               product_id,
+               sys.advertiser_id,
+               company_id,
+               account_name,
+               project_name,
+               advertiser_name,
+               company_name,
+               sale_id,
+               user_id,
+               org_code,
+               charge,
+               `show`,
+               photo_click,
+               aclick,
+               bclick,
+               photo_click_ratio,
+               play_3s_ratio,
+               action_ratio,
+               impression_1k_cost,
+               photo_click_cost,
+               action_cost,
+               share,
+               comment,
+               `like`,
+               follow,
+               cancel_follow,
+               report,
+               block,
+               negative,
+               submit,
+               download_started,
+               download_completed,
+               activation,
+               event_pay_first_day,
+               event_pay_purchase_amount_first_day,
+               event_pay_first_day_roi,
+               event_pay,
+               event_pay_purchase_amount,
+               event_pay_roi,
+               event_register,
+               event_register_cost,
+               event_register_ratio,
+               event_jin_jian_app,
+               event_jin_jian_app_cost,
+               event_credit_grant_app,
+               event_credit_grant_app_cost,
+               event_credit_grant_app_ratio,
+               event_order_paid,
+               event_order_paid_purchase_amount,
+               event_order_paid_cost,
+               event_next_day_stay,
+               event_next_day_stay_cost,
+               event_next_day_stay_ratio,
+               form_count,
+               form_cost,
+               form_action_ratio,
+               event_jin_jian_landing_page,
+               event_jin_jian_landing_page_cost,
+               event_credit_grant_landing_page,
+               event_credit_grant_landing_page_cost,
+               event_credit_grant_landing_page_ratio,
+               event_valid_clues,
+               event_valid_clues_cost,
+               event_add_wechat,
+               event_add_wechat_cost,
+               event_add_wechat_ratio,
+               event_get_through,
+               event_get_through_cost,
+               event_get_through_ratio,
+               event_app_invoked,
+               event_app_invoked_cost,
+               event_app_invoked_ratio,
+               event_credit_grant_landing_ratio,
+               play_5s_ratio,
+               play_end_ratio,
+               event_new_user_pay,
+               event_new_user_pay_cost,
+               event_new_user_pay_ratio,
+               click_1k_cost,
+               ad_product_cnt,
+               event_goods_view,
+               merchant_reco_fans,
+               event_order_amount_roi,
+               event_goods_view_cost,
+               merchant_reco_fans_cost,
+               event_button_click,
+               event_button_click_cost,
+               event_button_click_ratio,
+               event_order_paid_roi,
+               event_new_user_jinjian_app,
+               event_new_user_jinjian_app_cost,
+               event_new_user_jinjian_app_roi,
+               event_new_user_credit_grant_app,
+               event_new_user_credit_grant_app_cost,
+               event_new_user_credit_grant_app_roi,
+               event_new_user_jinjian_page,
+               event_new_user_jinjian_page_cost,
+               event_new_user_jinjian_page_roi,
+               event_new_user_credit_grant_page,
+               event_new_user_credit_grant_page_cost,
+               event_new_user_credit_grant_page_roi,
+               event_appoint_form,
+               event_appoint_form_cost,
+               event_appoint_form_ratio,
+               event_appoint_jump_click,
+               event_appoint_jump_click_cost,
+               event_appoint_jump_click_ratio,
+               union_event_pay_purchase_amount_7d,
+               union_event_pay_purchase_amount_7d_roi,
+               placement_type,
+               ad_scene,
+               event_order_successed,
+               ad_photo_played_10s_ratio,
+               key_action_cost,
+               event_add_shopping_cart_cost,
+               event_ad_watch_times_ratio,
+               event_outbound_call_cost,
+               event_outbound_call,
+               action_new_ratio,
+               event_watch_app_ad,
+               event_multi_conversion_cost,
+               event_phone_card_activate,
+               ad_photo_played_75percent_ratio,
+               key_action_ratio,
+               event_ad_watch_times_cost,
+               event_add_shopping_cart,
+               key_action,
+               event_multi_conversion,
+               event_wechat_connected,
+               event_dsp_gift_form,
+               event_intention_confirmed,
+               event_phone_get_through,
+               event_multi_conversion_ratio,
+               event_measurement_house,
+               ad_photo_played_2s_ratio,
+               event_outbound_call_ratio,
+               event_ad_watch_times,
+               event_pay_purchase_amount_one_day_by_conversion,
+                ad_photo_played_10s
+,ad_photo_played_2s
+,ad_photo_played_75percent
+,approx_pay_cost
+,approx_pay_count
+,approx_pay_ratio
+,cancel_like
+,click_conversion_ratio
+,conversion_cost
+,conversion_cost_by_impression_7d
+,conversion_num
+,conversion_num_by_impression_7d
+,conversion_num_cost
+,conversion_ratio
+,conversion_ratio_by_impression_7d
+,deep_conversion_cost
+,deep_conversion_cost_by_impression_7d
+,deep_conversion_num
+,deep_conversion_num_by_impression_7d
+,deep_conversion_ratio
+,deep_conversion_ratio_by_impression_7d
+,download_completed_cost
+,download_completed_ratio
+,download_conversion_ratio
+,download_installed
+,download_started_cost
+,download_started_ratio
+,event_24h_stay
+,event_24h_stay_by_conversion
+,event_24h_stay_by_conversion_cost
+,event_24h_stay_by_conversion_ratio
+,event_24h_stay_cost
+,event_24h_stay_ratio
+,event_ad_watch_10_times
+,event_ad_watch_10_times_cost
+,event_ad_watch_10_times_ratio
+,event_ad_watch_20_times
+,event_ad_watch_20_times_cost
+,event_ad_watch_20_times_ratio
+,event_ad_watch_5_times
+,event_ad_watch_5_times_cost
+,event_ad_watch_5_times_ratio
+,event_audition
+,event_consultation_valid_retained
+,event_consultation_valid_retained_cost
+,event_consultation_valid_retained_ratio
+,event_conversion_click_cost
+,event_conversion_click_ratio
+,event_credit_grant_first_day_app
+,event_credit_grant_first_day_app_cost
+,event_credit_grant_first_day_app_ratio
+,event_credit_grant_first_day_landing_page
+,event_credit_grant_first_day_landing_page_cost
+,event_credit_grant_first_day_landing_page_ratio
+,event_making_calls
+,event_making_calls_cost
+,event_making_calls_ratio
+,event_order_submit
+,event_pay_purchase_amount_one_day
+,event_pay_purchase_amount_one_day_by_conversion_roi
+,event_pay_purchase_amount_one_day_roi
+,event_pay_purchase_amount_three_day_by_conversion
+,event_pay_purchase_amount_three_day_by_conversion_roi
+,event_pay_purchase_amount_week_by_conversion
+,event_pay_purchase_amount_week_by_conversion_roi
+,event_pay_week_by_conversion
+,event_pay_week_by_conversion_cost
+,event_pay_weighted_purchase_amount
+,event_pay_weighted_purchase_amount_first_day
+,event_pre_component_consultation_valid_retained
+,event_three_day_stay_by_conversion
+,event_three_day_stay_by_conversion_cost
+,event_three_day_stay_by_conversion_ratio
+,event_wechat_qr_code_link_click
+,event_week_stay_by_conversion
+,event_week_stay_by_conversion_cost
+,event_week_stay_by_conversion_ratio
+,live_event_goods_view
+,live_played_3s
+,played_end
+,played_five_seconds
+,played_three_seconds
+,event_next_day_stay_new
+,event_next_day_stay_new_cost
+,event_next_day_stay_new_ratio
+,ad_show
+,event_credit_card_recheck
+,event_credit_card_recheck_first_day
+,event_no_intention
+,event_multi_pay_seven_day_by_conversion
+,event_multi_pay_seven_day_by_conversion_cost
+,mcb_left_time_perf_cpa_bid
+,mcb_left_time_perf_roi_ratio
+,mcb_all_day_perf_cpa_bid
+,mcb_all_day_perf_roi_ratio
+,live_room_avg_played_seconds
+,ad_live_share
+,ad_live_comment
+,live_played_started
+,live_played_started_cost
+,ad_live_follow
+,ad_live_follow_cost
+,simple_live_played_started
+,standard_live_played_started
+,conversion_component_impression
+,conversion_component_click
+,conversion_component_rate
+,ad_landing_page_impression
+,ad_app_download_half_impression
+,event_draw_credit_line
+  ,minigame_iaa_purchase_roi
+,minigame_iaa_purchase_amount
+
+        from media_api.kuaishou_account_report_daily report
+                 left join
+                          (SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+             ) sys
+             on report.advertiser_id = sys.account_id
+        where report.stat_date between {start_date} and {end_date}
+        and report.advertiser_id={account_id}
+            """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            print(account_id, '快手账户日报执行完成')
+        except Exception as e:
+            logging.error(account_id, '执行失败')
+
+    def hourly_report(self, account_id, start_date, end_date):
+        try:
+            sql = """
+            replace into application.kuaishou_account_report_hourly_dw
+(account_id, stat_date,`hour`, project_id, product_id, advertiser_id, company_id, account_name, project_name, advertiser_name,
+ company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio,
+ play_3s_ratio, action_ratio, impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow,
+ cancel_follow, report, block, negative, submit, download_started, download_completed, activation, event_pay_first_day,
+ event_pay_purchase_amount_first_day, event_pay_first_day_roi, event_pay, event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost, event_register_ratio, event_jin_jian_app, event_jin_jian_app_cost,
+ event_credit_grant_app, event_credit_grant_app_cost, event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount, event_order_paid_cost, event_next_day_stay, event_next_day_stay_cost,
+ event_next_day_stay_ratio, form_count, form_cost, form_action_ratio, event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost, event_credit_grant_landing_page, event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio, event_valid_clues, event_valid_clues_cost, event_add_wechat,
+ event_add_wechat_cost, event_add_wechat_ratio, event_get_through, event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost, event_app_invoked_ratio, event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay, event_new_user_pay_cost, event_new_user_pay_ratio, click_1k_cost, ad_product_cnt,
+ event_goods_view, merchant_reco_fans, event_order_amount_roi, event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost, event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app, event_new_user_jinjian_app_cost, event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app, event_new_user_credit_grant_app_cost, event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page, event_new_user_jinjian_page_cost, event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page, event_new_user_credit_grant_page_cost, event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost, event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost, event_appoint_jump_click_ratio, union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type, ad_scene, event_order_successed, ad_photo_played_10s_ratio,
+ key_action_cost, event_add_shopping_cart_cost, event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio, event_watch_app_ad, event_multi_conversion_cost, event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio, event_ad_watch_times_cost, event_add_shopping_cart, key_action,
+ event_multi_conversion, event_wechat_connected, event_dsp_gift_form, event_intention_confirmed,
+ event_phone_get_through, event_multi_conversion_ratio, event_measurement_house, ad_photo_played_2s_ratio,
+ event_outbound_call_ratio, event_ad_watch_times,event_pay_purchase_amount_one_day_by_conversion, ad_photo_played_10s
+,ad_photo_played_2s
+,ad_photo_played_75percent
+,approx_pay_cost
+,approx_pay_count
+,approx_pay_ratio
+,cancel_like
+,click_conversion_ratio
+,conversion_cost
+,conversion_cost_by_impression_7d
+,conversion_num
+,conversion_num_by_impression_7d
+,conversion_num_cost
+,conversion_ratio
+,conversion_ratio_by_impression_7d
+,deep_conversion_cost
+,deep_conversion_cost_by_impression_7d
+,deep_conversion_num
+,deep_conversion_num_by_impression_7d
+,deep_conversion_ratio
+,deep_conversion_ratio_by_impression_7d
+,download_completed_cost
+,download_completed_ratio
+,download_conversion_ratio
+,download_installed
+,download_started_cost
+,download_started_ratio
+,event_24h_stay
+,event_24h_stay_by_conversion
+,event_24h_stay_by_conversion_cost
+,event_24h_stay_by_conversion_ratio
+,event_24h_stay_cost
+,event_24h_stay_ratio
+,event_ad_watch_10_times
+,event_ad_watch_10_times_cost
+,event_ad_watch_10_times_ratio
+,event_ad_watch_20_times
+,event_ad_watch_20_times_cost
+,event_ad_watch_20_times_ratio
+,event_ad_watch_5_times
+,event_ad_watch_5_times_cost
+,event_ad_watch_5_times_ratio
+,event_audition
+,event_consultation_valid_retained
+,event_consultation_valid_retained_cost
+,event_consultation_valid_retained_ratio
+,event_conversion_click_cost
+,event_conversion_click_ratio
+,event_credit_grant_first_day_app
+,event_credit_grant_first_day_app_cost
+,event_credit_grant_first_day_app_ratio
+,event_credit_grant_first_day_landing_page
+,event_credit_grant_first_day_landing_page_cost
+,event_credit_grant_first_day_landing_page_ratio
+,event_making_calls
+,event_making_calls_cost
+,event_making_calls_ratio
+,event_order_submit
+,event_pay_purchase_amount_one_day
+,event_pay_purchase_amount_one_day_by_conversion_roi
+,event_pay_purchase_amount_one_day_roi
+,event_pay_purchase_amount_three_day_by_conversion
+,event_pay_purchase_amount_three_day_by_conversion_roi
+,event_pay_purchase_amount_week_by_conversion
+,event_pay_purchase_amount_week_by_conversion_roi
+,event_pay_week_by_conversion
+,event_pay_week_by_conversion_cost
+,event_pay_weighted_purchase_amount
+,event_pay_weighted_purchase_amount_first_day
+,event_pre_component_consultation_valid_retained
+,event_three_day_stay_by_conversion
+,event_three_day_stay_by_conversion_cost
+,event_three_day_stay_by_conversion_ratio
+,event_wechat_qr_code_link_click
+,event_week_stay_by_conversion
+,event_week_stay_by_conversion_cost
+,event_week_stay_by_conversion_ratio
+,live_event_goods_view
+,live_played_3s
+,played_end
+,played_five_seconds
+,played_three_seconds
+,event_next_day_stay_new
+,event_next_day_stay_new_cost
+,event_next_day_stay_new_ratio
+,ad_show
+,event_credit_card_recheck
+,event_credit_card_recheck_first_day
+,event_no_intention
+,event_multi_pay_seven_day_by_conversion
+,event_multi_pay_seven_day_by_conversion_cost
+,mcb_left_time_perf_cpa_bid
+,mcb_left_time_perf_roi_ratio
+,mcb_all_day_perf_cpa_bid
+,mcb_all_day_perf_roi_ratio
+,live_room_avg_played_seconds
+,ad_live_share
+,ad_live_comment
+,live_played_started
+,live_played_started_cost
+,ad_live_follow
+,ad_live_follow_cost
+,simple_live_played_started
+,standard_live_played_started
+,conversion_component_impression
+,conversion_component_click
+,conversion_component_rate
+,ad_landing_page_impression
+,ad_app_download_half_impression
+,event_draw_credit_line
+,minigame_iaa_purchase_amount
+,minigame_iaa_purchase_roi
+)
+        select report.advertiser_id as account_id,
+               stat_date,
+               `hour`,
+               project_id,
+               product_id,
+               sys.advertiser_id,
+               company_id,
+               account_name,
+               project_name,
+               advertiser_name,
+               company_name,
+               sale_id,
+               user_id,
+               org_code,
+               charge,
+               `show`,
+               photo_click,
+               aclick,
+               bclick,
+               photo_click_ratio,
+               play_3s_ratio,
+               action_ratio,
+               impression_1k_cost,
+               photo_click_cost,
+               action_cost,
+               share,
+               comment,
+               `like`,
+               follow,
+               cancel_follow,
+               report,
+               block,
+               negative,
+               submit,
+               download_started,
+               download_completed,
+               activation,
+               event_pay_first_day,
+               event_pay_purchase_amount_first_day,
+               event_pay_first_day_roi,
+               event_pay,
+               event_pay_purchase_amount,
+               event_pay_roi,
+               event_register,
+               event_register_cost,
+               event_register_ratio,
+               event_jin_jian_app,
+               event_jin_jian_app_cost,
+               event_credit_grant_app,
+               event_credit_grant_app_cost,
+               event_credit_grant_app_ratio,
+               event_order_paid,
+               event_order_paid_purchase_amount,
+               event_order_paid_cost,
+               event_next_day_stay,
+               event_next_day_stay_cost,
+               event_next_day_stay_ratio,
+               form_count,
+               form_cost,
+               form_action_ratio,
+               event_jin_jian_landing_page,
+               event_jin_jian_landing_page_cost,
+               event_credit_grant_landing_page,
+               event_credit_grant_landing_page_cost,
+               event_credit_grant_landing_page_ratio,
+               event_valid_clues,
+               event_valid_clues_cost,
+               event_add_wechat,
+               event_add_wechat_cost,
+               event_add_wechat_ratio,
+               event_get_through,
+               event_get_through_cost,
+               event_get_through_ratio,
+               event_app_invoked,
+               event_app_invoked_cost,
+               event_app_invoked_ratio,
+               event_credit_grant_landing_ratio,
+               play_5s_ratio,
+               play_end_ratio,
+               event_new_user_pay,
+               event_new_user_pay_cost,
+               event_new_user_pay_ratio,
+               click_1k_cost,
+               ad_product_cnt,
+               event_goods_view,
+               merchant_reco_fans,
+               event_order_amount_roi,
+               event_goods_view_cost,
+               merchant_reco_fans_cost,
+               event_button_click,
+               event_button_click_cost,
+               event_button_click_ratio,
+               event_order_paid_roi,
+               event_new_user_jinjian_app,
+               event_new_user_jinjian_app_cost,
+               event_new_user_jinjian_app_roi,
+               event_new_user_credit_grant_app,
+               event_new_user_credit_grant_app_cost,
+               event_new_user_credit_grant_app_roi,
+               event_new_user_jinjian_page,
+               event_new_user_jinjian_page_cost,
+               event_new_user_jinjian_page_roi,
+               event_new_user_credit_grant_page,
+               event_new_user_credit_grant_page_cost,
+               event_new_user_credit_grant_page_roi,
+               event_appoint_form,
+               event_appoint_form_cost,
+               event_appoint_form_ratio,
+               event_appoint_jump_click,
+               event_appoint_jump_click_cost,
+               event_appoint_jump_click_ratio,
+               union_event_pay_purchase_amount_7d,
+               union_event_pay_purchase_amount_7d_roi,
+               placement_type,
+               ad_scene,
+               event_order_successed,
+               ad_photo_played_10s_ratio,
+               key_action_cost,
+               event_add_shopping_cart_cost,
+               event_ad_watch_times_ratio,
+               event_outbound_call_cost,
+               event_outbound_call,
+               action_new_ratio,
+               event_watch_app_ad,
+               event_multi_conversion_cost,
+               event_phone_card_activate,
+               ad_photo_played_75percent_ratio,
+               key_action_ratio,
+               event_ad_watch_times_cost,
+               event_add_shopping_cart,
+               key_action,
+               event_multi_conversion,
+               event_wechat_connected,
+               event_dsp_gift_form,
+               event_intention_confirmed,
+               event_phone_get_through,
+               event_multi_conversion_ratio,
+               event_measurement_house,
+               ad_photo_played_2s_ratio,
+               event_outbound_call_ratio,
+               event_ad_watch_times,
+               event_pay_purchase_amount_one_day_by_conversion,
+                ad_photo_played_10s
+,ad_photo_played_2s
+,ad_photo_played_75percent
+,approx_pay_cost
+,approx_pay_count
+,approx_pay_ratio
+,cancel_like
+,click_conversion_ratio
+,conversion_cost
+,conversion_cost_by_impression_7d
+,conversion_num
+,conversion_num_by_impression_7d
+,conversion_num_cost
+,conversion_ratio
+,conversion_ratio_by_impression_7d
+,deep_conversion_cost
+,deep_conversion_cost_by_impression_7d
+,deep_conversion_num
+,deep_conversion_num_by_impression_7d
+,deep_conversion_ratio
+,deep_conversion_ratio_by_impression_7d
+,download_completed_cost
+,download_completed_ratio
+,download_conversion_ratio
+,download_installed
+,download_started_cost
+,download_started_ratio
+,event_24h_stay
+,event_24h_stay_by_conversion
+,event_24h_stay_by_conversion_cost
+,event_24h_stay_by_conversion_ratio
+,event_24h_stay_cost
+,event_24h_stay_ratio
+,event_ad_watch_10_times
+,event_ad_watch_10_times_cost
+,event_ad_watch_10_times_ratio
+,event_ad_watch_20_times
+,event_ad_watch_20_times_cost
+,event_ad_watch_20_times_ratio
+,event_ad_watch_5_times
+,event_ad_watch_5_times_cost
+,event_ad_watch_5_times_ratio
+,event_audition
+,event_consultation_valid_retained
+,event_consultation_valid_retained_cost
+,event_consultation_valid_retained_ratio
+,event_conversion_click_cost
+,event_conversion_click_ratio
+,event_credit_grant_first_day_app
+,event_credit_grant_first_day_app_cost
+,event_credit_grant_first_day_app_ratio
+,event_credit_grant_first_day_landing_page
+,event_credit_grant_first_day_landing_page_cost
+,event_credit_grant_first_day_landing_page_ratio
+,event_making_calls
+,event_making_calls_cost
+,event_making_calls_ratio
+,event_order_submit
+,event_pay_purchase_amount_one_day
+,event_pay_purchase_amount_one_day_by_conversion_roi
+,event_pay_purchase_amount_one_day_roi
+,event_pay_purchase_amount_three_day_by_conversion
+,event_pay_purchase_amount_three_day_by_conversion_roi
+,event_pay_purchase_amount_week_by_conversion
+,event_pay_purchase_amount_week_by_conversion_roi
+,event_pay_week_by_conversion
+,event_pay_week_by_conversion_cost
+,event_pay_weighted_purchase_amount
+,event_pay_weighted_purchase_amount_first_day
+,event_pre_component_consultation_valid_retained
+,event_three_day_stay_by_conversion
+,event_three_day_stay_by_conversion_cost
+,event_three_day_stay_by_conversion_ratio
+,event_wechat_qr_code_link_click
+,event_week_stay_by_conversion
+,event_week_stay_by_conversion_cost
+,event_week_stay_by_conversion_ratio
+,live_event_goods_view
+,live_played_3s
+,played_end
+,played_five_seconds
+,played_three_seconds
+,event_next_day_stay_new
+,event_next_day_stay_new_cost
+,event_next_day_stay_new_ratio
+,ad_show
+,event_credit_card_recheck
+,event_credit_card_recheck_first_day
+,event_no_intention
+,event_multi_pay_seven_day_by_conversion
+,event_multi_pay_seven_day_by_conversion_cost
+,mcb_left_time_perf_cpa_bid
+,mcb_left_time_perf_roi_ratio
+,mcb_all_day_perf_cpa_bid
+,mcb_all_day_perf_roi_ratio
+,live_room_avg_played_seconds
+,ad_live_share
+,ad_live_comment
+,live_played_started
+,live_played_started_cost
+,ad_live_follow
+,ad_live_follow_cost
+,simple_live_played_started
+,standard_live_played_started
+,conversion_component_impression
+,conversion_component_click
+,conversion_component_rate
+,ad_landing_page_impression
+,ad_app_download_half_impression
+,event_draw_credit_line
+,minigame_iaa_purchase_amount
+,minigame_iaa_purchase_roi
+
+        from media_api.kuaishou_account_report_hourly report
+                 left join
+                          (SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+             ) sys
+             on report.advertiser_id = sys.account_id
+        where report.stat_date between {start_date} and {end_date}
+        and  report.advertiser_id={account_id}
+     """.format(start_date=start_date, end_date=end_date, account_id=account_id)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            print(account_id, '快手账户时报执行完成')
+        except Exception as e:
+            logging.error(account_id, '执行失败')
+
+    def handler(self, date_type, account_id, start_date, end_date):
+        if date_type == "daily":
+            self.daily_report(account_id, start_date, end_date)
+        else:
+            self.hourly_report(account_id, start_date, end_date)

+ 366 - 0
Apietl/kwai_etl/report/AppKuaiShouAccountAdAceneReport.py

@@ -0,0 +1,366 @@
+# Author zhaohailiang
+# coding=utf-8
+# @Time    : 2021/12/20 2:20 下午
+# @Site    :
+# @File    : AppKuaishouAccountAdAceneReport.py
+# @Software: PyCharm
+# @contact: zhaohailiang@c-top.com.cn
+# @Tel 18201631162
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouAccountAdAceneReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+
+    def taskHandler(self):
+        try:
+            sql = """
+            SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+            replace into `application`.`app_kuaishou_account_ad_scene_report_d`
+            (
+            account_id,
+            stat_date,
+            project_id,
+            product_id,
+            advertiser_id,
+            company_id,
+            account_name,
+            project_name,
+            advertiser_name,
+            company_name,
+            sale_id,
+            sale_name,
+            user_id,
+            user_name,
+            leader_id,
+            leader_name,
+            org_code,
+            charge,
+            `show`,
+            photo_click,
+            aclick,
+            bclick,
+            photo_click_ratio,
+            play_3s_ratio,
+            action_ratio,
+            impression_1k_cost,
+            photo_click_cost,
+            action_cost,
+            `share`,
+            `comment`,
+            `like`,
+            follow,
+            cancel_follow,
+            report,
+            block,
+            negative,
+            submit,
+            download_started,
+            download_completed,
+            activation,
+            event_pay_first_day,
+            event_pay_purchase_amount_first_day,
+            event_pay_first_day_roi,
+            event_pay,
+            event_pay_purchase_amount,
+            event_pay_roi,
+            event_register,
+            event_register_cost,
+            event_register_ratio,
+            event_jin_jian_app,
+            event_jin_jian_app_cost,
+            event_credit_grant_app,
+            event_credit_grant_app_cost,
+            event_credit_grant_app_ratio,
+            event_order_paid,
+            event_order_paid_purchase_amount,
+            event_order_paid_cost,
+            event_next_day_stay,
+            event_next_day_stay_cost,
+            event_next_day_stay_ratio,
+            form_count,
+            form_cost,
+            form_action_ratio,
+            event_jin_jian_landing_page,
+            event_jin_jian_landing_page_cost,
+            event_credit_grant_landing_page,
+            event_credit_grant_landing_page_cost,
+            event_credit_grant_landing_page_ratio,
+            event_valid_clues,
+            event_valid_clues_cost,
+            event_add_wechat,
+            event_add_wechat_cost,
+            event_add_wechat_ratio,
+            event_get_through,
+            event_get_through_cost,
+            event_get_through_ratio,
+            event_app_invoked,
+            event_app_invoked_cost,
+            event_app_invoked_ratio,
+            event_credit_grant_landing_ratio,
+            play_5s_ratio,
+            play_end_ratio,
+            event_new_user_pay,
+            event_new_user_pay_cost,
+            event_new_user_pay_ratio,
+            click_1k_cost,
+            ad_product_cnt,
+            event_goods_view,
+            merchant_reco_fans,
+            event_order_amount_roi,
+            event_goods_view_cost,
+            merchant_reco_fans_cost,
+            event_button_click,
+            event_button_click_cost,
+            event_button_click_ratio,
+            event_order_paid_roi,
+            event_new_user_jinjian_app,
+            event_new_user_jinjian_app_cost,
+            event_new_user_jinjian_app_roi,
+            event_new_user_credit_grant_app,
+            event_new_user_credit_grant_app_cost,
+            event_new_user_credit_grant_app_roi,
+            event_new_user_jinjian_page,
+            event_new_user_jinjian_page_cost,
+            event_new_user_jinjian_page_roi,
+            event_new_user_credit_grant_page,
+            event_new_user_credit_grant_page_cost,
+            event_new_user_credit_grant_page_roi,
+            event_appoint_form,
+            event_appoint_form_cost,
+            event_appoint_form_ratio,
+            event_appoint_jump_click,
+            event_appoint_jump_click_cost,
+            event_appoint_jump_click_ratio,
+            union_event_pay_purchase_amount_7d,
+            union_event_pay_purchase_amount_7d_roi,
+            placement_type,
+            ad_scene,
+            event_order_successed,
+            ad_photo_played_10s_ratio,
+            key_action_cost,
+            event_add_shopping_cart_cost,
+            event_ad_watch_times_ratio,
+            event_outbound_call_cost,
+            event_outbound_call,
+            action_new_ratio,
+            event_watch_app_ad,
+            event_multi_conversion_cost,
+            event_phone_card_activate,
+            ad_photo_played_75percent_ratio,
+            key_action_ratio,
+            event_ad_watch_times_cost,
+            event_add_shopping_cart,
+            key_action,
+            event_multi_conversion,
+            event_wechat_connected,
+            event_dsp_gift_form,
+            event_intention_confirmed,
+            event_phone_get_through,
+            event_multi_conversion_ratio,
+            event_measurement_house,
+            ad_photo_played_2s_ratio,
+            event_outbound_call_ratio,
+            event_ad_watch_times
+            )
+            select 
+            report.advertiser_id account_id,
+            report.stat_date,
+            bei.project_id,
+            bei.product_id,
+            bei.advertiser_id,
+            bei.company_id,
+            bei.account_name,
+            bei.project_name,
+            bei.advertiser_name,
+            bei.company_name,
+            bei.sale_id,
+            bei.sale_name,
+            bei.user_id,
+            bei.user_name,
+            bei.leader_id,
+            bei.leader_name,
+            bei.org_code,
+            report.charge,
+            report.`show`,
+            report.photo_click,
+            report.aclick,
+            report.bclick,
+            report.photo_click_ratio,
+            report.play_3s_ratio,
+            report.action_ratio,
+            report.impression_1k_cost,
+            report.photo_click_cost,
+            report.action_cost,
+            report.`share`,
+            report.`comment`,
+            report.`like`,
+            report.follow,
+            report.cancel_follow,
+            report.report,
+            report.block,
+            report.negative,
+            report.submit,
+            report.download_started,
+            report.download_completed,
+            report.activation,
+            report.event_pay_first_day,
+            report.event_pay_purchase_amount_first_day,
+            report.event_pay_first_day_roi,
+            report.event_pay,
+            report.event_pay_purchase_amount,
+            report.event_pay_roi,
+            report.event_register,
+            report.event_register_cost,
+            report.event_register_ratio,
+            report.event_jin_jian_app,
+            report.event_jin_jian_app_cost,
+            report.event_credit_grant_app,
+            report.event_credit_grant_app_cost,
+            report.event_credit_grant_app_ratio,
+            report.event_order_paid,
+            report.event_order_paid_purchase_amount,
+            report.event_order_paid_cost,
+            report.event_next_day_stay,
+            report.event_next_day_stay_cost,
+            report.event_next_day_stay_ratio,
+            report.form_count,
+            report.form_cost,
+            report.form_action_ratio,
+            report.event_jin_jian_landing_page,
+            report.event_jin_jian_landing_page_cost,
+            report.event_credit_grant_landing_page,
+            report.event_credit_grant_landing_page_cost,
+            report.event_credit_grant_landing_page_ratio,
+            report.event_valid_clues,
+            report.event_valid_clues_cost,
+            report.event_add_wechat,
+            report.event_add_wechat_cost,
+            report.event_add_wechat_ratio,
+            report.event_get_through,
+            report.event_get_through_cost,
+            report.event_get_through_ratio,
+            report.event_app_invoked,
+            report.event_app_invoked_cost,
+            report.event_app_invoked_ratio,
+            report.event_credit_grant_landing_ratio,
+            report.play_5s_ratio,
+            report.play_end_ratio,
+            report.event_new_user_pay,
+            report.event_new_user_pay_cost,
+            report.event_new_user_pay_ratio,
+            report.click_1k_cost,
+            report.ad_product_cnt,
+            report.event_goods_view,
+            report.merchant_reco_fans,
+            report.event_order_amount_roi,
+            report.event_goods_view_cost,
+            report.merchant_reco_fans_cost,
+            report.event_button_click,
+            report.event_button_click_cost,
+            report.event_button_click_ratio,
+            report.event_order_paid_roi,
+            report.event_new_user_jinjian_app,
+            report.event_new_user_jinjian_app_cost,
+            report.event_new_user_jinjian_app_roi,
+            report.event_new_user_credit_grant_app,
+            report.event_new_user_credit_grant_app_cost,
+            report.event_new_user_credit_grant_app_roi,
+            report.event_new_user_jinjian_page,
+            report.event_new_user_jinjian_page_cost,
+            report.event_new_user_jinjian_page_roi,
+            report.event_new_user_credit_grant_page,
+            report.event_new_user_credit_grant_page_cost,
+            report.event_new_user_credit_grant_page_roi,
+            report.event_appoint_form,
+            report.event_appoint_form_cost,
+            report.event_appoint_form_ratio,
+            report.event_appoint_jump_click,
+            report.event_appoint_jump_click_cost,
+            report.event_appoint_jump_click_ratio,
+            report.union_event_pay_purchase_amount_7d,
+            report.union_event_pay_purchase_amount_7d_roi,
+            report.placement_type,
+            report.ad_scene,
+            report.event_order_successed,
+            report.ad_photo_played_10s_ratio,
+            report.key_action_cost,
+            report.event_add_shopping_cart_cost,
+            report.event_ad_watch_times_ratio,
+            report.event_outbound_call_cost,
+            report.event_outbound_call,
+            report.action_new_ratio,
+            report.event_watch_app_ad,
+            report.event_multi_conversion_cost,
+            report.event_phone_card_activate,
+            report.ad_photo_played_75percent_ratio,
+            report.key_action_ratio,
+            report.event_ad_watch_times_cost,
+            report.event_add_shopping_cart,
+            report.key_action,
+            report.event_multi_conversion,
+            report.event_wechat_connected,
+            report.event_dsp_gift_form,
+            report.event_intention_confirmed,
+            report.event_phone_get_through,
+            report.event_multi_conversion_ratio,
+            report.event_measurement_house,
+            report.ad_photo_played_2s_ratio,
+            report.event_outbound_call_ratio,
+            report.event_ad_watch_times
+            from (select * from `media_api`.kuaishou_account_ad_scene_report_daily where stat_date={date_time}) report
+            LEFT JOIN (
+            SELECT
+            cua.account_id id,
+            cua.account_id account_id,
+            cp.id project_id,
+            product.id product_id,
+            ca.id advertiser_id,
+            uc.company_id company_id,
+            cua.auth_name account_name,
+            cp.project_name project_name,
+            ca.`name` advertiser_name,
+            uc.company_name company_name,
+            cp.sale_id sale_id,
+            sale.realname sale_name,
+            su.id user_id,
+            su.realname user_name,
+            su.leader_id,
+            lea.realname leader_name,
+            su.org_code org_code
+            FROM
+            `jeecg-boot`.ctop_user_allocation cua
+            LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+            LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+            LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+            LEFT JOIN `jeecg-boot`.sys_user sale ON sale.id = cp.sale_id
+            LEFT JOIN `jeecg-boot`.sys_user lea ON lea.id = su.leader_id
+            LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+            LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+            WHERE
+            cua.account_id IS NOT NULL) bei
+ON report.advertiser_id = bei.account_id;
+ """.format(date_time=self.date_time)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouAccountAdAceneReport().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 164 - 0
Apietl/kwai_etl/report/AppKuaiShouAgentReport.py

@@ -0,0 +1,164 @@
+# Author zhaohailiang
+# coding=utf-8
+# @Time    : 2021/12/20 2:22 下午
+# @Site    :
+# @File    : AppKuaishouAgentReport.py
+# @Software: PyCharm
+# @contact: zhaohailiang@c-top.com.cn
+# @Tel 18201631162
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouAgentReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+
+    def taskHandler(self):
+        try:
+            sql = """
+    SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+replace into `application`.`app_kuaishou_agent_report_d`
+(
+account_id,
+agent_id,
+kuaishou_id,
+stat_date,
+project_id,
+product_id,
+advertiser_id,
+company_id,
+account_name,
+project_name,
+advertiser_name,
+company_name,
+sale_id,
+sale_name,
+user_id,
+user_name,
+leader_id,
+leader_name,
+org_code,
+total_charged_in_yuan,
+total_balance_in_yuan,
+real_charged_in_yuan,
+total_rebate_real_charged_in_yuan,
+contract_rebate_real_charged_in_yuan,
+direct_rebate_real_charged_in_yuan,
+credit_real_charged_in_yuan,
+charge_day_on_day_percent,
+ad_photo_impression,
+ad_photo_click,
+ad_item_impression,
+ad_item_click,
+photo_click_ratio,
+item_click_ratio,
+charged_campaign_count,
+product_name,
+corporation_name,
+first_cost_day,
+industry,
+second_industry,
+union_event_pay_purchase_amount_7d,
+union_event_pay_purchase_amount_7d_roi
+)
+select
+report.advertiser_id account_id,
+report.agent_id,
+report.user_id kuaishou_id,
+report.stat_date,
+bei.project_id,
+bei.product_id,
+bei.advertiser_id,
+bei.company_id,
+report.account_name,
+bei.project_name,
+bei.advertiser_name,
+bei.company_name,
+bei.sale_id,
+bei.sale_name,
+bei.user_id,
+bei.user_name,
+bei.leader_id,
+bei.leader_name,
+bei.org_code,
+report.total_charged_in_yuan,
+report.total_balance_in_yuan,
+report.real_charged_in_yuan,
+report.total_rebate_real_charged_in_yuan,
+report.contract_rebate_real_charged_in_yuan,
+report.direct_rebate_real_charged_in_yuan,
+report.credit_real_charged_in_yuan,
+report.charge_day_on_day_percent,
+report.ad_photo_impression,
+report.ad_photo_click,
+report.ad_item_impression,
+report.ad_item_click,
+report.photo_click_ratio,
+report.item_click_ratio,
+report.charged_campaign_count,
+report.product_name,
+report.corporation_name,
+report.first_cost_day,
+report.industry,
+report.second_industry,
+report.union_event_pay_purchase_amount_7d,
+report.union_event_pay_purchase_amount_7d_roi
+from (select * from media_api.kuaishou_agent_report_daily where stat_date={date_time}
+and  advertiser_id in(
+select advertiser_id from media_api.kuaishou_account_report_daily where stat_date={date_time}
+and charge>0)
+) report
+left JOIN (
+SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+sale.realname sale_name,
+su.id user_id,
+su.realname user_name,
+su.leader_id,
+lea.realname leader_name,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user sale ON sale.id = cp.sale_id
+LEFT JOIN `jeecg-boot`.sys_user lea ON lea.id = su.leader_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL) bei
+ON report.advertiser_id = bei.account_id;
+ """.format(date_time=self.date_time)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouAgentReport().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")

+ 253 - 0
Apietl/kwai_etl/report/CampaignDailyReport.py

@@ -0,0 +1,253 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : CampaignDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class CampaignDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+
+    def daily_report(self):
+        try:
+            sql = """select count(1) from media_api.kuaishou_campaign_report_daily where 
+            stat_date >= {start_time} """.format(
+                start_time=self.start_time)
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
+            pageCount = ceil(totalNum / 20000)
+            for i in range(pageCount):
+                sql = """
+                set @@tidb_mem_quota_query = 8 << 30;
+                replace into application.kuaishou_campaign_report_daily_dw
+(account_id, stat_date, campaign_id, campaign_name, project_id, product_id, advertiser_id,
+ company_id, account_name, project_name, advertiser_name, company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio, play_3s_ratio, action_ratio,
+ impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow, cancel_follow, report, block,
+ negative, submit, download_started, download_completed, activation, event_pay_first_day,
+ event_pay_purchase_amount_first_day, event_pay_first_day_roi, event_pay, event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost, event_register_ratio, event_jin_jian_app, event_jin_jian_app_cost,
+ event_credit_grant_app, event_credit_grant_app_cost, event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount, event_order_paid_cost, event_next_day_stay, event_next_day_stay_cost,
+ event_next_day_stay_ratio, form_count, form_cost, form_action_ratio, event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost, event_credit_grant_landing_page, event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio, event_valid_clues, event_valid_clues_cost, event_add_wechat,
+ event_add_wechat_cost, event_add_wechat_ratio, event_get_through, event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost, event_app_invoked_ratio, event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay, event_new_user_pay_cost, event_new_user_pay_ratio, click_1k_cost, ad_product_cnt,
+ event_goods_view, merchant_reco_fans, event_order_amount_roi, event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost, event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app, event_new_user_jinjian_app_cost, event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app, event_new_user_credit_grant_app_cost, event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page, event_new_user_jinjian_page_cost, event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page, event_new_user_credit_grant_page_cost, event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost, event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost, event_appoint_jump_click_ratio, union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type, ad_scene, event_order_successed, ad_photo_played_10s_ratio,
+ key_action_cost, event_add_shopping_cart_cost, event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio, event_watch_app_ad, event_multi_conversion_cost, event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio, event_ad_watch_times_cost, event_add_shopping_cart, key_action,
+ event_multi_conversion, event_wechat_connected, event_dsp_gift_form, event_intention_confirmed,
+ event_phone_get_through, event_multi_conversion_ratio, event_measurement_house, ad_photo_played_2s_ratio,
+ event_outbound_call_ratio, event_ad_watch_times)
+        select report.advertiser_id as account_id,
+               report.stat_date,
+               report.campaign_id,
+               report.campaign_name,
+               sys.project_id,
+               sys.product_id,
+               sys.advertiser_id,
+               sys.company_id,
+               sys.account_name,
+               sys.project_name,
+               sys.advertiser_name,
+               sys.company_name,
+               sys.sale_id,
+               sys.user_id,
+               sys.org_code,
+               report.charge,
+               report.`show`,
+               report.photo_click,
+               report.aclick,
+               report.bclick,
+               report.photo_click_ratio,
+               report.play_3s_ratio,
+               report.action_ratio,
+               report.impression_1k_cost,
+               report.photo_click_cost,
+               report.action_cost,
+               report.share,
+               report.comment,
+               report.`like`,
+               report.follow,
+               report.cancel_follow,
+               report.report,
+               report.block,
+               report.negative,
+               report.submit,
+               report.download_started,
+               report.download_completed,
+               report.activation,
+               report.event_pay_first_day,
+               report.event_pay_purchase_amount_first_day,
+               report.event_pay_first_day_roi,
+               report.event_pay,
+               report.event_pay_purchase_amount,
+               report.event_pay_roi,
+               report.event_register,
+               report.event_register_cost,
+               report.event_register_ratio,
+               report.event_jin_jian_app,
+               report.event_jin_jian_app_cost,
+               report.event_credit_grant_app,
+               report.event_credit_grant_app_cost,
+               report.event_credit_grant_app_ratio,
+               report.event_order_paid,
+               report.event_order_paid_purchase_amount,
+               report.event_order_paid_cost,
+               report.event_next_day_stay,
+               report.event_next_day_stay_cost,
+               report.event_next_day_stay_ratio,
+               report.form_count,
+               report.form_cost,
+               report.form_action_ratio,
+               report.event_jin_jian_landing_page,
+               report.event_jin_jian_landing_page_cost,
+               report.event_credit_grant_landing_page,
+               report.event_credit_grant_landing_page_cost,
+               report.event_credit_grant_landing_page_ratio,
+               report.event_valid_clues,
+               report.event_valid_clues_cost,
+               report.event_add_wechat,
+               report.event_add_wechat_cost,
+               report.event_add_wechat_ratio,
+               report.event_get_through,
+               report.event_get_through_cost,
+               report.event_get_through_ratio,
+               report.event_app_invoked,
+               report.event_app_invoked_cost,
+               report.event_app_invoked_ratio,
+               report.event_credit_grant_landing_ratio,
+               report.play_5s_ratio,
+               report.play_end_ratio,
+               report.event_new_user_pay,
+               report.event_new_user_pay_cost,
+               report.event_new_user_pay_ratio,
+               report.click_1k_cost,
+               report.ad_product_cnt,
+               report.event_goods_view,
+               report.merchant_reco_fans,
+               report.event_order_amount_roi,
+               report.event_goods_view_cost,
+               report.merchant_reco_fans_cost,
+               report.event_button_click,
+               report.event_button_click_cost,
+               report.event_button_click_ratio,
+               report.event_order_paid_roi,
+               report.event_new_user_jinjian_app,
+               report.event_new_user_jinjian_app_cost,
+               report.event_new_user_jinjian_app_roi,
+               report.event_new_user_credit_grant_app,
+               report.event_new_user_credit_grant_app_cost,
+               report.event_new_user_credit_grant_app_roi,
+               report.event_new_user_jinjian_page,
+               report.event_new_user_jinjian_page_cost,
+               report.event_new_user_jinjian_page_roi,
+               report.event_new_user_credit_grant_page,
+               report.event_new_user_credit_grant_page_cost,
+               report.event_new_user_credit_grant_page_roi,
+               report.event_appoint_form,
+               report.event_appoint_form_cost,
+               report.event_appoint_form_ratio,
+               report.event_appoint_jump_click,
+               report.event_appoint_jump_click_cost,
+               report.event_appoint_jump_click_ratio,
+               report.union_event_pay_purchase_amount_7d,
+               report.union_event_pay_purchase_amount_7d_roi,
+               report.placement_type,
+               report.ad_scene,
+               report.event_order_successed,
+               report.ad_photo_played_10s_ratio,
+               report.key_action_cost,
+               report.event_add_shopping_cart_cost,
+               report.event_ad_watch_times_ratio,
+               report.event_outbound_call_cost,
+               report.event_outbound_call,
+               report.action_new_ratio,
+               report.event_watch_app_ad,
+               report.event_multi_conversion_cost,
+               report.event_phone_card_activate,
+               report.ad_photo_played_75percent_ratio,
+               report.key_action_ratio,
+               report.event_ad_watch_times_cost,
+               report.event_add_shopping_cart,
+               report.key_action,
+               report.event_multi_conversion,
+               report.event_wechat_connected,
+               report.event_dsp_gift_form,
+               report.event_intention_confirmed,
+               report.event_phone_get_through,
+               report.event_multi_conversion_ratio,
+               report.event_measurement_house,
+               report.ad_photo_played_2s_ratio,
+               report.event_outbound_call_ratio,
+               report.event_ad_watch_times
+        from media_api.kuaishou_campaign_report_daily report
+                 left join
+                          (SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+             ) sys
+             on report.advertiser_id = sys.account_id
+        where report.stat_date  >={start_time} order by campaign_id,stat_date limit {num},20000""". \
+                    format(start_time=self.start_time, num=i * 20000)
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = CampaignDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 255 - 0
Apietl/kwai_etl/report/CampaignHourlyReport.py

@@ -0,0 +1,255 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : CampaignHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class CampaignHourlyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+
+    def hourly_report(self):
+        try:
+            sql = """
+            select advertiser_id from (
+            select advertiser_id ,sum(charge) as charge from media_api.kuaishou_campaign_report_hourly where 
+            stat_date = {start_time} group by advertiser_id)t 
+            where charge>0
+            """.format(
+                start_time=self.start_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """replace into application.kuaishou_campaign_report_hourly_dw
+(account_id, stat_date,`hour`, campaign_id, campaign_name, project_id, product_id, advertiser_id,
+ company_id, account_name, project_name, advertiser_name, company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio, play_3s_ratio, action_ratio,
+ impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow, cancel_follow, report, block,
+ negative, submit, download_started, download_completed, activation, event_pay_first_day,
+ event_pay_purchase_amount_first_day, event_pay_first_day_roi, event_pay, event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost, event_register_ratio, event_jin_jian_app, event_jin_jian_app_cost,
+ event_credit_grant_app, event_credit_grant_app_cost, event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount, event_order_paid_cost, event_next_day_stay, event_next_day_stay_cost,
+ event_next_day_stay_ratio, form_count, form_cost, form_action_ratio, event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost, event_credit_grant_landing_page, event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio, event_valid_clues, event_valid_clues_cost, event_add_wechat,
+ event_add_wechat_cost, event_add_wechat_ratio, event_get_through, event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost, event_app_invoked_ratio, event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay, event_new_user_pay_cost, event_new_user_pay_ratio, click_1k_cost, ad_product_cnt,
+ event_goods_view, merchant_reco_fans, event_order_amount_roi, event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost, event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app, event_new_user_jinjian_app_cost, event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app, event_new_user_credit_grant_app_cost, event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page, event_new_user_jinjian_page_cost, event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page, event_new_user_credit_grant_page_cost, event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost, event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost, event_appoint_jump_click_ratio, union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type, ad_scene, event_order_successed, ad_photo_played_10s_ratio,
+ key_action_cost, event_add_shopping_cart_cost, event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio, event_watch_app_ad, event_multi_conversion_cost, event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio, event_ad_watch_times_cost, event_add_shopping_cart, key_action,
+ event_multi_conversion, event_wechat_connected, event_dsp_gift_form, event_intention_confirmed,
+ event_phone_get_through, event_multi_conversion_ratio, event_measurement_house, ad_photo_played_2s_ratio,
+ event_outbound_call_ratio, event_ad_watch_times)
+        select report.advertiser_id as account_id,
+               report.stat_date,
+               report.hour,
+               report.campaign_id,
+               report.campaign_name,
+               sys.project_id,
+               sys.product_id,
+               sys.advertiser_id,
+               sys.company_id,
+               sys.account_name,
+               sys.project_name,
+               sys.advertiser_name,
+               sys.company_name,
+               sys.sale_id,
+               sys.user_id,
+               sys.org_code,
+               report.charge,
+               report.`show`,
+               report.photo_click,
+               report.aclick,
+               report.bclick,
+               report.photo_click_ratio,
+               report.play_3s_ratio,
+               report.action_ratio,
+               report.impression_1k_cost,
+               report.photo_click_cost,
+               report.action_cost,
+               report.share,
+               report.comment,
+               report.`like`,
+               report.follow,
+               report.cancel_follow,
+               report.report,
+               report.block,
+               report.negative,
+               report.submit,
+               report.download_started,
+               report.download_completed,
+               report.activation,
+               report.event_pay_first_day,
+               report.event_pay_purchase_amount_first_day,
+               report.event_pay_first_day_roi,
+               report.event_pay,
+               report.event_pay_purchase_amount,
+               report.event_pay_roi,
+               report.event_register,
+               report.event_register_cost,
+               report.event_register_ratio,
+               report.event_jin_jian_app,
+               report.event_jin_jian_app_cost,
+               report.event_credit_grant_app,
+               report.event_credit_grant_app_cost,
+               report.event_credit_grant_app_ratio,
+               report.event_order_paid,
+               report.event_order_paid_purchase_amount,
+               report.event_order_paid_cost,
+               report.event_next_day_stay,
+               report.event_next_day_stay_cost,
+               report.event_next_day_stay_ratio,
+               report.form_count,
+               report.form_cost,
+               report.form_action_ratio,
+               report.event_jin_jian_landing_page,
+               report.event_jin_jian_landing_page_cost,
+               report.event_credit_grant_landing_page,
+               report.event_credit_grant_landing_page_cost,
+               report.event_credit_grant_landing_page_ratio,
+               report.event_valid_clues,
+               report.event_valid_clues_cost,
+               report.event_add_wechat,
+               report.event_add_wechat_cost,
+               report.event_add_wechat_ratio,
+               report.event_get_through,
+               report.event_get_through_cost,
+               report.event_get_through_ratio,
+               report.event_app_invoked,
+               report.event_app_invoked_cost,
+               report.event_app_invoked_ratio,
+               report.event_credit_grant_landing_ratio,
+               report.play_5s_ratio,
+               report.play_end_ratio,
+               report.event_new_user_pay,
+               report.event_new_user_pay_cost,
+               report.event_new_user_pay_ratio,
+               report.click_1k_cost,
+               report.ad_product_cnt,
+               report.event_goods_view,
+               report.merchant_reco_fans,
+               report.event_order_amount_roi,
+               report.event_goods_view_cost,
+               report.merchant_reco_fans_cost,
+               report.event_button_click,
+               report.event_button_click_cost,
+               report.event_button_click_ratio,
+               report.event_order_paid_roi,
+               report.event_new_user_jinjian_app,
+               report.event_new_user_jinjian_app_cost,
+               report.event_new_user_jinjian_app_roi,
+               report.event_new_user_credit_grant_app,
+               report.event_new_user_credit_grant_app_cost,
+               report.event_new_user_credit_grant_app_roi,
+               report.event_new_user_jinjian_page,
+               report.event_new_user_jinjian_page_cost,
+               report.event_new_user_jinjian_page_roi,
+               report.event_new_user_credit_grant_page,
+               report.event_new_user_credit_grant_page_cost,
+               report.event_new_user_credit_grant_page_roi,
+               report.event_appoint_form,
+               report.event_appoint_form_cost,
+               report.event_appoint_form_ratio,
+               report.event_appoint_jump_click,
+               report.event_appoint_jump_click_cost,
+               report.event_appoint_jump_click_ratio,
+               report.union_event_pay_purchase_amount_7d,
+               report.union_event_pay_purchase_amount_7d_roi,
+               report.placement_type,
+               report.ad_scene,
+               report.event_order_successed,
+               report.ad_photo_played_10s_ratio,
+               report.key_action_cost,
+               report.event_add_shopping_cart_cost,
+               report.event_ad_watch_times_ratio,
+               report.event_outbound_call_cost,
+               report.event_outbound_call,
+               report.action_new_ratio,
+               report.event_watch_app_ad,
+               report.event_multi_conversion_cost,
+               report.event_phone_card_activate,
+               report.ad_photo_played_75percent_ratio,
+               report.key_action_ratio,
+               report.event_ad_watch_times_cost,
+               report.event_add_shopping_cart,
+               report.key_action,
+               report.event_multi_conversion,
+               report.event_wechat_connected,
+               report.event_dsp_gift_form,
+               report.event_intention_confirmed,
+               report.event_phone_get_through,
+               report.event_multi_conversion_ratio,
+               report.event_measurement_house,
+               report.ad_photo_played_2s_ratio,
+               report.event_outbound_call_ratio,
+               report.event_ad_watch_times
+        from media_api.kuaishou_campaign_report_hourly report
+                 left join
+                          (SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+             ) sys
+             on report.advertiser_id = sys.account_id
+        where report.stat_date = {start_time} and report.advertiser_id = {account_id}""". \
+                    format(start_time=self.start_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = CampaignHourlyReport().hourly_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 307 - 0
Apietl/kwai_etl/report/DeleteMateirialVideoDailyReport.py

@@ -0,0 +1,307 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialVideoDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from TidbConn import TidbConn
+
+
+class MateirialVideoDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """replace into application.kuaishou_material_video_report_daily_dw
+(account_id, project_id, product_id, advertiser_id,
+ company_id, account_name, project_name,
+ advertiser_name, company_name, sale_id, user_id,
+ clip_id, shot_id, plan_id, leader_id, org_code,
+ photo_url, photo_id, cover_url, image_token,
+ photo_caption, stat_date, charge, `show`, photo_click,
+ aclick, bclick, photo_click_ratio, play_3s_ratio,
+ action_ratio, impression_1k_cost, photo_click_cost,
+ action_cost, share, comment, `like`, follow,
+ cancel_follow, report, block, negative, submit,
+ download_started, download_completed, activation,
+ event_pay_first_day,
+ event_pay_purchase_amount_first_day,
+ event_pay_first_day_roi, event_pay,
+ event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost,
+ event_register_ratio, event_jin_jian_app,
+ event_jin_jian_app_cost, event_credit_grant_app,
+ event_credit_grant_app_cost,
+ event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount,
+ event_order_paid_cost, event_next_day_stay,
+ event_next_day_stay_cost, event_next_day_stay_ratio,
+ form_count, form_cost, form_action_ratio,
+ event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost,
+ event_credit_grant_landing_page,
+ event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio,
+ event_valid_clues, event_valid_clues_cost,
+ event_add_wechat, event_add_wechat_cost,
+ event_add_wechat_ratio, event_get_through,
+ event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost,
+ event_app_invoked_ratio,
+ event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay,
+ event_new_user_pay_cost, event_new_user_pay_ratio,
+ click_1k_cost, ad_product_cnt, event_goods_view,
+ merchant_reco_fans, event_order_amount_roi,
+ event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost,
+ event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app,
+ event_new_user_jinjian_app_cost,
+ event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app,
+ event_new_user_credit_grant_app_cost,
+ event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page,
+ event_new_user_jinjian_page_cost,
+ event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page,
+ event_new_user_credit_grant_page_cost,
+ event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost,
+ event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost,
+ event_appoint_jump_click_ratio,
+ union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type,
+ ad_scene, event_order_successed,
+ ad_photo_played_10s_ratio, key_action_cost,
+ event_add_shopping_cart_cost,
+ event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio,
+ event_watch_app_ad, event_multi_conversion_cost,
+ event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio,
+ event_ad_watch_times_cost, event_add_shopping_cart,
+ key_action, event_multi_conversion,
+ event_wechat_connected, event_dsp_gift_form,
+ event_intention_confirmed, event_phone_get_through,
+ event_multi_conversion_ratio, event_measurement_house,
+ ad_photo_played_2s_ratio, event_outbound_call_ratio,
+ event_ad_watch_times,
+ signature,
+         channel_type ,
+       clip_name,
+       shot_name,
+       plan_name,
+       leader_name,
+       video_name,
+       channel_name,
+       clip_company_id)
+select report.advertiser_id                                                                     as account_id,
+       cp.id                                                                                    as project_id,
+       product.id                                                                               as product_id,
+       ca.id                                                                                    as advertiser_id,
+       uc.company_id,
+       account_name,
+       cp.project_name                                                                          as project_name,
+       ca.name                                                                                  as advertiser_name,
+       uc.company_name,
+       cp.sale_id,
+       cua.user_id,
+       clip_id,
+       shot_id,
+       plan_id,
+       cma.leader_id,
+       su.org_code,
+       report.photo_url,
+       report.photo_id,
+       report.cover_url,
+       report.image_token,
+       report.photo_caption,
+       report.stat_date,
+       charge,
+       `show`,
+       photo_click,
+       aclick,
+       bclick,
+       photo_click_ratio,
+       play_3s_ratio,
+       action_ratio,
+       impression_1k_cost,
+       photo_click_cost,
+       action_cost,
+       share,
+       comment,
+       `like`,
+       follow,
+       cancel_follow,
+       report,
+       block,
+       negative,
+       submit,
+       download_started,
+       download_completed,
+       activation,
+       event_pay_first_day,
+       event_pay_purchase_amount_first_day,
+       event_pay_first_day_roi,
+       event_pay,
+       event_pay_purchase_amount,
+       event_pay_roi,
+       event_register,
+       event_register_cost,
+       event_register_ratio,
+       event_jin_jian_app,
+       event_jin_jian_app_cost,
+       event_credit_grant_app,
+       event_credit_grant_app_cost,
+       event_credit_grant_app_ratio,
+       event_order_paid,
+       event_order_paid_purchase_amount,
+       event_order_paid_cost,
+       event_next_day_stay,
+       event_next_day_stay_cost,
+       event_next_day_stay_ratio,
+       form_count,
+       form_cost,
+       form_action_ratio,
+       event_jin_jian_landing_page,
+       event_jin_jian_landing_page_cost,
+       event_credit_grant_landing_page,
+       event_credit_grant_landing_page_cost,
+       event_credit_grant_landing_page_ratio,
+       event_valid_clues,
+       event_valid_clues_cost,
+       event_add_wechat,
+       event_add_wechat_cost,
+       event_add_wechat_ratio,
+       event_get_through,
+       event_get_through_cost,
+       event_get_through_ratio,
+       event_app_invoked,
+       event_app_invoked_cost,
+       event_app_invoked_ratio,
+       event_credit_grant_landing_ratio,
+       play_5s_ratio,
+       play_end_ratio,
+       event_new_user_pay,
+       event_new_user_pay_cost,
+       event_new_user_pay_ratio,
+       click_1k_cost,
+       ad_product_cnt,
+       event_goods_view,
+       merchant_reco_fans,
+       event_order_amount_roi,
+       event_goods_view_cost,
+       merchant_reco_fans_cost,
+       event_button_click,
+       event_button_click_cost,
+       event_button_click_ratio,
+       event_order_paid_roi,
+       event_new_user_jinjian_app,
+       event_new_user_jinjian_app_cost,
+       event_new_user_jinjian_app_roi,
+       event_new_user_credit_grant_app,
+       event_new_user_credit_grant_app_cost,
+       event_new_user_credit_grant_app_roi,
+       event_new_user_jinjian_page,
+       event_new_user_jinjian_page_cost,
+       event_new_user_jinjian_page_roi,
+       event_new_user_credit_grant_page,
+       event_new_user_credit_grant_page_cost,
+       event_new_user_credit_grant_page_roi,
+       event_appoint_form,
+       event_appoint_form_cost,
+       event_appoint_form_ratio,
+       event_appoint_jump_click,
+       event_appoint_jump_click_cost,
+       event_appoint_jump_click_ratio,
+       union_event_pay_purchase_amount_7d,
+       union_event_pay_purchase_amount_7d_roi,
+       placement_type,
+       ad_scene,
+       event_order_successed,
+       ad_photo_played_10s_ratio,
+       key_action_cost,
+       event_add_shopping_cart_cost,
+       event_ad_watch_times_ratio,
+       event_outbound_call_cost,
+       event_outbound_call,
+       action_new_ratio,
+       event_watch_app_ad,
+       event_multi_conversion_cost,
+       event_phone_card_activate,
+       ad_photo_played_75percent_ratio,
+       key_action_ratio,
+       event_ad_watch_times_cost,
+       event_add_shopping_cart,
+       key_action,
+       event_multi_conversion,
+       event_wechat_connected,
+       event_dsp_gift_form,
+       event_intention_confirmed,
+       event_phone_get_through,
+       event_multi_conversion_ratio,
+       event_measurement_house,
+       ad_photo_played_2s_ratio,
+       event_outbound_call_ratio,
+       event_ad_watch_times,
+       video.signature,
+       video.channel_type,
+       clip.realname,
+       shot.realname,
+       plan.realname,
+       leader.leader_name,
+       case when cmi.material_name is not null then cmi.material_name else video.photo_name end as video_name,
+       case when video.channel_type = 1 then '素造' else '内部' end,
+       uc.company_id
+from media_api.kuaishou_material_video_report_daily report
+         inner join media_api.kuaishou_video_list video
+                    on report.advertiser_id = video.account_id and report.photo_id = video.photo_id
+         left join `jeecg-boot`.ctop_material_info cmi on video.signature = cmi.code
+         left join `jeecg-boot`.ctop_material_ascription cma on cma.code = video.signature
+         left join `jeecg-boot`.ctop_user_allocation cua on cua.account_id = video.account_id
+         left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+         left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+         left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+         left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+         left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+         left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+         left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+         left join `jeecg-boot`.user_company uc on uc.user_id = clip.id
+         left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+ where report.stat_date between {start_time} and {end_time} 
+ and report.photo_id = 0
+""".format(start_time=self.start_time, end_time=self.end_time)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = MateirialVideoDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 316 - 0
Apietl/kwai_etl/report/MateirialVideoDailyReport.py

@@ -0,0 +1,316 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialVideoDailyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class MateirialVideoDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.kuaishou_material_video_report_daily where 
+            stat_date between {start_time} and {end_time} group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """replace into application.kuaishou_material_video_report_daily_dw
+(account_id, project_id, product_id, advertiser_id,
+ company_id, account_name, project_name,
+ advertiser_name, company_name, sale_id, user_id,
+ clip_id, shot_id, plan_id, leader_id, org_code,
+ photo_url, photo_id, cover_url, image_token,
+ photo_caption, stat_date, charge, `show`, photo_click,
+ aclick, bclick, photo_click_ratio, play_3s_ratio,
+ action_ratio, impression_1k_cost, photo_click_cost,
+ action_cost, share, comment, `like`, follow,
+ cancel_follow, report, block, negative, submit,
+ download_started, download_completed, activation,
+ event_pay_first_day,
+ event_pay_purchase_amount_first_day,
+ event_pay_first_day_roi, event_pay,
+ event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost,
+ event_register_ratio, event_jin_jian_app,
+ event_jin_jian_app_cost, event_credit_grant_app,
+ event_credit_grant_app_cost,
+ event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount,
+ event_order_paid_cost, event_next_day_stay,
+ event_next_day_stay_cost, event_next_day_stay_ratio,
+ form_count, form_cost, form_action_ratio,
+ event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost,
+ event_credit_grant_landing_page,
+ event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio,
+ event_valid_clues, event_valid_clues_cost,
+ event_add_wechat, event_add_wechat_cost,
+ event_add_wechat_ratio, event_get_through,
+ event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost,
+ event_app_invoked_ratio,
+ event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay,
+ event_new_user_pay_cost, event_new_user_pay_ratio,
+ click_1k_cost, ad_product_cnt, event_goods_view,
+ merchant_reco_fans, event_order_amount_roi,
+ event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost,
+ event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app,
+ event_new_user_jinjian_app_cost,
+ event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app,
+ event_new_user_credit_grant_app_cost,
+ event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page,
+ event_new_user_jinjian_page_cost,
+ event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page,
+ event_new_user_credit_grant_page_cost,
+ event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost,
+ event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost,
+ event_appoint_jump_click_ratio,
+ union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type,
+ ad_scene, event_order_successed,
+ ad_photo_played_10s_ratio, key_action_cost,
+ event_add_shopping_cart_cost,
+ event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio,
+ event_watch_app_ad, event_multi_conversion_cost,
+ event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio,
+ event_ad_watch_times_cost, event_add_shopping_cart,
+ key_action, event_multi_conversion,
+ event_wechat_connected, event_dsp_gift_form,
+ event_intention_confirmed, event_phone_get_through,
+ event_multi_conversion_ratio, event_measurement_house,
+ ad_photo_played_2s_ratio, event_outbound_call_ratio,
+ event_ad_watch_times,
+ signature,
+         channel_type ,
+       clip_name,
+       shot_name,
+       plan_name,
+       leader_name,
+       video_name,
+       channel_name,
+       clip_company_id,
+       play_3s_count,
+      shot_leader_id,
+       shot_leader_name)
+select report.advertiser_id                                                                     as account_id,
+       cp.id                                                                                    as project_id,
+       product.id                                                                               as product_id,
+       ca.id                                                                                    as advertiser_id,
+       uc.company_id,
+      cua.auth_name as  account_name,
+       cp.project_name                                                                          as project_name,
+       ca.name                                                                                  as advertiser_name,
+       uc.company_name,
+       cp.sale_id,
+       cua.user_id,
+       clip_id,
+       shot_id,
+       plan_id,
+       cma.leader_id,
+       leader.org_code,
+       report.photo_url,
+       report.photo_id,
+       report.cover_url,
+       report.image_token,
+       report.photo_caption,
+       report.stat_date,
+       charge,
+       `show`,
+       photo_click,
+       aclick,
+       bclick,
+       photo_click_ratio,
+       play_3s_ratio,
+       action_ratio,
+       impression_1k_cost,
+       photo_click_cost,
+       action_cost,
+       share,
+       comment,
+       `like`,
+       follow,
+       cancel_follow,
+       report,
+       block,
+       negative,
+       submit,
+       download_started,
+       download_completed,
+       activation,
+       event_pay_first_day,
+       event_pay_purchase_amount_first_day,
+       event_pay_first_day_roi,
+       event_pay,
+       event_pay_purchase_amount,
+       event_pay_roi,
+       event_register,
+       event_register_cost,
+       event_register_ratio,
+       event_jin_jian_app,
+       event_jin_jian_app_cost,
+       event_credit_grant_app,
+       event_credit_grant_app_cost,
+       event_credit_grant_app_ratio,
+       event_order_paid,
+       event_order_paid_purchase_amount,
+       event_order_paid_cost,
+       event_next_day_stay,
+       event_next_day_stay_cost,
+       event_next_day_stay_ratio,
+       form_count,
+       form_cost,
+       form_action_ratio,
+       event_jin_jian_landing_page,
+       event_jin_jian_landing_page_cost,
+       event_credit_grant_landing_page,
+       event_credit_grant_landing_page_cost,
+       event_credit_grant_landing_page_ratio,
+       event_valid_clues,
+       event_valid_clues_cost,
+       event_add_wechat,
+       event_add_wechat_cost,
+       event_add_wechat_ratio,
+       event_get_through,
+       event_get_through_cost,
+       event_get_through_ratio,
+       event_app_invoked,
+       event_app_invoked_cost,
+       event_app_invoked_ratio,
+       event_credit_grant_landing_ratio,
+       play_5s_ratio,
+       play_end_ratio,
+       event_new_user_pay,
+       event_new_user_pay_cost,
+       event_new_user_pay_ratio,
+       click_1k_cost,
+       ad_product_cnt,
+       event_goods_view,
+       merchant_reco_fans,
+       event_order_amount_roi,
+       event_goods_view_cost,
+       merchant_reco_fans_cost,
+       event_button_click,
+       event_button_click_cost,
+       event_button_click_ratio,
+       event_order_paid_roi,
+       event_new_user_jinjian_app,
+       event_new_user_jinjian_app_cost,
+       event_new_user_jinjian_app_roi,
+       event_new_user_credit_grant_app,
+       event_new_user_credit_grant_app_cost,
+       event_new_user_credit_grant_app_roi,
+       event_new_user_jinjian_page,
+       event_new_user_jinjian_page_cost,
+       event_new_user_jinjian_page_roi,
+       event_new_user_credit_grant_page,
+       event_new_user_credit_grant_page_cost,
+       event_new_user_credit_grant_page_roi,
+       event_appoint_form,
+       event_appoint_form_cost,
+       event_appoint_form_ratio,
+       event_appoint_jump_click,
+       event_appoint_jump_click_cost,
+       event_appoint_jump_click_ratio,
+       union_event_pay_purchase_amount_7d,
+       union_event_pay_purchase_amount_7d_roi,
+       placement_type,
+       ad_scene,
+       event_order_successed,
+       ad_photo_played_10s_ratio,
+       key_action_cost,
+       event_add_shopping_cart_cost,
+       event_ad_watch_times_ratio,
+       event_outbound_call_cost,
+       event_outbound_call,
+       action_new_ratio,
+       event_watch_app_ad,
+       event_multi_conversion_cost,
+       event_phone_card_activate,
+       ad_photo_played_75percent_ratio,
+       key_action_ratio,
+       event_ad_watch_times_cost,
+       event_add_shopping_cart,
+       key_action,
+       event_multi_conversion,
+       event_wechat_connected,
+       event_dsp_gift_form,
+       event_intention_confirmed,
+       event_phone_get_through,
+       event_multi_conversion_ratio,
+       event_measurement_house,
+       ad_photo_played_2s_ratio,
+       event_outbound_call_ratio,
+       event_ad_watch_times,
+       video.signature,
+       video.channel_type,
+       clip.realname,
+       shot.realname,
+       plan.realname,
+       leader.realname,
+       case when cmi.material_name is not null then cmi.material_name else video.photo_name end as video_name,
+       case when video.channel_type = 1 then '素造' else '内部' end,
+       uc.company_id,
+       round(play_3s_ratio*aclick) as play_3s_count,
+       cma.shot_leader_id,
+       cma.shot_leader_name
+from media_api.kuaishou_material_video_report_daily report
+         left join media_api.signature_photo_id video
+                    on report.photo_id = video.photo_id
+         left join (select  code,material_name from `jeecg-boot`.ctop_material_info where status=1) cmi on video.signature = cmi.code
+         left join `jeecg-boot`.ctop_material_ascription cma on cma.material_id = video.signature
+         left join `jeecg-boot`.ctop_user_allocation cua on cua.account_id = report.advertiser_id
+         left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+         left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+         left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+         left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+         left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+         left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+         left join `jeecg-boot`.user_company uc on uc.user_id = clip.id
+         left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+ where report.stat_date between {start_time} and {end_time} 
+   and report.advertiser_id={account_id}""".format(
+                    start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = MateirialVideoDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 314 - 0
Apietl/kwai_etl/report/MateirialVideoHourlyReport.py

@@ -0,0 +1,314 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialVideoHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class MateirialVideoHourlyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def hourly_report(self):
+        try:
+            sql = """select advertiser_id from media_api.kuaishou_material_video_report_hourly where 
+            stat_date between {start_time} and {end_time} group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """replace into application.kuaishou_material_video_report_hourly_dw
+(account_id, project_id, product_id, advertiser_id,
+ company_id, account_name, project_name,
+ advertiser_name, company_name, sale_id, user_id,
+ clip_id, shot_id, plan_id, leader_id, org_code,
+ photo_url, photo_id, cover_url, image_token,
+ photo_caption, stat_date, `hour`,charge, `show`, photo_click,
+ aclick, bclick, photo_click_ratio, play_3s_ratio,
+ action_ratio, impression_1k_cost, photo_click_cost,
+ action_cost, share, comment, `like`, follow,
+ cancel_follow, report, block, negative, submit,
+ download_started, download_completed, activation,
+ event_pay_first_day,
+ event_pay_purchase_amount_first_day,
+ event_pay_first_day_roi, event_pay,
+ event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost,
+ event_register_ratio, event_jin_jian_app,
+ event_jin_jian_app_cost, event_credit_grant_app,
+ event_credit_grant_app_cost,
+ event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount,
+ event_order_paid_cost, event_next_day_stay,
+ event_next_day_stay_cost, event_next_day_stay_ratio,
+ form_count, form_cost, form_action_ratio,
+ event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost,
+ event_credit_grant_landing_page,
+ event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio,
+ event_valid_clues, event_valid_clues_cost,
+ event_add_wechat, event_add_wechat_cost,
+ event_add_wechat_ratio, event_get_through,
+ event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost,
+ event_app_invoked_ratio,
+ event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay,
+ event_new_user_pay_cost, event_new_user_pay_ratio,
+ click_1k_cost, ad_product_cnt, event_goods_view,
+ merchant_reco_fans, event_order_amount_roi,
+ event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost,
+ event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app,
+ event_new_user_jinjian_app_cost,
+ event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app,
+ event_new_user_credit_grant_app_cost,
+ event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page,
+ event_new_user_jinjian_page_cost,
+ event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page,
+ event_new_user_credit_grant_page_cost,
+ event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost,
+ event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost,
+ event_appoint_jump_click_ratio,
+ union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type,
+ ad_scene, event_order_successed,
+ ad_photo_played_10s_ratio, key_action_cost,
+ event_add_shopping_cart_cost,
+ event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio,
+ event_watch_app_ad, event_multi_conversion_cost,
+ event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio,
+ event_ad_watch_times_cost, event_add_shopping_cart,
+ key_action, event_multi_conversion,
+ event_wechat_connected, event_dsp_gift_form,
+ event_intention_confirmed, event_phone_get_through,
+ event_multi_conversion_ratio, event_measurement_house,
+ ad_photo_played_2s_ratio, event_outbound_call_ratio,
+ event_ad_watch_times,
+ signature,
+         channel_type ,
+       clip_name,
+       shot_name,
+       plan_name,
+       leader_name,
+       video_name,
+       channel_name,
+       clip_company_id)
+select 
+       report.advertiser_id                                                                     as account_id,
+       cp.id                                                                                    as project_id,
+       product.id                                                                               as product_id,
+       ca.id                                                                                    as advertiser_id,
+       uc.company_id,
+       account_name,
+       cp.project_name                                                                          as project_name,
+       ca.name                                                                                  as advertiser_name,
+       uc.company_name,
+       cp.sale_id,
+       cua.user_id,
+       clip_id,
+       shot_id,
+       plan_id,
+       cma.leader_id,
+       su.org_code,
+       report.photo_url,
+       report.photo_id,
+       report.cover_url,
+       report.image_token,
+       report.photo_caption,
+       report.stat_date,
+       report.`hour`,
+       charge,
+       `show`,
+       photo_click,
+       aclick,
+       bclick,
+       photo_click_ratio,
+       play_3s_ratio,
+       action_ratio,
+       impression_1k_cost,
+       photo_click_cost,
+       action_cost,
+       share,
+       comment,
+       `like`,
+       follow,
+       cancel_follow,
+       report,
+       block,
+       negative,
+       submit,
+       download_started,
+       download_completed,
+       activation,
+       event_pay_first_day,
+       event_pay_purchase_amount_first_day,
+       event_pay_first_day_roi,
+       event_pay,
+       event_pay_purchase_amount,
+       event_pay_roi,
+       event_register,
+       event_register_cost,
+       event_register_ratio,
+       event_jin_jian_app,
+       event_jin_jian_app_cost,
+       event_credit_grant_app,
+       event_credit_grant_app_cost,
+       event_credit_grant_app_ratio,
+       event_order_paid,
+       event_order_paid_purchase_amount,
+       event_order_paid_cost,
+       event_next_day_stay,
+       event_next_day_stay_cost,
+       event_next_day_stay_ratio,
+       form_count,
+       form_cost,
+       form_action_ratio,
+       event_jin_jian_landing_page,
+       event_jin_jian_landing_page_cost,
+       event_credit_grant_landing_page,
+       event_credit_grant_landing_page_cost,
+       event_credit_grant_landing_page_ratio,
+       event_valid_clues,
+       event_valid_clues_cost,
+       event_add_wechat,
+       event_add_wechat_cost,
+       event_add_wechat_ratio,
+       event_get_through,
+       event_get_through_cost,
+       event_get_through_ratio,
+       event_app_invoked,
+       event_app_invoked_cost,
+       event_app_invoked_ratio,
+       event_credit_grant_landing_ratio,
+       play_5s_ratio,
+       play_end_ratio,
+       event_new_user_pay,
+       event_new_user_pay_cost,
+       event_new_user_pay_ratio,
+       click_1k_cost,
+       ad_product_cnt,
+       event_goods_view,
+       merchant_reco_fans,
+       event_order_amount_roi,
+       event_goods_view_cost,
+       merchant_reco_fans_cost,
+       event_button_click,
+       event_button_click_cost,
+       event_button_click_ratio,
+       event_order_paid_roi,
+       event_new_user_jinjian_app,
+       event_new_user_jinjian_app_cost,
+       event_new_user_jinjian_app_roi,
+       event_new_user_credit_grant_app,
+       event_new_user_credit_grant_app_cost,
+       event_new_user_credit_grant_app_roi,
+       event_new_user_jinjian_page,
+       event_new_user_jinjian_page_cost,
+       event_new_user_jinjian_page_roi,
+       event_new_user_credit_grant_page,
+       event_new_user_credit_grant_page_cost,
+       event_new_user_credit_grant_page_roi,
+       event_appoint_form,
+       event_appoint_form_cost,
+       event_appoint_form_ratio,
+       event_appoint_jump_click,
+       event_appoint_jump_click_cost,
+       event_appoint_jump_click_ratio,
+       union_event_pay_purchase_amount_7d,
+       union_event_pay_purchase_amount_7d_roi,
+       placement_type,
+       ad_scene,
+       event_order_successed,
+       ad_photo_played_10s_ratio,
+       key_action_cost,
+       event_add_shopping_cart_cost,
+       event_ad_watch_times_ratio,
+       event_outbound_call_cost,
+       event_outbound_call,
+       action_new_ratio,
+       event_watch_app_ad,
+       event_multi_conversion_cost,
+       event_phone_card_activate,
+       ad_photo_played_75percent_ratio,
+       key_action_ratio,
+       event_ad_watch_times_cost,
+       event_add_shopping_cart,
+       key_action,
+       event_multi_conversion,
+       event_wechat_connected,
+       event_dsp_gift_form,
+       event_intention_confirmed,
+       event_phone_get_through,
+       event_multi_conversion_ratio,
+       event_measurement_house,
+       ad_photo_played_2s_ratio,
+       event_outbound_call_ratio,
+       event_ad_watch_times,
+       video.signature,
+       video.channel_type,
+       clip.realname,
+       shot.realname,
+       plan.realname,
+       leader.leader_name,
+       case when cmi.material_name is not null then cmi.material_name else video.photo_name end as video_name,
+       case when video.channel_type = 1 then '素造' else '内部' end as channel_name,
+       uc.company_id
+from media_api.kuaishou_material_video_report_hourly report
+         left join media_api.kuaishou_video_list video
+                    on report.advertiser_id = video.account_id and report.photo_id = video.photo_id
+         left join `jeecg-boot`.ctop_material_info cmi on video.signature = cmi.code
+         left join `jeecg-boot`.ctop_material_ascription cma on cma.code = video.signature
+         left join `jeecg-boot`.ctop_user_allocation cua on cua.account_id = report.advertiser_id
+         left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+         left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+         left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+         left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+         left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+         left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+         left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+         left join `jeecg-boot`.user_company uc on uc.user_id = clip.id
+         left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+ where report.stat_date between {start_time} and {end_time} and  report.photo_id!=0
+ and report.advertiser_id={account_id}
+""".format(
+                    start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = MateirialVideoHourlyReport().hourly_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 50 - 0
Apietl/kwai_etl/report/ReplaceAccountInfo.py

@@ -0,0 +1,50 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/10/26 10:15 上午
+# @Site    :
+# @File    : ReplaceAccountInfo.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+# @Tel 1501435553
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+
+
+class ReplaceAccountInfo:
+
+    @staticmethod
+    def handler():
+        try:
+            sql = """
+replace into `jeecg-boot`.ctop_account_attribution_info(id, account_id, project_id,product_id, advertiser_id, company_id,account_name, project_name, advertiser_name, company_name, sale_id, user_id,org_code)
+select cua.account_id,
+       cua.account_id,
+       cp.id,
+       product.id,
+       ca.id,
+       uc.company_id,
+       cua.auth_name,
+       cp.project_name,
+       ca.name,
+       uc.company_name,
+       cp.sale_id,
+       su.id,
+       su.org_code
+from `jeecg-boot`.ctop_user_allocation cua
+         left join `jeecg-boot`.ctop_project cp on cua.project_id = cp.id
+         left join `jeecg-boot`.ctop_advertiser ca on ca.id = cua.advertiser_id
+         left join `jeecg-boot`.sys_user su on su.id = cua.user_id
+         left join `jeecg-boot`.user_company uc on uc.user_id = cua.user_id
+         left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
+where cua.account_id is not null
+            """
+            TidbConn.upsert(conn=TidbConn.get_connection(), sql_buff=sql)
+            TidbConn.get_connection().commit()
+            TidbConn.conn_close(conn=TidbConn.get_connection())
+        except Exception as e:
+            logging.error(e)
+
+
+if __name__ == '__main__':
+    ReplaceAccountInfo.handler()

+ 258 - 0
Apietl/kwai_etl/report/UnitDailyReport.py

@@ -0,0 +1,258 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : UnitReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class UnitDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """
+            select advertiser_id from media_api.kuaishou_unit_report_daily where 
+            stat_date between {start_time} and {end_time} group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """
+                replace into application.kuaishou_unit_report_daily_dw
+(account_id, stat_date, campaign_id, campaign_name, unit_id, unit_name, project_id, product_id, advertiser_id,
+ company_id, account_name, project_name, advertiser_name, company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio, play_3s_ratio, action_ratio,
+ impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow, cancel_follow, report, block,
+ negative, submit, download_started, download_completed, activation, event_pay_first_day,
+ event_pay_purchase_amount_first_day, event_pay_first_day_roi, event_pay, event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost, event_register_ratio, event_jin_jian_app, event_jin_jian_app_cost,
+ event_credit_grant_app, event_credit_grant_app_cost, event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount, event_order_paid_cost, event_next_day_stay, event_next_day_stay_cost,
+ event_next_day_stay_ratio, form_count, form_cost, form_action_ratio, event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost, event_credit_grant_landing_page, event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio, event_valid_clues, event_valid_clues_cost, event_add_wechat,
+ event_add_wechat_cost, event_add_wechat_ratio, event_get_through, event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost, event_app_invoked_ratio, event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay, event_new_user_pay_cost, event_new_user_pay_ratio, click_1k_cost, ad_product_cnt,
+ event_goods_view, merchant_reco_fans, event_order_amount_roi, event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost, event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app, event_new_user_jinjian_app_cost, event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app, event_new_user_credit_grant_app_cost, event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page, event_new_user_jinjian_page_cost, event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page, event_new_user_credit_grant_page_cost, event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost, event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost, event_appoint_jump_click_ratio, union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type, ad_scene, event_order_successed, ad_photo_played_10s_ratio,
+ key_action_cost, event_add_shopping_cart_cost, event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio, event_watch_app_ad, event_multi_conversion_cost, event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio, event_ad_watch_times_cost, event_add_shopping_cart, key_action,
+ event_multi_conversion, event_wechat_connected, event_dsp_gift_form, event_intention_confirmed,
+ event_phone_get_through, event_multi_conversion_ratio, event_measurement_house, ad_photo_played_2s_ratio,
+ event_outbound_call_ratio, event_ad_watch_times,event_order_submit,
+ live_event_goods_view)
+        select report.advertiser_id as account_id,
+               report.stat_date,
+               report.campaign_id,
+               report.campaign_name,
+               report.unit_id,
+               report.unit_name,
+               sys.project_id,
+               sys.product_id,
+               sys.advertiser_id,
+               sys.company_id,
+               sys.account_name,
+               sys.project_name,
+               sys.advertiser_name,
+               sys.company_name,
+               sys.sale_id,
+               sys.user_id,
+               sys.org_code,
+               report.charge,
+               report.`show`,
+               report.photo_click,
+               report.aclick,
+               report.bclick,
+               report.photo_click_ratio,
+               report.play_3s_ratio,
+               report.action_ratio,
+               report.impression_1k_cost,
+               report.photo_click_cost,
+               report.action_cost,
+               report.share,
+               report.comment,
+               report.`like`,
+               report.follow,
+               report.cancel_follow,
+               report.report,
+               report.block,
+               report.negative,
+               report.submit,
+               report.download_started,
+               report.download_completed,
+               report.activation,
+               report.event_pay_first_day,
+               report.event_pay_purchase_amount_first_day,
+               report.event_pay_first_day_roi,
+               report.event_pay,
+               report.event_pay_purchase_amount,
+               report.event_pay_roi,
+               report.event_register,
+               report.event_register_cost,
+               report.event_register_ratio,
+               report.event_jin_jian_app,
+               report.event_jin_jian_app_cost,
+               report.event_credit_grant_app,
+               report.event_credit_grant_app_cost,
+               report.event_credit_grant_app_ratio,
+               report.event_order_paid,
+               report.event_order_paid_purchase_amount,
+               report.event_order_paid_cost,
+               report.event_next_day_stay,
+               report.event_next_day_stay_cost,
+               report.event_next_day_stay_ratio,
+               report.form_count,
+               report.form_cost,
+               report.form_action_ratio,
+               report.event_jin_jian_landing_page,
+               report.event_jin_jian_landing_page_cost,
+               report.event_credit_grant_landing_page,
+               report.event_credit_grant_landing_page_cost,
+               report.event_credit_grant_landing_page_ratio,
+               report.event_valid_clues,
+               report.event_valid_clues_cost,
+               report.event_add_wechat,
+               report.event_add_wechat_cost,
+               report.event_add_wechat_ratio,
+               report.event_get_through,
+               report.event_get_through_cost,
+               report.event_get_through_ratio,
+               report.event_app_invoked,
+               report.event_app_invoked_cost,
+               report.event_app_invoked_ratio,
+               report.event_credit_grant_landing_ratio,
+               report.play_5s_ratio,
+               report.play_end_ratio,
+               report.event_new_user_pay,
+               report.event_new_user_pay_cost,
+               report.event_new_user_pay_ratio,
+               report.click_1k_cost,
+               report.ad_product_cnt,
+               report.event_goods_view,
+               report.merchant_reco_fans,
+               report.event_order_amount_roi,
+               report.event_goods_view_cost,
+               report.merchant_reco_fans_cost,
+               report.event_button_click,
+               report.event_button_click_cost,
+               report.event_button_click_ratio,
+               report.event_order_paid_roi,
+               report.event_new_user_jinjian_app,
+               report.event_new_user_jinjian_app_cost,
+               report.event_new_user_jinjian_app_roi,
+               report.event_new_user_credit_grant_app,
+               report.event_new_user_credit_grant_app_cost,
+               report.event_new_user_credit_grant_app_roi,
+               report.event_new_user_jinjian_page,
+               report.event_new_user_jinjian_page_cost,
+               report.event_new_user_jinjian_page_roi,
+               report.event_new_user_credit_grant_page,
+               report.event_new_user_credit_grant_page_cost,
+               report.event_new_user_credit_grant_page_roi,
+               report.event_appoint_form,
+               report.event_appoint_form_cost,
+               report.event_appoint_form_ratio,
+               report.event_appoint_jump_click,
+               report.event_appoint_jump_click_cost,
+               report.event_appoint_jump_click_ratio,
+               report.union_event_pay_purchase_amount_7d,
+               report.union_event_pay_purchase_amount_7d_roi,
+               report.placement_type,
+               report.ad_scene,
+               report.event_order_successed,
+               report.ad_photo_played_10s_ratio,
+               report.key_action_cost,
+               report.event_add_shopping_cart_cost,
+               report.event_ad_watch_times_ratio,
+               report.event_outbound_call_cost,
+               report.event_outbound_call,
+               report.action_new_ratio,
+               report.event_watch_app_ad,
+               report.event_multi_conversion_cost,
+               report.event_phone_card_activate,
+               report.ad_photo_played_75percent_ratio,
+               report.key_action_ratio,
+               report.event_ad_watch_times_cost,
+               report.event_add_shopping_cart,
+               report.key_action,
+               report.event_multi_conversion,
+               report.event_wechat_connected,
+               report.event_dsp_gift_form,
+               report.event_intention_confirmed,
+               report.event_phone_get_through,
+               report.event_multi_conversion_ratio,
+               report.event_measurement_house,
+               report.ad_photo_played_2s_ratio,
+               report.event_outbound_call_ratio,
+               report.event_ad_watch_times,
+               report.event_order_submit,
+               report.live_event_goods_view
+        from media_api.kuaishou_unit_report_daily report
+                 left join
+                         (SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+             ) sys
+             on report.advertiser_id = sys.account_id
+        where report.stat_date between {start_time} and {end_time}
+        and report.advertiser_id={account_id}""".format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = UnitDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 253 - 0
Apietl/kwai_etl/report/UnitHourlyReport.py

@@ -0,0 +1,253 @@
+# Author renyupeng
+# coding=utf-8
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : UnitHourlyReport.py
+# @Software: PyCharm
+# @contact: renyupeng@c-top.com.cn
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class UnitHourlyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def hourly_report(self):
+        try:
+            sql = """select advertiser_id from media_api.kuaishou_unit_report_hourly where 
+            stat_date between {start_time} and {end_time} group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(self.conn, sql)
+            for account_id in account_list:
+                sql = """replace into application.kuaishou_unit_report_hourly_dw
+(account_id, stat_date,`hour`, campaign_id, campaign_name, unit_id, unit_name, project_id, product_id, advertiser_id,
+ company_id, account_name, project_name, advertiser_name, company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio, play_3s_ratio, action_ratio,
+ impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow, cancel_follow, report, block,
+ negative, submit, download_started, download_completed, activation, event_pay_first_day,
+ event_pay_purchase_amount_first_day, event_pay_first_day_roi, event_pay, event_pay_purchase_amount, event_pay_roi,
+ event_register, event_register_cost, event_register_ratio, event_jin_jian_app, event_jin_jian_app_cost,
+ event_credit_grant_app, event_credit_grant_app_cost, event_credit_grant_app_ratio, event_order_paid,
+ event_order_paid_purchase_amount, event_order_paid_cost, event_next_day_stay, event_next_day_stay_cost,
+ event_next_day_stay_ratio, form_count, form_cost, form_action_ratio, event_jin_jian_landing_page,
+ event_jin_jian_landing_page_cost, event_credit_grant_landing_page, event_credit_grant_landing_page_cost,
+ event_credit_grant_landing_page_ratio, event_valid_clues, event_valid_clues_cost, event_add_wechat,
+ event_add_wechat_cost, event_add_wechat_ratio, event_get_through, event_get_through_cost, event_get_through_ratio,
+ event_app_invoked, event_app_invoked_cost, event_app_invoked_ratio, event_credit_grant_landing_ratio, play_5s_ratio,
+ play_end_ratio, event_new_user_pay, event_new_user_pay_cost, event_new_user_pay_ratio, click_1k_cost, ad_product_cnt,
+ event_goods_view, merchant_reco_fans, event_order_amount_roi, event_goods_view_cost, merchant_reco_fans_cost,
+ event_button_click, event_button_click_cost, event_button_click_ratio, event_order_paid_roi,
+ event_new_user_jinjian_app, event_new_user_jinjian_app_cost, event_new_user_jinjian_app_roi,
+ event_new_user_credit_grant_app, event_new_user_credit_grant_app_cost, event_new_user_credit_grant_app_roi,
+ event_new_user_jinjian_page, event_new_user_jinjian_page_cost, event_new_user_jinjian_page_roi,
+ event_new_user_credit_grant_page, event_new_user_credit_grant_page_cost, event_new_user_credit_grant_page_roi,
+ event_appoint_form, event_appoint_form_cost, event_appoint_form_ratio, event_appoint_jump_click,
+ event_appoint_jump_click_cost, event_appoint_jump_click_ratio, union_event_pay_purchase_amount_7d,
+ union_event_pay_purchase_amount_7d_roi, placement_type, ad_scene, event_order_successed, ad_photo_played_10s_ratio,
+ key_action_cost, event_add_shopping_cart_cost, event_ad_watch_times_ratio, event_outbound_call_cost,
+ event_outbound_call, action_new_ratio, event_watch_app_ad, event_multi_conversion_cost, event_phone_card_activate,
+ ad_photo_played_75percent_ratio, key_action_ratio, event_ad_watch_times_cost, event_add_shopping_cart, key_action,
+ event_multi_conversion, event_wechat_connected, event_dsp_gift_form, event_intention_confirmed,
+ event_phone_get_through, event_multi_conversion_ratio, event_measurement_house, ad_photo_played_2s_ratio,
+ event_outbound_call_ratio, event_ad_watch_times)
+        select report.advertiser_id as account_id,
+               report.stat_date,
+               report.hour,
+               report.campaign_id,
+               report.campaign_name,
+               report.unit_id,
+               report.unit_name,
+               sys.project_id,
+               sys.product_id,
+               sys.advertiser_id,
+               sys.company_id,
+               sys.account_name,
+               sys.project_name,
+               sys.advertiser_name,
+               sys.company_name,
+               sys.sale_id,
+               sys.user_id,
+               sys.org_code,
+               report.charge,
+               report.`show`,
+               report.photo_click,
+               report.aclick,
+               report.bclick,
+               report.photo_click_ratio,
+               report.play_3s_ratio,
+               report.action_ratio,
+               report.impression_1k_cost,
+               report.photo_click_cost,
+               report.action_cost,
+               report.share,
+               report.comment,
+               report.`like`,
+               report.follow,
+               report.cancel_follow,
+               report.report,
+               report.block,
+               report.negative,
+               report.submit,
+               report.download_started,
+               report.download_completed,
+               report.activation,
+               report.event_pay_first_day,
+               report.event_pay_purchase_amount_first_day,
+               report.event_pay_first_day_roi,
+               report.event_pay,
+               report.event_pay_purchase_amount,
+               report.event_pay_roi,
+               report.event_register,
+               report.event_register_cost,
+               report.event_register_ratio,
+               report.event_jin_jian_app,
+               report.event_jin_jian_app_cost,
+               report.event_credit_grant_app,
+               report.event_credit_grant_app_cost,
+               report.event_credit_grant_app_ratio,
+               report.event_order_paid,
+               report.event_order_paid_purchase_amount,
+               report.event_order_paid_cost,
+               report.event_next_day_stay,
+               report.event_next_day_stay_cost,
+               report.event_next_day_stay_ratio,
+               report.form_count,
+               report.form_cost,
+               report.form_action_ratio,
+               report.event_jin_jian_landing_page,
+               report.event_jin_jian_landing_page_cost,
+               report.event_credit_grant_landing_page,
+               report.event_credit_grant_landing_page_cost,
+               report.event_credit_grant_landing_page_ratio,
+               report.event_valid_clues,
+               report.event_valid_clues_cost,
+               report.event_add_wechat,
+               report.event_add_wechat_cost,
+               report.event_add_wechat_ratio,
+               report.event_get_through,
+               report.event_get_through_cost,
+               report.event_get_through_ratio,
+               report.event_app_invoked,
+               report.event_app_invoked_cost,
+               report.event_app_invoked_ratio,
+               report.event_credit_grant_landing_ratio,
+               report.play_5s_ratio,
+               report.play_end_ratio,
+               report.event_new_user_pay,
+               report.event_new_user_pay_cost,
+               report.event_new_user_pay_ratio,
+               report.click_1k_cost,
+               report.ad_product_cnt,
+               report.event_goods_view,
+               report.merchant_reco_fans,
+               report.event_order_amount_roi,
+               report.event_goods_view_cost,
+               report.merchant_reco_fans_cost,
+               report.event_button_click,
+               report.event_button_click_cost,
+               report.event_button_click_ratio,
+               report.event_order_paid_roi,
+               report.event_new_user_jinjian_app,
+               report.event_new_user_jinjian_app_cost,
+               report.event_new_user_jinjian_app_roi,
+               report.event_new_user_credit_grant_app,
+               report.event_new_user_credit_grant_app_cost,
+               report.event_new_user_credit_grant_app_roi,
+               report.event_new_user_jinjian_page,
+               report.event_new_user_jinjian_page_cost,
+               report.event_new_user_jinjian_page_roi,
+               report.event_new_user_credit_grant_page,
+               report.event_new_user_credit_grant_page_cost,
+               report.event_new_user_credit_grant_page_roi,
+               report.event_appoint_form,
+               report.event_appoint_form_cost,
+               report.event_appoint_form_ratio,
+               report.event_appoint_jump_click,
+               report.event_appoint_jump_click_cost,
+               report.event_appoint_jump_click_ratio,
+               report.union_event_pay_purchase_amount_7d,
+               report.union_event_pay_purchase_amount_7d_roi,
+               report.placement_type,
+               report.ad_scene,
+               report.event_order_successed,
+               report.ad_photo_played_10s_ratio,
+               report.key_action_cost,
+               report.event_add_shopping_cart_cost,
+               report.event_ad_watch_times_ratio,
+               report.event_outbound_call_cost,
+               report.event_outbound_call,
+               report.action_new_ratio,
+               report.event_watch_app_ad,
+               report.event_multi_conversion_cost,
+               report.event_phone_card_activate,
+               report.ad_photo_played_75percent_ratio,
+               report.key_action_ratio,
+               report.event_ad_watch_times_cost,
+               report.event_add_shopping_cart,
+               report.key_action,
+               report.event_multi_conversion,
+               report.event_wechat_connected,
+               report.event_dsp_gift_form,
+               report.event_intention_confirmed,
+               report.event_phone_get_through,
+               report.event_multi_conversion_ratio,
+               report.event_measurement_house,
+               report.ad_photo_played_2s_ratio,
+               report.event_outbound_call_ratio,
+               report.event_ad_watch_times
+        from media_api.kuaishou_unit_report_hourly report
+                 left join
+                          (SELECT
+cua.account_id id,
+cua.account_id account_id,
+cp.id project_id,
+product.id product_id,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.leader_id,
+su.org_code org_code
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+WHERE
+cua.account_id IS NOT NULL
+             ) sys
+             on report.advertiser_id = sys.account_id
+        where report.stat_date between {start_time} and {end_time} and report.advertiser_id={account_id}""". \
+                    format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = UnitHourlyReport().hourly_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 0 - 0
Apietl/ruixuan/mgs/JyOrderItemList.py


Nem az összes módosított fájl került megjelenítésre, mert túl sok fájl változott