renyupeng 3 سال پیش
والد
کامیت
24f07799f0

+ 3 - 5
Apietl/bytedance_etl/report/AdDailyReport.py

@@ -8,10 +8,8 @@
 # @Tel 1501435553
 import logging
 from math import ceil
-
-from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class AdDailyReport:
@@ -24,7 +22,7 @@ class AdDailyReport:
     def daily_report(self):
         try:
             sql = """select count(1) from media_api.bytedance_ad_report_daily where 
-            stat_date between {start_time} and {end_time}""".format(
+            stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             pageCount = ceil(totalNum / 20000)
@@ -282,7 +280,7 @@ from `media_api`.bytedance_ad_report_daily a
      on a.advertiser_id = b.account_id
 where a.stat_datetime between {start_time} and {end_time}
 order by a.advertiser_id, a.stat_datetime,a.ad_id
-limit {num}:20000;
+limit {num},20000;
         """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 3 - 6
Apietl/bytedance_etl/report/AdHourlyReport.py

@@ -7,11 +7,8 @@
 # @contact: renyupeng@c-top.com.cn
 # @Tel 1501435553
 import logging
-from math import ceil
-
-from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class AdHourlyReport:
@@ -24,7 +21,7 @@ class AdHourlyReport:
     def Hourly_report(self):
         try:
             sql = """select advertiser_id from media_api.bytedance_ad_report_hourly where 
-            stat_date between {start_time} and {end_time} group by advertiser_id""".format(
+            stat_datetime between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
             account_list = TidbConn.quary(sql)
             for account_id in account_list:
@@ -281,7 +278,7 @@ from `media_api`.bytedance_ad_report_hourly a
       where cua.account_id is not null) b
      on a.advertiser_id = b.account_id
 where a.stat_datetime between {start_time} and {end_time}
-and a.advertiser={account_id}
+and a.advertiser_id={account_id}
         """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 3 - 3
Apietl/bytedance_etl/report/AdvertiserDailyReport.py

@@ -11,7 +11,7 @@ from math import ceil
 
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class AdvertiserDailyReport:
@@ -24,7 +24,7 @@ class AdvertiserDailyReport:
     def daily_report(self):
         try:
             sql = """select count(1) from media_api.bytedance_advertiser_report_daily where 
-            stat_date between {start_time} and {end_time}""".format(
+            stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             pageCount = ceil(totalNum / 20000)
@@ -287,7 +287,7 @@ from `media_api`.bytedance_advertiser_report_daily a
       where cua.account_id is not null) b
      on a.advertiser_id = b.account_id
 where a.stat_datetime between {start_time} and {end_time}
-order by account_id limit {num}:20000;
+order by account_id limit {num},20000;
         """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 3 - 3
Apietl/bytedance_etl/report/AdvertiserHourlyReport.py

@@ -11,7 +11,7 @@ from math import ceil
 
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class AdvertiserHourlyReport:
@@ -24,7 +24,7 @@ class AdvertiserHourlyReport:
     def hourly_report(self):
         try:
             sql = """select count(1) from media_api.bytedance_advertiser_report_hourly where 
-            stat_date between {start_time} and {end_time}""".format(
+            stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             pageCount = ceil(totalNum / 20000)
@@ -288,7 +288,7 @@ from `media_api`.bytedance_advertiser_report_hourly a
       where cua.account_id is not null) b
      on a.advertiser_id = b.account_id
 where a.stat_datetime between {start_time} and {end_time}
-order by a.advertiser_id,a.stat_datetime,a.hour limit {num}:20000;
+order by a.advertiser_id,a.stat_datetime,a.hour limit {num},20000;
         """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 0 - 1
Apietl/bytedance_etl/report/AdvertiserReport.py

@@ -8,7 +8,6 @@
 
 import logging
 from Apietl.conn.TidbConn import TidbConn
-from Apietl.utlis.Utils import Utils
 
 
 class AdvertiserReport:

+ 4 - 6
Apietl/bytedance_etl/report/CampaignDailyReport.py

@@ -8,10 +8,8 @@
 # @Tel 1501435553
 import logging
 from math import ceil
-
-from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class CampaignDailyReport:
@@ -24,7 +22,7 @@ class CampaignDailyReport:
     def daily_report(self):
         try:
             sql = """select count(1) from media_api.bytedance_campaign_report_daily where 
-            stat_date between {start_time} and {end_time}""".format(
+            stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             pageCount = ceil(totalNum / 20000)
@@ -288,8 +286,8 @@ from `media_api`.bytedance_campaign_report_daily a
       where cua.account_id is not null) b
      on a.advertiser_id = b.account_id
 where a.stat_datetime between {start_time} and {end_time}
-a.advertiser_id,a.campaign_id a.stat_datetime
-limit {num}:20000;
+order by a.advertiser_id,a.campaign_id,a.stat_datetime
+limit {num},20000;
         """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 4 - 6
Apietl/bytedance_etl/report/CampaignHourlyReport.py

@@ -8,10 +8,8 @@
 # @Tel 1501435553
 import logging
 from math import ceil
-
-from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class CampaignHourlyReport:
@@ -24,7 +22,7 @@ class CampaignHourlyReport:
     def hourly_report(self):
         try:
             sql = """select count(1) from media_api.bytedance_campaign_report_hourly where 
-            stat_date between {start_time} and {end_time}""".format(
+            stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             pageCount = ceil(totalNum / 20000)
@@ -290,8 +288,8 @@ from `media_api`.bytedance_campaign_report_hourly a
       where cua.account_id is not null) b
      on a.advertiser_id = b.account_id
 where a.stat_datetime between {start_time} and {end_time}
-order by a.advertiser_id,a.campaign_id a.stat_datetime,a.hour
-limit {num}:20000;
+order by a.advertiser_id,a.campaign_id,a.stat_datetime,a.hour
+limit {num},20000;
         """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 3 - 6
Apietl/bytedance_etl/report/CreativeDailyReport.py

@@ -7,11 +7,8 @@
 # @contact: renyupeng@c-top.com.cn
 # @Tel 1501435553
 import logging
-from math import ceil
-
-from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class CreativeDailyReport:
@@ -24,7 +21,7 @@ class CreativeDailyReport:
     def daily_report(self):
         try:
             sql = """select advertiser_id from media_api.bytedance_creative_report_daily where 
-            stat_date between {start_time} and {end_time} group by advertiser_id""".format(
+            stat_datetime between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
             account_list = TidbConn.quary(sql)
             for account_id in account_list:
@@ -290,7 +287,7 @@ from `media_api`.bytedance_creative_report_daily a
       where cua.account_id is not null) b
      on a.advertiser_id = b.account_id
 where a.stat_datetime between {start_time} and {end_time}
-and a.advertiser={account_id}
+and a.advertiser_id={account_id}
         """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)

+ 343 - 4
Apietl/bytedance_etl/report/MaterialImageDailyReport.py

@@ -1,8 +1,347 @@
 # Author renyupeng
 # coding=utf-8
-# @Time    : 2021/10/21 5:53 下午
-# @Site    : 
-# @File    : MaterialImageDailyReport.py
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialImageDailyReport.py
 # @Software: PyCharm
 # @contact: renyupeng@c-top.com.cn
-# @Tel 1501435553
+
+import logging
+from math import ceil
+
+from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
+from Apietl.utlis.Logger import Logger
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class MateirialImageDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_material_report_daily where
+             stat_datetime between {start_time} and {end_time} and image_mode in
+      ('CREATIVE_IMAGE_MODE_SMALL', 'CREATIVE_IMAGE_MODE_LARGE', 'CREATIVE_IMAGE_MODE_LARGE_VERTICAL',
+       'CREATIVE_IMAGE_MODE_GROUP', 'CREATIVE_IMAGE_MODE_GIF', 'TOUTIAO_SEARCH_AD_IMAGE', 'SEARCH_AD_SMALL_IMAGE',
+       'CREATIVE_IMAGE_MODE_UNION_SPLASH') 
+           group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(sql)
+            for account_id in account_list:
+                sql = """
+replace into application.bytedance_material_image_report_daily_dw(account_id, material_id, image_mode, inventory,
+                                                                  stat_datetime, project_id, product_id, advertiser_id,
+                                                                  company_id, account_name, project_name,
+                                                                  advertiser_name, company_name, sale_id, user_id,
+                                                                  image_name,
+                                                                  cost, `show`, avg_show_cost, click,
+                                                                  avg_click_cost, ctr, `convert`, convert_cost,
+                                                                  convert_rate, deep_convert, deep_convert_cost,
+                                                                  deep_convert_rate, attribution_convert,
+                                                                  attribution_convert_cost, attribution_deep_convert,
+                                                                  attribution_deep_convert_cost, download_start,
+                                                                  download_start_cost, download_start_rate,
+                                                                  download_finish, download_finish_cost,
+                                                                  download_finish_rate, click_install, install_finish,
+                                                                  install_finish_cost, install_finish_rate, active,
+                                                                  active_cost, active_rate, register,
+                                                                  active_register_cost, active_register_rate,
+                                                                  active_pay_amount, next_day_open, next_day_open_cost,
+                                                                  next_day_open_rate, attribution_next_day_open_cnt,
+                                                                  attribution_next_day_open_cost,
+                                                                  attribution_next_day_open_rate, game_addiction,
+                                                                  game_addiction_cost, game_addiction_rate, pay_count,
+                                                                  active_pay_cost, active_pay_rate, loan_completion,
+                                                                  loan_completion_cost, loan_completion_rate,
+                                                                  pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                                  loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                                  in_app_detail_uv, in_app_cart, in_app_pay,
+                                                                  in_app_order, attribution_game_pay_7d_count,
+                                                                  attribution_game_pay_7d_cost,
+                                                                  attribution_active_pay_7d_per_count, game_pay_cost,
+                                                                  game_pay_count, phone, form, map_search, button, view,
+                                                                  download, qq, lottery, vote, message, redirect,
+                                                                  shopping, consult, wechat, phone_confirm,
+                                                                  phone_connect, consult_effective, coupon,
+                                                                  coupon_single_page, redirect_to_shop, poi_collect,
+                                                                  poi_address_click, luban_order_cnt,
+                                                                  luban_order_stat_amount, luban_order_roi,
+                                                                  luban_live_enter_cnt, live_watch_one_minute_count,
+                                                                  luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                                  luban_live_comment_cnt, luban_live_share_cnt,
+                                                                  luban_live_gift_cnt, luban_live_gift_amount,
+                                                                  luban_live_slidecart_click_cnt,
+                                                                  luban_live_click_product_cnt,
+                                                                  luban_live_pay_order_count,
+                                                                  luban_live_pay_order_stat_cost, wechat_login_count,
+                                                                  attribution_wechat_login_30d_count, wechat_login_cost,
+                                                                  attribution_wechat_login_30d_cost,
+                                                                  wechat_first_pay_count,
+                                                                  attribution_wechat_first_pay_30d_count,
+                                                                  wechat_first_pay_cost,
+                                                                  attribution_wechat_first_pay_30d_cost,
+                                                                  wechat_first_pay_rate,
+                                                                  attribution_wechat_first_pay_30d_rate,
+                                                                  wechat_pay_amount, attribution_wechat_pay_30d_amount,
+                                                                  attribution_wechat_pay_30d_roi, phone_effective,
+                                                                  total_play, valid_play, valid_play_cost,
+                                                                  valid_play_rate, play_25_feed_break,
+                                                                  play_50_feed_break, play_75_feed_break,
+                                                                  play_100_feed_break, average_play_time_per_play,
+                                                                  play_over_rate, wifi_play_rate, wifi_play,
+                                                                  play_duration_sum, advanced_creative_phone_click,
+                                                                  advanced_creative_counsel_click,
+                                                                  advanced_creative_form_click,
+                                                                  advanced_creative_coupon_addition,
+                                                                  advanced_creative_form_submit, card_show, share,
+                                                                  comment, `like`, follow, home_visited,
+                                                                  ies_challenge_click, ies_music_click, location_click,
+                                                                  message_action, click_landing_page, click_shopwindow,
+                                                                  click_website, click_download, click_call_dy, cpc,
+                                                                  cpm, cpa, interact_per_cost, convert_show_rate,
+                                                                  play_duration_10s_rate, play_50_feed_break_rate,
+                                                                  play_25_feed_break_rate, play_duration_5s_rate,
+                                                                  play_duration_2s_rate, average_play_progress,
+                                                                  play_100_feed_break_rate, play_duration_3s_rate,
+                                                                  play_75_feed_break_rate, play_duration,
+                                                                  play_duration_3s, play_duration_2s, play_duration_10s,
+                                                                  average_video_play, avg_rank,
+                                                                  submit_certification_count, approval_count,
+                                                                  first_order_count, first_rental_order_count,
+                                                                  commute_first_pay_count)
+
+select report.advertiser_id account_id,      #'账户ID'
+       report.material_id,                   #头条素材id
+       report.image_mode,                    #素材类型
+       report.inventory,                     #广告位
+       report.stat_datetime,                 #数据起始时间
+       cua.project_id,                       #项目id
+       product.id           product_id,      #项目中产品id
+       cua.advertiser_id,                    #广告主id
+       uc.company_id,                        #运营人的公司id
+       cua.auth_name        account_name,    #账户名称
+       cp.project_name,                      #项目名称
+       ca.name              advertiser_name, #广告主名称
+       uc.company_name,                      #运营人的公司名称
+       cp.sale_id,                           #项目里的销售id
+       cua.user_id,                          #运营id
+       image.filename,
+       report.`cost`,
+       report.`show`,
+       report.`avg_show_cost`,
+       report.`click`,
+       report.`avg_click_cost`,
+       report.`ctr`,
+       report.`convert`,
+       report.`convert_cost`,
+       report.`convert_rate`,
+       report.`deep_convert`,
+       report.`deep_convert_cost`,
+       report.`deep_convert_rate`,
+       report.`attribution_convert`,
+       report.`attribution_convert_cost`,
+       report.`attribution_deep_convert`,
+       report.`attribution_deep_convert_cost`,
+       report.`download_start`,
+       report.`download_start_cost`,
+       report.`download_start_rate`,
+       report.`download_finish`,
+       report.`download_finish_cost`,
+       report.`download_finish_rate`,
+       report.`click_install`,
+       report.`install_finish`,
+       report.`install_finish_cost`,
+       report.`install_finish_rate`,
+       report.`active`,
+       report.`active_cost`,
+       report.`active_rate`,
+       report.`register`,
+       report.`active_register_cost`,
+       report.`active_register_rate`,
+       report.`active_pay_amount`,
+       report.`next_day_open`,
+       report.`next_day_open_cost`,
+       report.`next_day_open_rate`,
+       report.`attribution_next_day_open_cnt`,
+       report.`attribution_next_day_open_cost`,
+       report.`attribution_next_day_open_rate`,
+       report.`game_addiction`,
+       report.`game_addiction_cost`,
+       report.`game_addiction_rate`,
+       report.`pay_count`,
+       report.`active_pay_cost`,
+       report.`active_pay_rate`,
+       report.`loan_completion`,
+       report.`loan_completion_cost`,
+       report.`loan_completion_rate`,
+       report.`pre_loan_credit`,
+       report.`pre_loan_credit_cost`,
+       report.`loan_credit`,
+       report.`loan_credit_cost`,
+       report.`loan_credit_rate`,
+       report.`in_app_uv`,
+       report.`in_app_detail_uv`,
+       report.`in_app_cart`,
+       report.`in_app_pay`,
+       report.`in_app_order`,
+       report.`attribution_game_pay_7d_count`,
+       report.`attribution_game_pay_7d_cost`,
+       report.`attribution_active_pay_7d_per_count`,
+       report.`game_pay_cost`,
+       report.`game_pay_count`,
+       report.`phone`,
+       report.`form`,
+       report.`map_search`,
+       report.`button`,
+       report.`view`,
+       report.`download`,
+       report.`qq`,
+       report.`lottery`,
+       report.`vote`,
+       report.`message`,
+       report.`redirect`,
+       report.`shopping`,
+       report.`consult`,
+       report.`wechat`,
+       report.`phone_confirm`,
+       report.`phone_connect`,
+       report.`consult_effective`,
+       report.`coupon`,
+       report.`coupon_single_page`,
+       report.`redirect_to_shop`,
+       report.`poi_collect`,
+       report.`poi_address_click`,
+       report.`luban_order_cnt`,
+       report.`luban_order_stat_amount`,
+       report.`luban_order_roi`,
+       report.`luban_live_enter_cnt`,
+       report.`live_watch_one_minute_count`,
+       report.`luban_live_follow_cnt`,
+       report.`live_fans_club_join_cnt`,
+       report.`luban_live_comment_cnt`,
+       report.`luban_live_share_cnt`,
+       report.`luban_live_gift_cnt`,
+       report.`luban_live_gift_amount`,
+       report.`luban_live_slidecart_click_cnt`,
+       report.`luban_live_click_product_cnt`,
+       report.`luban_live_pay_order_count`,
+       report.`luban_live_pay_order_stat_cost`,
+       report.`wechat_login_count`,
+       report.`attribution_wechat_login_30d_count`,
+       report.`wechat_login_cost`,
+       report.`attribution_wechat_login_30d_cost`,
+       report.`wechat_first_pay_count`,
+       report.`attribution_wechat_first_pay_30d_count`,
+       report.`wechat_first_pay_cost`,
+       report.`attribution_wechat_first_pay_30d_cost`,
+       report.`wechat_first_pay_rate`,
+       report.`attribution_wechat_first_pay_30d_rate`,
+       report.`wechat_pay_amount`,
+       report.`attribution_wechat_pay_30d_amount`,
+       report.`attribution_wechat_pay_30d_roi`,
+       report.`phone_effective`,
+       report.`total_play`,
+       report.`valid_play`,
+       report.`valid_play_cost`,
+       report.`valid_play_rate`,
+       report.`play_25_feed_break`,
+       report.`play_50_feed_break`,
+       report.`play_75_feed_break`,
+       report.`play_100_feed_break`,
+       report.`average_play_time_per_play`,
+       report.`play_over_rate`,
+       report.`wifi_play_rate`,
+       report.`wifi_play`,
+       report.`play_duration_sum`,
+       report.`advanced_creative_phone_click`,
+       report.`advanced_creative_counsel_click`,
+       report.`advanced_creative_form_click`,
+       report.`advanced_creative_coupon_addition`,
+       report.`advanced_creative_form_submit`,
+       report.`card_show`,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.`follow`,
+       report.`home_visited`,
+       report.`ies_challenge_click`,
+       report.`ies_music_click`,
+       report.`location_click`,
+       report.`message_action`,
+       report.`click_landing_page`,
+       report.`click_shopwindow`,
+       report.`click_website`,
+       report.`click_download`,
+       report.`click_call_dy`,
+       report.`cpc`,
+       report.`cpm`,
+       report.`cpa`,
+       report.`interact_per_cost`,
+       report.`convert_show_rate`,
+       report.`play_duration_10s_rate`,
+       report.`play_50_feed_break_rate`,
+       report.`play_25_feed_break_rate`,
+       report.`play_duration_5s_rate`,
+       report.`play_duration_2s_rate`,
+       report.`average_play_progress`,
+       report.`play_100_feed_break_rate`,
+       report.`play_duration_3s_rate`,
+       report.`play_75_feed_break_rate`,
+       report.`play_duration`,
+       report.`play_duration_3s`,
+       report.`play_duration_2s`,
+       report.`play_duration_10s`,
+       report.`average_video_play`,
+       report.`avg_rank`,
+       report.`submit_certification_count`,
+       report.`approval_count`,
+       report.`first_order_count`,
+       report.`first_rental_order_count`,
+       report.`commute_first_pay_count`
+from media_api.bytedance_material_report_daily report
+         left join
+     `media_api`.bytedance_file_image_get image
+     on report.material_id = image.material_id
+         and report.advertiser_id = image.advertiser_id
+         left join `jeecg-boot`.ctop_user_allocation cua
+                   on report.advertiser_id = cua.account_id
+         left join `jeecg-boot`.ctop_advertiser ca
+                   on cua.advertiser_id = ca.id
+         left join `jeecg-boot`.ctop_project cp
+                   on cua.project_id = cp.id
+         left join `jeecg-boot`.user_company uc
+                   on cua.user_id = uc.user_id
+         left join `jeecg-boot`.ctop_product product
+                   on cp.product_id = product.id
+         left join `jeecg-boot`.sys_user su
+                   on cua.user_id = su.id
+where report.image_mode in
+      ('CREATIVE_IMAGE_MODE_SMALL', 'CREATIVE_IMAGE_MODE_LARGE', 'CREATIVE_IMAGE_MODE_LARGE_VERTICAL',
+       'CREATIVE_IMAGE_MODE_GROUP', 'CREATIVE_IMAGE_MODE_GIF', 'TOUTIAO_SEARCH_AD_IMAGE', 'SEARCH_AD_SMALL_IMAGE',
+       'CREATIVE_IMAGE_MODE_UNION_SPLASH')
+  and report.stat_datetime between {start_time} and {end_time}
+  and report.advertiser_id = {account_id}
+ """.format( start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = MateirialImageDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 356 - 4
Apietl/bytedance_etl/report/MaterialVideoDailyReport.py

@@ -1,8 +1,360 @@
 # Author renyupeng
 # coding=utf-8
-# @Time    : 2021/10/21 5:54 下午
-# @Site    : 
-# @File    : MaterialVideoDailyReport.py
+# @Time    : 2021/9/15 5:10 下午
+# @Site    :
+# @File    : MateirialVideoDailyReport.py
 # @Software: PyCharm
 # @contact: renyupeng@c-top.com.cn
-# @Tel 1501435553
+
+import logging
+from Apietl.utlis.Utils import Utils
+from Apietl.conn.TidbConn import TidbConn
+
+
+class MateirialVideoDailyReport:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_args()
+        self.start_time = self.parm[0]
+        self.end_time = self.parm[1]
+
+    def daily_report(self):
+        try:
+            sql = """select advertiser_id from media_api.bytedance_material_report_daily where 
+         stat_datetime between {start_time} and {end_time} 
+         and image_mode in ('CREATIVE_IMAGE_MODE_VIDEO', 'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL', 'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+         group by advertiser_id""".format(
+                start_time=self.start_time, end_time=self.end_time)
+            account_list = TidbConn.quary(sql)
+            for account_id in account_list:
+                sql = """
+                replace into application.bytedance_material_video_report_daily_dw(account_id, material_id, image_mode, inventory,
+                                                                  stat_datetime, project_id, product_id, advertiser_id,
+                                                                  company_id, account_name, project_name,
+                                                                  advertiser_name, company_name, sale_id, user_id,
+                                                                  clip_id, shot_id, plan_id, leader_id, org_code,
+                                                                  signature, channel_type, clip_name, shot_name,
+                                                                  plan_name, leader_name, video_name, channel_name,
+                                                                  clip_company_id, cost, `show`, avg_show_cost, click,
+                                                                  avg_click_cost, ctr, `convert`, convert_cost,
+                                                                  convert_rate, deep_convert, deep_convert_cost,
+                                                                  deep_convert_rate, attribution_convert,
+                                                                  attribution_convert_cost, attribution_deep_convert,
+                                                                  attribution_deep_convert_cost, download_start,
+                                                                  download_start_cost, download_start_rate,
+                                                                  download_finish, download_finish_cost,
+                                                                  download_finish_rate, click_install, install_finish,
+                                                                  install_finish_cost, install_finish_rate, active,
+                                                                  active_cost, active_rate, register,
+                                                                  active_register_cost, active_register_rate,
+                                                                  active_pay_amount, next_day_open, next_day_open_cost,
+                                                                  next_day_open_rate, attribution_next_day_open_cnt,
+                                                                  attribution_next_day_open_cost,
+                                                                  attribution_next_day_open_rate, game_addiction,
+                                                                  game_addiction_cost, game_addiction_rate, pay_count,
+                                                                  active_pay_cost, active_pay_rate, loan_completion,
+                                                                  loan_completion_cost, loan_completion_rate,
+                                                                  pre_loan_credit, pre_loan_credit_cost, loan_credit,
+                                                                  loan_credit_cost, loan_credit_rate, in_app_uv,
+                                                                  in_app_detail_uv, in_app_cart, in_app_pay,
+                                                                  in_app_order, attribution_game_pay_7d_count,
+                                                                  attribution_game_pay_7d_cost,
+                                                                  attribution_active_pay_7d_per_count, game_pay_cost,
+                                                                  game_pay_count, phone, form, map_search, button, view,
+                                                                  download, qq, lottery, vote, message, redirect,
+                                                                  shopping, consult, wechat, phone_confirm,
+                                                                  phone_connect, consult_effective, coupon,
+                                                                  coupon_single_page, redirect_to_shop, poi_collect,
+                                                                  poi_address_click, luban_order_cnt,
+                                                                  luban_order_stat_amount, luban_order_roi,
+                                                                  luban_live_enter_cnt, live_watch_one_minute_count,
+                                                                  luban_live_follow_cnt, live_fans_club_join_cnt,
+                                                                  luban_live_comment_cnt, luban_live_share_cnt,
+                                                                  luban_live_gift_cnt, luban_live_gift_amount,
+                                                                  luban_live_slidecart_click_cnt,
+                                                                  luban_live_click_product_cnt,
+                                                                  luban_live_pay_order_count,
+                                                                  luban_live_pay_order_stat_cost, wechat_login_count,
+                                                                  attribution_wechat_login_30d_count, wechat_login_cost,
+                                                                  attribution_wechat_login_30d_cost,
+                                                                  wechat_first_pay_count,
+                                                                  attribution_wechat_first_pay_30d_count,
+                                                                  wechat_first_pay_cost,
+                                                                  attribution_wechat_first_pay_30d_cost,
+                                                                  wechat_first_pay_rate,
+                                                                  attribution_wechat_first_pay_30d_rate,
+                                                                  wechat_pay_amount, attribution_wechat_pay_30d_amount,
+                                                                  attribution_wechat_pay_30d_roi, phone_effective,
+                                                                  total_play, valid_play, valid_play_cost,
+                                                                  valid_play_rate, play_25_feed_break,
+                                                                  play_50_feed_break, play_75_feed_break,
+                                                                  play_100_feed_break, average_play_time_per_play,
+                                                                  play_over_rate, wifi_play_rate, wifi_play,
+                                                                  play_duration_sum, advanced_creative_phone_click,
+                                                                  advanced_creative_counsel_click,
+                                                                  advanced_creative_form_click,
+                                                                  advanced_creative_coupon_addition,
+                                                                  advanced_creative_form_submit, card_show, share,
+                                                                  comment, `like`, follow, home_visited,
+                                                                  ies_challenge_click, ies_music_click, location_click,
+                                                                  message_action, click_landing_page, click_shopwindow,
+                                                                  click_website, click_download, click_call_dy, cpc,
+                                                                  cpm, cpa, interact_per_cost, convert_show_rate,
+                                                                  play_duration_10s_rate, play_50_feed_break_rate,
+                                                                  play_25_feed_break_rate, play_duration_5s_rate,
+                                                                  play_duration_2s_rate, average_play_progress,
+                                                                  play_100_feed_break_rate, play_duration_3s_rate,
+                                                                  play_75_feed_break_rate, play_duration,
+                                                                  play_duration_3s, play_duration_2s, play_duration_10s,
+                                                                  average_video_play, avg_rank,
+                                                                  submit_certification_count, approval_count,
+                                                                  first_order_count, first_rental_order_count,
+                                                                  commute_first_pay_count)
+        select
+       report.advertiser_id account_id,   #'账户ID'
+       report.material_id,                #头条素材id
+       report.image_mode,                 #素材类型
+       report.inventory,                  #广告位
+       report.stat_datetime,              #数据起始时间
+       cua.project_id,                    #项目id
+       product.id product_id,             #项目中产品id
+       cua.advertiser_id,                 #广告主id
+       uc.company_id,                     #运营人的公司id
+       cua.auth_name account_name,        #账户名称
+       cp.project_name,                   #项目名称
+       ca.name advertiser_name,           #广告主名称
+       uc.company_name,                   #运营人的公司名称
+       cp.sale_id,                        #项目里的销售id
+       cua.user_id,                       #运营id
+       cma.clip_id,                       #剪辑id
+       cma.shot_id,                       #拍摄id
+       cma.plan_id,                       #编导id
+       cma.leader_id,                     #负责人id
+       su.org_code,                       #机构编码
+       cmi.code signature,                #素材id
+       null chl_type,                       #渠道类型
+       clip.realname clip_name,           #剪辑名称
+       shot.realname shot_name,           #拍摄名称
+       plan.realname plan_name,           #编导名称
+       leader.realname lead_name,         #负责人名称
+       case when cmi.material_name is not null then cmi.material_name else video.filename end as video_name, ##视频名称
+       '' chl_name,                       #渠道名称
+       up.company_id clip_company_id,     #剪辑公司
+       report.`cost`,
+       report.`show`,
+       report.`avg_show_cost`,
+       report.`click`,
+       report.`avg_click_cost`,
+       report.`ctr`,
+       report.`convert`,
+       report.`convert_cost`,
+       report.`convert_rate`,
+       report.`deep_convert`,
+       report.`deep_convert_cost`,
+       report.`deep_convert_rate`,
+       report.`attribution_convert`,
+       report.`attribution_convert_cost`,
+       report.`attribution_deep_convert`,
+       report.`attribution_deep_convert_cost`,
+       report.`download_start`,
+       report.`download_start_cost`,
+       report.`download_start_rate`,
+       report.`download_finish`,
+       report.`download_finish_cost`,
+       report.`download_finish_rate`,
+       report.`click_install`,
+       report.`install_finish`,
+       report.`install_finish_cost`,
+       report.`install_finish_rate`,
+       report.`active`,
+       report.`active_cost`,
+       report.`active_rate`,
+       report.`register`,
+       report.`active_register_cost`,
+       report.`active_register_rate`,
+       report.`active_pay_amount`,
+       report.`next_day_open`,
+       report.`next_day_open_cost`,
+       report.`next_day_open_rate`,
+       report.`attribution_next_day_open_cnt`,
+       report.`attribution_next_day_open_cost`,
+       report.`attribution_next_day_open_rate`,
+       report.`game_addiction`,
+       report.`game_addiction_cost`,
+       report.`game_addiction_rate`,
+       report.`pay_count`,
+       report.`active_pay_cost`,
+       report.`active_pay_rate`,
+       report.`loan_completion`,
+       report.`loan_completion_cost`,
+       report.`loan_completion_rate`,
+       report.`pre_loan_credit`,
+       report.`pre_loan_credit_cost`,
+       report.`loan_credit`,
+       report.`loan_credit_cost`,
+       report.`loan_credit_rate`,
+       report.`in_app_uv`,
+       report.`in_app_detail_uv`,
+       report.`in_app_cart`,
+       report.`in_app_pay`,
+       report.`in_app_order`,
+       report.`attribution_game_pay_7d_count`,
+       report.`attribution_game_pay_7d_cost`,
+       report.`attribution_active_pay_7d_per_count`,
+       report.`game_pay_cost`,
+       report.`game_pay_count`,
+       report.`phone`,
+       report.`form`,
+       report.`map_search`,
+       report.`button`,
+       report.`view`,
+       report.`download`,
+       report.`qq`,
+       report.`lottery`,
+       report.`vote`,
+       report.`message`,
+       report.`redirect`,
+       report.`shopping`,
+       report.`consult`,
+       report.`wechat`,
+       report.`phone_confirm`,
+       report.`phone_connect`,
+       report.`consult_effective`,
+       report.`coupon`,
+       report.`coupon_single_page`,
+       report.`redirect_to_shop`,
+       report.`poi_collect`,
+       report.`poi_address_click`,
+       report.`luban_order_cnt`,
+       report.`luban_order_stat_amount`,
+       report.`luban_order_roi`,
+       report.`luban_live_enter_cnt`,
+       report.`live_watch_one_minute_count`,
+       report.`luban_live_follow_cnt`,
+       report.`live_fans_club_join_cnt`,
+       report.`luban_live_comment_cnt`,
+       report.`luban_live_share_cnt`,
+       report.`luban_live_gift_cnt`,
+       report.`luban_live_gift_amount`,
+       report.`luban_live_slidecart_click_cnt`,
+       report.`luban_live_click_product_cnt`,
+       report.`luban_live_pay_order_count`,
+       report.`luban_live_pay_order_stat_cost`,
+       report.`wechat_login_count`,
+       report.`attribution_wechat_login_30d_count`,
+       report.`wechat_login_cost`,
+       report.`attribution_wechat_login_30d_cost`,
+       report.`wechat_first_pay_count`,
+       report.`attribution_wechat_first_pay_30d_count`,
+       report.`wechat_first_pay_cost`,
+       report.`attribution_wechat_first_pay_30d_cost`,
+       report.`wechat_first_pay_rate`,
+       report.`attribution_wechat_first_pay_30d_rate`,
+       report.`wechat_pay_amount`,
+       report.`attribution_wechat_pay_30d_amount`,
+       report.`attribution_wechat_pay_30d_roi`,
+       report.`phone_effective`,
+       report.`total_play`,
+       report.`valid_play`,
+       report.`valid_play_cost`,
+       report.`valid_play_rate`,
+       report.`play_25_feed_break`,
+       report.`play_50_feed_break`,
+       report.`play_75_feed_break`,
+       report.`play_100_feed_break`,
+       report.`average_play_time_per_play`,
+       report.`play_over_rate`,
+       report.`wifi_play_rate`,
+       report.`wifi_play`,
+       report.`play_duration_sum`,
+       report.`advanced_creative_phone_click`,
+       report.`advanced_creative_counsel_click`,
+       report.`advanced_creative_form_click`,
+       report.`advanced_creative_coupon_addition`,
+       report.`advanced_creative_form_submit`,
+       report.`card_show`,
+       report.`share`,
+       report.`comment`,
+       report.`like`,
+       report.`follow`,
+       report.`home_visited`,
+       report.`ies_challenge_click`,
+       report.`ies_music_click`,
+       report.`location_click`,
+       report.`message_action`,
+       report.`click_landing_page`,
+       report.`click_shopwindow`,
+       report.`click_website`,
+       report.`click_download`,
+       report.`click_call_dy`,
+       report.`cpc`,
+       report.`cpm`,
+       report.`cpa`,
+       report.`interact_per_cost`,
+       report.`convert_show_rate`,
+       report.`play_duration_10s_rate`,
+       report.`play_50_feed_break_rate`,
+       report.`play_25_feed_break_rate`,
+       report.`play_duration_5s_rate`,
+       report.`play_duration_2s_rate`,
+       report.`average_play_progress`,
+       report.`play_100_feed_break_rate`,
+       report.`play_duration_3s_rate`,
+       report.`play_75_feed_break_rate`,
+       report.`play_duration`,
+       report.`play_duration_3s`,
+       report.`play_duration_2s`,
+       report.`play_duration_10s`,
+       report.`average_video_play`,
+       report.`avg_rank`,
+       report.`submit_certification_count`,
+       report.`approval_count`,
+       report.`first_order_count`,
+       report.`first_rental_order_count`,
+       report.`commute_first_pay_count`
+from media_api.bytedance_material_report_daily report
+inner join
+`media_api`.bytedance_file_video_get video
+on report.advertiser_id=video.advertiser_id and report.material_id=video.material_id
+left join `jeecg-boot`.ctop_material_info cmi
+on video.signature=cmi.code
+left join `jeecg-boot`.ctop_material_ascription cma
+on video.signature=cma.code
+left join `jeecg-boot`.ctop_user_allocation cua
+on report.advertiser_id=cua.account_id
+left join `jeecg-boot`.ctop_advertiser ca
+on cua.advertiser_id = ca.id
+left join `jeecg-boot`.ctop_project cp
+on cua.project_id = cp.id
+left join `jeecg-boot`.user_company uc
+on cua.user_id=uc.user_id
+left join `jeecg-boot`.ctop_product product
+on cp.product_id=product.id
+left join `jeecg-boot`.sys_user su
+on cua.user_id = su.id
+left join `jeecg-boot`.sys_user clip on clip.id = cma.clip_id
+left join `jeecg-boot`.sys_user shot on shot.id = cma.shot_id
+left join `jeecg-boot`.sys_user plan on plan.id = cma.plan_id
+left join `jeecg-boot`.sys_user leader on leader.id = cma.leader_id
+left join `jeecg-boot`.user_company up on up.user_id=cma.clip_id where
+report.image_mode in ('CREATIVE_IMAGE_MODE_VIDEO', 'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL', 'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
+and report.stat_datetime between {start_time} and {end_time}  and report.advertiser_id={account_id}
+                """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
+                print(sql)
+                TidbConn.upsert(conn=self.conn, sql_buff=sql)
+                self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = MateirialVideoDailyReport().daily_report()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 8 - 0
Apietl/config.ini

@@ -7,6 +7,14 @@ tidb_port = 3390
 tidb_user = hcst
 tidb_password = hcst@2021
 
+
+
+[tidb_pro]
+tidb_host = 139.186.27.96
+tidb_port = 3390
+tidb_user = data
+tidb_password = hcst@2021
+
 [webhook]
 url = 192.168.0.202
 

+ 12 - 0
Apietl/conn/TidbConn.py

@@ -28,6 +28,18 @@ class TidbConn:
                                passwd=ConfConstant.TIDB_PASSWORD,
                                port=ConfConstant.TIDB_PORT, charset="utf8")
 
+
+    @staticmethod
+    def get_pro_connection():
+        """
+        获取Tidb连接
+        :return: 连接
+        """
+        return pymysql.connect(host=ConfConstant.TIDB_PRO_HOST, user=ConfConstant.TIDB_PRO_USER,
+                               passwd=ConfConstant.TIDB_PRO_PASSWORD,
+                               port=ConfConstant.TIDB_PRO_PORT, charset="utf8")
+
+
     @classmethod
     def _reConn(cls, num=28800, stime=5):  # 重试连接总次数为1天,这里根据实际情况自己设置,如果服务器宕机1天都没发现就......
         _number = 0

+ 14 - 0
Apietl/utlis/webHook.py

@@ -9,6 +9,8 @@ from flask import Flask  # To be able to start the application
 from ConfConstant import ConfConstant
 from report.AccountReport import AccountReport
 
+from Apietl.bytedance_etl.report import AdvertiserReport
+
 app = Flask(__name__)
 
 
@@ -29,5 +31,17 @@ def api_webhook_messages():
     return '200'
 
 
+@app.route('/webhook/bytedance_account_report', methods=['POST'])
+def api_webhook_bytedance_messages():
+    print(json.loads(request.data))
+    my_info = json.loads(request.data)
+    account_id = my_info["adveriser_id"]
+    start_date = my_info["start_date"]
+    end_date = my_info["end_date"]
+    date_type = my_info["date_type"]
+    AdvertiserReport().handler(date_type, account_id, start_date, end_date)
+    return '200'
+
+
 if __name__ == '__main__':
     app.run(port=8765, host=ConfConstant.URL, debug=True)

+ 27 - 4
monitor/ApplicationCDCMonitor.py

@@ -6,6 +6,8 @@
 # @Software: PyCharm
 # @contact: renyupeng@c-top.com.cn
 # @Tel 1501435553
+import logging
+
 from Apietl.constant.ConfConstant import ConfConstant
 import requests
 import os
@@ -30,17 +32,38 @@ class ApplicationCDCMonitor:
         print(message)
         for json in message:
             task_id = json['id']
-            state = json['state']
             tso = json['checkpoint_tso']
             error = json['error']
             if error is not None:
                 requests.delete(url='http://{}:{}{}/{}'.format(self.cdc_url, self.cdc_port,
                                                                self.cdc_changefeeds_url, task_id))
-
                 os.system(
                     'cd /data/tidb-deploy/cdc-8300/bin; ./cdc cli changefeed create --pd=http://192.168.0.184:2379 '
-                    '--sink-uri="mysql://data:hcst@2021@139.186.27.96:3390" --config ../conf/application.toml '
-                    '--changefeed-id=application-task --start-ts {tso}'.format(tso=tso))
+                    '--sink-uri="mysql://data:hcst@2021@139.186.27.96:3390" --config ../conf/{task_id}.toml '
+                    '--changefeed-id={task_id} --start-ts {tso}'.format(tso=tso, task_id=task_id))
+
+                backResult = requests.get(
+                    url='http://{}:{}{}/{}'.format(self.cdc_url, self.cdc_port, self.cdc_changefeeds_url, task_id),
+                    headers=headers)
+                if backResult is None:
+                    os.system(
+                        'cd /data/tidb-deploy/cdc-8300/bin; ./cdc cli changefeed create --pd=http://192.168.0.184:2379 '
+                        '--sink-uri="mysql://data:hcst@2021@139.186.27.96:3390" --config ../conf/{task_id}.toml '
+                        '--changefeed-id={task_id} '.format(task_id=task_id))
+                else:
+                    result = backResult.json()
+                    if result['error'] is not None:
+                        requests.delete(url='http://{}:{}{}/{}'.format(self.cdc_url, self.cdc_port,
+                                                                       self.cdc_changefeeds_url, task_id))
+                        os.system(
+                            'cd /data/tidb-deploy/cdc-8300/bin; ./cdc cli changefeed create '
+                            '--pd=http://192.168.0.184:2379 '
+                            '--sink-uri="mysql://data:hcst@2021@139.186.27.96:3390" --config ../conf/{task_id}.toml '
+                            '--changefeed-id={task_id} '.format(task_id=task_id))
+                    else:
+                        logging.info('创新创建没有问题')
+            else:
+                logging.info("无异常")
 
 
 if __name__ == '__main__':