renyupeng 4 年之前
父节点
当前提交
183f12cf25
共有 1 个文件被更改,包括 9 次插入10 次删除
  1. 9 10
      Apietl/kwai_etl/report/MateirialVideoHourlyReport.py

+ 9 - 10
Apietl/kwai_etl/report/MateirialVideoHourlyReport.py

@@ -11,7 +11,7 @@ from math import ceil
 
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class MateirialVideoHourlyReport:
@@ -23,12 +23,11 @@ class MateirialVideoHourlyReport:
 
     def hourly_report(self):
         try:
-            sql = """select count(1) from media_api.kuaishou_material_video_report_hourly where 
-            stat_date between {start_time} and {end_time}""".format(
+            sql = """select advertiser_id from media_api.kuaishou_material_video_report_hourly where 
+            stat_date between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
-            pageCount = ceil(totalNum / 10000)
-            for i in range(pageCount):
+            account_list = TidbConn.quary(sql)
+            for account_id in account_list:
                 sql = """replace into application.kuaishou_material_video_report_hourly_dw
 (account_id, project_id, product_id, advertiser_id,
  company_id, account_name, project_name,
@@ -115,7 +114,7 @@ class MateirialVideoHourlyReport:
        video_name,
        channel_name,
        clip_company_id)
-select /*+ read_from_storage(tiflash[media_api.kuaishou_material_video_report_hourly]) */
+select 
        report.advertiser_id                                                                     as account_id,
        cp.id                                                                                    as project_id,
        product.id                                                                               as product_id,
@@ -276,7 +275,6 @@ select /*+ read_from_storage(tiflash[media_api.kuaishou_material_video_report_ho
        case when video.channel_type = 1 then '素造' else '内部' end as channel_name,
        uc.company_id
 from media_api.kuaishou_material_video_report_hourly report
-        /*+ read_from_storage(tiflash[media_api.kuaishou_video_list]) */
          inner join media_api.kuaishou_video_list video
                     on report.advertiser_id = video.account_id and report.photo_id = video.photo_id
          left join `jeecg-boot`.ctop_material_info cmi on video.signature = cmi.code
@@ -292,8 +290,9 @@ from media_api.kuaishou_material_video_report_hourly report
          left join `jeecg-boot`.user_company uc on uc.user_id = clip.id
          left join `jeecg-boot`.ctop_product product on product.id = cp.product_id
  where report.stat_date between {start_time} and {end_time} and  report.photo_id!=0
-  order by advertiser_id,photo_id ,stat_date,`hour` limit {num},10000""".format(
-                    start_time=self.start_time, end_time=self.end_time, num=i * 10000)
+ and report.advertiser_id={account_id}
+""".format(
+                    start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)
                 self.conn.commit()