renyupeng 3 роки тому
батько
коміт
3adfcbc0a6
1 змінених файлів з 7 додано та 4 видалено
  1. 7 4
      Apietl/kwai_etl/report/UnitDailyReport.py

+ 7 - 4
Apietl/kwai_etl/report/UnitDailyReport.py

@@ -21,13 +21,16 @@ class UnitDailyReport:
 
     def daily_report(self):
         try:
-            sql = """select count(1) from media_api.kuaishou_unit_report_daily where 
+            sql = """
+            select count(1) from media_api.kuaishou_unit_report_daily where 
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
-            pageCount = ceil(totalNum / 3000)
+            pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
-                sql = """replace into application.kuaishou_unit_report_daily_dw
+                sql = """
+                set @@tidb_mem_quota_query = 8 << 30;
+                replace into application.kuaishou_unit_report_daily_dw
 (account_id, stat_date, campaign_id, campaign_name, unit_id, unit_name, project_id, product_id, advertiser_id,
  company_id, account_name, project_name, advertiser_name, company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio, play_3s_ratio, action_ratio,
  impression_1k_cost, photo_click_cost, action_cost, share, comment, `like`, follow, cancel_follow, report, block,
@@ -206,7 +209,7 @@ class UnitDailyReport:
              `jeecg-boot`.ctop_account_attribution_info sys
              on report.advertiser_id = sys.account_id
         where report.stat_date between {start_time} and {end_time} order by unit_id,stat_date limit {num},3000""".\
-                    format(start_time=self.start_time, end_time=self.end_time, num=i * 3000)
+                    format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 TidbConn.exec_sql(self.conn, sql)
                 self.conn.commit()
             return 0