renyupeng 2 年之前
父节点
当前提交
8a26df7869
共有 2 个文件被更改,包括 14 次插入19 次删除
  1. 5 12
      Apietl/kwai_etl/report/AccountDailyReport.py
  2. 9 7
      Apietl/kwai_etl/report/AccountHourlyReport.py

+ 5 - 12
Apietl/kwai_etl/report/AccountDailyReport.py

@@ -25,13 +25,7 @@ class AccountDailyReport:
     def daily_report(self):
         ReplaceAccountInfo.handler()
         try:
-            sql = """select count(1) from media_api.kuaishou_account_report_daily where 
-            stat_date between {start_time} and {end_time}""".format(
-                start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(self.conn, sql)[0][0]
-            pageCount = ceil(totalNum / 20000)
-            for i in range(pageCount):
-                sql = """
+            sql = """
             replace into application.kuaishou_account_report_daily_dw
 (account_id, stat_date, project_id, product_id, advertiser_id, company_id, account_name, project_name, advertiser_name,
  company_name, sale_id, user_id, org_code, charge, `show`, photo_click, aclick, bclick, photo_click_ratio,
@@ -328,11 +322,10 @@ class AccountDailyReport:
              `jeecg-boot`.ctop_account_attribution_info sys
              on report.advertiser_id = sys.account_id
         where report.stat_date between {start_time} and {end_time}
-        order by account_id,stat_date limit {num},20000
-        """.format(start_time=self.start_time, end_time=self.end_time,num=i * 20000)
-                print(sql)
-                TidbConn.upsert(conn=self.conn, sql_buff=sql)
-                self.conn.commit()
+        """.format(start_time=self.start_time, end_time=self.end_time)
+            print(sql)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
             TidbConn.conn_close(conn=self.conn)
             return 0
         except Exception as e:

+ 9 - 7
Apietl/kwai_etl/report/AccountHourlyReport.py

@@ -25,12 +25,14 @@ class AccountHourlyReport:
     def daily_report(self):
         ReplaceAccountInfo.handler()
         try:
-            sql = """select count(1) from media_api.kuaishou_account_report_hourly where 
-            stat_date between {start_time} and {end_time}""".format(
+            sql = """select advertiser_id from media_api.kuaishou_account_report_hourly where 
+            stat_date between {start_time} and {end_time}
+            group by advertiser_id
+            """.format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(self.conn, sql)[0][0]
-            pageCount = ceil(totalNum / 20000)
-            for i in range(pageCount):
+            adv_list = TidbConn.quary(self.conn, sql)
+
+            for account_id in adv_list:
                 sql = """
             replace into application.kuaishou_account_report_hourly_dw
 (account_id, stat_date, `hour`, project_id, product_id, advertiser_id, company_id, account_name, project_name, advertiser_name,
@@ -427,8 +429,8 @@ class AccountHourlyReport:
              `jeecg-boot`.ctop_account_attribution_info sys
              on report.advertiser_id = sys.account_id
         where report.stat_date between {start_time} and {end_time}
-               order by account_id,hour,stat_date limit {num},20000
-        """.format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
+    and  report.advertiser_id = {account_id}
+        """.format(start_time=self.start_time, end_time=self.end_time, account_id=account_id[0])
                 print(sql)
                 TidbConn.upsert(conn=self.conn, sql_buff=sql)
                 self.conn.commit()