瀏覽代碼

"etl 脚本"

renyupeng 3 年之前
父節點
當前提交
78dba7bd2b

+ 3 - 0
Apietl/config.ini

@@ -10,6 +10,9 @@ tidb_password = hcst@2021
 [webhook]
 [webhook]
 url = 192.168.0.202
 url = 192.168.0.202
 
 
+[Log]
+Log_home=/data/etl_logs
+
 ;[Log]
 ;[Log]
 ;log_dir = /opt/log/
 ;log_dir = /opt/log/
 ;log_level =INFO
 ;log_level =INFO

+ 1 - 1
Apietl/constant/ConfConstant.py

@@ -31,7 +31,7 @@ class ConfConstant:
 
 
     # ### log
     # ### log
     # # 日志存放路径
     # # 日志存放路径
-    # LOG_DIR = _cfp.get('log', 'log_dir')
+    Log_home = _cfp.get('log', 'Log_home')
     # # 日志等级
     # # 日志等级
     # LOG_LEVEL = _cfp.get('log', 'log_level')
     # LOG_LEVEL = _cfp.get('log', 'log_level')
 
 

+ 9 - 5
Apietl/kwai_etl/report/CampaignDailyReport.py

@@ -8,6 +8,8 @@
 
 
 import logging
 import logging
 from math import ceil
 from math import ceil
+
+from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
 from Apietl.utlis.Utils import Utils
 from TidbConn import TidbConn
 from TidbConn import TidbConn
 
 
@@ -25,7 +27,7 @@ class CampaignDailyReport:
             stat_date between {start_time} and {end_time}""".format(
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             totalNum = TidbConn.quary(sql)[0][0]
-            pageCount = ceil(totalNum / 5000)
+            pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
             for i in range(pageCount):
                 sql = """
                 sql = """
                 set @@tidb_mem_quota_query = 8 << 30;
                 set @@tidb_mem_quota_query = 8 << 30;
@@ -205,8 +207,8 @@ class CampaignDailyReport:
                  left join
                  left join
              `jeecg-boot`.ctop_account_attribution_info sys
              `jeecg-boot`.ctop_account_attribution_info sys
              on report.advertiser_id = sys.account_id
              on report.advertiser_id = sys.account_id
-        where report.stat_date between {start_time} and {end_time} order by campaign_id,stat_date limit {num},5000""". \
-                    format(start_time=self.start_time, end_time=self.end_time, num=i * 5000)
+        where report.stat_date between {start_time} and {end_time} order by campaign_id,stat_date limit {num},20000""". \
+                    format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 TidbConn.exec_sql(self.conn, sql)
                 TidbConn.exec_sql(self.conn, sql)
                 self.conn.commit()
                 self.conn.commit()
             return 0
             return 0
@@ -217,8 +219,10 @@ class CampaignDailyReport:
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
     status = CampaignDailyReport().daily_report()
     status = CampaignDailyReport().daily_report()
+    media_name = 'kwai'
+    trace_log = Logger().etl_logger(media_name, CampaignDailyReport().start_time)
 
 
     if status == 0:
     if status == 0:
-        logging.info("作业执行成功")
+        trace_log.info("作业执行成功")
     else:
     else:
-        logging.info("作业执行失败")
+        trace_log.error("作业执行失败")

+ 9 - 5
Apietl/kwai_etl/report/CampaignHourlyReport.py

@@ -8,6 +8,8 @@
 
 
 import logging
 import logging
 from math import ceil
 from math import ceil
+
+from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
 from Apietl.utlis.Utils import Utils
 from TidbConn import TidbConn
 from TidbConn import TidbConn
 
 
@@ -25,7 +27,7 @@ class CampaignHourlyReport:
             stat_date between {start_time} and {end_time}""".format(
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             totalNum = TidbConn.quary(sql)[0][0]
-            pageCount = ceil(totalNum / 5000)
+            pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
             for i in range(pageCount):
                 sql = """replace into application.kuaishou_campaign_report_hourly_dw
                 sql = """replace into application.kuaishou_campaign_report_hourly_dw
 (account_id, stat_date,`hour`, campaign_id, campaign_name, project_id, product_id, advertiser_id,
 (account_id, stat_date,`hour`, campaign_id, campaign_name, project_id, product_id, advertiser_id,
@@ -204,8 +206,8 @@ class CampaignHourlyReport:
                  left join
                  left join
              `jeecg-boot`.ctop_account_attribution_info sys
              `jeecg-boot`.ctop_account_attribution_info sys
              on report.advertiser_id = sys.account_id
              on report.advertiser_id = sys.account_id
-        where report.stat_date between {start_time} and {end_time} order by campaign_id,stat_date,`hour` limit {num},5000""". \
-                    format(start_time=self.start_time, end_time=self.end_time, num=i * 5000)
+        where report.stat_date between {start_time} and {end_time} order by campaign_id,stat_date,`hour` limit {num},20000""". \
+                    format(start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 TidbConn.exec_sql(self.conn, sql)
                 TidbConn.exec_sql(self.conn, sql)
                 self.conn.commit()
                 self.conn.commit()
             return 0
             return 0
@@ -216,7 +218,9 @@ class CampaignHourlyReport:
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
     status = CampaignHourlyReport().hourly_report()
     status = CampaignHourlyReport().hourly_report()
+    media_name = 'kwai'
+    trace_log = Logger().etl_logger(media_name, CampaignHourlyReport().start_time)
     if status == 0:
     if status == 0:
-        logging.info("作业执行成功")
+        trace_log.info("作业执行成功")
     else:
     else:
-        logging.info("作业执行失败")
+        trace_log.error("作业执行失败")

+ 9 - 5
Apietl/kwai_etl/report/MateirialVideoDailyReport.py

@@ -8,6 +8,8 @@
 
 
 import logging
 import logging
 from math import ceil
 from math import ceil
+
+from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
 from Apietl.utlis.Utils import Utils
 from TidbConn import TidbConn
 from TidbConn import TidbConn
 
 
@@ -25,7 +27,7 @@ class MateirialVideoDailyReport:
             stat_date between {start_time} and {end_time}""".format(
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             totalNum = TidbConn.quary(sql)[0][0]
-            pageCount = ceil(totalNum / 5000)
+            pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
             for i in range(pageCount):
                 sql = """replace into application.kuaishou_material_video_report_daily_dw
                 sql = """replace into application.kuaishou_material_video_report_daily_dw
 (account_id, project_id, product_id, advertiser_id,
 (account_id, project_id, product_id, advertiser_id,
@@ -270,8 +272,8 @@ from media_api.kuaishou_material_video_report_daily report
      `jeecg-boot`.ctop_material_attribution_info sys
      `jeecg-boot`.ctop_material_attribution_info sys
      on report.advertiser_id = sys.account_id
      on report.advertiser_id = sys.account_id
  where report.stat_date between {start_time} and {end_time} and  report.photo_id!=0
  where report.stat_date between {start_time} and {end_time} and  report.photo_id!=0
-  order by advertiser_id,photo_id ,stat_date limit {num},5000""".format(
-                    start_time=self.start_time, end_time=self.end_time, num=i * 5000)
+  order by advertiser_id,photo_id ,stat_date limit {num},20000""".format(
+                    start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 TidbConn.exec_sql(self.conn, sql)
                 TidbConn.exec_sql(self.conn, sql)
                 self.conn.commit()
                 self.conn.commit()
             return 0
             return 0
@@ -282,7 +284,9 @@ from media_api.kuaishou_material_video_report_daily report
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
     status = MateirialVideoDailyReport().daily_report()
     status = MateirialVideoDailyReport().daily_report()
+    media_name = 'kwai'
+    trace_log = Logger().etl_logger(media_name, MateirialVideoDailyReport().start_time)
     if status == 0:
     if status == 0:
-        logging.info("作业执行成功")
+        trace_log.info("作业执行成功")
     else:
     else:
-        logging.info("作业执行失败")
+        trace_log.error("作业执行失败")

+ 9 - 6
Apietl/kwai_etl/report/MateirialVideoHourlyReport.py

@@ -8,6 +8,8 @@
 
 
 import logging
 import logging
 from math import ceil
 from math import ceil
+
+from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
 from Apietl.utlis.Utils import Utils
 from TidbConn import TidbConn
 from TidbConn import TidbConn
 
 
@@ -25,7 +27,7 @@ class MateirialVideoHourlyReport:
             stat_date between {start_time} and {end_time}""".format(
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             totalNum = TidbConn.quary(sql)[0][0]
-            pageCount = ceil(totalNum / 5000)
+            pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
             for i in range(pageCount):
                 sql = """replace into application.kuaishou_material_video_report_daily_dw
                 sql = """replace into application.kuaishou_material_video_report_daily_dw
 (account_id, project_id, product_id, advertiser_id,
 (account_id, project_id, product_id, advertiser_id,
@@ -270,8 +272,8 @@ from media_api.kuaishou_material_video_report_hourly report
      `jeecg-boot`.ctop_material_attribution_info sys
      `jeecg-boot`.ctop_material_attribution_info sys
      on report.advertiser_id = sys.account_id
      on report.advertiser_id = sys.account_id
  where report.stat_date between {start_time} and {end_time} and  report.photo_id!=0
  where report.stat_date between {start_time} and {end_time} and  report.photo_id!=0
-  order by advertiser_id,photo_id ,stat_date,`hour` limit {num},5000""".format(
-                    start_time=self.start_time, end_time=self.end_time, num=i * 5000)
+  order by advertiser_id,photo_id ,stat_date,`hour` limit {num},20000""".format(
+                    start_time=self.start_time, end_time=self.end_time, num=i * 20000)
                 TidbConn.exec_sql(self.conn, sql)
                 TidbConn.exec_sql(self.conn, sql)
                 self.conn.commit()
                 self.conn.commit()
             return 0
             return 0
@@ -282,8 +284,9 @@ from media_api.kuaishou_material_video_report_hourly report
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
     status = MateirialVideoHourlyReport().hourly_report()
     status = MateirialVideoHourlyReport().hourly_report()
-
+    media_name = 'kwai'
+    trace_log = Logger().etl_logger(media_name, MateirialVideoHourlyReport().start_time)
     if status == 0:
     if status == 0:
-        logging.info("作业执行成功")
+        trace_log.info("作业执行成功")
     else:
     else:
-        logging.info("作业执行失败")
+        trace_log.error("作业执行失败")

+ 6 - 2
Apietl/kwai_etl/report/UnitDailyReport.py

@@ -8,6 +8,8 @@
 
 
 import logging
 import logging
 from math import ceil
 from math import ceil
+
+from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
 from Apietl.utlis.Utils import Utils
 from TidbConn import TidbConn
 from TidbConn import TidbConn
 
 
@@ -219,7 +221,9 @@ class UnitDailyReport:
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
     status = UnitDailyReport().daily_report()
     status = UnitDailyReport().daily_report()
+    media_name = 'kwai'
+    trace_log = Logger().etl_logger(media_name, UnitDailyReport().start_time)
     if status == 0:
     if status == 0:
-        logging.info("作业执行成功")
+        trace_log.info("作业执行成功")
     else:
     else:
-        logging.info("作业执行失败")
+        trace_log.error("作业执行失败")

+ 9 - 5
Apietl/kwai_etl/report/UnitHourlyReport.py

@@ -11,6 +11,8 @@ from math import ceil
 
 
 import pandas as pd
 import pandas as pd
 from sqlalchemy import create_engine
 from sqlalchemy import create_engine
+
+from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
 from Apietl.utlis.Utils import Utils
 from TidbConn import TidbConn
 from TidbConn import TidbConn
 
 
@@ -28,7 +30,7 @@ class UnitHourlyReport:
             stat_date between {start_time} and {end_time}""".format(
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
                 start_time=self.start_time, end_time=self.end_time)
             totalNum = TidbConn.quary(sql)[0][0]
             totalNum = TidbConn.quary(sql)[0][0]
-            pageCount = ceil(totalNum / 3000)
+            pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
             for i in range(pageCount):
                 sql = """replace into application.kuaishou_unit_report_hourly_dw
                 sql = """replace into application.kuaishou_unit_report_hourly_dw
 (account_id, stat_date,`hour`, campaign_id, campaign_name, unit_id, unit_name, project_id, product_id, advertiser_id,
 (account_id, stat_date,`hour`, campaign_id, campaign_name, unit_id, unit_name, project_id, product_id, advertiser_id,
@@ -209,8 +211,8 @@ class UnitHourlyReport:
                  left join
                  left join
              `jeecg-boot`.ctop_account_attribution_info sys
              `jeecg-boot`.ctop_account_attribution_info sys
              on report.advertiser_id = sys.account_id
              on report.advertiser_id = sys.account_id
-        where report.stat_date between {start_time} and {end_time} order by unit_id,hour,stat_date limit {num},3000""".\
-                    format(start_time=self.start_time, end_time=self.end_time,num=i * 3000)
+        where report.stat_date between {start_time} and {end_time} order by unit_id,hour,stat_date limit {num},20000""".\
+                    format(start_time=self.start_time, end_time=self.end_time,num=i * 20000)
                 TidbConn.exec_sql(self.conn, sql)
                 TidbConn.exec_sql(self.conn, sql)
                 self.conn.commit()
                 self.conn.commit()
             return 0
             return 0
@@ -221,7 +223,9 @@ class UnitHourlyReport:
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
     status = UnitHourlyReport().hourly_report()
     status = UnitHourlyReport().hourly_report()
+    media_name = 'kwai'
+    trace_log = Logger().etl_logger(media_name, UnitHourlyReport().start_time)
     if status == 0:
     if status == 0:
-        logging.info("作业执行成功")
+        trace_log.info("作业执行成功")
     else:
     else:
-        logging.info("作业执行失败")
+        trace_log.error("作业执行失败")