renyupeng 3 anos atrás
pai
commit
e8fcd601ea

+ 2 - 1
Apietl/bytedance_etl/Q4Task/AppBytedanceAdReportBusi.py

@@ -23,7 +23,7 @@ class AppBytedanceAdReportBusi:
         try:
             sql_porject = """select project_id from `jeecg-boot`.ctop_user_allocation where media_id=1
 group by project_id """
-            project_list = TidbConn.quary(sql_porject)
+            project_list = TidbConn.quary(self.conn, sql_porject)
             for project_id in project_list:
                 try:
                     sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
@@ -573,6 +573,7 @@ group by project_id """
                     ) bei ON report.account_id = bei.account_id
                     where bei.project_id={project_id}""".format(date_time=self.date_time,
                                                                 day_count=self.day_count, project_id=project_id[0])
+                    print(sql)
                     TidbConn.upsert(conn=self.conn, sql_buff=sql)
                     self.conn.commit()
                 except Exception as e:

+ 3 - 2
Apietl/bytedance_etl/Q4Task/AppBytedanceCompanyUserReportBusi.py

@@ -566,15 +566,16 @@ pa.org_type,
 pa.id dep_id,
 pa.depart_name,
 cua.user_id,
-su.realname,
+su.realname ,
 cp.sale_id,
 su.leader_id,
-su.leader_name
+se.realname as leader_name
 FROM
 `jeecg-boot`.ctop_user_allocation cua
 LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
 LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
 LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user se ON se.id = su.leader_id
 LEFT JOIN `jeecg-boot`.sys_user_depart de ON su.leader_id = de.user_id
 LEFT JOIN `jeecg-boot`.sys_depart pa ON pa.id = de.dep_id
 ) dep ON report.advertiser_id = dep.account_id

+ 1 - 1
Apietl/bytedance_etl/Q4Task/AppBytedanceImageReportBusi.py

@@ -23,7 +23,7 @@ class AppBytedanceImageReportBusi:
         try:
             sql_porject = """select project_id from `jeecg-boot`.ctop_user_allocation where media_id=1
 group by project_id """
-            project_list = TidbConn.quary(sql_porject)
+            project_list = TidbConn.quary(self.conn,sql_porject)
             for project_id in project_list:
                 try:
                     sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';

+ 1 - 1
Apietl/bytedance_etl/Q4Task/AppBytedanceVideoReportBusi.py

@@ -23,7 +23,7 @@ class AppBytedanceVideoReportBusi:
         try:
             sql_porject = """select project_id from `jeecg-boot`.ctop_user_allocation where media_id=1
 group by project_id """
-            project_list = TidbConn.quary(sql_porject)
+            project_list = TidbConn.quary(self.conn,sql_porject)
             for project_id in project_list:
                 try:
                     sql = """SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';

+ 1 - 1
Apietl/bytedance_etl/report/AdDailyReport.py

@@ -24,7 +24,7 @@ class AdDailyReport:
             sql = """select count(1) from media_api.bytedance_ad_report_daily where 
             stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 1 - 1
Apietl/bytedance_etl/report/AdHourlyReport.py

@@ -23,7 +23,7 @@ class AdHourlyReport:
             sql = """select advertiser_id from media_api.bytedance_ad_report_hourly where 
             stat_datetime between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn,sql)
             for account_id in account_list:
                 sql = """
 replace into application.bytedance_ad_report_hourly_dw(account_id, campaign_id, campaign_name, ad_id, ad_name,

+ 1 - 1
Apietl/bytedance_etl/report/AdvertiserDailyReport.py

@@ -26,7 +26,7 @@ class AdvertiserDailyReport:
             sql = """select count(1) from media_api.bytedance_advertiser_report_daily where 
             stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 1 - 1
Apietl/bytedance_etl/report/AdvertiserHourlyReport.py

@@ -26,7 +26,7 @@ class AdvertiserHourlyReport:
             sql = """select count(1) from media_api.bytedance_advertiser_report_hourly where 
             stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 1 - 1
Apietl/bytedance_etl/report/CampaignDailyReport.py

@@ -24,7 +24,7 @@ class CampaignDailyReport:
             sql = """select count(1) from media_api.bytedance_campaign_report_daily where 
             stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 1 - 1
Apietl/bytedance_etl/report/CampaignHourlyReport.py

@@ -24,7 +24,7 @@ class CampaignHourlyReport:
             sql = """select count(1) from media_api.bytedance_campaign_report_hourly where 
             stat_datetime between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 1 - 1
Apietl/bytedance_etl/report/CreativeDailyReport.py

@@ -23,7 +23,7 @@ class CreativeDailyReport:
             sql = """select advertiser_id from media_api.bytedance_creative_report_daily where 
             stat_datetime between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 sql = """
 replace into application.bytedance_creative_report_daily_dw(account_id, campaign_id, campaign_name, ad_id, ad_name,

+ 1 - 1
Apietl/bytedance_etl/report/MaterialImageDailyReport.py

@@ -31,7 +31,7 @@ class MateirialImageDailyReport:
        'CREATIVE_IMAGE_MODE_UNION_SPLASH') 
            group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 sql = """
 replace into application.bytedance_material_image_report_daily_dw(account_id, material_id, image_mode, inventory,

+ 1 - 1
Apietl/bytedance_etl/report/MaterialVideoDailyReport.py

@@ -25,7 +25,7 @@ class MateirialVideoDailyReport:
          and image_mode in ('CREATIVE_IMAGE_MODE_VIDEO', 'CREATIVE_IMAGE_MODE_VIDEO_VERTICAL', 'CREATIVE_IMAGE_MODE_UNION_SPLASH_VIDEO')
          group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
 
                 try:

+ 15 - 12
Apietl/conn/TidbConn.py

@@ -28,7 +28,6 @@ class TidbConn:
                                passwd=ConfConstant.TIDB_PASSWORD,
                                port=ConfConstant.TIDB_PORT, charset="utf8")
 
-
     @staticmethod
     def get_pro_connection():
         """
@@ -39,7 +38,6 @@ class TidbConn:
                                passwd=ConfConstant.TIDB_PRO_PASSWORD,
                                port=ConfConstant.TIDB_PRO_PORT, charset="utf8")
 
-
     @classmethod
     def _reConn(cls, num=28800, stime=5):  # 重试连接总次数为1天,这里根据实际情况自己设置,如果服务器宕机1天都没发现就......
         _number = 0
@@ -56,10 +54,10 @@ class TidbConn:
                 time.sleep(stime)  # 连接不成功,休眠3秒钟,继续循环,知道成功或重试次数结束
 
     @classmethod
-    def quary(cls, sql):
-        mysql_conn = TidbConn.get_connection()
-        TidbConn._reCon()
-        cur = mysql_conn.cursor()
+    def quary(cls, conn, sql):
+        # mysql_conn = TidbConn.get_connection()
+        TidbConn._reCon(conn)
+        cur = conn.cursor()
         cur.execute(sql)
         result = cur.fetchall()
         return result
@@ -86,23 +84,28 @@ class TidbConn:
 
     @staticmethod
     def upsert(conn, sql_buff):
-        TidbConn._reCon()
+        while True:
+            try:
+                conn.ping()
+                break
+            except OperationalError:
+                conn.ping(True)
         try:
             TidbConn.exec_sql(conn, sql_buff)
         except Exception as e:
             logging.error('失败详细信息:', repr(e))
-        TidbConn.get_connection().commit()
+        conn.commit()
 
-    @classmethod
-    def _reCon(cls):
+    @staticmethod
+    def _reCon(conn):
         """ MySQLdb.OperationalError异常"""
         # self.con.close()
         while True:
             try:
-                TidbConn.get_connection().ping()
+                conn.ping()
                 break
             except OperationalError:
-                TidbConn.get_connection().ping(True)
+                conn.ping(True)
 
 
 if __name__ == '__main__':

+ 1 - 1
Apietl/kwai_etl/Q4Task/AppKuaishouVideoList.py

@@ -22,7 +22,7 @@ class AppKuaishouVideoList:
         try:
             sql = """select account_id from media_api.kuaishou_video_list where  date_format(stat_date, '%Y-%m-%d') >= date_sub('{date_time}', INTERVAL 13 DAY)
               group by account_id """.format(date_time=self.date_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 try:
                     sql = """

+ 1 - 1
Apietl/kwai_etl/Q4Task/AppKuaishouVideoListAll.py

@@ -18,7 +18,7 @@ class AppKuaishouVideoListAll:
     def execSql(self):
         try:
             sql = """select account_id from media_api.kuaishou_video_list  group by account_id """
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 try:
                     sql = """

+ 1 - 1
Apietl/kwai_etl/report/AccountDailyReport.py

@@ -28,7 +28,7 @@ class AccountDailyReport:
             sql = """select count(1) from media_api.kuaishou_account_report_daily where 
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 2 - 2
Apietl/kwai_etl/report/AccountHourlyReport.py

@@ -12,7 +12,7 @@ from math import ceil
 from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class AccountHourlyReport:
@@ -28,7 +28,7 @@ class AccountHourlyReport:
             sql = """select count(1) from media_api.kuaishou_account_report_hourly where 
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 2 - 2
Apietl/kwai_etl/report/CampaignDailyReport.py

@@ -12,7 +12,7 @@ from math import ceil
 from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class CampaignDailyReport:
@@ -28,7 +28,7 @@ class CampaignDailyReport:
             sql = """select count(1) from media_api.kuaishou_campaign_report_daily where 
             stat_date between {start_time} and {end_time}""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            totalNum = TidbConn.quary(sql)[0][0]
+            totalNum = TidbConn.quary(self.conn, sql)[0][0]
             pageCount = ceil(totalNum / 20000)
             for i in range(pageCount):
                 sql = """

+ 2 - 2
Apietl/kwai_etl/report/CampaignHourlyReport.py

@@ -12,7 +12,7 @@ from math import ceil
 from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class CampaignHourlyReport:
@@ -28,7 +28,7 @@ class CampaignHourlyReport:
             sql = """select advertiser_id from media_api.kuaishou_campaign_report_hourly where 
             stat_date between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 sql = """replace into application.kuaishou_campaign_report_hourly_dw
 (account_id, stat_date,`hour`, campaign_id, campaign_name, project_id, product_id, advertiser_id,

+ 2 - 2
Apietl/kwai_etl/report/MateirialVideoDailyReport.py

@@ -12,7 +12,7 @@ from math import ceil
 from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class MateirialVideoDailyReport:
@@ -27,7 +27,7 @@ class MateirialVideoDailyReport:
             sql = """select advertiser_id from media_api.kuaishou_material_video_report_daily where 
             stat_date between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 sql = """replace into application.kuaishou_material_video_report_daily_dw
 (account_id, project_id, product_id, advertiser_id,

+ 1 - 1
Apietl/kwai_etl/report/MateirialVideoHourlyReport.py

@@ -27,7 +27,7 @@ class MateirialVideoHourlyReport:
             sql = """select advertiser_id from media_api.kuaishou_material_video_report_hourly where 
             stat_date between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 sql = """replace into application.kuaishou_material_video_report_hourly_dw
 (account_id, project_id, product_id, advertiser_id,

+ 2 - 2
Apietl/kwai_etl/report/UnitDailyReport.py

@@ -12,7 +12,7 @@ from math import ceil
 from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
 from Apietl.utlis.Logger import Logger
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class UnitDailyReport:
@@ -29,7 +29,7 @@ class UnitDailyReport:
             select advertiser_id from media_api.kuaishou_unit_report_daily where 
             stat_date between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 sql = """
                 replace into application.kuaishou_unit_report_daily_dw

+ 2 - 2
Apietl/kwai_etl/report/UnitHourlyReport.py

@@ -11,7 +11,7 @@ from math import ceil
 
 from Apietl.kwai_etl.report.ReplaceAccountInfo import ReplaceAccountInfo
 from Apietl.utlis.Utils import Utils
-from TidbConn import TidbConn
+from Apietl.conn.TidbConn import TidbConn
 
 
 class UnitHourlyReport:
@@ -27,7 +27,7 @@ class UnitHourlyReport:
             sql = """select advertiser_id from media_api.kuaishou_unit_report_hourly where 
             stat_date between {start_time} and {end_time} group by advertiser_id""".format(
                 start_time=self.start_time, end_time=self.end_time)
-            account_list = TidbConn.quary(sql)
+            account_list = TidbConn.quary(self.conn, sql)
             for account_id in account_list:
                 sql = """replace into application.kuaishou_unit_report_hourly_dw
 (account_id, stat_date,`hour`, campaign_id, campaign_name, unit_id, unit_name, project_id, product_id, advertiser_id,