MonitorAppTask.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156
  1. # Author renyupeng
  2. # coding=utf-8
  3. # @Time : 2021/11/17 6:48 下午
  4. # @Site :
  5. # @File : MonitorAppTask.py
  6. # @Software: PyCharm
  7. # @contact: renyupeng@c-top.com.cn
  8. # @Tel 1501435553
  9. import datetime
  10. import logging
  11. import os
  12. from Apietl.conn.TidbConn import TidbConn
  13. from Apietl.conn.TidbProConn import TidbProConn
  14. from OuterDataCompareInnerData.Utils.SchedulerList import SchedulerList
  15. from OuterDataCompareInnerData.Utils.sendMessage import feishuMsg
  16. class MonitorAppTask:
  17. def __init__(self):
  18. self.InnerConn = TidbConn.get_connection()
  19. self.OuterConn = TidbProConn.get_connection()
  20. self.table_list = [
  21. 'bytedance_ad_report_daily_dw'
  22. , 'bytedance_advertiser_report_daily_dw'
  23. , 'bytedance_campaign_report_daily_dw'
  24. , 'bytedance_creative_report_daily_dw'
  25. , 'bytedance_material_image_report_daily_dw'
  26. , 'bytedance_material_video_report_daily_dw'
  27. , 'kuaishou_account_report_daily_dw'
  28. , 'kuaishou_campaign_report_daily_dw'
  29. , 'kuaishou_material_video_report_daily_dw'
  30. , 'kuaishou_unit_report_daily_dw'
  31. ]
  32. self.stat_date = (datetime.date.today() + datetime.timedelta(-1)).strftime("%Y%m%d")
  33. def CheckTask(self):
  34. try:
  35. for table in self.table_list:
  36. try:
  37. if 'app' in table:
  38. sql = """select count(1) from `application`.{table}""".format(table=table)
  39. InnertotalNum = TidbConn.quary(self.InnerConn, sql)[0][0]
  40. OutertotalNum = TidbConn.quary(self.OuterConn, sql)[0][0]
  41. print("""{stat_date}日{table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum}""".format(
  42. stat_date=self.stat_date,
  43. table=table,
  44. InnertotalNum=InnertotalNum,
  45. OutertotalNum=OutertotalNum))
  46. elif 'kuaishou' in table:
  47. sql = """select count(1) from `application`.{table} where stat_date={stat_date}""".format(
  48. table=table, stat_date=self.stat_date)
  49. InnertotalNum = TidbConn.quary(self.InnerConn, sql)[0][0]
  50. OutertotalNum = TidbConn.quary(self.OuterConn, sql)[0][0]
  51. print("""{stat_date}日 {table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum}""".format(
  52. stat_date=self.stat_date,
  53. table=table,
  54. InnertotalNum=InnertotalNum,
  55. OutertotalNum=OutertotalNum))
  56. if InnertotalNum == 0:
  57. feishuMsg.send_robot_msg(
  58. """{stat_date}日 {table} 数据为0""".format(stat_date=self.stat_date, table=table))
  59. elif InnertotalNum != OutertotalNum:
  60. if table == 'kuaishou_material_video_report_daily_dw':
  61. sql = """select account_id,photo_id from
  62. application.{table} where stat_date={stat_date} """.format(
  63. table=table, stat_date=self.stat_date)
  64. InnertotalItem = TidbConn.quary(self.InnerConn, sql)
  65. OutertotalItem = TidbConn.quary(self.OuterConn, sql)
  66. set1 = set(InnertotalItem)
  67. set2 = set(OutertotalItem)
  68. diff_set = set1 - set2
  69. account_list = []
  70. for diff in diff_set:
  71. account_id = diff[0]
  72. account_list.append(account_id)
  73. acc_list = tuple(account_list)
  74. acclist = ','.join(str(i) for i in acc_list)
  75. feishuMsg.send_robot_msg(
  76. """{stat_date}日 {table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum} 数据不相等 datax
  77. 数据同步启动""".format(
  78. stat_date=self.stat_date,
  79. table=table,
  80. InnertotalNum=InnertotalNum,
  81. OutertotalNum=OutertotalNum))
  82. datax_json = SchedulerList.SchedulerList(table)
  83. os.system(
  84. 'sh /data/datax/application/push_pro1.sh {stat_date} {table} {acclist} {datax_json}'.format(
  85. stat_date=self.stat_date, table=table, acclist=acclist, datax_json=datax_json))
  86. else:
  87. feishuMsg.send_robot_msg(
  88. """{stat_date}日 {table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum} 数据不相等 datax
  89. 数据同步启动""".format(
  90. stat_date=self.stat_date,
  91. table=table,
  92. InnertotalNum=InnertotalNum,
  93. OutertotalNum=OutertotalNum))
  94. datax_json = SchedulerList.SchedulerList(table)
  95. os.system(
  96. 'sh /data/datax/application/push_pro.sh {stat_date} {table} {datax_json}'.format(
  97. stat_date=self.stat_date, table=table, datax_json=datax_json))
  98. else:
  99. feishuMsg.send_robot_msg(
  100. """{stat_date}日 {table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum} 数据相等 """.format(
  101. stat_date=self.stat_date,
  102. table=table,
  103. InnertotalNum=InnertotalNum,
  104. OutertotalNum=OutertotalNum))
  105. else:
  106. sql = """select count(1) from `application`.{table} where stat_datetime={stat_date}""".format(
  107. table=table, stat_date=self.stat_date)
  108. InnertotalNum = TidbConn.quary(self.InnerConn, sql)[0][0]
  109. OutertotalNum = TidbConn.quary(self.OuterConn, sql)[0][0]
  110. print("""{stat_date}日 {table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum}""".format(
  111. stat_date=self.stat_date,
  112. table=table,
  113. InnertotalNum=InnertotalNum,
  114. OutertotalNum=OutertotalNum))
  115. if InnertotalNum == 0:
  116. feishuMsg.send_robot_msg(
  117. """{stat_date}日 {table} 数据为0""".format(stat_date=self.stat_date, table=table))
  118. elif InnertotalNum != OutertotalNum:
  119. feishuMsg.send_robot_msg(
  120. """{stat_date}日 {table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum} 数据不相等 datax
  121. 数据同步启动""".format(
  122. stat_date=self.stat_date,
  123. table=table,
  124. InnertotalNum=InnertotalNum,
  125. OutertotalNum=OutertotalNum))
  126. datax_json = SchedulerList.SchedulerList(table)
  127. os.system('sh /data/datax/application/push_pro.sh {stat_date} {table} {datax_json}'.format(
  128. stat_date=self.stat_date, table=table, datax_json=datax_json))
  129. else:
  130. feishuMsg.send_robot_msg(
  131. """{stat_date}日 {table} 内网数量为{InnertotalNum},外网数量为{OutertotalNum} 数据相等 """.format(
  132. stat_date=self.stat_date,
  133. table=table,
  134. InnertotalNum=InnertotalNum,
  135. OutertotalNum=OutertotalNum))
  136. except Exception as e:
  137. logging.error(e)
  138. return -1
  139. TidbConn.conn_close(conn=self.InnerConn)
  140. TidbProConn.conn_close(conn=self.OuterConn)
  141. except Exception as e:
  142. logging.error(e)
  143. return -1
  144. if __name__ == '__main__':
  145. MonitorAppTask().CheckTask()