瀏覽代碼

新增基建

zhl 3 年之前
父節點
當前提交
3837957e35
共有 2 個文件被更改,包括 473 次插入0 次删除
  1. 241 0
      Apietl/task/AppBytedanceAccountBaseInfoDsD.py
  2. 232 0
      Apietl/task/AppKuaishouAccountBaseInfoDsD.py

+ 241 - 0
Apietl/task/AppBytedanceAccountBaseInfoDsD.py

@@ -0,0 +1,241 @@
+# Author zhaohailiang
+# coding=utf-8
+# @Time    : 2022/01/19 21:10 下午
+# @Site    :
+# @File    : AppBytedanceAccountBaseInfoDsD.py
+# @Software: PyCharm
+# @contact: zhaohailiang@c-top.com.cn
+# @Tel 18201631162
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppBytedanceAccountBaseInfoDsD:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+
+    def taskHandler(self):
+        try:
+            sql = """
+         SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+         replace into `application`.`app_bytedance_account_base_info_d`
+            (
+            stat_date,
+            account_id,
+            account_name,
+            project_id,
+            project_name,
+            product_id,
+            product_name,
+            user_id,
+            user_name,
+            leader_id,
+            leader_name,
+            company_id,
+            company_name,
+            dept_id,
+            dept_name,
+            all_campaign_num,
+            new_campaign_num,
+            all_ad_num,
+            new_ad_num,
+            all_video_num,
+            new_video_num,
+            effect_num,
+            hot_num,
+            put_campaign_num,
+            put_unit_num,
+            trust_campaign_num,
+            trust_unit_num
+            )
+            select 
+            {date_time} stat_date,
+            t1.account_id,
+            bei.account_name,
+            bei.project_id,
+            bei.project_name,
+            bei.product_id,
+            bei.product_name,
+            bei.user_id,
+            bei.user_name,
+            bei.leader_id,
+            bei.leader_name,
+            bei.company_id,
+            bei.company_name,
+            bei.dept_id,
+            bei.dept_name,
+            (t1.all_campaign_num+ifnull(t2.cam_num,0)) as all_campaign_num ,
+            t2.cam_num new_campaign_num,
+            (t1.all_ad_num+ifnull(t2.ad_num,0)) as all_ad_num,
+            t2.ad_num new_ad_num,
+            (t1.all_video_num+ifnull(t3.day_num,0)) as all_video_num,
+            t3.day_num new_video_num,
+            t4.effect_num,
+            t4.hot_num,
+            t1.put_campaign_num,
+            t1.put_unit_num,
+            t1.trust_campaign_num,
+            t1.trust_unit_num
+            from (select * from `application`.`app_bytedance_account_base_info_d` where stat_date= date_sub({date_time}, INTERVAL 1 DAY)) t1
+            left join 
+            (select account_id,count(distinct campaign_id) cam_num,count(id) ad_num from  `media_api`.bytedance_ad_get 
+            where date_format(ad_create_time,'%Y%m%d')={date_time} group by account_id) t2
+            on t1.account_id=t2.account_id
+            left join 
+            (select  advertiser_id account_id,count(distinct signature) day_num from `media_api`.bytedance_file_video_get  
+            where date_format(create_time,'%Y%m%d')={date_time} group by advertiser_id) t3
+            on t1.account_id=t3.account_id
+            left join 
+            (select a.account_id,count(distinct ifnull(b.signature,null)) effect_num,count(distinct ifnull(c.signature,null)) hot_num from 
+            (select distinct advertiser_id account_id,signature from `media_api`.bytedance_file_video_get where date_format(create_time,'%Y%m%d')<={date_time}) a
+            left join (select  distinct signature from application.effect_material  where media_id=1) b
+            on a.signature=b.signature
+            left join (select  distinct signature from application.hot_material  where media_id=1) c
+            on a.signature=c.signature 
+            group by a.account_id) t4
+            on t1.account_id=t4.account_id
+            left join 
+            (
+            SELECT
+            cua.account_id id,
+            cua.account_id,
+            cp.id project_id,
+            product.id product_id,
+            product.product_name,
+            ca.id advertiser_id,
+            uc.company_id company_id,
+            cua.auth_name account_name,
+            cp.project_name project_name,
+            ca.`name` advertiser_name,
+            uc.company_name company_name,
+            cp.sale_id sale_id,
+            su.id user_id,
+            su.realname user_name,
+            su.leader_id,
+            lea.realname leader_name,
+            su.org_code org_code,
+            dep.dep_id dept_id,
+            part.depart_name dept_name
+            FROM
+            `jeecg-boot`.ctop_user_allocation cua
+            LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+            LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+            LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+            LEFT JOIN `jeecg-boot`.sys_user lea ON lea.id = su.leader_id 
+            LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+            LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+            LEFT JOIN `jeecg-boot`.sys_user_depart dep ON cua.user_id=dep.user_id
+            LEFT JOIN `jeecg-boot`.sys_depart part  ON dep.dep_id=part.id
+            WHERE
+            cua.account_id IS NOT NULL and cua.account_status=0 and cua.media_id=1
+            ) bei
+            ON t1.account_id = bei.account_id  where bei.account_id is not null
+            union all
+            select 
+            {date_time} stat_date,
+            t1.account_id,
+            bei.account_name,
+            bei.project_id,
+            bei.project_name,
+            bei.product_id,
+            bei.product_name,
+            bei.user_id,
+            bei.user_name,
+            bei.leader_id,
+            bei.leader_name,
+            bei.company_id,
+            bei.company_name,
+            bei.dept_id,
+            bei.dept_name,
+            t2.cam_num as all_campaign_num ,
+            t2.cam_num new_campaign_num,
+            t2.ad_num as all_ad_num,
+            t2.ad_num as new_ad_num,
+            t3.day_num as all_video_num,
+            t3.day_num new_video_num,
+            t4.effect_num,
+            t4.hot_num,
+            0 put_campaign_num,
+            0 put_unit_num,
+            0 trust_campaign_num,
+            0 trust_unit_num
+            from 
+            (select distinct advertiser_id account_id,signature from `media_api`.bytedance_file_video_get where date_format(create_time,'%Y%m%d')={date_time}) t1
+            left join (select distinct account_id from `application`.`app_bytedance_account_base_info_d`) tt
+            on t1.account_id=tt.account_id 
+            left join 
+            (select account_id,count(distinct campaign_id) cam_num,count(id) ad_num from  `media_api`.bytedance_ad_get 
+            where date_format(ad_create_time,'%Y%m%d')={date_time} group by account_id) t2
+            on t1.account_id=t2.account_id
+            left join 
+            (select  advertiser_id account_id,count(distinct signature) day_num from `media_api`.bytedance_file_video_get  
+            where date_format(create_time,'%Y%m%d')={date_time} group by advertiser_id) t3
+            on t1.account_id=t3.account_id
+            left join 
+            (select a.account_id,count(distinct ifnull(b.signature,null)) effect_num,count(distinct ifnull(c.signature,null)) hot_num from 
+            (select distinct advertiser_id account_id,signature from `media_api`.bytedance_file_video_get where date_format(create_time,'%Y%m%d')<={date_time}) a
+            left join (select  distinct signature from application.effect_material  where media_id=1) b
+            on a.signature=b.signature
+            left join (select  distinct signature from application.hot_material  where media_id=1) c
+            on a.signature=c.signature 
+            group by a.account_id) t4
+            on t1.account_id=t4.account_id
+            left join 
+            (
+            SELECT
+            cua.account_id id,
+            cua.account_id,
+            cp.id project_id,
+            product.id product_id,
+            product.product_name,
+            ca.id advertiser_id,
+            uc.company_id company_id,
+            cua.auth_name account_name,
+            cp.project_name project_name,
+            ca.`name` advertiser_name,
+            uc.company_name company_name,
+            cp.sale_id sale_id,
+            su.id user_id,
+            su.realname user_name,
+            su.leader_id,
+            lea.realname leader_name,
+            su.org_code org_code,
+            dep.dep_id dept_id,
+            part.depart_name dept_name
+            FROM
+            `jeecg-boot`.ctop_user_allocation cua
+            LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+            LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+            LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+            LEFT JOIN `jeecg-boot`.sys_user lea ON lea.id = su.leader_id 
+            LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+            LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+            LEFT JOIN `jeecg-boot`.sys_user_depart dep ON cua.user_id=dep.user_id
+            LEFT JOIN `jeecg-boot`.sys_depart part  ON dep.dep_id=part.id
+            WHERE
+            cua.account_id IS NOT NULL and cua.account_status=0 and cua.media_id=1
+            ) bei
+            ON t1.account_id = bei.account_id  where tt.account_id is null and bei.account_id is not null
+            group by t1.account_id;
+            """.format(date_time=self.date_time)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppBytedanceAccountBaseInfoDsD().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)

+ 232 - 0
Apietl/task/AppKuaishouAccountBaseInfoDsD.py

@@ -0,0 +1,232 @@
+# Author zhaohailiang
+# coding=utf-8
+# @Time    : 2022/01/19 21:10 下午
+# @Site    :
+# @File    : AppKuaishouAccountBaseInfoDsD.py
+# @Software: PyCharm
+# @contact: zhaohailiang@c-top.com.cn
+# @Tel 18201631162
+import logging
+
+from Apietl.conn.TidbConn import TidbConn
+from Apietl.utlis.Utils import Utils
+
+
+class AppKuaishouAccountBaseInfoDsD:
+    def __init__(self):
+        self.conn = TidbConn.get_connection()
+        self.parm = Utils.get_task_args()
+        self.date_time = self.parm[0]
+
+    def taskHandler(self):
+        try:
+            sql = """
+          SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+          replace into `application`.`app_kuaishou_account_base_info_d`
+(
+stat_date,
+account_id,
+account_name,
+project_id,
+project_name,
+product_id,
+product_name,
+user_id,
+user_name,
+leader_id,
+leader_name,
+company_id,
+company_name,
+dept_id,
+dept_name,
+all_campaign_num,
+new_campaign_num,
+all_unit_num,
+new_unit_num,
+all_video_num,
+new_video_num,
+effect_num,
+hot_num,
+put_campaign_num,
+put_unit_num,
+trust_campaign_num,
+trust_unit_num
+)
+select 
+{date_time} stat_date,
+t1.account_id,
+bei.account_name,
+bei.project_id,
+bei.project_name,
+bei.product_id,
+bei.product_name,
+bei.user_id,
+bei.user_name,
+bei.leader_id,
+bei.leader_name,
+bei.company_id,
+bei.company_name,
+bei.dept_id,
+bei.dept_name,
+(t1.all_campaign_num+ifnull(t2.cam_num,0)) as all_campaign_num ,
+t2.cam_num new_campaign_num,
+(t1.all_unit_num+ifnull(t2.unit_num,0)) as all_ad_num,
+t2.unit_num new_ad_num,
+(t1.all_video_num+ifnull(t3.day_num,0)) as all_video_num,
+t3.day_num new_video_num,
+t4.effect_num,
+t4.hot_num,
+t1.put_campaign_num,
+t1.put_unit_num,
+t1.trust_campaign_num,
+t1.trust_unit_num
+from (select * from `application`.`app_kuaishou_account_base_info_d` where stat_date= date_sub({date_time}, INTERVAL 1 DAY)) t1
+left join 
+(select advertiser_id account_id,count(distinct campaign_id) cam_num ,count(unit_id) unit_num   from  `media_api`.kuaishou_ad_unit_list where date_format(create_time,'%Y%m%d')={date_time}
+group by advertiser_id) t2
+on t1.account_id=t2.account_id
+left join 
+(select account_id,count(distinct signature) day_num from `media_api`.kuaishou_video_list  
+where date_format(stat_date,'%Y%m%d')={date_time} group by account_id) t3
+on t1.account_id=t3.account_id
+left join 
+(select a.account_id,count(distinct ifnull(b.signature,null)) effect_num,count(distinct ifnull(c.signature,null)) hot_num from 
+(select distinct account_id,signature from `media_api`.kuaishou_video_list where date_format(stat_date,'%Y%m%d')<={date_time}) a
+left join (select  distinct signature from application.effect_material  where media_id=2) b
+on a.signature=b.signature
+left join (select  distinct signature from application.hot_material  where media_id=2) c
+on a.signature=c.signature 
+group by a.account_id) t4
+on t1.account_id=t4.account_id
+left join 
+(
+SELECT
+cua.account_id id,
+cua.account_id,
+cp.id project_id,
+product.id product_id,
+product.product_name,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.realname user_name,
+su.leader_id,
+lea.realname leader_name,
+su.org_code org_code,
+dep.dep_id dept_id,
+part.depart_name dept_name
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user lea ON lea.id = su.leader_id 
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+LEFT JOIN `jeecg-boot`.sys_user_depart dep ON cua.user_id=dep.user_id
+LEFT JOIN `jeecg-boot`.sys_depart part  ON dep.dep_id=part.id
+WHERE
+cua.account_id IS NOT NULL and cua.account_status=0 and cua.media_id=2
+) bei
+ON t1.account_id = bei.account_id  where bei.account_id is not null
+union all
+select 
+{date_time} stat_date,
+t1.account_id,
+bei.account_name,
+bei.project_id,
+bei.project_name,
+bei.product_id,
+bei.product_name,
+bei.user_id,
+bei.user_name,
+bei.leader_id,
+bei.leader_name,
+bei.company_id,
+bei.company_name,
+bei.dept_id,
+bei.dept_name,
+t2.cam_num as all_campaign_num ,
+t2.cam_num new_campaign_num,
+t2.unit_num as all_ad_num,
+t2.unit_num new_ad_num,
+t3.day_num  as all_video_num,
+t3.day_num new_video_num,
+0 effect_num,
+0 hot_num,
+0 put_campaign_num,
+0 put_unit_num,
+0 trust_campaign_num,
+0 trust_unit_num
+from 
+(select distinct account_id,signature from `media_api`.kuaishou_video_list where date_format(stat_date,'%Y%m%d')={date_time}) t1
+left join 
+(select * from `application`.`app_kuaishou_account_base_info_d` where stat_date= date_sub({date_time}, INTERVAL 1 DAY)) tt
+on t1.account_id=tt.account_id
+left join 
+(select advertiser_id account_id,count(distinct campaign_id) cam_num ,count(unit_id) unit_num   from  `media_api`.kuaishou_ad_unit_list where date_format(create_time,'%Y%m%d')={date_time}
+group by advertiser_id) t2
+on t1.account_id=t2.account_id
+left join 
+(select account_id,count(distinct signature) day_num from `media_api`.kuaishou_video_list  
+where date_format(stat_date,'%Y%m%d')={date_time} group by account_id) t3
+on t1.account_id=t3.account_id
+left join 
+(
+SELECT
+cua.account_id id,
+cua.account_id,
+cp.id project_id,
+product.id product_id,
+product.product_name,
+ca.id advertiser_id,
+uc.company_id company_id,
+cua.auth_name account_name,
+cp.project_name project_name,
+ca.`name` advertiser_name,
+uc.company_name company_name,
+cp.sale_id sale_id,
+su.id user_id,
+su.realname user_name,
+su.leader_id,
+lea.realname leader_name,
+su.org_code org_code,
+dep.dep_id dept_id,
+part.depart_name dept_name
+FROM
+`jeecg-boot`.ctop_user_allocation cua
+LEFT JOIN `jeecg-boot`.ctop_project cp ON cua.project_id = cp.id
+LEFT JOIN `jeecg-boot`.ctop_advertiser ca ON ca.id = cua.advertiser_id
+LEFT JOIN `jeecg-boot`.sys_user su ON su.id = cua.user_id
+LEFT JOIN `jeecg-boot`.sys_user lea ON lea.id = su.leader_id 
+LEFT JOIN `jeecg-boot`.user_company uc ON uc.user_id = cua.user_id
+LEFT JOIN `jeecg-boot`.ctop_product product ON product.id = cp.product_id
+LEFT JOIN `jeecg-boot`.sys_user_depart dep ON cua.user_id=dep.user_id
+LEFT JOIN `jeecg-boot`.sys_depart part  ON dep.dep_id=part.id
+WHERE
+cua.account_id IS NOT NULL and cua.account_status=0 and cua.media_id=2
+) bei
+ ON t1.account_id = bei.account_id  where tt.account_id is null and bei.account_id is not null;
+            """.format(date_time=self.date_time)
+            TidbConn.upsert(conn=self.conn, sql_buff=sql)
+            self.conn.commit()
+            TidbConn.conn_close(conn=self.conn)
+            return 0
+        except Exception as e:
+            logging.error(e)
+            return -1
+
+
+if __name__ == '__main__':
+    status = AppKuaishouAccountBaseInfoDsD().taskHandler()
+    if status == 0:
+        print("作业执行成功")
+        exit(0)
+    else:
+        print("作业执行失败")
+        exit(-1)