Преглед на файлове

release of '设计绩效清洗定时任务' on 2021.3.25

jiequan.bi преди 4 години
родител
ревизия
b4a34d5d5b

+ 28 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ETLReportDesignJob.java

@@ -0,0 +1,28 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.toutiao.modules.link.service.IEtlDesignDailyReportService;
+import com.xxl.job.core.context.XxlJobHelper;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Date;
+
+public class ETLReportDesignJob {
+
+    @Autowired
+    IEtlDesignDailyReportService etlDesignDailyReportService;
+
+    @XxlJob("ETLReportDesignJob")
+    public void execute() throws Exception {
+        String statDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
+        try {
+            etlDesignDailyReportService.cleanEtl(statDate);
+        } catch (Exception e) {
+            XxlJobHelper.log("头条视频报表基础数据拉取异常:{}", e.getMessage());
+        } finally {
+            XxlJobHelper.log("ETLReportVideoCleanJob is finish");
+        }
+    }
+
+}

+ 36 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/entity/EtlDesignDailyReport.java

@@ -0,0 +1,36 @@
+package cn.com.ctop.toutiao.modules.link.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+/**
+ * 设计人员日报
+ */
+@Data
+@TableName("etl_design_daily_report")
+public class EtlDesignDailyReport {
+    
+	/**id*/
+	@TableId(type = IdType.AUTO)
+	private Integer id;
+	/**userId*/
+	private String userId;
+	/**userName*/
+	private String userName;
+	/**cost*/
+	private java.math.BigDecimal cost;
+	/**statDate*/
+	private String statDate;
+	/**roleName*/
+	private String roleName;
+	/**leaderId*/
+	private String leaderId;
+	/**leaderName*/
+	private String leaderName;
+	/**depId*/
+	private String depId;
+	/**depName*/
+	private String depName;
+}

+ 18 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLDesignDailyReportMapper.java

@@ -0,0 +1,18 @@
+package cn.com.ctop.toutiao.modules.link.mapper;
+
+import cn.com.ctop.toutiao.modules.link.entity.EtlDesignDailyReport;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+public interface ETLDesignDailyReportMapper extends BaseMapper<EtlDesignDailyReport> {
+
+    List<JSONObject> queryMaterialCostGroupMd5ByStatDate(@Param("statDate") String statDate);
+
+    List<EtlDesignDailyReport> queryAllDesign(@Param("statDate") String statDate);
+
+    void insertBatch(@Param("records") List<EtlDesignDailyReport> records);
+
+}

+ 88 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/EtlDesignDailyReportMapper.xml

@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.toutiao.modules.link.mapper.ETLDesignDailyReportMapper">
+
+    <select id="queryMaterialCostGroupMd5ByStatDate" resultType="com.alibaba.fastjson.JSONObject">
+        SELECT
+            sum(cost) cost,
+            clip_id clipId,
+            plan_id planId,
+            shot_id shotId
+        FROM
+            etl_report_bytedance_video
+        WHERE
+            stat_datetime = #{statDate}
+          AND cover_url!=''
+        GROUP BY md5 HAVING sum(cost)>0
+        union all
+        SELECT
+            t1.cost,
+            t2.clip_id clipId,
+            t2.plan_id planId,
+            t2.shot_id shotId
+        FROM
+            (select sum(charge) cost,
+                    t1.signature
+             from ctop_etl_kuaishou_account_material_report_daily t1
+             where
+                 t1.stat_date = #{statDate}
+             GROUP BY t1.signature HAVING  sum(charge)>0
+            ) t1
+                LEFT JOIN ctop_etl_kuaishou_video_info t2
+                          ON t1.signature = t2.video_code
+        where t2.clip_id  is not null
+    </select>
+
+    <select id="queryAllDesign" resultType="cn.com.ctop.toutiao.modules.link.entity.EtlDesignDailyReport">
+        SELECT
+            t1.id as userId,
+            t1.realname as userName,
+            t5.role_name as roleName,
+            t1.leader_id,
+            t1.leader_name,
+            t3.id as depId,
+            t3.depart_name as depName,
+            #{statDate} as statDate
+        FROM
+            sys_user t1
+                LEFT JOIN sys_user_depart t2 on t1.id=t2.user_id
+                LEFT JOIN sys_depart t3 on t2.dep_id=t3.id
+                LEFT JOIN sys_user_role t4 on t1.id=t4.user_id
+                LEFT JOIN sys_role t5 on t4.role_id=t5.id
+-- where t5.role_name in ('设计组长','剪辑','策划')
+        where t3.depart_name like '快手设计%'
+           or t3.depart_name like '设计-%'
+           or t3.depart_name like '头条设计%'
+            and t1.leader_id is not null
+            and t1.`status`=1
+            and t1.del_flag=0
+    </select>
+
+    <insert id="insertBatch">
+        insert into etl_design_daily_report(
+        user_id,
+        user_name,
+        cost,
+        stat_date,
+        role_name,
+        leader_id,
+        leader_name,
+        dep_id,
+        dep_name
+        )
+        values
+        <foreach collection="records" item="record" separator=",">
+            (#{record.userId},
+            #{record.userName},
+            #{record.cost},
+            #{record.statDate},
+            #{record.roleName},
+            #{record.leaderId},
+            #{record.leaderName},
+            #{record.depId},
+            #{record.depName}
+            )
+        </foreach>
+    </insert>
+
+</mapper>

+ 13 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IEtlDesignDailyReportService.java

@@ -0,0 +1,13 @@
+package cn.com.ctop.toutiao.modules.link.service;
+
+import cn.com.ctop.toutiao.modules.link.entity.EtlDesignDailyReport;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ *  设计人员日报
+ */
+public interface IEtlDesignDailyReportService extends IService<EtlDesignDailyReport> {
+
+    void cleanEtl(String statDate);
+
+}

+ 56 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/EtlDesignDailyReportServiceImpl.java

@@ -0,0 +1,56 @@
+package cn.com.ctop.toutiao.modules.link.service.impl;
+
+import cn.com.ctop.toutiao.modules.link.entity.EtlDesignDailyReport;
+import cn.com.ctop.toutiao.modules.link.mapper.ETLDesignDailyReportMapper;
+import cn.com.ctop.toutiao.modules.link.service.IEtlDesignDailyReportService;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 设计人员日报
+ */
+@Service
+public class EtlDesignDailyReportServiceImpl extends ServiceImpl<ETLDesignDailyReportMapper, EtlDesignDailyReport> implements IEtlDesignDailyReportService {
+
+    @Autowired
+    ETLDesignDailyReportMapper etlDesignDailyReportMapper;
+
+    @Override
+    public void cleanEtl(String statDate) {
+        //查询t-1所有头条、快手素材产生的报表数据
+        List<JSONObject> materialReport=etlDesignDailyReportMapper.queryMaterialCostGroupMd5ByStatDate(statDate);
+        Map<String, BigDecimal> userMapCost= new HashMap<>();
+        materialReport.forEach(m -> {
+            if (userMapCost.get(m.getString("clipId")) == null) {
+                userMapCost.put(m.getString("clipId"), userMapCost.get(m.getString("clipId"))==null?BigDecimal.ZERO:userMapCost.get(m.getString("clipId")));
+            } else {
+                userMapCost.put(m.getString("clipId"), userMapCost.get(m.getString("clipId"))==null?BigDecimal.ZERO:userMapCost.get(m.getString("clipId")).add(m.getBigDecimal("cost")));
+            }
+            if (userMapCost.get(m.getString("planId")) == null) {
+                userMapCost.put(m.getString("planId"), userMapCost.get(m.getString("planId"))==null?BigDecimal.ZERO:userMapCost.get(m.getString("planId")));
+            } else {
+                userMapCost.put(m.getString("planId"), userMapCost.get(m.getString("planId"))==null?BigDecimal.ZERO:userMapCost.get(m.getString("planId")).add(m.getBigDecimal("cost")));
+            }
+            if (m.getString("shotId") != null) {
+                if (userMapCost.get(m.getString("shotId")) == null) {
+                    userMapCost.put(m.getString("shotId"), userMapCost.get(m.getString("shotId"))==null?BigDecimal.ZERO:userMapCost.get(m.getString("shotId")));
+                } else {
+                    userMapCost.put(m.getString("shotId"), userMapCost.get(m.getString("shotId"))==null?BigDecimal.ZERO:userMapCost.get(m.getString("shotId")).add(m.getBigDecimal("cost")));
+                }
+            }
+        });
+        //查询所有需要清洗数据的设计人员
+        List<EtlDesignDailyReport> allDesign = etlDesignDailyReportMapper.queryAllDesign(statDate);
+        allDesign.forEach(it->{
+            it.setCost(userMapCost.get(it.getUserId())==null?BigDecimal.ZERO:userMapCost.get(it.getUserId()));
+        });
+        etlDesignDailyReportMapper.insertBatch(allDesign);
+    }
+}