Kaynağa Gözat

添加爆款视频数据清洗定时任务

syh 4 yıl önce
ebeveyn
işleme
7b695c366c
20 değiştirilmiş dosya ile 581 ekleme ve 30 silme
  1. 0 3
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java
  2. 2 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaiShouReportDailyMaterialMapper.java
  3. 38 20
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouReportDailyMaterialMapper.xml
  4. 3 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouReportDailyMaterialService.java
  5. 7 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouReportDailyMaterialServiceImpl.java
  6. 4 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaiShouReportDailyMaterial.java
  7. 83 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouProjectVideoEtlInfo.java
  8. 14 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouProjectVideoEtlInfoMapper.java
  9. 5 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouProjectVideoEtlInfoMapper.xml
  10. 15 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouProjectVideoEtlInfoService.java
  11. 2 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouVideoEtlInfoService.java
  12. 31 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouProjectVideoEtlInfoServiceImpl.java
  13. 115 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouVideoEtlInfoServiceImpl.java
  14. 76 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceProjectVideoEtlInfo.java
  15. 14 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceProjectVideoEtlInfoMapper.java
  16. 5 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceProjectVideoEtlInfoMapper.xml
  17. 15 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceProjectVideoEtlInfoService.java
  18. 2 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceVideoEtlInfoService.java
  19. 18 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceProjectVideoEtlInfoServiceImpl.java
  20. 132 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoEtlInfoServiceImpl.java

+ 0 - 3
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceEtlVideoJob.java

@@ -1,15 +1,12 @@
 package cn.com.ctop.job.bytedance.handler;
 
-import cn.com.ctop.common.module.entity.CtopOauthToken;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
-import com.xxl.job.core.context.XxlJobHelper;
 import com.xxl.job.core.handler.annotation.XxlJob;
 import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import java.util.Date;
-import java.util.List;
 
 @Component
 public class BytedanceEtlVideoJob {

+ 2 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaiShouReportDailyMaterialMapper.java

@@ -30,9 +30,9 @@ public interface KuaiShouReportDailyMaterialMapper extends BaseMapper<KuaiShouRe
 
     void batchReplace(@Param("addList") List<KuaiShouReportDailyMaterial> addList);
 
-    KuaiShouReportDailyMaterial getEtlDataByParams(@Param("signature")String signature,@Param("date") String date);
+    KuaiShouReportDailyMaterial getEtlDataByParams(@Param("signature")String signature, @Param("date") String date,@Param("projectId")Long projectId);
 
     List<KuaiShouReportDailyMaterial> getSignatureByDate(@Param("date")String date);
 
-
+    List<KuaiShouReportDailyMaterial> getSignatureByDateGroupByProjectId(@Param("date")String date);
 }

+ 38 - 20
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouReportDailyMaterialMapper.xml

@@ -277,6 +277,30 @@
     </select>
 
 
+    <select id="getEtlDataByParams"
+            resultType="cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial">
+        select
+        report.channel_type,
+        report.signature,
+        report.cover_url,
+        report.photo_url,
+        min(report.stat_date)as 'stat_date',
+        sum(report.charge)as 'charge',
+        sum(report.photo_click)as 'photo_click',
+        sum(report.photo_show)as 'photo_show',
+        sum(report.aclick)as 'aclick',
+        sum(report.bclick)as 'bclick'
+        from ctop_kuaishou_report_daily_material report
+        left join ctop_user_allocation allocation on allocation.account_id = report.account_id
+        where
+        report.signature = #{signature}
+        and
+        report.stat_date&lt;=#{date}
+        <if test="projectId!=null">
+            and allocation.project_id = #{projectId}
+        </if>
+        group by report.signature
+    </select>
     <select id="getVideoList" parameterType="java.util.Map"
             resultType="cn.com.ctop.kuaishou.modules.batch.entity.vo.KuaiShouVideoGetVo">
 
@@ -316,26 +340,6 @@
         ORDER BY
         sum(t1.charge) DESC
     </select>
-    <select id="getEtlDataByParams"
-            resultType="cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial">
-        select
-            channel_type,
-            signature,
-            cover_url,
-            photo_url,
-            min(stat_date)as 'stat_date',
-            sum(charge)as 'charge',
-            sum(photo_click)as 'photo_click',
-            sum(photo_show)as 'photo_show',
-            sum(aclick)as 'aclick',
-            sum(bclick)as 'bclick'
-        from ctop_kuaishou_report_daily_material
-        where
-            signature = #{signature}
-        and
-            stat_date&lt;=#{date}
-        group by signature
-    </select>
     <select id="getSignatureByDate"
             resultType="cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial">
         select
@@ -349,5 +353,19 @@
         and
             stat_date = #{date}
     </select>
+    <select id="getSignatureByDateGroupByProjectId"
+            resultType="cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial">
+        select
+    distinct signature,project_id
+from
+    ctop_kuaishou_report_daily_material report
+    left join ctop_user_allocation allocation on report.account_id = allocation.account_id
+where
+    signature is not null
+    and
+    signature != ''
+    and
+    stat_date = #{date}
+    </select>
 </mapper>
 

+ 3 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouReportDailyMaterialService.java

@@ -17,10 +17,11 @@ public interface IKuaiShouReportDailyMaterialService extends IService<KuaiShouRe
 
     List<KuaiShouVideoGetVo> getVideoList(Map<String, Object> requestMap);
 
-
     void getMaterialReportByAccountIdAndStatDate(Long accountId, String token, String startDate, String endDate, Integer page);
 
     List<KuaiShouReportDailyMaterial> getSignatureByDate(String date);
 
-    KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date);
+    KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date,Long projectId);
+
+    List<KuaiShouReportDailyMaterial> getSignatureByDateGroupByProjectId(String date);
 }

+ 7 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouReportDailyMaterialServiceImpl.java

@@ -105,7 +105,12 @@ public class KuaiShouReportDailyMaterialServiceImpl extends ServiceImpl<KuaiShou
     }
 
     @Override
-    public KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date) {
-        return dailyMaterialMapper.getEtlDataByParams(signature,date);
+    public KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date,Long projectId) {
+        return dailyMaterialMapper.getEtlDataByParams(signature,date,projectId);
+    }
+
+    @Override
+    public List<KuaiShouReportDailyMaterial> getSignatureByDateGroupByProjectId(String date) {
+        return dailyMaterialMapper.getSignatureByDateGroupByProjectId(date);
     }
 }

+ 4 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaiShouReportDailyMaterial.java

@@ -11,6 +11,7 @@ import lombok.EqualsAndHashCode;
 import lombok.experimental.Accessors;
 import org.jeecgframework.poi.excel.annotation.Excel;
 import org.springframework.format.annotation.DateTimeFormat;
+import com.baomidou.mybatisplus.annotation.TableField;
 
 import java.util.Date;
 
@@ -416,4 +417,7 @@ public class KuaiShouReportDailyMaterial {
     @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
     @ApiModelProperty(value = "修改时间")
     private Date updateTime;
+
+    @TableField(exist = false)
+    private Long projectId;
 }

+ 83 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouProjectVideoEtlInfo.java

@@ -0,0 +1,83 @@
+package cn.com.ctop.kuaishou.modules.report.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+/**
+ * 快手视频项目消耗数据清洗
+ * @author jeecg-boot
+ * @date   2021-01-08
+ * @version V1.0
+ */
+@Data
+@TableName("ctop_kuaishou_project_video_etl_info")
+public class KuaishouProjectVideoEtlInfo {
+	/**id*/
+	@TableId(type = IdType.AUTO)
+	private Long id;
+	/**视频唯一标识*/
+	private String videoCode;
+	/**项目id*/
+	private Long projectId;
+	/**初次产生消耗时间*/
+	private String stateDate;
+	/**url*/
+	private String url;
+	/**coverUrl*/
+	private String coverUrl;
+	/**渠道类型*/
+	private String channelType;
+	/**companyName*/
+	private String companyName;
+	/**planId*/
+	private String planId;
+	/**planName*/
+	private String planName;
+	/**clipId*/
+	private String clipId;
+	/**clipName*/
+	private String clipName;
+	/**shotId*/
+	private String shotId;
+	/**shotName*/
+	private String shotName;
+	/**singleWeekCost*/
+	private BigDecimal singleWeekCost;
+	/**twoWeekCost*/
+	private BigDecimal twoWeekCost;
+	/**fourWeekCost*/
+	private BigDecimal fourWeekCost;
+	/**总消耗*/
+	private BigDecimal totalCost;
+	/**photoShow*/
+	private Long photoShow;
+	/**photoClick*/
+	private Long photoClick;
+	/**aclick*/
+	private Long aclick;
+	/**bclick*/
+	private Long bclick;
+	/**createTime*/
+	private Date createTime;
+	/**updateTime*/
+	private Date updateTime;
+
+	public KuaishouProjectVideoEtlInfo(KuaiShouReportDailyMaterial nowData,Long projectId) {
+		this.url = nowData.getPhotoUrl();
+		this.videoCode  = nowData.getSignature();
+		this.stateDate = nowData.getStatDate();
+		this.channelType = nowData.getChannelType()+"";
+		this.coverUrl = nowData.getCoverUrl();
+		this.totalCost = nowData.getCharge();
+		this.photoClick = nowData.getPhotoClick();
+		this.photoShow =  nowData.getPhotoShow();
+		this.aclick = nowData.getAclick();
+		this.bclick = nowData.getBclick();
+		this.projectId = projectId;
+	}
+}

+ 14 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouProjectVideoEtlInfoMapper.java

@@ -0,0 +1,14 @@
+package cn.com.ctop.kuaishou.modules.report.mapper;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouProjectVideoEtlInfo;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * 快手视频项目消耗数据清洗
+ * @author: jeecg-boot
+ * @date:   2021-01-08
+ * @cersion: V1.0
+ */
+public interface KuaishouProjectVideoEtlInfoMapper extends BaseMapper<KuaishouProjectVideoEtlInfo> {
+
+}

+ 5 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouProjectVideoEtlInfoMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.kuaishou.modules.report.mapper.KuaishouProjectVideoEtlInfoMapper">
+
+</mapper>

+ 15 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouProjectVideoEtlInfoService.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.kuaishou.modules.report.service;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouProjectVideoEtlInfo;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * @Description: 快手视频项目消耗数据清洗
+ * @Author: jeecg-boot
+ * @Date:   2021-01-08
+ * @Version: V1.0
+ */
+public interface IKuaishouProjectVideoEtlInfoService extends IService<KuaishouProjectVideoEtlInfo> {
+
+    KuaishouProjectVideoEtlInfo getByProjectVideoEtlInfoByParams(String signature, Long projectId);
+}

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouVideoEtlInfoService.java

@@ -14,5 +14,7 @@ import java.util.Date;
 public interface IKuaishouVideoEtlInfoService extends IService<KuaishouVideoEtlInfo> {
     void etlKuaishouVideoInfo(Date date);
 
+    void etlKuaishouVideoInfoGroupByProjectId(Date date);
+
     KuaishouVideoEtlInfo getBySignature(String signature);
 }

+ 31 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouProjectVideoEtlInfoServiceImpl.java

@@ -0,0 +1,31 @@
+package cn.com.ctop.kuaishou.modules.report.service.impl;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouProjectVideoEtlInfo;
+import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouProjectVideoEtlInfoMapper;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouProjectVideoEtlInfoService;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * 快手视频项目消耗数据清洗
+ * @author jeecg-boot
+ * @date   2021-01-08
+ * @version V1.0
+ */
+@Service
+public class KuaishouProjectVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouProjectVideoEtlInfoMapper, KuaishouProjectVideoEtlInfo> implements IKuaishouProjectVideoEtlInfoService {
+
+    @Override
+    public KuaishouProjectVideoEtlInfo getByProjectVideoEtlInfoByParams(String signature, Long projectId) {
+        QueryWrapper<KuaishouProjectVideoEtlInfo>queryWrapper = new QueryWrapper<>();
+        if(null!=signature&&!signature.trim().equals("")){
+            queryWrapper.eq("video_code",signature);
+        }
+        if(null!=projectId&&projectId!=0){
+            queryWrapper.eq("project_id",projectId);
+        }
+        queryWrapper.last("limit 1");
+        return this.getOne(queryWrapper);
+    }
+}

+ 115 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouVideoEtlInfoServiceImpl.java

@@ -3,8 +3,10 @@ package cn.com.ctop.kuaishou.modules.report.service.impl;
 import cn.com.ctop.common.module.service.IMaterialAscriptionService;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouReportDailyMaterialService;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial;
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouProjectVideoEtlInfo;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouVideoEtlInfo;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouVideoEtlInfoMapper;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouProjectVideoEtlInfoService;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouVideoEtlInfoService;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -51,7 +53,7 @@ public class KuaishouVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouVideoEt
 
                         String signature = material.getSignature();
                         KuaishouVideoEtlInfo etlInfo = this.getBySignature(signature);
-                        KuaiShouReportDailyMaterial nowData = dailyMaterialService.getEtlDataByParams(signature,date);
+                        KuaiShouReportDailyMaterial nowData = dailyMaterialService.getEtlDataByParams(signature,date,null);
                         String minDate = nowData.getStatDate();
                         Integer days = DateUtils.getdaysOfTwoDate(minDate,date);
                         if(null == etlInfo){
@@ -169,6 +171,118 @@ public class KuaishouVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouVideoEt
     }
 
     @Autowired
+    private IKuaishouProjectVideoEtlInfoService kuaishouProjectVideoEtlInfoService;
+    @Override
+    public void etlKuaishouVideoInfoGroupByProjectId(Date getDate) {
+        String date = DateUtils.formatDate(getDate);
+        //1:查询单当日视频信息--以项目汇总
+        List<KuaiShouReportDailyMaterial> materials = dailyMaterialService.getSignatureByDateGroupByProjectId(date);
+        if(null!=materials&&!materials.isEmpty()){
+            CountDownLatch countDownLatch= new CountDownLatch(materials.size());
+            materials.forEach(material->{
+                executorService.submit(()->{
+                    try {
+                        if(null == material.getSignature()||material.getSignature().trim().equals("")){
+                            return;
+                        }
+                        if(null == material.getProjectId()||material.getProjectId() == 0){
+                            return;
+                        }
+
+                        String signature = material.getSignature();
+                        Long projectId = material.getProjectId();
+                        KuaishouProjectVideoEtlInfo etlInfo = kuaishouProjectVideoEtlInfoService.getByProjectVideoEtlInfoByParams(signature,projectId);
+                        KuaiShouReportDailyMaterial nowData = dailyMaterialService.getEtlDataByParams(signature,date,projectId);
+                        String minDate = nowData.getStatDate();
+                        Integer days = DateUtils.getdaysOfTwoDate(minDate,date);
+                        if(null == etlInfo){
+                            //第一次查询的数据
+                            etlInfo = new KuaishouProjectVideoEtlInfo(nowData,projectId);
+                            JSONObject ascription =  materialAscriptionService.getDetailByCode(signature);
+                            String planId = null;
+                            if(null!=ascription){
+                                planId = ascription.getString("planId");
+                                if(null!=planId&&!planId.trim().equals("")){
+                                    etlInfo.setPlanId(planId);
+                                }
+                                String planName =  ascription.getString("planName");
+                                if(null!=planName&&!planName.trim().equals("")){
+                                    etlInfo.setPlanName(planName);
+                                }
+                                String clipId = ascription.getString("clipId");
+                                if(null!=clipId&&!clipId.trim().equals("")){
+                                    etlInfo.setClipId(clipId);
+                                }
+                                String clipName = ascription.getString("clipName");
+                                if(null!=clipName&&!clipName.trim().equals("")){
+                                    etlInfo.setClipName(clipName);
+                                }
+                                String shotId = ascription.getString("shotId");
+                                if(null!=shotId&&!shotId.trim().equals("")){
+                                    etlInfo.setShotId(shotId);
+                                }
+                                String shotName = ascription.getString("shotName");
+                                if(null!=shotName&&!shotName.trim().equals("")){
+                                    etlInfo.setShotName(shotName);
+                                }
+                            }
+                            if(null!=planId){
+                                String companyName = kuaishouVideoEtlInfoMapper.getCompanyNameByUserId(planId);
+                                if (null!=companyName&&!companyName.trim().equals("")){
+                                    etlInfo.setCompanyName(companyName);
+                                }
+                            }
+                            //计算投放总天数
+                            if(days <= 7){
+                                etlInfo.setSingleWeekCost(nowData.getCharge());
+                            }
+                            if(days <= 14){
+                                etlInfo.setTwoWeekCost(nowData.getCharge());
+                            }
+                            if(days <= 28){
+                                etlInfo.setFourWeekCost(nowData.getCharge());
+                            }
+                            etlInfo.setCreateTime(new Date());
+                            etlInfo.setUpdateTime(new Date());
+                            kuaishouProjectVideoEtlInfoService.save(etlInfo);
+                        }else{
+                            //只需要更新汇总数据
+                            etlInfo.setTotalCost(nowData.getCharge());
+                            etlInfo.setAclick(nowData.getAclick());
+                            etlInfo.setBclick(nowData.getBclick());
+                            etlInfo.setPhotoClick(nowData.getPhotoClick());
+                            etlInfo.setPhotoShow(nowData.getPhotoShow());
+                            etlInfo.setUpdateTime(new Date());
+                            //计算投放总天数
+                            if(days <= 7){
+                                etlInfo.setSingleWeekCost(nowData.getCharge());
+                            }
+                            if(days <= 14){
+                                etlInfo.setTwoWeekCost(nowData.getCharge());
+                            }
+                            if(days <= 28){
+                                etlInfo.setFourWeekCost(nowData.getCharge());
+                            }
+                            kuaishouProjectVideoEtlInfoService.updateById(etlInfo);
+                        }
+                    }catch (Exception e){
+
+                    }finally {
+                        countDownLatch.countDown();
+                    }
+                });
+            });
+
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            log.info("{}数据获取完成",date);
+        }
+    }
+
+    @Autowired
     private IMaterialAscriptionService materialAscriptionService;
     @Override
     public KuaishouVideoEtlInfo getBySignature(String signature) {

+ 76 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceProjectVideoEtlInfo.java

@@ -0,0 +1,76 @@
+package cn.com.ctop.toutiao.modules.report.entity;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.Date;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.baomidou.mybatisplus.annotation.TableField;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.jeecgframework.poi.excel.annotation.Excel;
+
+/**
+ * 快手视频项目消耗数据清洗
+ * @author jeecg-boot
+ * @date   2021-01-08
+ * @version V1.0
+ */
+@Data
+@TableName("ctop_bytedance_project_video_etl_info")
+public class BytedanceProjectVideoEtlInfo {
+
+	/**id*/
+	@TableId(type = IdType.AUTO)
+	private Long id;
+	/**视频唯一标识*/
+	private String videoCode;
+	/**项目id*/
+	private Long projectId;
+	/**初次产生消耗时间*/
+	private String stateDate;
+	/**url*/
+	private String url;
+	/**coverUrl*/
+	private String coverUrl;
+	/**companyName*/
+	private String companyName;
+	/**planId*/
+	private String planId;
+	/**planName*/
+	private String planName;
+	/**clipId*/
+	private String clipId;
+	/**clipName*/
+	private String clipName;
+	/**shotId*/
+	private String shotId;
+	/**shotName*/
+	private String shotName;
+	/**singleWeekCost*/
+	private java.math.BigDecimal singleWeekCost;
+	/**twoWeekCost*/
+	private BigDecimal twoWeekCost;
+	/**fourWeekCost*/
+	private BigDecimal fourWeekCost;
+	/**总消耗*/
+	private BigDecimal totalCost;
+	/**clickNum*/
+	private Integer clickNum;
+	/**showNum*/
+	private Integer showNum;
+	/**activeNum*/
+	private Integer activeNum;
+	/**convertNum*/
+	private Integer convertNum;
+	/**createTime*/
+	private Date createTime;
+	/**updateTime*/
+	private Date updateTime;
+}

+ 14 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceProjectVideoEtlInfoMapper.java

@@ -0,0 +1,14 @@
+package cn.com.ctop.toutiao.modules.report.mapper;
+
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceProjectVideoEtlInfo;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * 快手视频项目消耗数据清洗
+ * @author: jeecg-boot
+ * @date:   2021-01-08
+ * @cersion: V1.0
+ */
+public interface BytedanceProjectVideoEtlInfoMapper extends BaseMapper<BytedanceProjectVideoEtlInfo> {
+
+}

+ 5 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceProjectVideoEtlInfoMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.toutiao.modules.report.mapper.BytedanceProjectVideoEtlInfoMapper">
+
+</mapper>

+ 15 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceProjectVideoEtlInfoService.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.toutiao.modules.report.service;
+
+import cn.com.ctop.ctop.entity.BytedanceProjectVideoEtlInfo;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceProjectVideoEtlInfo;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * @Description: 快手视频项目消耗数据清洗
+ * @Author: jeecg-boot
+ * @Date:   2021-01-08
+ * @Version: V1.0
+ */
+public interface IBytedanceProjectVideoEtlInfoService extends IService<BytedanceProjectVideoEtlInfo> {
+
+}

+ 2 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceVideoEtlInfoService.java

@@ -15,4 +15,6 @@ public interface IBytedanceVideoEtlInfoService extends IService<BytedanceVideoEt
     void etlBytedanceVideoInfo(Date date);
 
     BytedanceVideoEtlInfo getBySignature(String signature);
+
+    void etlBytedanceVideoInfoByProjectId(Date getDate);
 }

+ 18 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceProjectVideoEtlInfoServiceImpl.java

@@ -0,0 +1,18 @@
+package cn.com.ctop.toutiao.modules.report.service.impl;
+
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceProjectVideoEtlInfo;
+import cn.com.ctop.toutiao.modules.report.mapper.BytedanceProjectVideoEtlInfoMapper;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceProjectVideoEtlInfoService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * 快手视频项目消耗数据清洗
+ * @author jeecg-boot
+ * @date   2021-01-08
+ * @version V1.0
+ */
+@Service
+public class BytedanceProjectVideoEtlInfoServiceImpl extends ServiceImpl<BytedanceProjectVideoEtlInfoMapper, BytedanceProjectVideoEtlInfo> implements IBytedanceProjectVideoEtlInfoService {
+
+}

+ 132 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoEtlInfoServiceImpl.java

@@ -176,4 +176,136 @@ public class BytedanceVideoEtlInfoServiceImpl extends ServiceImpl<BytedanceVideo
         queryWrapper.eq("video_code",signature).last("limit 1");
         return this.getOne(queryWrapper);
     }
+
+    @Override
+    public void etlBytedanceVideoInfoByProjectId(Date getDate) {
+        String date = DateUtils.formatDate(getDate);
+        //1:查询单当日视频信息
+        List<BytedanceVideoEtlInfo> materials = materialDailyService.getSignatureByDate(date);
+        if(null!=materials&&!materials.isEmpty()){
+            CountDownLatch countDownLatch= new CountDownLatch(materials.size());
+            materials.forEach(material->{
+                executorService.submit(()->{
+                    try {
+                        if(null == material.getVideoCode()||material.getVideoCode().trim().equals("")){
+                            return;
+                        }
+                        String signature = material.getVideoCode();
+                        BytedanceVideoEtlInfo etlInfo = this.getBySignature(signature);
+                        BytedanceVideoEtlInfo nowData = materialDailyService.getEtlDataByParams(signature,date);
+                        String minDate = nowData.getStateDate();
+                        Integer days = DateUtils.getdaysOfTwoDate(minDate,date);
+                        if(null == etlInfo){
+                            //第一次查询的数据
+                            etlInfo = nowData;
+                            BigDecimal totalCost = etlInfo.getTotalCost();
+                            JSONObject ascription =  materialAscriptionService.getDetailByCode(signature);
+                            String planId = null;
+                            if(null!=ascription){
+                                planId = ascription.getString("planId");
+                                if(null!=planId&&!planId.trim().equals("")){
+                                    etlInfo.setPlanId(planId);
+                                }
+                                String planName =  ascription.getString("planName");
+                                if(null!=planName&&!planName.trim().equals("")){
+                                    etlInfo.setPlanName(planName);
+                                }
+                                String clipId = ascription.getString("clipId");
+                                if(null!=clipId&&!clipId.trim().equals("")){
+                                    etlInfo.setClipId(clipId);
+                                }
+                                String clipName = ascription.getString("clipName");
+                                if(null!=clipName&&!clipName.trim().equals("")){
+                                    etlInfo.setClipName(clipName);
+                                }
+                                String shotId = ascription.getString("shotId");
+                                if(null!=shotId&&!shotId.trim().equals("")){
+                                    etlInfo.setShotId(shotId);
+                                }
+                                String shotName = ascription.getString("shotName");
+                                if(null!=shotName&&!shotName.trim().equals("")){
+                                    etlInfo.setShotName(shotName);
+                                }
+                            }
+                            if(null!=planId){
+                                String companyName = userAllocationMapper.getCompanyNameByUserId(planId);
+                                if (null!=companyName&&!companyName.trim().equals("")){
+                                    etlInfo.setCompanyName(companyName);
+                                }
+                            }
+                            //计算投放总天数
+                            if(days <= 7){
+                                etlInfo.setSingleWeekCost(nowData.getTotalCost());
+                            }
+                            if(days <= 14){
+                                etlInfo.setTwoWeekCost(nowData.getTotalCost());
+                            }
+                            if(days <= 28){
+                                etlInfo.setFourWeekCost(nowData.getTotalCost());
+                            }
+                            //计算是否有效,爆款
+                            if(totalCost.compareTo(new BigDecimal("5000"))>0){
+                                //此刻成为有效
+                                etlInfo.setEffectDate(date);
+                                etlInfo.setEffectDayNum(days);
+                            }
+                            if(totalCost.compareTo(new BigDecimal("100000"))>0){
+                                //此刻成为爆款
+                                etlInfo.setFaddishDate(date);
+                                etlInfo.setFaddishDayNum(days);
+                            }
+                            etlInfo.setCreateTime(new Date());
+                            etlInfo.setUpdateTime(new Date());
+                            this.save(etlInfo);
+                        }else{
+                            //只需要更新汇总数据
+                            etlInfo.setTotalCost(nowData.getTotalCost());
+                            BigDecimal totalCost = etlInfo.getTotalCost();
+                            etlInfo.setClickNum(nowData.getClickNum());
+                            etlInfo.setShowNum(nowData.getShowNum());
+                            etlInfo.setActiveNum(nowData.getActiveNum());
+                            etlInfo.setConvertNum(nowData.getConvertNum());
+                            etlInfo.setUpdateTime(new Date());
+                            //计算投放总天数
+                            if(days <= 7){
+                                etlInfo.setSingleWeekCost(nowData.getTotalCost());
+                            }
+                            if(days <= 14){
+                                etlInfo.setTwoWeekCost(nowData.getTotalCost());
+                            }
+                            if(days <= 28){
+                                etlInfo.setFourWeekCost(nowData.getTotalCost());
+                            }
+                            if(null == etlInfo.getEffectDayNum()){
+                                if(totalCost.compareTo(new BigDecimal("5000"))>0){
+                                    //此刻成为有效
+                                    etlInfo.setEffectDate(date);
+                                    etlInfo.setEffectDayNum(days);
+                                }
+                            }
+                            if(null == etlInfo.getFaddishDayNum()){
+                                if(totalCost.compareTo(new BigDecimal("100000"))>0){
+                                    //此刻成为爆款
+                                    etlInfo.setFaddishDate(date);
+                                    etlInfo.setFaddishDayNum(days);
+                                }
+                            }
+                            this.updateById(etlInfo);
+                        }
+                    }catch (Exception e){
+
+                    }finally {
+                        countDownLatch.countDown();
+                    }
+                });
+            });
+
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            log.info("{}数据获取完成",date);
+        }
+    }
 }