فهرست منبع

快手爆款视频数据清洗

syh 4 سال پیش
والد
کامیت
84501698f0
13فایلهای تغییر یافته به همراه391 افزوده شده و 14 حذف شده
  1. 7 0
      jeecg-boot-base-common/src/main/java/org/jeecg/common/util/DateUtils.java
  2. 12 7
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  3. 0 2
      module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAudienceReportJob.java
  4. 4 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaiShouReportDailyMaterialMapper.java
  5. 34 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouReportDailyMaterialMapper.xml
  6. 3 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouReportDailyMaterialService.java
  7. 11 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouReportDailyMaterialServiceImpl.java
  8. 4 4
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaiShouReportDailyMaterial.java
  9. 88 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouVideoEtlInfo.java
  10. 15 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouVideoEtlInfoMapper.java
  11. 8 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouVideoEtlInfoMapper.xml
  12. 18 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouVideoEtlInfoService.java
  13. 187 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouVideoEtlInfoServiceImpl.java

+ 7 - 0
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/DateUtils.java

@@ -839,6 +839,13 @@ public class DateUtils extends PropertyEditorSupport {
         System.err.println(s1);
     }
 
+    public static int getdaysOfTwoDate(String minDate, String date) {
+        Date bigDate = parseDate(date,"yyyy-MM-dd");
+        Date smallDate = parseDate(minDate,"yyyy-MM-dd");
+        int days = (int) ((bigDate.getTime() - smallDate.getTime()) / (1000*3600*24));
+        return days+1;
+    }
+
     /**
      * String类型 转换为Date, 如果参数长度为10 转换格式”yyyy-MM-dd“ 如果参数长度为19 转换格式”yyyy-MM-dd
      * HH:mm:ss“ * @param text String类型的时间值

+ 12 - 7
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -15,6 +15,7 @@ import cn.com.ctop.kuaishou.modules.batch.service.IKuaishouInterfaceService;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouCostGroupService;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyAgentService;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouVideoEtlInfoService;
 import cn.com.ctop.kuaishou.modules.report.service.IRuleKuaiShouPlanService;
 import cn.com.ctop.oa.modules.service.IWechatCheckinDataService;
 import cn.com.ctop.oa.modules.service.IWechatNoListService;
@@ -330,11 +331,6 @@ public class SampleTest {
 //        }
     }
 
-
-    @Autowired
-    private IKuaishouCostGroupService kuaishouCostGroupService;
-    @Autowired
-    private IAlarmEventSendService alarmEventSendService;
     @Autowired
     private IWechatCheckinDataService wechatCheckinDataService;
     @Autowired
@@ -392,6 +388,15 @@ public class SampleTest {
         CtopOauthToken token = tokenService.getTokenByAccountId(9589905L);
         ruleKuaiShouPlanService.cleanRuleDataTarget(token.getAccountId(), 1);
     }
-
-
+    @Autowired
+    private IKuaishouVideoEtlInfoService etlInfoService;
+    @Test
+    public void etlKuaishouData(){
+        String dateString = "2020-03-17";
+        Date startDate = DateUtils.parseDate(dateString,"yyyy-MM-dd");
+        for(int i=0;i<=290;i++){
+            Date getDate = DateUtils.addDay(startDate,i);
+            etlInfoService.etlKuaishouVideoInfo(getDate);
+        }
+    }
 }

+ 0 - 2
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouDailyAudienceReportJob.java

@@ -28,8 +28,6 @@ public class KuaishouDailyAudienceReportJob {
     @Autowired
     private IKuaishouAudienceReportDailyService kuaishouAudienceReportDailyService;
     @Autowired
-    private IKuaiShouCampaignService kuaiShouCampaignService;
-    @Autowired
     private IKuaiShouGroupService kuaiShouGroupService;
 
     static ExecutorService executorService = Executors.newFixedThreadPool(6);

+ 4 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaiShouReportDailyMaterialMapper.java

@@ -29,4 +29,8 @@ public interface KuaiShouReportDailyMaterialMapper extends BaseMapper<KuaiShouRe
     List<KuaiShouVideoGetVo> getVideoList(Map<String, Object> requestMap);
 
     void batchReplace(@Param("addList") List<KuaiShouReportDailyMaterial> addList);
+
+    KuaiShouReportDailyMaterial getEtlDataByParams(@Param("signature")String signature,@Param("date") String date);
+
+    List<KuaiShouReportDailyMaterial> getSignatureByDate(@Param("date")String date);
 }

+ 34 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouReportDailyMaterialMapper.xml

@@ -316,6 +316,39 @@
         ORDER BY
         sum(t1.charge) DESC
     </select>
+    <select id="getEtlDataByParams"
+            resultType="cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial">
+        select
+            channel_type,
+            signature,
+            cover_url,
+            photo_url,
+            min(stat_date)as 'stat_date',
+            sum(charge)as 'charge',
+            sum(photo_click)as 'photo_click',
+            sum(photo_show)as 'photo_show',
+            sum(aclick)as 'aclick',
+            sum(bclick)as 'bclick'
+        from ctop_kuaishou_report_daily_material
+        where
+            signature = #{signature}
+        and
+            stat_date&lt;=#{date}
+        group by signature
+    </select>
+    <select id="getSignatureByDate"
+            resultType="cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial">
+        select
+            distinct signature
+        from
+        ctop_kuaishou_report_daily_material
+        where
+            signature is not null
+        and
+            signature != ''
+        and
+            stat_date = #{date}
+    </select>
 
 
-</mapper>
+</mapper>

+ 3 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaiShouReportDailyMaterialService.java

@@ -20,4 +20,7 @@ public interface IKuaiShouReportDailyMaterialService extends IService<KuaiShouRe
 
     void getMaterialReportByAccountIdAndStatDate(Long accountId, String token, String startDate, String endDate, Integer page);
 
+    List<KuaiShouReportDailyMaterial> getSignatureByDate(String date);
+
+    KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date);
 }

+ 11 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouReportDailyMaterialServiceImpl.java

@@ -9,6 +9,7 @@ import cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial;
 import cn.com.ctop.kuaishou.modules.report.entity.VideoGetvo;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -93,4 +94,14 @@ public class KuaiShouReportDailyMaterialServiceImpl extends ServiceImpl<KuaiShou
         dailyMaterialMapper.batchReplace(addList);
         getMaterialReportByAccountIdAndStatDate(accountId, token, startDate, endDate, page + 1);
     }
+
+    @Override
+    public List<KuaiShouReportDailyMaterial> getSignatureByDate(String date) {
+        return dailyMaterialMapper.getSignatureByDate(date);
+    }
+
+    @Override
+    public KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date) {
+        return dailyMaterialMapper.getEtlDataByParams(signature,date);
+    }
 }

+ 4 - 4
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaiShouReportDailyMaterial.java

@@ -99,19 +99,19 @@ public class KuaiShouReportDailyMaterial {
      */
     @Excel(name = "曝光数", width = 15)
     @ApiModelProperty(value = "曝光数")
-    private Integer photoShow;
+    private Long photoShow;
     /**
      * 素材曝光数
      */
     @Excel(name = "素材曝光数", width = 15)
     @ApiModelProperty(value = "素材曝光数")
-    private Integer aclick;
+    private Long aclick;
     /**
      * 行为数
      */
     @Excel(name = "行为数", width = 15)
     @ApiModelProperty(value = "行为数")
-    private Integer bclick;
+    private Long bclick;
     /**
      * 分享数
      */
@@ -195,7 +195,7 @@ public class KuaiShouReportDailyMaterial {
      */
     @Excel(name = "封面点击数", width = 15)
     @ApiModelProperty(value = "封面点击数")
-    private Integer photoClick;
+    private Long photoClick;
     /**
      * 封面点击率
      */

+ 88 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouVideoEtlInfo.java

@@ -0,0 +1,88 @@
+package cn.com.ctop.kuaishou.modules.report.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+/**
+ * 快手视频信息清洗表(有效,爆款)
+ * @author jeecg-boot
+ * @date   2020-12-17
+ * @version V1.0
+ */
+@Data
+@TableName("ctop_kuaishou_video_etl_info")
+public class KuaishouVideoEtlInfo {
+
+	@TableId(type = IdType.AUTO)
+	private Long id;
+	/**视频唯一标识*/
+	private String videoCode;
+	/**初次产生消耗时间*/
+	private String stateDate;
+	/**url*/
+	private String url;
+	/**coverUrl*/
+	private String coverUrl;
+	/**渠道类型*/
+	private String channelType;
+	/**companyName*/
+	private String companyName;
+	/**planId*/
+	private String planId;
+	/**planName*/
+	private String planName;
+	/**clipId*/
+	private String clipId;
+	/**clipName*/
+	private String clipName;
+	/**shotId*/
+	private String shotId;
+	/**shotName*/
+	private String shotName;
+	/**singleWeekCost*/
+	private BigDecimal singleWeekCost;
+	/**twoWeekCost*/
+	private BigDecimal twoWeekCost;
+	/**fourWeekCost*/
+	private BigDecimal fourWeekCost;
+	/**总消耗*/
+	private BigDecimal totalCost;
+	/**photoShow*/
+	private Long photoShow;
+	/**photoClick*/
+	private Long photoClick;
+	/**aclick*/
+	private Long aclick;
+	/**bclick*/
+	private Long bclick;
+	private String effectDate;
+	private String faddishDate;
+	private Integer effectDayNum;
+	private Integer faddishDayNum;
+	/**createTime*/
+	private Date createTime;
+	/**updateTime*/
+	private Date updateTime;
+
+	public KuaishouVideoEtlInfo() {
+	}
+
+	public KuaishouVideoEtlInfo(KuaiShouReportDailyMaterial nowData) {
+		this.url = nowData.getPhotoUrl();
+		this.videoCode  = nowData.getSignature();
+		this.stateDate = nowData.getStatDate();
+		this.channelType = nowData.getChannelType()+"";
+		this.coverUrl = nowData.getCoverUrl();
+		this.totalCost = nowData.getCharge();
+		this.photoClick = nowData.getPhotoClick();
+		this.photoShow =  nowData.getPhotoShow();
+		this.aclick = nowData.getAclick();
+		this.bclick = nowData.getBclick();
+
+	}
+}

+ 15 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouVideoEtlInfoMapper.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.kuaishou.modules.report.mapper;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouVideoEtlInfo;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * 快手视频信息清洗表(有效,爆款)
+ * @author: jeecg-boot
+ * @date:   2020-12-17
+ * @cersion: V1.0
+ */
+public interface KuaishouVideoEtlInfoMapper extends BaseMapper<KuaishouVideoEtlInfo> {
+
+    String getCompanyNameByUserId(String planId);
+}

+ 8 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouVideoEtlInfoMapper.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.kuaishou.modules.report.mapper.KuaishouVideoEtlInfoMapper">
+
+    <select id="getCompanyNameByUserId" resultType="java.lang.String">
+        select company_name from user_company where user_id = #{planId} limit 1
+    </select>
+</mapper>

+ 18 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouVideoEtlInfoService.java

@@ -0,0 +1,18 @@
+package cn.com.ctop.kuaishou.modules.report.service;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouVideoEtlInfo;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+import java.util.Date;
+
+/**
+ * @Description: 快手视频信息清洗表(有效,爆款)
+ * @Author: jeecg-boot
+ * @Date:   2020-12-17
+ * @Version: V1.0
+ */
+public interface IKuaishouVideoEtlInfoService extends IService<KuaishouVideoEtlInfo> {
+    void etlKuaishouVideoInfo(Date date);
+
+    KuaishouVideoEtlInfo getBySignature(String signature);
+}

+ 187 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouVideoEtlInfoServiceImpl.java

@@ -0,0 +1,187 @@
+package cn.com.ctop.kuaishou.modules.report.service.impl;
+
+import cn.com.ctop.common.module.entity.MaterialAscription;
+import cn.com.ctop.common.module.entity.MaterialInfo;
+import cn.com.ctop.common.module.service.IMaterialAscriptionService;
+import cn.com.ctop.common.module.service.IMaterialInfoService;
+import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouReportDailyMaterialService;
+import cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial;
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouVideoEtlInfo;
+import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouVideoEtlInfoMapper;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouVideoEtlInfoService;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.opencv.core.Mat;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.logging.Logger;
+
+/**
+ * 快手视频信息清洗表(有效,爆款)
+ * @author jeecg-boot
+ * @date   2020-12-17
+ * @version V1.0
+ */
+@Slf4j
+@Service
+public class KuaishouVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouVideoEtlInfoMapper, KuaishouVideoEtlInfo> implements IKuaishouVideoEtlInfoService {
+    @Autowired
+    private KuaishouVideoEtlInfoMapper kuaishouVideoEtlInfoMapper;
+    @Autowired
+    private IKuaiShouReportDailyMaterialService dailyMaterialService;
+    static ExecutorService executorService = Executors.newFixedThreadPool(50);
+    @Override
+    public void etlKuaishouVideoInfo(Date getDate) {
+        String date = DateUtils.formatDate(getDate);
+        //1:查询单当日视频信息
+        List<KuaiShouReportDailyMaterial> materials = dailyMaterialService.getSignatureByDate(date);
+        if(null!=materials&&!materials.isEmpty()){
+            CountDownLatch countDownLatch= new CountDownLatch(materials.size());
+            materials.forEach(material->{
+                executorService.submit(()->{
+                    try {
+                        if(null == material.getSignature()||material.getSignature().trim().equals("")){
+                            return;
+                        }
+
+                        String signature = material.getSignature();
+                        KuaishouVideoEtlInfo etlInfo = this.getBySignature(signature);
+                        KuaiShouReportDailyMaterial nowData = dailyMaterialService.getEtlDataByParams(signature,date);
+                        System.out.println("code:"+signature+";nowDataCode:"+nowData.getSignature());
+                        String minDate = nowData.getStatDate();
+                        Integer days = DateUtils.getdaysOfTwoDate(minDate,date);
+                        if(null == etlInfo){
+                            //第一次查询的数据
+                            etlInfo = new KuaishouVideoEtlInfo(nowData);
+                            BigDecimal totalCost = etlInfo.getTotalCost();
+                            JSONObject ascription =  materialAscriptionService.getDetailByCode(signature);
+                            String planId = null;
+                            if(null!=ascription){
+                                planId = ascription.getString("planId");
+                                if(null!=planId&&!planId.trim().equals("")){
+                                    etlInfo.setPlanId(planId);
+                                }
+                                String planName =  ascription.getString("planName");
+                                if(null!=planName&&!planName.trim().equals("")){
+                                    etlInfo.setPlanName(planName);
+                                }
+                                String clipId = ascription.getString("clipId");
+                                if(null!=clipId&&!clipId.trim().equals("")){
+                                    etlInfo.setClipId(clipId);
+                                }
+                                String clipName = ascription.getString("clipName");
+                                if(null!=clipName&&!clipName.trim().equals("")){
+                                    etlInfo.setClipName(clipName);
+                                }
+                                String shotId = ascription.getString("shotId");
+                                if(null!=shotId&&!shotId.trim().equals("")){
+                                    etlInfo.setShotId(shotId);
+                                }
+                                String shotName = ascription.getString("shotName");
+                                if(null!=shotName&&!shotName.trim().equals("")){
+                                    etlInfo.setShotName(shotName);
+                                }
+                            }
+                            if(null!=planId){
+                                String companyName = kuaishouVideoEtlInfoMapper.getCompanyNameByUserId(planId);
+                                if (null!=companyName&&!companyName.trim().equals("")){
+                                    etlInfo.setCompanyName(companyName);
+                                }
+                            }
+                            //计算投放总天数
+                            if(days == 7){
+                                etlInfo.setSingleWeekCost(nowData.getCharge());
+                            }
+                            if(days == 14){
+                                etlInfo.setTwoWeekCost(nowData.getCharge());
+                            }
+                            if(days == 28){
+                                etlInfo.setFourWeekCost(nowData.getCharge());
+                            }
+                            //计算是否有效,爆款
+                            if(totalCost.compareTo(new BigDecimal("5000"))>0){
+                                //此刻成为有效
+                                etlInfo.setEffectDate(date);
+                                etlInfo.setEffectDayNum(days);
+                            }
+                            if(totalCost.compareTo(new BigDecimal("100000"))>0){
+                                //此刻成为爆款
+                                etlInfo.setEffectDate(date);
+                                etlInfo.setEffectDayNum(days);
+                            }
+                            etlInfo.setCreateTime(new Date());
+                            etlInfo.setUpdateTime(new Date());
+                            this.save(etlInfo);
+                        }else{
+                            BigDecimal totalCost = etlInfo.getTotalCost();
+                            //只需要更新汇总数据
+                            etlInfo.setTotalCost(nowData.getCharge());
+                            etlInfo.setAclick(nowData.getAclick());
+                            etlInfo.setBclick(nowData.getBclick());
+                            etlInfo.setPhotoClick(nowData.getPhotoClick());
+                            etlInfo.setPhotoShow(nowData.getPhotoShow());
+                            etlInfo.setUpdateTime(new Date());
+                            //计算投放总天数
+                            if(days == 7){
+                                etlInfo.setSingleWeekCost(nowData.getCharge());
+                            }
+                            if(days == 14){
+                                etlInfo.setTwoWeekCost(nowData.getCharge());
+                            }
+                            if(days == 28){
+                                etlInfo.setFourWeekCost(nowData.getCharge());
+                            }
+                            if(null == etlInfo.getEffectDayNum()){
+                                if(totalCost.compareTo(new BigDecimal("5000"))>0){
+                                    //此刻成为有效
+                                    etlInfo.setEffectDate(date);
+                                    etlInfo.setEffectDayNum(days);
+                                }
+                            }
+                            if(null == etlInfo.getFaddishDayNum()){
+                                if(totalCost.compareTo(new BigDecimal("100000"))>0){
+                                    //此刻成为爆款
+                                    etlInfo.setEffectDate(date);
+                                    etlInfo.setEffectDayNum(days);
+                                }
+                            }
+                            this.updateById(etlInfo);
+                        }
+                    }catch (Exception e){
+
+                    }finally {
+                        countDownLatch.countDown();
+                    }
+                });
+            });
+
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            log.info("{}数据获取完成",date);
+        }
+    }
+
+    @Autowired
+    private IMaterialAscriptionService materialAscriptionService;
+    @Autowired
+    private IMaterialInfoService materialInfoService;
+    @Override
+    public KuaishouVideoEtlInfo getBySignature(String signature) {
+        QueryWrapper<KuaishouVideoEtlInfo> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("video_code",signature).last("limit 1");
+        return this.getOne(queryWrapper);
+    }
+}