Pārlūkot izejas kodu

修改运营商报表爬虫获取逻辑

syh 4 gadi atpakaļ
vecāks
revīzija
2e9feac202
15 mainītis faili ar 545 papildinājumiem un 11 dzēšanām
  1. 18 8
      jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java
  2. 2 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/UserAllocationMapper.java
  3. 3 0
      module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/UserAllocationMapper.xml
  4. 22 0
      module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouEffectVideoJob.java
  5. 1 3
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouVideoEtlInfoServiceImpl.java
  6. 150 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceVideoEtlInfoController.java
  7. 74 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceVideoEtlInfo.java
  8. 5 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceReportMaterialDailyMapper.java
  9. 16 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceVideoEtlInfoMapper.java
  10. 34 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceReportMaterialDailyMapper.xml
  11. 8 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceVideoEtlInfoMapper.xml
  12. 4 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceReportMaterialDailyService.java
  13. 18 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceVideoEtlInfoService.java
  14. 11 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceReportMaterialDailyServiceImpl.java
  15. 179 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoEtlInfoServiceImpl.java

+ 18 - 8
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -22,10 +22,7 @@ import cn.com.ctop.oa.modules.service.IWechatNoListService;
 import cn.com.ctop.oa.modules.service.IWechatUserListService;
 import cn.com.ctop.toutiao.modules.material.entity.ByteDanceAdvertisePlan;
 import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataService;
-import cn.com.ctop.toutiao.modules.report.service.IByteDanceVideoReportDailyService;
-import cn.com.ctop.toutiao.modules.report.service.IBytedanceFundDailyService;
-import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportService;
-import cn.com.ctop.toutiao.modules.report.service.IReportService;
+import cn.com.ctop.toutiao.modules.report.service.*;
 import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
@@ -389,14 +386,27 @@ public class SampleTest {
         ruleKuaiShouPlanService.cleanRuleDataTarget(token.getAccountId(), 1);
     }
     @Autowired
-    private IKuaishouVideoEtlInfoService etlInfoService;
+    private IKuaishouVideoEtlInfoService kuaishouVideoEtlInfoService;
     @Test
     public void etlKuaishouData(){
-        String dateString = "2020-05-25";
+        String dateString = "2020-01-01";
         Date startDate = DateUtils.parseDate(dateString,"yyyy-MM-dd");
-        for(int i=0;i<=277;i++){
+        for(int i=0;i<=365;i++){
             Date getDate = DateUtils.addDay(startDate,i);
-            etlInfoService.etlKuaishouVideoInfo(getDate);
+            kuaishouVideoEtlInfoService.etlKuaishouVideoInfo(getDate);
+        }
+    }
+
+    @Autowired
+    private IBytedanceVideoEtlInfoService bytedanceVideoEtlInfoService;
+    @Test
+    public void etlBytedanceData(){
+        String dateString = "2020-01-01";
+        Date startDate = DateUtils.parseDate(dateString,"yyyy-MM-dd");
+        for(int i=0;i<=365;i++){
+            Date getDate = DateUtils.addDay(startDate,i);
+            bytedanceVideoEtlInfoService.etlBytedanceVideoInfo(getDate);
+            kuaishouVideoEtlInfoService.etlKuaishouVideoInfo(getDate);
         }
     }
 

+ 2 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/UserAllocationMapper.java

@@ -33,4 +33,6 @@ public interface UserAllocationMapper extends BaseMapper<UserAllocation> {
     JSONObject getSwitchStatusByAccount(@Param("accountId") Long accountId);
 
     List<Long>  queryAccountsByDeleteComment();
+
+    String getCompanyNameByUserId(@Param("planId") String planId);
 }

+ 3 - 0
module-common/src/main/java/cn/com/ctop/common/module/mapper/xml/UserAllocationMapper.xml

@@ -95,6 +95,9 @@
           AND account_status = 0
           AND delete_comment = 0
     </select>
+    <select id="getCompanyNameByUserId" resultType="java.lang.String">
+        select company_name from user_company where user_id = #{planId} limit 1
+    </select>
 
 
 </mapper>

+ 22 - 0
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/KuaishouEffectVideoJob.java

@@ -0,0 +1,22 @@
+package cn.com.ctop.job.kuaishou.handler;
+
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouVideoEtlInfoService;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.text.ParseException;
+import java.util.Date;
+
+@Component
+public class KuaishouEffectVideoJob {
+    @Autowired
+    private IKuaishouVideoEtlInfoService etlInfoService;
+
+    @XxlJob("kuaishouGroupDailyReportJob")
+    public void execute() throws ParseException {
+        Date getDate = DateUtils.addDay(new Date(),-1);
+        etlInfoService.etlKuaishouVideoInfo(getDate);
+    }
+}

+ 1 - 3
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouVideoEtlInfoServiceImpl.java

@@ -122,7 +122,6 @@ public class KuaishouVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouVideoEt
                             etlInfo.setUpdateTime(new Date());
                             this.save(etlInfo);
                         }else{
-                            BigDecimal totalCost = etlInfo.getTotalCost();
                             //只需要更新汇总数据
                             etlInfo.setTotalCost(nowData.getCharge());
                             etlInfo.setAclick(nowData.getAclick());
@@ -130,6 +129,7 @@ public class KuaishouVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouVideoEt
                             etlInfo.setPhotoClick(nowData.getPhotoClick());
                             etlInfo.setPhotoShow(nowData.getPhotoShow());
                             etlInfo.setUpdateTime(new Date());
+                            BigDecimal totalCost = etlInfo.getTotalCost();
                             //计算投放总天数
                             if(days == 7){
                                 etlInfo.setSingleWeekCost(nowData.getCharge());
@@ -175,8 +175,6 @@ public class KuaishouVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouVideoEt
 
     @Autowired
     private IMaterialAscriptionService materialAscriptionService;
-    @Autowired
-    private IMaterialInfoService materialInfoService;
     @Override
     public KuaishouVideoEtlInfo getBySignature(String signature) {
         QueryWrapper<KuaishouVideoEtlInfo> queryWrapper = new QueryWrapper<>();

+ 150 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/controller/BytedanceVideoEtlInfoController.java

@@ -0,0 +1,150 @@
+package cn.com.ctop.toutiao.modules.report.controller;
+
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.system.query.QueryGenerator;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Arrays;
+
+ /**
+ * 头条视频数据清洗信息
+ * @author jeecg-boot
+ * @date   2020-12-21
+ * @version V1.0
+ */
+@Slf4j
+@Api(tags="头条视频数据清洗信息")
+@RestController
+@RequestMapping("/ctop/bytedanceVideoEtlInfo")
+public class BytedanceVideoEtlInfoController {
+	@Autowired
+	private IBytedanceVideoEtlInfoService bytedanceVideoEtlInfoService;
+
+	/**
+	  * 分页列表查询
+	 * @param bytedanceVideoEtlInfo
+	 * @param pageNo
+	 * @param pageSize
+	 * @param req
+	 * @return
+	 */
+	@ApiOperation(value="头条视频数据清洗信息-分页列表查询", notes="头条视频数据清洗信息-分页列表查询")
+	@GetMapping(value = "/list")
+	public Result<IPage<BytedanceVideoEtlInfo>> queryPageList(BytedanceVideoEtlInfo bytedanceVideoEtlInfo,
+															  @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
+															  @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
+															  HttpServletRequest req) {
+		Result<IPage<BytedanceVideoEtlInfo>> result = new Result<>();
+		QueryWrapper<BytedanceVideoEtlInfo> queryWrapper = QueryGenerator.initQueryWrapper(bytedanceVideoEtlInfo, req.getParameterMap());
+		Page<BytedanceVideoEtlInfo> page = new Page<BytedanceVideoEtlInfo>(pageNo, pageSize);
+		IPage<BytedanceVideoEtlInfo> pageList = bytedanceVideoEtlInfoService.page(page, queryWrapper);
+		result.setSuccess(true);
+		result.setResult(pageList);
+		return result;
+	}
+
+	/**
+	  *   添加
+	 * @param bytedanceVideoEtlInfo
+	 * @return
+	 */
+	@ApiOperation(value="头条视频数据清洗信息-添加", notes="头条视频数据清洗信息-添加")
+	@PostMapping(value = "/add")
+	public Result<BytedanceVideoEtlInfo> add(@RequestBody BytedanceVideoEtlInfo bytedanceVideoEtlInfo) {
+		Result<BytedanceVideoEtlInfo> result = new Result<>();
+		try {
+			bytedanceVideoEtlInfoService.save(bytedanceVideoEtlInfo);
+			result.success("添加成功!");
+		} catch (Exception e) {
+			log.error(e.getMessage(),e);
+			result.error500("操作失败");
+		}
+		return result;
+	}
+
+	/**
+	  *  编辑
+	 * @param bytedanceVideoEtlInfo
+	 * @return
+	 */
+	@ApiOperation(value="头条视频数据清洗信息-编辑", notes="头条视频数据清洗信息-编辑")
+	@PutMapping(value = "/edit")
+	public Result<BytedanceVideoEtlInfo> edit(@RequestBody BytedanceVideoEtlInfo bytedanceVideoEtlInfo) {
+		Result<BytedanceVideoEtlInfo> result = new Result<BytedanceVideoEtlInfo>();
+		BytedanceVideoEtlInfo bytedanceVideoEtlInfoEntity = bytedanceVideoEtlInfoService.getById(bytedanceVideoEtlInfo.getId());
+		if(bytedanceVideoEtlInfoEntity==null) {
+			result.error500("未找到对应实体");
+		}else {
+			boolean ok = bytedanceVideoEtlInfoService.updateById(bytedanceVideoEtlInfo);
+			if(ok) {
+				result.success("修改成功!");
+			}
+		}
+
+		return result;
+	}
+
+	/**
+	  *   通过id删除
+	 * @param id
+	 * @return
+	 */
+	@ApiOperation(value="头条视频数据清洗信息-通过id删除", notes="头条视频数据清洗信息-通过id删除")
+	@DeleteMapping(value = "/delete")
+	public Result<?> delete(@RequestParam(name="id",required=true) String id) {
+		try {
+			bytedanceVideoEtlInfoService.removeById(id);
+		} catch (Exception e) {
+			log.error("删除失败",e.getMessage());
+			return Result.error("删除失败!");
+		}
+		return Result.ok("删除成功!");
+	}
+
+	/**
+	  *  批量删除
+	 * @param ids
+	 * @return
+	 */
+	@ApiOperation(value="头条视频数据清洗信息-批量删除", notes="头条视频数据清洗信息-批量删除")
+	@DeleteMapping(value = "/deleteBatch")
+	public Result<BytedanceVideoEtlInfo> deleteBatch(@RequestParam(name="ids",required=true) String ids) {
+		Result<BytedanceVideoEtlInfo> result = new Result<>();
+		if(ids==null || "".equals(ids.trim())) {
+			result.error500("参数不识别!");
+		}else {
+			this.bytedanceVideoEtlInfoService.removeByIds(Arrays.asList(ids.split(",")));
+			result.success("删除成功!");
+		}
+		return result;
+	}
+
+	/**
+	  * 通过id查询
+	 * @param id
+	 * @return
+	 */
+	@ApiOperation(value="头条视频数据清洗信息-通过id查询", notes="头条视频数据清洗信息-通过id查询")
+	@GetMapping(value = "/queryById")
+	public Result<BytedanceVideoEtlInfo> queryById(@RequestParam(name="id",required=true) String id) {
+		Result<BytedanceVideoEtlInfo> result = new Result<>();
+		BytedanceVideoEtlInfo bytedanceVideoEtlInfo = bytedanceVideoEtlInfoService.getById(id);
+		if(bytedanceVideoEtlInfo==null) {
+			result.error500("未找到对应实体");
+		}else {
+			result.setResult(bytedanceVideoEtlInfo);
+			result.setSuccess(true);
+		}
+		return result;
+	}
+}

+ 74 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/entity/BytedanceVideoEtlInfo.java

@@ -0,0 +1,74 @@
+package cn.com.ctop.toutiao.modules.report.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+/**
+ * 头条视频数据清洗信息
+ * @author jeecg-boot
+ * @date   2020-12-21
+ * @version V1.0
+ */
+@Data
+@TableName("ctop_bytedance_video_etl_info")
+public class BytedanceVideoEtlInfo {
+
+	/**id*/
+	@TableId(type = IdType.AUTO)
+	private Long id;
+	/**视频唯一标识*/
+	private String videoCode;
+	/**初次产生消耗时间*/
+	private String stateDate;
+	/**url*/
+	private String url;
+	/**coverUrl*/
+	private String coverUrl;
+	/**companyName*/
+	private String companyName;
+	/**planId*/
+	private String planId;
+	/**planName*/
+	private String planName;
+	/**clipId*/
+	private String clipId;
+	/**clipName*/
+	private String clipName;
+	/**shotId*/
+	private String shotId;
+	/**shotName*/
+	private String shotName;
+	/**singleWeekCost*/
+	private BigDecimal singleWeekCost;
+	/**twoWeekCost*/
+	private BigDecimal twoWeekCost;
+	/**fourWeekCost*/
+	private BigDecimal fourWeekCost;
+	/**总消耗*/
+	private BigDecimal totalCost;
+	/**clickNum*/
+	private Long clickNum;
+	/**showNum*/
+	private Long showNum;
+	/**activeNum*/
+	private Long activeNum;
+	/**convertNum*/
+	private Long convertNum;
+	/**createTime*/
+	private Date createTime;
+	/**updateTime*/
+	private Date updateTime;
+	/**effectDate*/
+	private String effectDate;
+	/**faddishDate*/
+	private String faddishDate;
+	/**effectDayNum*/
+	private Integer effectDayNum;
+	/**faddishDayNum*/
+	private Integer faddishDayNum;
+}

+ 5 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceReportMaterialDailyMapper.java

@@ -6,6 +6,7 @@ import cn.com.ctop.toutiao.modules.material.vo.BytedanceVideoVo;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialDaily;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialRetry;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportVideoMaterialDaily;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
 import com.alibaba.fastjson.JSONArray;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
@@ -58,4 +59,8 @@ public interface BytedanceReportMaterialDailyMapper extends BaseMapper<Bytedance
                                                               @Param("code")String code,
                                                               @Param("order")String order,
                                                               @Param("target")String target);
+
+    List<BytedanceVideoEtlInfo> getSignatureByDate(@Param(value = "date") String date);
+
+    BytedanceVideoEtlInfo getEtlDataByParams(@Param("signature")String signature, @Param("date")String date);
 }

+ 16 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/BytedanceVideoEtlInfoMapper.java

@@ -0,0 +1,16 @@
+package cn.com.ctop.toutiao.modules.report.mapper;
+
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+
+/**
+ * 头条视频数据清洗信息
+ * @author: jeecg-boot
+ * @date:   2020-12-21
+ * @cersion: V1.0
+ */
+public interface BytedanceVideoEtlInfoMapper extends BaseMapper<BytedanceVideoEtlInfo> {
+
+    BytedanceVideoEtlInfo getBySignature(@Param(value = "signature") String signature);
+}

+ 34 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceReportMaterialDailyMapper.xml

@@ -416,6 +416,40 @@ where a.plane_id is not null and a.plane_name is null
         </if>
         group by report.material_id)a order by a.${target} ${order}
     </select>
+    <select id="getSignatureByDate"
+            resultType="cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo">
+        select
+            distinct signature as 'video_code'
+        from
+        ctop_bytedance_report_material_daily report
+        left join ctop_bytedance_video_info video on video.material_id = report.material_id and video.account_id = report.account_id
+        where
+            video.signature is not null
+        and
+            video.signature != ''
+        and
+            report.stat_datetime = #{date}
+    </select>
+    <select id="getEtlDataByParams"
+            resultType="cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo">
+        select
+        video.signature as 'video_code',
+        '' as cover_url,
+        video.video_url as url,
+        min(report.stat_datetime)as 'state_date',
+        sum(report.cost)as 'total_cost',
+        sum(report.click)as 'click_num',
+        sum(report.active)as 'active_num',
+        sum(report.show_material)as 'show_num',
+        sum(report.convert_material)as 'convert_num'
+        from ctop_bytedance_report_material_daily report
+        left join ctop_bytedance_video_info video on report.account_id = video.account_id and video.material_id = report.material_id
+        where
+            video.signature = #{signature}
+        and
+            report.stat_datetime&lt;= #{date}
+        group by video.signature;
+    </select>
 
     <insert id="replaceIntoVideoMaterialBatch">
         REPLACE INTO ctop_bytedance_report_video_material_daily

+ 8 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceVideoEtlInfoMapper.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.toutiao.modules.report.mapper.BytedanceVideoEtlInfoMapper">
+
+    <select id="getBySignature" resultType="cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo">
+
+    </select>
+</mapper>

+ 4 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceReportMaterialDailyService.java

@@ -2,6 +2,7 @@ package cn.com.ctop.toutiao.modules.report.service;
 
 import cn.com.ctop.toutiao.modules.material.vo.BytedanceVideoVo;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialDaily;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
 import com.baomidou.mybatisplus.extension.service.IService;
 
 import java.util.List;
@@ -15,4 +16,7 @@ import java.util.List;
 public interface IBytedanceReportMaterialDailyService extends IService<BytedanceReportMaterialDaily> {
     List<BytedanceReportMaterialDaily> getListByParams(String date);
     List<BytedanceVideoVo> getVideoVoByDate(String date);
+    List<BytedanceVideoEtlInfo> getSignatureByDate(String date);
+
+    BytedanceVideoEtlInfo getEtlDataByParams(String signature, String date);
 }

+ 18 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceVideoEtlInfoService.java

@@ -0,0 +1,18 @@
+package cn.com.ctop.toutiao.modules.report.service;
+
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+import java.util.Date;
+
+/**
+ * @Description: 头条视频数据清洗信息
+ * @Author: jeecg-boot
+ * @Date:   2020-12-21
+ * @Version: V1.0
+ */
+public interface IBytedanceVideoEtlInfoService extends IService<BytedanceVideoEtlInfo> {
+    void etlBytedanceVideoInfo(Date date);
+
+    BytedanceVideoEtlInfo getBySignature(String signature);
+}

+ 11 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceReportMaterialDailyServiceImpl.java

@@ -2,6 +2,7 @@ package cn.com.ctop.toutiao.modules.report.service.impl;
 
 import cn.com.ctop.toutiao.modules.material.vo.BytedanceVideoVo;
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialDaily;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceReportMaterialDailyMapper;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportMaterialDailyService;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -39,5 +40,15 @@ public class BytedanceReportMaterialDailyServiceImpl extends ServiceImpl<Bytedan
         return bytedanceReportMaterialDailyMapper.getVideoVoByDate(date);
     }
 
+    @Override
+    public List<BytedanceVideoEtlInfo> getSignatureByDate(String date) {
+        return bytedanceReportMaterialDailyMapper.getSignatureByDate(date);
+    }
+
+    @Override
+    public BytedanceVideoEtlInfo getEtlDataByParams(String signature, String date) {
+        return bytedanceReportMaterialDailyMapper.getEtlDataByParams(signature,date);
+    }
+
 
 }

+ 179 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoEtlInfoServiceImpl.java

@@ -0,0 +1,179 @@
+package cn.com.ctop.toutiao.modules.report.service.impl;
+
+import cn.com.ctop.common.module.mapper.UserAllocationMapper;
+import cn.com.ctop.common.module.service.IMaterialAscriptionService;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceReportMaterialDaily;
+import cn.com.ctop.toutiao.modules.report.entity.BytedanceVideoEtlInfo;
+import cn.com.ctop.toutiao.modules.report.mapper.BytedanceVideoEtlInfoMapper;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceReportMaterialDailyService;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoEtlInfoService;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 头条视频数据清洗信息
+ * @author jeecg-boot
+ * @date   2020-12-21
+ * @version V1.0
+ */
+@Slf4j
+@Service
+public class BytedanceVideoEtlInfoServiceImpl extends ServiceImpl<BytedanceVideoEtlInfoMapper, BytedanceVideoEtlInfo> implements IBytedanceVideoEtlInfoService {
+    @Autowired
+    private IBytedanceReportMaterialDailyService materialDailyService;
+    @Autowired
+    private UserAllocationMapper userAllocationMapper;
+    static ExecutorService executorService = Executors.newFixedThreadPool(50);
+    @Autowired
+    private IMaterialAscriptionService materialAscriptionService;
+    @Override
+    public void etlBytedanceVideoInfo(Date getDate) {
+        String date = DateUtils.formatDate(getDate);
+        //1:查询单当日视频信息
+        List<BytedanceVideoEtlInfo> materials = materialDailyService.getSignatureByDate(date);
+        if(null!=materials&&!materials.isEmpty()){
+            CountDownLatch countDownLatch= new CountDownLatch(materials.size());
+            materials.forEach(material->{
+                executorService.submit(()->{
+                    try {
+                        if(null == material.getVideoCode()||material.getVideoCode().trim().equals("")){
+                            return;
+                        }
+                        String signature = material.getVideoCode();
+                        BytedanceVideoEtlInfo etlInfo = this.getBySignature(signature);
+                        BytedanceVideoEtlInfo nowData = materialDailyService.getEtlDataByParams(signature,date);
+                        String minDate = nowData.getStateDate();
+                        Integer days = DateUtils.getdaysOfTwoDate(minDate,date);
+                        if(null == etlInfo){
+                            //第一次查询的数据
+                            etlInfo = nowData;
+                            BigDecimal totalCost = etlInfo.getTotalCost();
+                            JSONObject ascription =  materialAscriptionService.getDetailByCode(signature);
+                            String planId = null;
+                            if(null!=ascription){
+                                planId = ascription.getString("planId");
+                                if(null!=planId&&!planId.trim().equals("")){
+                                    etlInfo.setPlanId(planId);
+                                }
+                                String planName =  ascription.getString("planName");
+                                if(null!=planName&&!planName.trim().equals("")){
+                                    etlInfo.setPlanName(planName);
+                                }
+                                String clipId = ascription.getString("clipId");
+                                if(null!=clipId&&!clipId.trim().equals("")){
+                                    etlInfo.setClipId(clipId);
+                                }
+                                String clipName = ascription.getString("clipName");
+                                if(null!=clipName&&!clipName.trim().equals("")){
+                                    etlInfo.setClipName(clipName);
+                                }
+                                String shotId = ascription.getString("shotId");
+                                if(null!=shotId&&!shotId.trim().equals("")){
+                                    etlInfo.setShotId(shotId);
+                                }
+                                String shotName = ascription.getString("shotName");
+                                if(null!=shotName&&!shotName.trim().equals("")){
+                                    etlInfo.setShotName(shotName);
+                                }
+                            }
+                            if(null!=planId){
+                                String companyName = userAllocationMapper.getCompanyNameByUserId(planId);
+                                if (null!=companyName&&!companyName.trim().equals("")){
+                                    etlInfo.setCompanyName(companyName);
+                                }
+                            }
+                            //计算投放总天数
+                            if(days == 7){
+                                etlInfo.setSingleWeekCost(nowData.getTotalCost());
+                            }
+                            if(days == 14){
+                                etlInfo.setTwoWeekCost(nowData.getTotalCost());
+                            }
+                            if(days == 28){
+                                etlInfo.setFourWeekCost(nowData.getTotalCost());
+                            }
+                            //计算是否有效,爆款
+                            if(totalCost.compareTo(new BigDecimal("5000"))>0){
+                                //此刻成为有效
+                                etlInfo.setEffectDate(date);
+                                etlInfo.setEffectDayNum(days);
+                            }
+                            if(totalCost.compareTo(new BigDecimal("100000"))>0){
+                                //此刻成为爆款
+                                etlInfo.setEffectDate(date);
+                                etlInfo.setEffectDayNum(days);
+                            }
+                            etlInfo.setCreateTime(new Date());
+                            etlInfo.setUpdateTime(new Date());
+                            this.save(etlInfo);
+                        }else{
+                            //只需要更新汇总数据
+                            etlInfo.setTotalCost(nowData.getTotalCost());
+                            BigDecimal totalCost = etlInfo.getTotalCost();
+                            etlInfo.setClickNum(nowData.getClickNum());
+                            etlInfo.setShowNum(nowData.getShowNum());
+                            etlInfo.setActiveNum(nowData.getActiveNum());
+                            etlInfo.setConvertNum(nowData.getConvertNum());
+                            etlInfo.setUpdateTime(new Date());
+                            //计算投放总天数
+                            if(days == 7){
+                                etlInfo.setSingleWeekCost(nowData.getTotalCost());
+                            }
+                            if(days == 14){
+                                etlInfo.setTwoWeekCost(nowData.getTotalCost());
+                            }
+                            if(days == 28){
+                                etlInfo.setFourWeekCost(nowData.getTotalCost());
+                            }
+                            if(null == etlInfo.getEffectDayNum()){
+                                if(totalCost.compareTo(new BigDecimal("5000"))>0){
+                                    //此刻成为有效
+                                    etlInfo.setEffectDate(date);
+                                    etlInfo.setEffectDayNum(days);
+                                }
+                            }
+                            if(null == etlInfo.getFaddishDayNum()){
+                                if(totalCost.compareTo(new BigDecimal("100000"))>0){
+                                    //此刻成为爆款
+                                    etlInfo.setEffectDate(date);
+                                    etlInfo.setEffectDayNum(days);
+                                }
+                            }
+                            this.updateById(etlInfo);
+                        }
+                    }catch (Exception e){
+
+                    }finally {
+                        countDownLatch.countDown();
+                    }
+                });
+            });
+
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            log.info("{}数据获取完成",date);
+        }
+    }
+
+    @Override
+    public BytedanceVideoEtlInfo getBySignature(String signature) {
+        QueryWrapper<BytedanceVideoEtlInfo> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("video_code",signature).last("limit 1");
+        return this.getOne(queryWrapper);
+    }
+}