Prechádzať zdrojové kódy

fix of '头条视频报表逻辑调整' on 2021.2.2

jiequan.bi 4 rokov pred
rodič
commit
f7a564ff71
19 zmenil súbory, kde vykonal 1073 pridanie a 53 odobranie
  1. 1 1
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/orderplatform/mapper/xml/PlatformScriptMapper.xml
  2. 50 0
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BaseReportVideoDailyLoadJob.java
  3. 1 1
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceMaterialsLoadJob.java
  4. 30 0
      module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ETLReportVideoCleanJob.java
  5. 196 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/entity/BaseReportBytedanceVideoDaily.java
  6. 436 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/entity/ETLReportBytedanceVideo.java
  7. 11 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/BaseReportBytedanceVideoDailyMapper.java
  8. 29 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLReportBytedanceVideoMapper.java
  9. 5 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/BaseReportBytedanceVideoDailyMapper.xml
  10. 79 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml
  11. 11 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IBaseReportBytedanceVideoDailyService.java
  12. 17 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IETLReportBytedanceVideoService.java
  13. 15 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/BaseReportBytedanceVideoDailyServiceImpl.java
  14. 95 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java
  15. 25 43
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceVideoReportMapper.xml
  16. 3 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceVideoReportService.java
  17. 0 5
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceReportServiceImpl.java
  18. 66 0
      module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoReportServiceImpl.java
  19. 3 3
      module-toutiao/src/main/resources/json_template/bytedance_video_report_mapping.json

+ 1 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/orderplatform/mapper/xml/PlatformScriptMapper.xml

@@ -8,7 +8,7 @@
             t1.`name`,
             t1.size,
             t1.create_time,
-            FROM_UNIXTIME(CAST(t1.create_time/1000 AS INT)) as createTime,
+        FROM_UNIXTIME(ROUND((t1.create_time/1000),0)) as createTime,
             (SELECT realname from sys_user where id=t1.creator limit 1) as creatorName,
             (SELECT project_name from ctop_project where id=t1.project_id limit 1) as projectName,
             t2.material_name as materialName

+ 50 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BaseReportVideoDailyLoadJob.java

@@ -0,0 +1,50 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.service.ICtopOauthTokenService;
+import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoReportService;
+import com.xxl.job.core.context.XxlJobHelper;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@Component
+public class BaseReportVideoDailyLoadJob {
+
+    @Autowired
+    ICtopOauthTokenService oauthTokenService;
+    @Autowired
+    IBytedanceVideoReportService bytedanceVideoReportService;
+    private ExecutorService executorService = Executors.newFixedThreadPool(10);
+    private CountDownLatch countDownLatch;
+    @XxlJob("baseReportVideoDailyLoadJob")
+    public void execute() throws Exception {
+        List<CtopOauthToken> tokens = oauthTokenService.selectToutiaoToken();
+        countDownLatch = new CountDownLatch(tokens.size());
+        String statDate= DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
+        tokens.forEach(token -> executorService.submit(() -> {
+            try {
+                bytedanceVideoReportService.pullVideoReport(token,statDate,statDate,1);
+            } catch (Exception e) {
+                XxlJobHelper.log("头条视频报表基础数据拉取异常:{}",e.getMessage());
+            } finally {
+                countDownLatch.countDown();
+            }
+        }));
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        if(!executorService.isShutdown()){
+            executorService.shutdown();
+        }
+    }
+}

+ 1 - 1
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/BytedanceMaterialsLoadJob.java

@@ -17,7 +17,7 @@ import java.util.concurrent.Executors;
 public class BytedanceMaterialsLoadJob {
     @Autowired
     private ICtopOauthTokenService tokenService;
-    static ExecutorService executorService = Executors.newFixedThreadPool(4);
+    static ExecutorService executorService = Executors.newFixedThreadPool(10);
     //线程计数器
     static CountDownLatch countDownLatch = null;
     @Autowired

+ 30 - 0
module-job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/ETLReportVideoCleanJob.java

@@ -0,0 +1,30 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
+import com.xxl.job.core.context.XxlJobHelper;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Date;
+
+@Component
+public class ETLReportVideoCleanJob {
+
+    @Autowired
+    IETLReportBytedanceVideoService etlReportBytedanceVideoService;
+
+    @XxlJob("ETLReportVideoCleanJob")
+    public void execute() throws Exception {
+        String statDate = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
+        try {
+            etlReportBytedanceVideoService.cleanETL(statDate);
+        } catch (Exception e) {
+            XxlJobHelper.log("头条视频报表基础数据拉取异常:{}", e.getMessage());
+        } finally {
+            XxlJobHelper.log("ETLReportVideoCleanJob is finish");
+        }
+    }
+}
+

+ 196 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/entity/BaseReportBytedanceVideoDaily.java

@@ -0,0 +1,196 @@
+package cn.com.ctop.toutiao.modules.link.entity;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+/**
+ * 头条视频报表源数据
+ */
+@Data
+@TableName("base_report_bytedance_video_daily")
+public class BaseReportBytedanceVideoDaily {
+    
+	/**id*/
+	@TableId(type = IdType.AUTO)
+	private Long id;
+	/**账户id*/
+	private Long accountId;
+	/**md5*/
+	private Long materialId;
+	/**数据起始时间*/
+	private String statDatetime;
+	/**总花费*/
+	private BigDecimal cost;
+	/**点击量*/
+	private Long click;
+	/**应用下载-激活*/
+	private Long active;
+	/**应用下载-安卓下载完成*/
+	private Long downloadFinish;
+	/**应用下载-安卓下载开始*/
+	private Long downloadStart;
+	/**应用下载-安卓安装完成*/
+	private Long installFinish;
+	/**应用下载-注册*/
+	private Long register;
+	/**应用下载-付费数*/
+	private Long payCount;
+	/**应用下载-到达uv*/
+	private Long inAppUv;
+	/**应用下载-详情页到站uv*/
+	private Long inAppDetailUv;
+	/**应用下载-加入购物车*/
+	private Long inAppCart;
+	/**应用下载-提交订单*/
+	private Long inAppOrder;
+	/**应用下载-付费*/
+	private Long inAppPay;
+	/**落地页-电话拨打数*/
+	private Long phone;
+	/**表单提交数*/
+	private Long form;
+	/**落地页-地图搜索*/
+	private Long mapSearch;
+	/**落地页-按钮button*/
+	private Long button;
+	/**落地页-关键页面浏览*/
+	private Long viewMaterial;
+	/**落地页-QQ咨询数*/
+	private Long qq;
+	/**落地页-抽奖数*/
+	private Long lottery;
+	/**落地页-投票*/
+	private Long vote;
+	/**落地页-页面跳转*/
+	private Long redirect;
+	/**落地页-商品购买*/
+	private Long shopping;
+	/**落地页-在线咨询*/
+	private Long consult;
+	/**落地页-微信*/
+	private Long wechat;
+	/**落地页-智能电话确认拨打*/
+	private Long phoneConfirm;
+	/**落地页-智能电话确认接通*/
+	private Long phoneConnect;
+	/**落地页-智能电话有效咨询*/
+	private Long consultEffective;
+	/**视频-总播放*/
+	private Long totalPlay;
+	/**视频-有效播放*/
+	private Long validPlay;
+	/**视频-wifi播放*/
+	private Long wifiPlay;
+	/**视频-播放25%进度率*/
+	private BigDecimal play25FeedBreakRate;
+	/**视频-播放25%进度总数*/
+	private Long play25FeedBreak;
+	/**视频-播放50%进度总数*/
+	private Long play50FeedBreak;
+	/**视频-播放75%进度总数*/
+	private Long play75FeedBreak;
+	/**视频-播放100%进度总数*/
+	private Long play100FeedBreak;
+	/**附加创意-电话按钮*/
+	private Long advancedCreativePhoneClick;
+	/**附加创意-在线咨询*/
+	private Long advancedCreativeCounselClick;
+	/**附加创意-表单提交*/
+	private Long advancedCreativeFormClick;
+	/**互动数据-分享数*/
+	private Long shareMaterial;
+	/**互动数据-评论数*/
+	private Long commentMaterial;
+	/**互动数据-新增关注数*/
+	private Long follow;
+	/**互动数据-主页访问量*/
+	private Long homeVisited;
+	/**互动数据-挑战赛查看数*/
+	private Long iesChallengeClick;
+	/**互动数据-音乐查看数*/
+	private Long iesMusicClick;
+	/**次留数*/
+	private Long nextDayOpen;
+	/**次留率*/
+	private BigDecimal nextDayOpenRate;
+	/**次留成本*/
+	private BigDecimal nextDayOpenCost;
+	/**activePayAmount*/
+	private Long activePayAmount;
+	/**视频数据-有效播放成本*/
+	private BigDecimal validPlayCost;
+	/**附加创意-附加创意卡券领取*/
+	private Long advancedCreativeCouponAddition;
+	/**convertMaterial*/
+	private Long convertMaterial;
+	/**应用下载广告数据-付费成本*/
+	private BigDecimal activePayCost;
+	/**落地页转化数据-下载开始*/
+	private Long download;
+	/**cpc*/
+	private BigDecimal cpc;
+	/**互动数据-POI点击数*/
+	private Long locationClick;
+	/**	视频数据-播完率*/
+	private BigDecimal playOverRate;
+	/**展现数据-点击率*/
+	private BigDecimal ctr;
+	/**cpm*/
+	private BigDecimal cpm;
+	/**视频数据-WiFi播放占比*/
+	private BigDecimal wifiPlayRate;
+	/**互动数据-点赞数*/
+	private Long likeMaterial;
+	/**应用下载广告数据-激活成本*/
+	private BigDecimal activeCost;
+	/**应用下载广告数据-关键行为成本*/
+	private BigDecimal gameAddictionCost;
+	/**应用下载广告数据-关键行为数*/
+	private Long gameAddiction;
+	/**应用下载广告数据-激活率*/
+	private BigDecimal activeRate;
+	/**	应用下载广告数据-关键行为率*/
+	private BigDecimal gameAddictionRate;
+	/**应用下载广告数据-注册率*/
+	private BigDecimal activeRegisterRate;
+	/**应用下载广告数据-安卓下载完成成本*/
+	private BigDecimal downloadFinishCost;
+	/**应用下载广告数据-注册成本*/
+	private BigDecimal activeRegisterCost;
+	/**展示数*/
+	private Long materialShow;
+	/**转化数据-转化率*/
+	private BigDecimal convertRate;
+	/**应用下载广告数据-安卓下载完成率*/
+	private BigDecimal downloadFinishRate;
+	/**应用下载广告数据-安卓安装完成率*/
+	private BigDecimal installFinishRate;
+	/**	落地页转化数据-建站卡券领取*/
+	private Long coupon;
+	/**落地页转化数据-卡券页领取*/
+	private Long couponSinglePage;
+	/**playOver*/
+	private Long playOver;
+	/**应用下载广告数据-安卓下载开始成本*/
+	private BigDecimal downloadStartCost;
+	/**落地页转化数据-短信咨询*/
+	private Long message;
+	/**视频数据-有效播放率*/
+	private BigDecimal validPlayRate;
+	/**视频数据-平均单次播放时长*/
+	private BigDecimal averagePlayTimePerPlay;
+	/**转化数据-转化成本*/
+	private BigDecimal convertCost;
+	/**应用下载广告数据-安卓安装完成成本*/
+	private BigDecimal installFinishCost;
+	/**应用下载广告数据-安卓下载开始率*/
+	private BigDecimal downloadStartRate;
+	/**createTime*/
+	private Date createTime;
+	/**updateTime*/
+	private Date updateTime;
+}

+ 436 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/entity/ETLReportBytedanceVideo.java

@@ -0,0 +1,436 @@
+package cn.com.ctop.toutiao.modules.link.entity;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+
+import org.jeecgframework.poi.excel.annotation.Excel;
+
+/**
+ * 头条视频报表ETL
+ */
+@Data
+@TableName("etl_report_bytedance_video")
+public class ETLReportBytedanceVideo {
+    
+	/**id*/
+	@TableId(type = IdType.AUTO)
+	private Long id;
+	/**materialId*/
+	@Excel(name = "materialId", width = 15)
+    @ApiModelProperty(value = "materialId")
+	private Long materialId;
+
+	private String materialName;
+	/**视频唯一标识*/
+	private String md5;
+
+	private String url;
+
+	private String toutiaoUrl;
+
+	private String coverUrl;
+
+	/**账户id*/
+	@Excel(name = "账户id", width = 15)
+    @ApiModelProperty(value = "账户id")
+	private Long accountId;
+
+	private String accountName;
+	/**项目id*/
+	@Excel(name = "项目id", width = 15)
+    @ApiModelProperty(value = "项目id")
+	private Long projectId;
+	/**项目名称*/
+	@Excel(name = "项目名称", width = 15)
+    @ApiModelProperty(value = "项目名称")
+	private String projectName;
+	/**项目id*/
+	@Excel(name = "项目id", width = 15)
+    @ApiModelProperty(value = "项目id")
+	private Long productId;
+	/**产品名称*/
+	@Excel(name = "产品名称", width = 15)
+    @ApiModelProperty(value = "产品名称")
+	private String productName;
+	/**广告主id*/
+	@Excel(name = "广告主id", width = 15)
+    @ApiModelProperty(value = "广告主id")
+	private String advertiserId;
+	/**广告主名称*/
+	@Excel(name = "广告主名称", width = 15)
+    @ApiModelProperty(value = "广告主名称")
+	private String advertiserName;
+	/**剪辑id*/
+	@Excel(name = "剪辑id", width = 15)
+    @ApiModelProperty(value = "剪辑id")
+	private String clipId;
+	/**剪辑名称*/
+	@Excel(name = "剪辑名称", width = 15)
+    @ApiModelProperty(value = "剪辑名称")
+	private String clipName;
+	/**拍摄id*/
+	@Excel(name = "拍摄id", width = 15)
+    @ApiModelProperty(value = "拍摄id")
+	private String shotId;
+	/**拍摄名称*/
+	@Excel(name = "拍摄名称", width = 15)
+    @ApiModelProperty(value = "拍摄名称")
+	private String shotName;
+	/**编导id*/
+	@Excel(name = "编导id", width = 15)
+    @ApiModelProperty(value = "编导id")
+	private String planId;
+	/**编导名称*/
+	@Excel(name = "编导名称", width = 15)
+    @ApiModelProperty(value = "编导名称")
+	private String planName;
+	/**materialCreateTime*/
+	private String materialCreateTime;
+	/**数据起始时间*/
+	@Excel(name = "数据起始时间", width = 15)
+    @ApiModelProperty(value = "数据起始时间")
+	private String statDatetime;
+	/**点击量*/
+	@Excel(name = "点击量", width = 15)
+    @ApiModelProperty(value = "点击量")
+	private Long click;
+	/**总花费*/
+	@Excel(name = "总花费", width = 15)
+    @ApiModelProperty(value = "总花费")
+	private BigDecimal cost;
+	/**应用下载-激活*/
+	@Excel(name = "应用下载-激活", width = 15)
+    @ApiModelProperty(value = "应用下载-激活")
+	private Long active;
+	/**应用下载-安卓下载完成*/
+	@Excel(name = "应用下载-安卓下载完成", width = 15)
+    @ApiModelProperty(value = "应用下载-安卓下载完成")
+	private Long downloadFinish;
+	/**应用下载-安卓下载开始*/
+	@Excel(name = "应用下载-安卓下载开始", width = 15)
+    @ApiModelProperty(value = "应用下载-安卓下载开始")
+	private Long downloadStart;
+	/**应用下载-安卓安装完成*/
+	@Excel(name = "应用下载-安卓安装完成", width = 15)
+    @ApiModelProperty(value = "应用下载-安卓安装完成")
+	private Long installFinish;
+	/**应用下载-注册*/
+	@Excel(name = "应用下载-注册", width = 15)
+    @ApiModelProperty(value = "应用下载-注册")
+	private Long register;
+	/**应用下载-付费数*/
+	@Excel(name = "应用下载-付费数", width = 15)
+    @ApiModelProperty(value = "应用下载-付费数")
+	private Long payCount;
+	/**应用下载-到达uv*/
+	@Excel(name = "应用下载-到达uv", width = 15)
+    @ApiModelProperty(value = "应用下载-到达uv")
+	private Long inAppUv;
+	/**应用下载-详情页到站uv*/
+	@Excel(name = "应用下载-详情页到站uv", width = 15)
+    @ApiModelProperty(value = "应用下载-详情页到站uv")
+	private Long inAppDetailUv;
+	/**应用下载-加入购物车*/
+	@Excel(name = "应用下载-加入购物车", width = 15)
+    @ApiModelProperty(value = "应用下载-加入购物车")
+	private Long inAppCart;
+	/**应用下载-提交订单*/
+	@Excel(name = "应用下载-提交订单", width = 15)
+    @ApiModelProperty(value = "应用下载-提交订单")
+	private Long inAppOrder;
+	/**应用下载-付费*/
+	@Excel(name = "应用下载-付费", width = 15)
+    @ApiModelProperty(value = "应用下载-付费")
+	private Long inAppPay;
+	/**落地页-电话拨打数*/
+	@Excel(name = "落地页-电话拨打数", width = 15)
+    @ApiModelProperty(value = "落地页-电话拨打数")
+	private Long phone;
+	/**表单提交数*/
+	@Excel(name = "表单提交数", width = 15)
+    @ApiModelProperty(value = "表单提交数")
+	private Long form;
+	/**落地页-地图搜索*/
+	@Excel(name = "落地页-地图搜索", width = 15)
+    @ApiModelProperty(value = "落地页-地图搜索")
+	private Long mapSearch;
+	/**落地页-按钮button*/
+	@Excel(name = "落地页-按钮button", width = 15)
+    @ApiModelProperty(value = "落地页-按钮button")
+	private Long button;
+	/**落地页-关键页面浏览*/
+	@Excel(name = "落地页-关键页面浏览", width = 15)
+    @ApiModelProperty(value = "落地页-关键页面浏览")
+	private Long viewMaterial;
+	/**落地页-QQ咨询数*/
+	@Excel(name = "落地页-QQ咨询数", width = 15)
+    @ApiModelProperty(value = "落地页-QQ咨询数")
+	private Long qq;
+	/**落地页-抽奖数*/
+	@Excel(name = "落地页-抽奖数", width = 15)
+    @ApiModelProperty(value = "落地页-抽奖数")
+	private Long lottery;
+	/**落地页-投票*/
+	@Excel(name = "落地页-投票", width = 15)
+    @ApiModelProperty(value = "落地页-投票")
+	private Long vote;
+	/**落地页-页面跳转*/
+	@Excel(name = "落地页-页面跳转", width = 15)
+    @ApiModelProperty(value = "落地页-页面跳转")
+	private Long redirect;
+	/**落地页-商品购买*/
+	@Excel(name = "落地页-商品购买", width = 15)
+    @ApiModelProperty(value = "落地页-商品购买")
+	private Long shopping;
+	/**落地页-在线咨询*/
+	@Excel(name = "落地页-在线咨询", width = 15)
+    @ApiModelProperty(value = "落地页-在线咨询")
+	private Long consult;
+	/**落地页-微信*/
+	@Excel(name = "落地页-微信", width = 15)
+    @ApiModelProperty(value = "落地页-微信")
+	private Long wechat;
+	/**落地页-智能电话确认拨打*/
+	@Excel(name = "落地页-智能电话确认拨打", width = 15)
+    @ApiModelProperty(value = "落地页-智能电话确认拨打")
+	private Long phoneConfirm;
+	/**落地页-智能电话确认接通*/
+	@Excel(name = "落地页-智能电话确认接通", width = 15)
+    @ApiModelProperty(value = "落地页-智能电话确认接通")
+	private Long phoneConnect;
+	/**落地页-智能电话有效咨询*/
+	@Excel(name = "落地页-智能电话有效咨询", width = 15)
+    @ApiModelProperty(value = "落地页-智能电话有效咨询")
+	private Long consultEffective;
+	/**视频-总播放*/
+	@Excel(name = "视频-总播放", width = 15)
+    @ApiModelProperty(value = "视频-总播放")
+	private Long totalPlay;
+	/**视频-有效播放*/
+	@Excel(name = "视频-有效播放", width = 15)
+    @ApiModelProperty(value = "视频-有效播放")
+	private Long validPlay;
+	/**视频-wifi播放*/
+	@Excel(name = "视频-wifi播放", width = 15)
+    @ApiModelProperty(value = "视频-wifi播放")
+	private Long wifiPlay;
+	/**视频-播放25%进度率*/
+	@Excel(name = "视频-播放25%进度率", width = 15)
+    @ApiModelProperty(value = "视频-播放25%进度率")
+	private BigDecimal play25FeedBreakRate;
+	/**视频-播放25%进度总数*/
+	@Excel(name = "视频-播放25%进度总数", width = 15)
+    @ApiModelProperty(value = "视频-播放25%进度总数")
+	private Long play25FeedBreak;
+	/**视频-播放50%进度总数*/
+	@Excel(name = "视频-播放50%进度总数", width = 15)
+    @ApiModelProperty(value = "视频-播放50%进度总数")
+	private Long play50FeedBreak;
+	/**视频-播放75%进度总数*/
+	@Excel(name = "视频-播放75%进度总数", width = 15)
+    @ApiModelProperty(value = "视频-播放75%进度总数")
+	private Long play75FeedBreak;
+	/**视频-播放100%进度总数*/
+	@Excel(name = "视频-播放100%进度总数", width = 15)
+    @ApiModelProperty(value = "视频-播放100%进度总数")
+	private Long play100FeedBreak;
+	/**附加创意-电话按钮*/
+	@Excel(name = "附加创意-电话按钮", width = 15)
+    @ApiModelProperty(value = "附加创意-电话按钮")
+	private Long advancedCreativePhoneClick;
+	/**附加创意-在线咨询*/
+	@Excel(name = "附加创意-在线咨询", width = 15)
+    @ApiModelProperty(value = "附加创意-在线咨询")
+	private Long advancedCreativeCounselClick;
+	/**附加创意-表单提交*/
+	@Excel(name = "附加创意-表单提交", width = 15)
+    @ApiModelProperty(value = "附加创意-表单提交")
+	private Long advancedCreativeFormClick;
+	/**互动数据-分享数*/
+	@Excel(name = "互动数据-分享数", width = 15)
+    @ApiModelProperty(value = "互动数据-分享数")
+	private Long shareMaterial;
+	/**互动数据-评论数*/
+	@Excel(name = "互动数据-评论数", width = 15)
+    @ApiModelProperty(value = "互动数据-评论数")
+	private Long commentMaterial;
+	/**互动数据-新增关注数*/
+	@Excel(name = "互动数据-新增关注数", width = 15)
+    @ApiModelProperty(value = "互动数据-新增关注数")
+	private Long follow;
+	/**互动数据-主页访问量*/
+	@Excel(name = "互动数据-主页访问量", width = 15)
+    @ApiModelProperty(value = "互动数据-主页访问量")
+	private Long homeVisited;
+	/**互动数据-挑战赛查看数*/
+	@Excel(name = "互动数据-挑战赛查看数", width = 15)
+    @ApiModelProperty(value = "互动数据-挑战赛查看数")
+	private Long iesChallengeClick;
+	/**互动数据-音乐查看数*/
+	@Excel(name = "互动数据-音乐查看数", width = 15)
+    @ApiModelProperty(value = "互动数据-音乐查看数")
+	private Long iesMusicClick;
+	/**次留数*/
+	@Excel(name = "次留数", width = 15)
+    @ApiModelProperty(value = "次留数")
+	private Long nextDayOpen;
+	/**次留率*/
+	@Excel(name = "次留率", width = 15)
+    @ApiModelProperty(value = "次留率")
+	private BigDecimal nextDayOpenRate;
+	/**次留成本*/
+	@Excel(name = "次留成本", width = 15)
+    @ApiModelProperty(value = "次留成本")
+	private BigDecimal nextDayOpenCost;
+	/**activePayAmount*/
+	@Excel(name = "activePayAmount", width = 15)
+    @ApiModelProperty(value = "activePayAmount")
+	private Long activePayAmount;
+	/**视频数据-有效播放成本*/
+	@Excel(name = "视频数据-有效播放成本", width = 15)
+    @ApiModelProperty(value = "视频数据-有效播放成本")
+	private BigDecimal validPlayCost;
+	/**附加创意-附加创意卡券领取*/
+	@Excel(name = "附加创意-附加创意卡券领取", width = 15)
+    @ApiModelProperty(value = "附加创意-附加创意卡券领取")
+	private Long advancedCreativeCouponAddition;
+	/**convertMaterial*/
+	@Excel(name = "convertMaterial", width = 15)
+    @ApiModelProperty(value = "convertMaterial")
+	private Long convertMaterial;
+	/**应用下载广告数据-付费成本*/
+	@Excel(name = "应用下载广告数据-付费成本", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-付费成本")
+	private BigDecimal activePayCost;
+	/**落地页转化数据-下载开始*/
+	@Excel(name = "落地页转化数据-下载开始", width = 15)
+    @ApiModelProperty(value = "落地页转化数据-下载开始")
+	private Long download;
+	/**cpc*/
+	@Excel(name = "cpc", width = 15)
+    @ApiModelProperty(value = "cpc")
+	private BigDecimal cpc;
+	/**互动数据-POI点击数*/
+	@Excel(name = "互动数据-POI点击数", width = 15)
+    @ApiModelProperty(value = "互动数据-POI点击数")
+	private Long locationClick;
+	/**	视频数据-播完率*/
+	@Excel(name = "	视频数据-播完率", width = 15)
+    @ApiModelProperty(value = "	视频数据-播完率")
+	private BigDecimal playOverRate;
+	/**展现数据-点击率*/
+	@Excel(name = "展现数据-点击率", width = 15)
+    @ApiModelProperty(value = "展现数据-点击率")
+	private BigDecimal ctr;
+	/**cpm*/
+	@Excel(name = "cpm", width = 15)
+    @ApiModelProperty(value = "cpm")
+	private BigDecimal cpm;
+	/**视频数据-WiFi播放占比*/
+	@Excel(name = "视频数据-WiFi播放占比", width = 15)
+    @ApiModelProperty(value = "视频数据-WiFi播放占比")
+	private BigDecimal wifiPlayRate;
+	/**互动数据-点赞数*/
+	@Excel(name = "互动数据-点赞数", width = 15)
+    @ApiModelProperty(value = "互动数据-点赞数")
+	private Long likeMaterial;
+	/**应用下载广告数据-激活成本*/
+	@Excel(name = "应用下载广告数据-激活成本", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-激活成本")
+	private BigDecimal activeCost;
+	/**应用下载广告数据-关键行为成本*/
+	@Excel(name = "应用下载广告数据-关键行为成本", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-关键行为成本")
+	private BigDecimal gameAddictionCost;
+	/**应用下载广告数据-关键行为数*/
+	@Excel(name = "应用下载广告数据-关键行为数", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-关键行为数")
+	private Long gameAddiction;
+	/**应用下载广告数据-激活率*/
+	@Excel(name = "应用下载广告数据-激活率", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-激活率")
+	private BigDecimal activeRate;
+	/**	应用下载广告数据-关键行为率*/
+	@Excel(name = "	应用下载广告数据-关键行为率", width = 15)
+    @ApiModelProperty(value = "	应用下载广告数据-关键行为率")
+	private BigDecimal gameAddictionRate;
+	/**应用下载广告数据-注册率*/
+	@Excel(name = "应用下载广告数据-注册率", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-注册率")
+	private BigDecimal activeRegisterRate;
+	/**应用下载广告数据-安卓下载完成成本*/
+	@Excel(name = "应用下载广告数据-安卓下载完成成本", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-安卓下载完成成本")
+	private BigDecimal downloadFinishCost;
+	/**应用下载广告数据-注册成本*/
+	@Excel(name = "应用下载广告数据-注册成本", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-注册成本")
+	private BigDecimal activeRegisterCost;
+	/**materailShow*/
+	@Excel(name = "materailShow", width = 15)
+    @ApiModelProperty(value = "materailShow")
+	private Long materailShow;
+	/**转化数据-转化率*/
+	@Excel(name = "转化数据-转化率", width = 15)
+    @ApiModelProperty(value = "转化数据-转化率")
+	private BigDecimal convertRate;
+	/**应用下载广告数据-安卓下载完成率*/
+	@Excel(name = "应用下载广告数据-安卓下载完成率", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-安卓下载完成率")
+	private BigDecimal downloadFinishRate;
+	/**应用下载广告数据-安卓安装完成率*/
+	@Excel(name = "应用下载广告数据-安卓安装完成率", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-安卓安装完成率")
+	private BigDecimal installFinishRate;
+	/**	落地页转化数据-建站卡券领取*/
+	@Excel(name = "	落地页转化数据-建站卡券领取", width = 15)
+    @ApiModelProperty(value = "	落地页转化数据-建站卡券领取")
+	private Long coupon;
+	/**落地页转化数据-卡券页领取*/
+	@Excel(name = "落地页转化数据-卡券页领取", width = 15)
+    @ApiModelProperty(value = "落地页转化数据-卡券页领取")
+	private Long couponSinglePage;
+	/**playOver*/
+	@Excel(name = "playOver", width = 15)
+    @ApiModelProperty(value = "playOver")
+	private Long playOver;
+	/**应用下载广告数据-安卓下载开始成本*/
+	@Excel(name = "应用下载广告数据-安卓下载开始成本", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-安卓下载开始成本")
+	private BigDecimal downloadStartCost;
+	/**落地页转化数据-短信咨询*/
+	@Excel(name = "落地页转化数据-短信咨询", width = 15)
+    @ApiModelProperty(value = "落地页转化数据-短信咨询")
+	private Long message;
+	/**视频数据-有效播放率*/
+	@Excel(name = "视频数据-有效播放率", width = 15)
+    @ApiModelProperty(value = "视频数据-有效播放率")
+	private BigDecimal validPlayRate;
+	/**视频数据-平均单次播放时长*/
+	@Excel(name = "视频数据-平均单次播放时长", width = 15)
+    @ApiModelProperty(value = "视频数据-平均单次播放时长")
+	private BigDecimal averagePlayTimePerPlay;
+	/**转化数据-转化成本*/
+	@Excel(name = "转化数据-转化成本", width = 15)
+    @ApiModelProperty(value = "转化数据-转化成本")
+	private BigDecimal convertCost;
+	/**应用下载广告数据-安卓安装完成成本*/
+	@Excel(name = "应用下载广告数据-安卓安装完成成本", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-安卓安装完成成本")
+	private BigDecimal installFinishCost;
+	/**应用下载广告数据-安卓下载开始率*/
+	@Excel(name = "应用下载广告数据-安卓下载开始率", width = 15)
+    @ApiModelProperty(value = "应用下载广告数据-安卓下载开始率")
+	private BigDecimal downloadStartRate;
+	/**createTime*/
+    @ApiModelProperty(value = "createTime")
+	private Date createTime;
+	/**updateTime*/
+    @ApiModelProperty(value = "updateTime")
+	private Date updateTime;
+}

+ 11 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/BaseReportBytedanceVideoDailyMapper.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.toutiao.modules.link.mapper;
+
+import cn.com.ctop.toutiao.modules.link.entity.BaseReportBytedanceVideoDaily;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * 头条视频报表源数据
+ */
+public interface BaseReportBytedanceVideoDailyMapper extends BaseMapper<BaseReportBytedanceVideoDaily> {
+
+}

+ 29 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/ETLReportBytedanceVideoMapper.java

@@ -0,0 +1,29 @@
+package cn.com.ctop.toutiao.modules.link.mapper;
+
+import cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+/**
+ * 头条视频报表ETL
+ */
+public interface ETLReportBytedanceVideoMapper extends BaseMapper<ETLReportBytedanceVideo> {
+
+    //根据material_id查询MD5、头条视频链接
+    JSONObject queryVideoInfoByMaterialId(@Param("materialId") Long materialId);
+    //根据MD5查询视频信息
+    JSONObject queryMaterialVideoByMd5(@Param("md5") String md5);
+
+    JSONObject queryMaterialVideoByMd5Old(@Param("md5") String md5);
+
+    //根据MD5查询素材关联人信息
+    JSONObject queryRelatePersonByMd5(@Param("md5") String md5);
+
+    JSONObject queryRelatePersonByMd5Old(@Param("md5") String md5);
+
+    //根据accountId 查询项目、产品、广告主信息
+    JSONObject queryAccountDetailByAccountId(@Param("accountId") Long accountId);
+}

+ 5 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/BaseReportBytedanceVideoDailyMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.toutiao.modules.link.mapper.BaseReportBytedanceVideoDailyMapper">
+
+</mapper>

+ 79 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/mapper/xml/ETLReportBytedanceVideoMapper.xml

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.toutiao.modules.link.mapper.ETLReportBytedanceVideoMapper">
+    <select id="queryVideoInfoByMaterialId"  resultType="com.alibaba.fastjson.JSONObject">
+        SELECT
+            signature AS md5,
+            video_url AS url
+        FROM
+            ctop_bytedance_video_info
+        WHERE
+            material_id = #{materialId}
+        limit 1
+    </select>
+    <select id="queryMaterialVideoByMd5"  resultType="com.alibaba.fastjson.JSONObject">
+        SELECT
+            `name`  as materialName,
+            url,
+            cover_url coverUrl,
+            create_time as createTime
+        FROM
+            ctop_material_video_info
+        WHERE
+            md5 = #{md5}
+        LIMIT 1
+    </select>
+    <select id="queryMaterialVideoByMd5Old"  resultType="com.alibaba.fastjson.JSONObject">
+        SELECT
+            material_name as materialName,
+            url,
+            cover_url coverUrl,
+            create_time as createTime
+        FROM
+            ctop_material_info
+        WHERE
+            code = #{md5}
+        LIMIT 1
+    </select>
+    <select id="queryRelatePersonByMd5"  resultType="com.alibaba.fastjson.JSONObject">
+        SELECT
+            clip_id clipId,
+            clip_name clipaname,
+            shot_id shotId,
+            shot_name shotName,
+            plan_id planId,
+            plan_name planName
+        FROM
+            ctop_material_video_author
+        WHERE
+            video_md5 = #{md5}
+        LIMIT 1
+    </select>
+    <select id="queryRelatePersonByMd5Old"  resultType="com.alibaba.fastjson.JSONObject">
+        SELECT
+            clip_id clipId,
+            (select realname from sys_user where id=clip_id) as clipaname,
+            shot_id shotId,
+            (select realname from sys_user where id=shot_id) as shotName,
+            plan_id planId,
+            (select realname from sys_user where id=plan_id) as planName
+        FROM
+            ctop_material_ascription
+        WHERE
+            code = #{md5}
+        LIMIT 1
+    </select>
+    <select id="queryAccountDetailByAccountId"  resultType="com.alibaba.fastjson.JSONObject">
+        SELECT
+            t1.advertiser_id as advertiserId,
+            (SELECT `name` from ctop_advertiser where id=t1.advertiser_id limit 1) as advertiserName,
+            project_id as projectId,
+            (SELECT `project_name` from ctop_project where id=t1.project_id limit 1) as projectName,
+            auth_name as accountName
+        FROM
+            ctop_user_allocation t1
+        WHERE
+            account_id = #{accountId}
+        LIMIT 1
+    </select>
+</mapper>

+ 11 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IBaseReportBytedanceVideoDailyService.java

@@ -0,0 +1,11 @@
+package cn.com.ctop.toutiao.modules.link.service;
+
+import cn.com.ctop.toutiao.modules.link.entity.BaseReportBytedanceVideoDaily;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ *  头条视频报表源数据
+ */
+public interface IBaseReportBytedanceVideoDailyService extends IService<BaseReportBytedanceVideoDaily> {
+
+}

+ 17 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/IETLReportBytedanceVideoService.java

@@ -0,0 +1,17 @@
+package cn.com.ctop.toutiao.modules.link.service;
+
+import cn.com.ctop.toutiao.modules.link.entity.BaseReportBytedanceVideoDaily;
+import cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+import java.util.List;
+
+/**
+ * 头条视频报表ETL
+ */
+public interface IETLReportBytedanceVideoService extends IService<ETLReportBytedanceVideo> {
+
+    List<BaseReportBytedanceVideoDaily> listByStatDate(String statDate);
+
+    void cleanETL(String statDate);
+}

+ 15 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/BaseReportBytedanceVideoDailyServiceImpl.java

@@ -0,0 +1,15 @@
+package cn.com.ctop.toutiao.modules.link.service.impl;
+
+import cn.com.ctop.toutiao.modules.link.entity.BaseReportBytedanceVideoDaily;
+import cn.com.ctop.toutiao.modules.link.mapper.BaseReportBytedanceVideoDailyMapper;
+import cn.com.ctop.toutiao.modules.link.service.IBaseReportBytedanceVideoDailyService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * 头条视频报表源数据
+ */
+@Service
+public class BaseReportBytedanceVideoDailyServiceImpl extends ServiceImpl<BaseReportBytedanceVideoDailyMapper, BaseReportBytedanceVideoDaily> implements IBaseReportBytedanceVideoDailyService {
+
+}

+ 95 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/link/service/impl/ETLReportBytedanceVideoServiceImpl.java

@@ -0,0 +1,95 @@
+package cn.com.ctop.toutiao.modules.link.service.impl;
+
+import cn.com.ctop.toutiao.modules.link.entity.BaseReportBytedanceVideoDaily;
+import cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo;
+import cn.com.ctop.toutiao.modules.link.mapper.ETLReportBytedanceVideoMapper;
+import cn.com.ctop.toutiao.modules.link.service.IBaseReportBytedanceVideoDailyService;
+import cn.com.ctop.toutiao.modules.link.service.IETLReportBytedanceVideoService;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.xxl.job.core.context.XxlJobHelper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 头条视频报表ETL
+ */
+@Slf4j
+@Service
+public class ETLReportBytedanceVideoServiceImpl extends ServiceImpl<ETLReportBytedanceVideoMapper, ETLReportBytedanceVideo> implements IETLReportBytedanceVideoService {
+
+    @Autowired
+    IBaseReportBytedanceVideoDailyService baseReportBytedanceVideoDailyService;
+
+    @Autowired
+    ETLReportBytedanceVideoMapper etlReportBytedanceVideoMapper;
+
+    @Override
+    public List<BaseReportBytedanceVideoDaily> listByStatDate(String statDate) {
+        QueryWrapper<BaseReportBytedanceVideoDaily>  queryWrapper= new QueryWrapper<>();
+        queryWrapper.eq("stat_datetime",statDate);
+        return baseReportBytedanceVideoDailyService.list(queryWrapper);
+    }
+
+    @Override
+    public void cleanETL(String statDate) {
+        List<BaseReportBytedanceVideoDaily> source = listByStatDate(statDate);
+        if(source.isEmpty()){
+            XxlJobHelper.log("头条数据报表清洗ETL异常,请检查数据源");
+        }
+        List<ETLReportBytedanceVideo> batch = new ArrayList<>();
+        source.forEach(it->{
+            JSONObject videoInfo= etlReportBytedanceVideoMapper.queryVideoInfoByMaterialId(it.getMaterialId());
+            if(null==videoInfo||videoInfo.isEmpty()){
+                XxlJobHelper.log("头条数据报表清洗ETL异常,未关联到video_info,materialId:{}",it.getMaterialId());
+            }else {
+                //TODO 暂时从老数据库查询,新素材库怕同步数据有偏差
+                //JSONObject materialVideo= etlReportBytedanceVideoMapper.queryMaterialVideoByMd5(videoInfo.getString("md5"));
+                JSONObject materialVideo= etlReportBytedanceVideoMapper.queryMaterialVideoByMd5Old(videoInfo.getString("md5"));
+                //JSONObject relatePerson=etlReportBytedanceVideoMapper.queryRelatePersonByMd5(videoInfo.getString("md5"));
+                JSONObject relatePerson=etlReportBytedanceVideoMapper.queryRelatePersonByMd5Old(videoInfo.getString("md5"));
+
+                JSONObject accountDetail=etlReportBytedanceVideoMapper.queryAccountDetailByAccountId(it.getAccountId());
+                ETLReportBytedanceVideo temp = JSONObject.toJavaObject(JSONObject.parseObject(JSON.toJSON(it).toString()),ETLReportBytedanceVideo.class);
+                temp.setMd5(videoInfo.getString("md5"));
+                temp.setToutiaoUrl(videoInfo.getString("url"));
+                if(null!=materialVideo&&!materialVideo.isEmpty()) {
+                    temp.setUrl(materialVideo.getString("url"));
+                    temp.setCoverUrl(materialVideo.getString("coverUrl"));
+                    temp.setMaterialCreateTime(materialVideo.getDate("creteTime").toString());
+                }
+                if(null!=accountDetail&&!accountDetail.isEmpty()){
+                    temp.setAccountName(accountDetail.getString("accountName"));
+                    temp.setProjectId(accountDetail.getLong("projectId"));
+                    temp.setProjectName(accountDetail.getString("projectName"));
+                    temp.setProductId(accountDetail.getLong("productId"));
+                    temp.setProductName(accountDetail.getString("productName"));
+                    temp.setAdvertiserId(accountDetail.getString("advertiserId"));
+                    temp.setAdvertiserName(accountDetail.getString("advertiserName"));
+                }
+                if(null!=relatePerson&&!relatePerson.isEmpty()){
+                    temp.setClipId(relatePerson.getString("clipId"));
+                    temp.setClipName(relatePerson.getString("clipName"));
+                    temp.setShotId(relatePerson.getString("shotId"));
+                    temp.setShotName(relatePerson.getString("shotName"));
+                    temp.setPlanId(relatePerson.getString("planId"));
+                    temp.setPlanName(relatePerson.getString("planName"));
+                }
+                batch.add(temp);
+            }
+        });
+        boolean result = this.saveBatch(batch);
+        if(result){
+            XxlJobHelper.log("头条数据报表清洗成功,数据条数:{}",batch.size());
+        }else {
+            XxlJobHelper.log("头条数据报表清洗入库失败请重试,数据条数:{}",batch.size());
+        }
+    }
+
+}

+ 25 - 43
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/BytedanceVideoReportMapper.xml

@@ -3,20 +3,19 @@
 <mapper namespace="cn.com.ctop.toutiao.modules.report.mapper.BytedanceVideoReportMapper">
 
     <select id="queryVideoReportBy" resultType="com.alibaba.fastjson.JSONObject">
-        SELECT * from (SELECT
-        t1.signature,
-        t1.video_url as url,
-        IFNULL((SELECT u.realname FROM ctop_material_ascription t3 LEFT JOIN sys_user u on t3.clip_id=u.id where t3.material_id=t1.signature limit 1),'') AS clip,
-        IFNULL((SELECT u.realname FROM ctop_material_ascription t3 LEFT JOIN sys_user u on t3.shot_id=u.id where t3.material_id=t1.signature limit 1),'') AS shot,
-        IFNULL((SELECT u.realname FROM ctop_material_ascription t3 LEFT JOIN sys_user u on t3.plan_id=u.id where t3.material_id=t1.signature limit 1),'') AS plan,
-        (select t2.cover_url from ctop_material_info t2 where t2.code=t1.signature limit 1) AS converUrl,
-        (select t2.material_name from ctop_material_info t2 where t2.code=t1.signature limit 1) AS materialName,
-        (SELECT project_name FROM ctop_project t4 WHERE t4.id = t3.project_id) AS projectName,
-        (select t2.create_time from ctop_material_info t2 where t2.code=t1.signature limit 1) as createTime,
+        SELECT
+        t1.md5 signature,
+        IFNULL(t1.toutiao_url, '') AS url,
+        t1.cover_url AS converUrl,
+        t1.clip_name AS clip,
+        t1.shot_name AS shot,
+        t1.plan_name AS plan,
+        t1.material_name AS materialName,
+        t1.project_name AS projectName,
+        t1.matertial_create_time AS createTime,
         ${filed}
         FROM
-        ctop_bytedance_video_report_daily t1
-        LEFT JOIN ctop_user_allocation t3 ON t3.account_id=t1.account_id
+        etl_report_bytedance_video t1
         WHERE
         t1.stat_datetime >= #{startDate}
         AND t1.stat_datetime &lt;= #{endDate}
@@ -27,44 +26,27 @@
                 #{item}
             </foreach>
         </if>
-        AND t1.video_url is not null
-        GROUP BY t1.signature
-        ) t
-        WHERE (t.clip LIKE '%${clip}%')
-        AND (t.shot LIKE '%${shot}%')
-        AND (t.plan LIKE '%${plan}%')
+        AND (t1.clip_name LIKE '%${clip}%')
+        AND (t1.shot_name LIKE '%${shot}%')
+        AND (t1.plan_name LIKE '%${plan}%')
+        GROUP BY t1.md5
         ORDER BY ${target} ${order}
     </select>
 
     <select id="queryVideoDetailBy" resultType="com.alibaba.fastjson.JSONObject">
         SELECT
         t1.stat_datetime as statDate,
-        t1.signature,
-        t1.video_url as url,
-        (select t2.cover_url from ctop_material_info t2 where t2.code=t1.signature limit 1) AS converUrl,
-        (select t2.material_name from ctop_material_info t2 where t2.code=t1.signature limit 1) AS materialName,
-        (
-        SELECT
-        project_name
-        FROM
-        ctop_project t4
-        WHERE
-        t4.id = t3.project_id
-        ) AS projectName,
-        (
-        SELECT
-        advertiser_name
-        FROM
-        ctop_project t4
-        WHERE
-        t4.id = t3.project_id limit 1
-        ) AS advertiserName,
-        (select t2.create_time from ctop_material_info t2 where t2.code=t1.signature limit 1) as createTime,
+        t1.md5 signature,
+        IFNULL(t1.toutiao_url, '') AS url,
+        t1.cover_url AS converUrl,
+        t1.material_name AS materialName,
+        t1.project_name AS projectName,
+        t1.advertiser_name as advertiserName,
+        t1.matertial_create_time AS createTime,
         ${filed}
         FROM
-        ctop_bytedance_video_report_daily t1
-        LEFT JOIN ctop_user_allocation t3 ON t3.account_id=t1.account_id
-        WHERE t1.signature=#{signature}
+        etl_report_bytedance_video t1
+        WHERE t1.md5=#{signature}
         AND t1.stat_datetime >= #{startDate}
         AND t1.stat_datetime &lt;= #{endDate}
         <if test="accountIds != null">
@@ -74,7 +56,7 @@
                 #{item}
             </foreach>
         </if>
-        GROUP BY t1.signature,t1.stat_datetime
+        GROUP BY t1.md5,t1.stat_datetime
         order by t1.stat_datetime desc
     </select>
 </mapper>

+ 3 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IBytedanceVideoReportService.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.toutiao.modules.report.service;
 
+import cn.com.ctop.common.module.entity.CtopOauthToken;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.github.pagehelper.PageInfo;
@@ -13,4 +14,6 @@ public interface IBytedanceVideoReportService {
     List<JSONObject> getVideoDetailBy(JSONArray accountIds, String startDate, String endDate, String filed, String signature);
 
     List<JSONObject> exportExcelReportBy(JSONArray accountIds, String startDate, String endDate, String filed, String target, String order, String clip, String shot, String plan);
+
+    void pullVideoReport(CtopOauthToken oauthToken,String startDate, String endDate,int page);
 }

+ 0 - 5
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceReportServiceImpl.java

@@ -544,11 +544,6 @@ public class BytedanceReportServiceImpl implements IBytedanceReportService {
             record.setUpdateTime(new Date());
             recordService.updateById(record);
         }
-//        bytedanceReportMaterialDailyMapper.updateImageReportCode();
-//        bytedanceReportMaterialDailyMapper.updateImageReportProjectId();
-//        bytedanceReportMaterialDailyMapper.updateImageReportProjectName();
-//        bytedanceReportMaterialDailyMapper.updateImageReportPlaneId();
-//        bytedanceReportMaterialDailyMapper.updateImageReportPlaneName();
         log.info("头条素材报表结束 当前accountId为:{}  {}~{}" , accountId, record.getStartDate(),record.getEndDate());
     }
 

+ 66 - 0
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/BytedanceVideoReportServiceImpl.java

@@ -1,22 +1,35 @@
 package cn.com.ctop.toutiao.modules.report.service.impl;
 
+import cn.com.ctop.common.module.entity.CtopOauthToken;
+import cn.com.ctop.common.module.utils.Check;
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.toutiao.modules.link.entity.BaseReportBytedanceVideoDaily;
+import cn.com.ctop.toutiao.modules.link.service.IBaseReportBytedanceVideoDailyService;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceVideoReportMapper;
 import cn.com.ctop.toutiao.modules.report.service.IBytedanceVideoReportService;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
+@Slf4j
 @Service
 public class BytedanceVideoReportServiceImpl implements IBytedanceVideoReportService {
 
     @Autowired
     BytedanceVideoReportMapper bytedanceVideoReportMapper;
 
+    @Autowired
+    IBaseReportBytedanceVideoDailyService baseReportBytedanceVideoDailyService;
+
     @Override
     public PageInfo<JSONObject> getVideoReportBy(JSONArray accountIds, String startDate, String endDate, String filed, String target, String order, String clip, String shot, String plan, int pageSize, int pageNo) {
         PageHelper.startPage(pageNo,pageSize);
@@ -34,4 +47,57 @@ public class BytedanceVideoReportServiceImpl implements IBytedanceVideoReportSer
         return bytedanceVideoReportMapper.queryVideoReportBy(accountIds,startDate,endDate,filed,target,order,clip,shot,plan);
     }
 
+    @Override
+    public void pullVideoReport(CtopOauthToken oauthToken, String startDate, String endDate, int page) {
+
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Content-Type", " application/json");
+        headers.put("Access-Token", oauthToken.getAccessToken());
+
+        JSONObject param = new JSONObject();
+        param.put("advertiser_id", oauthToken.getAccountId());
+        param.put("start_date",startDate);
+        param.put("end_date",endDate);
+        param.put("page_size", 200);
+        param.put("page", page);
+
+        JSONObject resultJson = JSONObject.parseObject(HttpUtils.httpGetRequest("https://ad.oceanengine.com/open_api/2/report/video/get/", headers, param));
+        if (Check.isNull(resultJson)) {
+            log.warn("拉取头条视频报表返回结果为空,advertiserId:{}", oauthToken.getAccountId());
+            return;
+        }
+        if(resultJson.getInteger("code")!=0){
+            log.error("拉取头条视频表异常,advertiserId:{},message:{}", oauthToken.getAccountId(),resultJson.getString("message"));
+            return;
+        }
+        JSONObject data = resultJson.getJSONObject("data");
+        if (null == data || data.size() <= 0) {
+            log.error("拉取头条视频表异常,data is null");
+            return;
+        }
+
+        JSONArray list= data.getJSONArray("list");
+        if(list.isEmpty()){
+            log.warn("拉取头条视频表,数据为空advertiserId:{}", oauthToken.getAccountId());
+            return;
+        }
+        List<BaseReportBytedanceVideoDaily> batch =new ArrayList<>();
+        list.forEach(videoReport->{
+            JSONObject videoReportJSONObject= (JSONObject)videoReport;
+            Long materialId= videoReportJSONObject.getJSONObject("dimensions").getLong("material_id");
+            JSONObject metrics= videoReportJSONObject.getJSONObject("metrics");
+            BaseReportBytedanceVideoDaily temp= JSONObject.toJavaObject(metrics,BaseReportBytedanceVideoDaily.class);
+            temp.setMaterialShow(videoReportJSONObject.getLong("show"));
+            temp.setAccountId(oauthToken.getAccountId());
+            temp.setMaterialId(materialId);
+            temp.setStatDatetime(startDate);
+            batch.add(temp);
+        });
+        baseReportBytedanceVideoDailyService.saveBatch(batch);
+
+        if(resultJson.getJSONObject("data").getJSONObject("page_info").getInteger("total_page")>page){
+            pullVideoReport(oauthToken, startDate, endDate, page + 1);
+        }
+    }
+
 }

+ 3 - 3
module-toutiao/src/main/resources/json_template/bytedance_video_report_mapping.json

@@ -4,11 +4,11 @@
     "comment": "消耗"
   },
   "showMaterial": {
-    "filed": "IFNULL(sum(t1.show_material),0) as showMaterial ",
+    "filed": "IFNULL(sum(t1.material_show),0) as showMaterial ",
     "comment": "展示数"
   },
   "cpm": {
-    "filed": "CASE WHEN sum(t1.cost) / sum(t1.show_material) IS NULL THEN 0 ELSE ROUND((sum(t1.cost) / sum(t1.show_material)) * 1000, 2 ) END AS cpm ",
+    "filed": "CASE WHEN sum(t1.cost) / sum(t1.material_show) IS NULL THEN 0 ELSE ROUND((sum(t1.cost) / sum(t1.material_show)) * 1000, 2 ) END AS cpm ",
     "comment": "平均千次展现费用"
   },
   "clickMaterial": {
@@ -16,7 +16,7 @@
     "comment": "点击数"
   },
   "ctr": {
-    "filed": "CASE WHEN sum(t1.click) / sum(t1.show_material) IS NULL THEN 0 ELSE CONCAT( ROUND((sum(t1.click) / sum(t1.show_material)) * 100,2),'%') END AS ctr ",
+    "filed": "CASE WHEN sum(t1.click) / sum(t1.material_show) IS NULL THEN 0 ELSE CONCAT( ROUND((sum(t1.click) / sum(t1.material_show)) * 100,2),'%') END AS ctr ",
     "comment": "点击率"
   },
   "cpc": {