syh %!s(int64=4) %!d(string=hai) anos
pai
achega
0be69fb91e

+ 11 - 3
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -7,11 +7,10 @@ import cn.com.ctop.common.module.service.IBindAccountLoginService;
 import cn.com.ctop.common.module.service.ICtopOauthTokenService;
 import cn.com.ctop.common.module.service.IUserAllocationService;
 import cn.com.ctop.common.module.utils.CtopAdConstant;
-import cn.com.ctop.kuaishou.modules.ai.entity.AiKuaishouAccountTargetTemplate;
-import cn.com.ctop.kuaishou.modules.ai.service.IAiKuaishouAccountTargetTemplateService;
 import cn.com.ctop.kuaishou.modules.ai.service.IAiKuaishouNewCreateCampaign;
 import cn.com.ctop.kuaishou.modules.batch.service.*;
 import cn.com.ctop.kuaishou.modules.graphql.service.IKuaishouWebInterfaceService;
+import cn.com.ctop.kuaishou.modules.material.service.IEtlKuaishouVideoInfoService;
 import cn.com.ctop.kuaishou.modules.report.service.*;
 import cn.com.ctop.oa.modules.service.IWechatCheckinDataService;
 import cn.com.ctop.oa.modules.service.IWechatNoListService;
@@ -21,7 +20,6 @@ import cn.com.ctop.toutiao.modules.material.service.IByteDanceAdvertiserDataServ
 import cn.com.ctop.toutiao.modules.report.entity.BytedanceAccountReportTaskRecord;
 import cn.com.ctop.toutiao.modules.report.mapper.BytedanceReportMaterialDailyMapper;
 import cn.com.ctop.toutiao.modules.report.service.*;
-import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
 import org.junit.Test;
@@ -585,5 +583,15 @@ public class SampleTest {
             dailyService.etlKuaishouAccountMaterialReportDailyData(DateUtils.formatDate(DateUtils.addDay(startDate,i)));
         }
     }
+    @Autowired
+    private IEtlKuaishouVideoInfoService videoEtlInfoService;
+    @Test
+    public void etlKuaishouVideoInfo(){
+        Date startDate = DateUtils.parseDate("2020-09-11","yyyy-MM-dd");
+        for(int i=0;i<365;i++){
+//            dailyService.etlKuaishouAccountMaterialReportDailyData(DateUtils.formatDate(DateUtils.addDay(startDate,i)));
+            videoEtlInfoService.etlKuaishouVideoInfo(DateUtils.formatDate(DateUtils.addDay(startDate,i)));
+        }
+    }
 
 }

+ 1 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/mapper/EtlKuaishouVideoInfoMapper.java

@@ -14,5 +14,5 @@ public interface EtlKuaishouVideoInfoMapper extends BaseMapper<EtlKuaishouVideoI
 
     String getVideoMinDate(@Param(value = "signature") String signature);
 
-    void replaceVideoInfo(@Param(value = "video")EtlKuaishouVideoInfo etlKuaishouVideoInfo);
+    void replaceVideoInfo(@Param(value = "etlKuaishouVideoInfo")EtlKuaishouVideoInfo etlKuaishouVideoInfo);
 }

+ 38 - 94
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/mapper/xml/EtlKuaishouVideoInfoMapper.xml

@@ -3,100 +3,44 @@
 <mapper namespace="cn.com.ctop.kuaishou.modules.material.mapper.EtlKuaishouVideoInfoMapper">
     <insert id="replaceVideoInfo" parameterType="cn.com.ctop.kuaishou.modules.material.entity.EtlKuaishouVideoInfo">
         replace into ctop_etl_kuaishou_video_info
-        <trim prefix="(" suffix=")" suffixOverrides="," >
-            <if test="id != null" >
-                id,
-            </if>
-            <if test="videoCode != null" >
-                video_code,
-            </if>
-            <if test="stateDate != null" >
-                state_date,
-            </if>
-            <if test="url != null" >
-                url,
-            </if>
-            <if test="coverUrl != null" >
-                cover_url,
-            </if>
-            <if test="channelType != null" >
-                channel_type,
-            </if>
-            <if test="companyName != null" >
-                company_name,
-            </if>
-            <if test="planId != null" >
-                plan_id,
-            </if>
-            <if test="planName != null" >
-                plan_name,
-            </if>
-            <if test="clipId != null" >
-                clip_id,
-            </if>
-            <if test="clipName != null" >
-                clip_name,
-            </if>
-            <if test="shotId != null" >
-                shot_id,
-            </if>
-            <if test="shotName != null" >
-                shot_name,
-            </if>
-            <if test="designTeamLeaderId != null" >
-                design_team_leader_id,
-            </if>
-            <if test="designTeamLeaderName != null" >
-                design_team_leader_name,
-            </if>
-        </trim>
-        <trim prefix="values (" suffix=")" suffixOverrides="," >
-            <if test="id != null" >
-                #{id},
-            </if>
-            <if test="videoCode != null" >
-                #{videoCode},
-            </if>
-            <if test="stateDate != null" >
-                #{stateDate},
-            </if>
-            <if test="url != null" >
-                #{url},
-            </if>
-            <if test="coverUrl != null" >
-                #{coverUrl},
-            </if>
-            <if test="channelType != null" >
-                #{channelType},
-            </if>
-            <if test="companyName != null" >
-                #{companyName},
-            </if>
-            <if test="planId != null" >
-                #{planId},
-            </if>
-            <if test="planName != null" >
-                #{planName},
-            </if>
-            <if test="clipId != null" >
-                #{clipId},
-            </if>
-            <if test="clipName != null" >
-                #{clipName},
-            </if>
-            <if test="shotId != null" >
-                #{shotId},
-            </if>
-            <if test="shotName != null" >
-                #{shotName},
-            </if>
-            <if test="designTeamLeaderId != null" >
-                #{designTeamLeaderId},
-            </if>
-            <if test="designTeamLeaderName != null" >
-                #{designTeamLeaderName},
-            </if>
-        </trim>
+            (
+            video_code,
+            video_name,
+            state_date,
+            url,
+            cover_url,
+            channel_type,
+            channel_name,
+            plan_id,
+            plan_name,
+            clip_id,
+            clip_name,
+            shot_id,
+            shot_name,
+            design_team_leader_id,
+            design_team_leader_name)
+            values
+            (
+           #{etlKuaishouVideoInfo.videoCode},
+           #{etlKuaishouVideoInfo.videoName},
+           #{etlKuaishouVideoInfo.stateDate},
+                #{etlKuaishouVideoInfo.url},
+                #{etlKuaishouVideoInfo.coverUrl},
+                #{etlKuaishouVideoInfo.channelType},
+                #{etlKuaishouVideoInfo.channelName},
+                #{etlKuaishouVideoInfo.planId},
+                #{etlKuaishouVideoInfo.planName},
+                #{etlKuaishouVideoInfo.clipId},
+                #{etlKuaishouVideoInfo.clipName},
+                #{etlKuaishouVideoInfo.shotId},
+                #{etlKuaishouVideoInfo.shotName},
+                #{etlKuaishouVideoInfo.designTeamLeaderId},
+                #{etlKuaishouVideoInfo.designTeamLeaderName}
+        )
+
+
+
+
     </insert>
 
     <select id="getVideoMinDate" resultType="java.lang.String">

+ 82 - 61
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/material/service/impl/EtlKuaishouVideoInfoServiceImpl.java

@@ -1,14 +1,10 @@
 package cn.com.ctop.kuaishou.modules.material.service.impl;
 
-import cn.com.ctop.common.module.entity.MaterialAscription;
 import cn.com.ctop.common.module.entity.MaterialInfo;
 import cn.com.ctop.common.module.entity.SysUser;
-import cn.com.ctop.common.module.entity.UserCompany;
-import cn.com.ctop.common.module.mapper.CtopOauthTokenMapper;
 import cn.com.ctop.common.module.service.IMaterialAscriptionService;
 import cn.com.ctop.common.module.service.IMaterialInfoService;
 import cn.com.ctop.common.module.service.ISysUserService;
-import cn.com.ctop.common.module.service.IUserCompanyService;
 import cn.com.ctop.kuaishou.modules.batch.entity.KuaiShouVideoGet;
 import cn.com.ctop.kuaishou.modules.batch.service.IKuaiShouVideoGetService;
 import cn.com.ctop.kuaishou.modules.material.entity.EtlKuaishouVideoInfo;
@@ -17,6 +13,7 @@ import cn.com.ctop.kuaishou.modules.material.service.IEtlKuaishouVideoInfoServic
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -24,6 +21,9 @@ import org.springframework.stereotype.Service;
 import javax.annotation.Resource;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 /**
  * 快手视频报表信息
@@ -32,6 +32,7 @@ import java.util.List;
  * @version V1.0
  */
 @Service
+@Slf4j
 public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVideoInfoMapper, EtlKuaishouVideoInfo> implements IEtlKuaishouVideoInfoService {
     @Resource
     private EtlKuaishouVideoInfoMapper etlKuaishouVideoInfoMapper;
@@ -43,72 +44,92 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
     private ISysUserService sysUserService;
     @Autowired
     private IMaterialInfoService materialInfoService;
+
+    ExecutorService executorService = Executors.newFixedThreadPool(200);
+
     @Override
     public void etlKuaishouVideoInfo(String date) {
         //1:查询当前日期内的所有存在md5码信息的视频数据
         List<KuaiShouVideoGet> videoGets = videoGetService.getVideoInfoGroupBySignature(date);
         if(null!=videoGets&&!videoGets.isEmpty()){
+            CountDownLatch countDownLatch = new CountDownLatch(videoGets.size());
             videoGets.forEach(videoGet -> {
-                //1:查询素材名称
-                EtlKuaishouVideoInfo etlKuaishouVideoInfo = new EtlKuaishouVideoInfo();
-                MaterialInfo materialInfo = materialInfoService.getMaterialInfoByCode(videoGet.getSignature());
-                if(null == materialInfo){
-                    etlKuaishouVideoInfo.setVideoName(materialInfo.getMaterialName());
-                }else{
-                    etlKuaishouVideoInfo.setVideoName(videoGet.getPhotoName());
-                }
-                etlKuaishouVideoInfo.setVideoCode(videoGet.getSignature());
-                etlKuaishouVideoInfo.setUrl(videoGet.getUrl());
-                etlKuaishouVideoInfo.setCoverUrl(videoGet.getCoverUrl());
-                etlKuaishouVideoInfo.setChannelType(videoGet.getChannelType().toString());
-                if(videoGet.getChannelType() == 0){
-                    etlKuaishouVideoInfo.setChannelName("内部");
-                }else {
-                    etlKuaishouVideoInfo.setChannelName("素造");
-                }
-                //查询相关设计人员信息
-                JSONObject ascription = ascriptionService.getDetailByCode(videoGet.getSignature());
-                if(null!=ascription){
-                    String clipId = ascription.getString("clipId");
-                    if(null!=clipId&&!clipId.trim().equals("")){
-                        etlKuaishouVideoInfo.setClipId(clipId);
-                        //根据id查询剪辑所属的设计负责人
-                        SysUser user = sysUserService.getById(clipId);
-                        SysUser designTeamLeader = sysUserService.getDesignerTeamLeaderBy(user.getOrgCode());
-                        if(null!=designTeamLeader){
-                            etlKuaishouVideoInfo.setDesignTeamLeaderId(designTeamLeader.getId());
-                            etlKuaishouVideoInfo.setDesignTeamLeaderName(designTeamLeader.getRealname());
+                executorService.submit(()->{
+                    try {
+                        //1:查询素材名称
+                        EtlKuaishouVideoInfo etlKuaishouVideoInfo = new EtlKuaishouVideoInfo();
+                        MaterialInfo materialInfo = materialInfoService.getMaterialInfoByCode(videoGet.getSignature());
+                        if(null != materialInfo){
+                            etlKuaishouVideoInfo.setVideoName(materialInfo.getMaterialName());
+                        }else{
+                            etlKuaishouVideoInfo.setVideoName(videoGet.getPhotoName());
                         }
+                        etlKuaishouVideoInfo.setVideoCode(videoGet.getSignature());
+                        etlKuaishouVideoInfo.setUrl(videoGet.getUrl());
+                        etlKuaishouVideoInfo.setCoverUrl(videoGet.getCoverUrl());
+                        etlKuaishouVideoInfo.setChannelType(videoGet.getChannelType().toString());
+                        if(videoGet.getChannelType() == 0){
+                            etlKuaishouVideoInfo.setChannelName("内部");
+                        }else {
+                            etlKuaishouVideoInfo.setChannelName("素造");
+                        }
+                        //查询相关设计人员信息
+                        JSONObject ascription = ascriptionService.getDetailByCode(videoGet.getSignature());
+                        if(null!=ascription){
+                            String clipId = ascription.getString("clipId");
+                            if(null!=clipId&&!clipId.trim().equals("")){
+                                etlKuaishouVideoInfo.setClipId(clipId);
+                                //根据id查询剪辑所属的设计负责人
+                                SysUser user = sysUserService.getById(clipId);
+                                if(null!=user){
+                                    SysUser designTeamLeader = sysUserService.getDesignerTeamLeaderBy(user.getOrgCode());
+                                    if(null!=designTeamLeader){
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderId(designTeamLeader.getId());
+                                        etlKuaishouVideoInfo.setDesignTeamLeaderName(designTeamLeader.getRealname());
+                                    }
+                                }
+                            }
+                            String clipName = ascription.getString("clipName");
+                            if(null!=clipName&&!clipName.trim().equals("")){
+                                etlKuaishouVideoInfo.setClipName(clipName);
+                            }
+                            String planId = ascription.getString("planId");
+                            if(null!=planId&&!planId.trim().equals("")){
+                                etlKuaishouVideoInfo.setPlanId(planId);
+                            }
+                            String planName = ascription.getString("planName");
+                            if(null!=planName&&!planName.trim().equals("")){
+                                etlKuaishouVideoInfo.setPlanName(planName);
+                            }
+                            String shotId = ascription.getString("shotId");
+                            if(null!=shotId&&!shotId.trim().equals("")){
+                                etlKuaishouVideoInfo.setShotId(shotId);
+                            }
+                            String shotName = ascription.getString("shotName");
+                            if(null!=shotName&&!shotName.trim().equals("")){
+                                etlKuaishouVideoInfo.setShotName(shotName);
+                            }
+                        }
+                        //查询视频初次消耗时间
+                        String minDate = etlKuaishouVideoInfoMapper.getVideoMinDate(videoGet.getSignature());
+                        if(null!=minDate&&!"".equals(minDate)){
+                            Date getDate = DateUtils.parseDate(minDate,"yyyy-MM-dd");
+                            etlKuaishouVideoInfo.setStateDate(getDate);
+                        }
+                        etlKuaishouVideoInfoMapper.replaceVideoInfo(etlKuaishouVideoInfo);
+                    }catch (Exception e){
+                        e.printStackTrace();
+                    }finally {
+                        countDownLatch.countDown();
                     }
-                    String clipName = ascription.getString("clipName");
-                    if(null!=clipName&&!clipName.trim().equals("")){
-                        etlKuaishouVideoInfo.setClipName(clipName);
-                    }
-                    String planId = ascription.getString("planId");
-                    if(null!=planId&&!planId.trim().equals("")){
-                        etlKuaishouVideoInfo.setPlanId(planId);
-                    }
-                    String planName = ascription.getString("planName");
-                    if(null!=planName&&!planName.trim().equals("")){
-                        etlKuaishouVideoInfo.setPlanName(planName);
-                    }
-                    String shotId = ascription.getString("shotId");
-                    if(null!=shotId&&!shotId.trim().equals("")){
-                        etlKuaishouVideoInfo.setShotId(shotId);
-                    }
-                    String shotName = ascription.getString("shotName");
-                    if(null!=shotName&&!shotName.trim().equals("")){
-                        etlKuaishouVideoInfo.setShotName(shotName);
-                    }
-                }
-                //查询视频初次消耗时间
-                String minDate = etlKuaishouVideoInfoMapper.getVideoMinDate(videoGet.getSignature());
-                if(null!=minDate&&!"".equals(minDate)){
-                    Date getDate = DateUtils.parseDate(minDate,"yyyy-MM-dd");
-                    etlKuaishouVideoInfo.setStateDate(getDate);
-                }
-                etlKuaishouVideoInfoMapper.replaceVideoInfo(etlKuaishouVideoInfo);
+                });
             });
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            log.info("{}数据获取完成",date);
         }
     }
     @Override

+ 2 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouVideoEtlInfoServiceImpl.java

@@ -16,6 +16,7 @@ import org.jeecg.common.util.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Resource;
 import java.math.BigDecimal;
 import java.util.Date;
 import java.util.List;
@@ -32,7 +33,7 @@ import java.util.concurrent.Executors;
 @Slf4j
 @Service
 public class KuaishouVideoEtlInfoServiceImpl extends ServiceImpl<KuaishouVideoEtlInfoMapper, KuaishouVideoEtlInfo> implements IKuaishouVideoEtlInfoService {
-    @Autowired
+    @Resource
     private KuaishouVideoEtlInfoMapper kuaishouVideoEtlInfoMapper;
     @Autowired
     private IKuaiShouReportDailyMaterialService dailyMaterialService;