Prechádzať zdrojové kódy

快手素材报表提高数据库插入效率

yumeng 4 rokov pred
rodič
commit
248af34a08

+ 3 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaiShouReportDailyMaterialMapper.java

@@ -4,6 +4,7 @@ import cn.com.ctop.kuaishou.modules.batch.entity.vo.KuaiShouVideoGetVo;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterialVo;
 import cn.com.ctop.kuaishou.modules.report.entity.VideoGetvo;
+import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import org.apache.ibatis.annotations.Param;
 
@@ -35,4 +36,6 @@ public interface KuaiShouReportDailyMaterialMapper extends BaseMapper<KuaiShouRe
     List<KuaiShouReportDailyMaterial> getSignatureByDate(@Param("date")String date);
 
     List<KuaiShouReportDailyMaterial> getSignatureByDateGroupByProjectId(@Param("date")String date);
+
+    List<JSONObject> getVideoInfoByList(@Param("photoIdList") List<Long> photoIdList);
 }

+ 14 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaiShouReportDailyMaterialMapper.xml

@@ -367,5 +367,19 @@ where
     and
     stat_date = #{date}
     </select>
+    <select id="getVideoInfoByList" resultType="com.alibaba.fastjson.JSONObject">
+        select
+        photo_id as 'photoId',
+        signature as 'signature',
+        channel_type as 'channelType'
+        from ctop_kuaishou_video_get
+        where photo_id in
+        <foreach collection="photoIdList" item="item" separator=","
+                 open="(" close=")">
+            #{item}
+        </foreach>
+        and signature is not null
+        group by photo_id
+    </select>
 </mapper>
 

+ 24 - 16
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouReportDailyMaterialServiceImpl.java

@@ -9,7 +9,6 @@ import cn.com.ctop.kuaishou.modules.report.entity.KuaiShouReportDailyMaterial;
 import cn.com.ctop.kuaishou.modules.report.entity.VideoGetvo;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -57,16 +56,13 @@ public class KuaiShouReportDailyMaterialServiceImpl extends ServiceImpl<KuaiShou
         param.put("advertiser_id", accountId);
         param.put("temporal_granularity", "DAILY");
         param.put("view_type", 5);
-        param.put("page_size", 1000);
+        param.put("page_size", 2000);
         param.put("page", page);
-
         String result = HttpUtils.httpPostRequest(url, param, headers);
         JSONObject resultJson = JSONObject.parseObject(result);
-    //    log.info("快手返回数据:{}",resultJson);
         if (Check.isNull(resultJson)) {
             return;
         }
-
         Integer total_count = resultJson.getInteger("total_count");
         if (total_count == 0) {
             return;
@@ -75,9 +71,8 @@ public class KuaiShouReportDailyMaterialServiceImpl extends ServiceImpl<KuaiShou
         if (Check.isNull(details)) {
             return;
         }
-
         List<KuaiShouReportDailyMaterial> addList = new ArrayList<>();
-        long l1 = System.currentTimeMillis();
+        List<Long> photoIdList = new ArrayList<>();
         for (int i = 0; i < details.size(); i++) {
             JSONObject detailJson = details.getJSONObject(i);
             detailJson.put("photo_like", detailJson.getInteger("like"));
@@ -85,28 +80,41 @@ public class KuaiShouReportDailyMaterialServiceImpl extends ServiceImpl<KuaiShou
             detailJson.put("photo_comment", detailJson.getInteger("comment"));
             detailJson.put("photo_show", detailJson.getInteger("show"));
             KuaiShouReportDailyMaterial material = JSONObject.toJavaObject(detailJson, KuaiShouReportDailyMaterial.class);
-            VideoGetvo videoGetvo = dailyMaterialMapper.getVideoGetByPhotoId(material.getPhotoId());
-            if (!Check.isNull(videoGetvo)) {
-                material.setChannelType(videoGetvo.getChannelType());
-                material.setSignature(videoGetvo.getSignature());
-            }
             material.setAccountId(accountId);
-
+            photoIdList.add(material.getPhotoId());
             addList.add(material);
         }
-  //      log.info("addList:{}",addList);
+        addList = this.getVideoInfoByList(addList, photoIdList);
         dailyMaterialMapper.batchReplace(addList);
         getMaterialReportByAccountIdAndStatDate(accountId, token, startDate, endDate, page + 1);
     }
 
+    private List<KuaiShouReportDailyMaterial> getVideoInfoByList(List<KuaiShouReportDailyMaterial> addList, List<Long> photoIdList) {
+        List<JSONObject> videoInfoList = dailyMaterialMapper.getVideoInfoByList(photoIdList);
+        Map<Long, JSONObject> videoMap = new HashMap<>();
+        for (int i = 0; i < videoInfoList.size(); i++) {
+            JSONObject jsonObject = videoInfoList.get(i);
+            videoMap.put(jsonObject.getLong("photoId"), jsonObject);
+        }
+        for (KuaiShouReportDailyMaterial dailyMaterial : addList) {
+            JSONObject jsonObject = videoMap.get(dailyMaterial.getPhotoId());
+            if (Check.isNull(jsonObject)) {
+                continue;
+            }
+            dailyMaterial.setSignature(jsonObject.getString("signature"));
+            dailyMaterial.setChannelType(jsonObject.getInteger("channelType"));
+        }
+        return addList;
+    }
+
     @Override
     public List<KuaiShouReportDailyMaterial> getSignatureByDate(String date) {
         return dailyMaterialMapper.getSignatureByDate(date);
     }
 
     @Override
-    public KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date,Long projectId) {
-        return dailyMaterialMapper.getEtlDataByParams(signature,date,projectId);
+    public KuaiShouReportDailyMaterial getEtlDataByParams(String signature, String date, Long projectId) {
+        return dailyMaterialMapper.getEtlDataByParams(signature, date, projectId);
     }
 
     @Override