浏览代码

添加头条视频消耗数据清洗任务

songyh 4 年之前
父节点
当前提交
5b3902c502

+ 7 - 0
jeecg-boot-module-system/src/test/java/org/jeecg/SampleTest.java

@@ -218,4 +218,11 @@ public class SampleTest {
         }
     }
 
+    @Autowired
+    private IEtlBytedanceReportVideoDailyService etlBytedanceReportVideoDailyService;
+
+    @Test
+    public void etlBytedanceVideoReportInfo()throws Exception{
+        etlBytedanceReportVideoDailyService.etlBytedanceVideo();
+    }
 }

+ 3 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/EtlBytedanceReportVideoDailyMapper.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.toutiao.modules.report.mapper;
 
+import cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo;
 import cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo;
 import cn.com.ctop.toutiao.modules.report.entity.EtlBytedanceReportVideoDaily;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
@@ -21,7 +22,7 @@ public interface EtlBytedanceReportVideoDailyMapper extends BaseMapper<EtlByteda
     void updBytedanceVideoByMd5(String md5,String coverUrl,String url);
 
 
-    List<EtlBytedanceVideoInfo> etlBytedanceVideoInfo(@Param(value = "date") String date);
+    List<ETLReportBytedanceVideo> etlBytedanceVideoInfo(@Param(value = "date") String date);
 
-    void replaceEtlVideoInfo(@Param(value = "videoInfo") EtlBytedanceVideoInfo videoInfo);
+    void replaceEtlVideoInfo(@Param(value = "videoInfo") ETLReportBytedanceVideo videoInfo);
 }

+ 2 - 2
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/mapper/xml/EtlBytedanceReportVideoDailyMapper.xml

@@ -25,11 +25,11 @@
             total DESC;
     </select>
     <select id="etlBytedanceVideoInfo"
-            resultType="cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo">
+            resultType="cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo">
              select
                       (case when cmi.url is not null then cmi.url else '' end) 'url',
                           (CASE when cmi.cover_url is null then ''
-                                ELSE cmi.cover_url end) 'conver_url',
+                                ELSE cmi.cover_url end) 'cover_url',
                           (case when cmi.material_name is null then '' else cmi.material_name end)  'material_name',
                           (case when cbvi.signature is null then '' else cbvi.signature end)'md5',
                           report.material_id 'material_id',

+ 1 - 1
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/IEtlBytedanceReportVideoDailyService.java

@@ -16,5 +16,5 @@ public interface IEtlBytedanceReportVideoDailyService extends IService<EtlByteda
     Result updBytedanceVideo() throws Exception;
 
 
-    Result etlBytedanceVideo() throws Exception;
+    void etlBytedanceVideo() throws Exception;
 }

+ 21 - 31
module-toutiao/src/main/java/cn/com/ctop/toutiao/modules/report/service/impl/EtlBytedanceReportVideoDailyServiceImpl.java

@@ -1,5 +1,6 @@
 package cn.com.ctop.toutiao.modules.report.service.impl;
 
+import cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo;
 import cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo;
 import cn.com.ctop.toutiao.modules.report.entity.EtlBytedanceReportVideoDaily;
 import cn.com.ctop.toutiao.modules.report.mapper.EtlBytedanceReportVideoDailyMapper;
@@ -7,6 +8,7 @@ import cn.com.ctop.toutiao.modules.report.service.IEtlBytedanceReportVideoDailyS
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.google.api.client.util.Data;
 import com.google.common.base.Function;
+import lombok.extern.slf4j.Slf4j;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.jeecg.common.api.vo.Result;
 import org.jeecg.common.util.DateUtils;
@@ -32,6 +34,7 @@ import java.util.function.BiFunction;
  * @date 2021-02-20
  */
 @Service
+@Slf4j
 public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlBytedanceReportVideoDailyMapper, EtlBytedanceReportVideoDaily> implements IEtlBytedanceReportVideoDailyService {
 
 
@@ -63,43 +66,30 @@ public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlByte
     static ExecutorService executorService = Executors.newFixedThreadPool(20);
     static CountDownLatch countDownLatch = null;
     @Override
-    public Result etlBytedanceVideo() throws Exception{
-        String date = DateUtils.formatDate();
-        CompletableFuture future = CompletableFuture.supplyAsync(()->{
-            return etlBytedanceReportVideoDailyMapper.etlBytedanceVideoInfo(date);
-        }).handle(new BiFunction<List<EtlBytedanceVideoInfo>, Throwable, Object>() {
-            @Override
-            public Object apply(List<EtlBytedanceVideoInfo> list, Throwable throwable) {
-                if (throwable != null){
-                    return Result.error(throwable.getMessage());
-                }
-                if(null!=list&&!list.isEmpty()){
-                    countDownLatch = new CountDownLatch(list.size());
-                    list.forEach(videoInfo ->{
-                        executorService.submit(()->{
-                            try {
-                                etlBytedanceReportVideoDailyMapper.replaceEtlVideoInfo(videoInfo);
-                            }catch (Exception e){
-                            }finally {
-                                countDownLatch.countDown();
-                            }
-                        });
-                    });
-                }
+    public void etlBytedanceVideo()throws Exception {
+        String date = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
+        List<ETLReportBytedanceVideo> list = etlBytedanceReportVideoDailyMapper.etlBytedanceVideoInfo(date);
+        if (null != list && !list.isEmpty()) {
+            countDownLatch = new CountDownLatch(list.size());
+            list.forEach(videoInfo -> executorService.submit(() -> {
                 try {
-                    countDownLatch.await();
-                } catch (InterruptedException e) {
-                    e.printStackTrace();
+                    etlBytedanceReportVideoDailyMapper.replaceEtlVideoInfo(videoInfo);
+                } catch (Exception e) {
+                } finally {
+                    countDownLatch.countDown();
                 }
-                return Result.ok("数据清洗完成。");
+            }));
+
+            try {
+                countDownLatch.await();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
             }
-        });
-        return Result.ok(future.get());
+            log.info("数据清洗完成");
+        }
     }
 
 
 
 
-
-
 }