|
@@ -1,5 +1,6 @@
|
|
package cn.com.ctop.toutiao.modules.report.service.impl;
|
|
package cn.com.ctop.toutiao.modules.report.service.impl;
|
|
|
|
|
|
|
|
+import cn.com.ctop.toutiao.modules.link.entity.ETLReportBytedanceVideo;
|
|
import cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo;
|
|
import cn.com.ctop.toutiao.modules.material.entity.EtlBytedanceVideoInfo;
|
|
import cn.com.ctop.toutiao.modules.report.entity.EtlBytedanceReportVideoDaily;
|
|
import cn.com.ctop.toutiao.modules.report.entity.EtlBytedanceReportVideoDaily;
|
|
import cn.com.ctop.toutiao.modules.report.mapper.EtlBytedanceReportVideoDailyMapper;
|
|
import cn.com.ctop.toutiao.modules.report.mapper.EtlBytedanceReportVideoDailyMapper;
|
|
@@ -7,6 +8,7 @@ import cn.com.ctop.toutiao.modules.report.service.IEtlBytedanceReportVideoDailyS
|
|
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
|
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
|
import com.google.api.client.util.Data;
|
|
import com.google.api.client.util.Data;
|
|
import com.google.common.base.Function;
|
|
import com.google.common.base.Function;
|
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
import org.checkerframework.checker.nullness.qual.Nullable;
|
|
import org.checkerframework.checker.nullness.qual.Nullable;
|
|
import org.jeecg.common.api.vo.Result;
|
|
import org.jeecg.common.api.vo.Result;
|
|
import org.jeecg.common.util.DateUtils;
|
|
import org.jeecg.common.util.DateUtils;
|
|
@@ -32,6 +34,7 @@ import java.util.function.BiFunction;
|
|
* @date 2021-02-20
|
|
* @date 2021-02-20
|
|
*/
|
|
*/
|
|
@Service
|
|
@Service
|
|
|
|
+@Slf4j
|
|
public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlBytedanceReportVideoDailyMapper, EtlBytedanceReportVideoDaily> implements IEtlBytedanceReportVideoDailyService {
|
|
public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlBytedanceReportVideoDailyMapper, EtlBytedanceReportVideoDaily> implements IEtlBytedanceReportVideoDailyService {
|
|
|
|
|
|
|
|
|
|
@@ -63,43 +66,30 @@ public class EtlBytedanceReportVideoDailyServiceImpl extends ServiceImpl<EtlByte
|
|
static ExecutorService executorService = Executors.newFixedThreadPool(20);
|
|
static ExecutorService executorService = Executors.newFixedThreadPool(20);
|
|
static CountDownLatch countDownLatch = null;
|
|
static CountDownLatch countDownLatch = null;
|
|
@Override
|
|
@Override
|
|
- public Result etlBytedanceVideo() throws Exception{
|
|
|
|
- String date = DateUtils.formatDate();
|
|
|
|
- CompletableFuture future = CompletableFuture.supplyAsync(()->{
|
|
|
|
- return etlBytedanceReportVideoDailyMapper.etlBytedanceVideoInfo(date);
|
|
|
|
- }).handle(new BiFunction<List<EtlBytedanceVideoInfo>, Throwable, Object>() {
|
|
|
|
- @Override
|
|
|
|
- public Object apply(List<EtlBytedanceVideoInfo> list, Throwable throwable) {
|
|
|
|
- if (throwable != null){
|
|
|
|
- return Result.error(throwable.getMessage());
|
|
|
|
- }
|
|
|
|
- if(null!=list&&!list.isEmpty()){
|
|
|
|
- countDownLatch = new CountDownLatch(list.size());
|
|
|
|
- list.forEach(videoInfo ->{
|
|
|
|
- executorService.submit(()->{
|
|
|
|
- try {
|
|
|
|
- etlBytedanceReportVideoDailyMapper.replaceEtlVideoInfo(videoInfo);
|
|
|
|
- }catch (Exception e){
|
|
|
|
- }finally {
|
|
|
|
- countDownLatch.countDown();
|
|
|
|
- }
|
|
|
|
- });
|
|
|
|
- });
|
|
|
|
- }
|
|
|
|
|
|
+ public void etlBytedanceVideo()throws Exception {
|
|
|
|
+ String date = DateUtils.formatDate(DateUtils.addDay(new Date(), -1));
|
|
|
|
+ List<ETLReportBytedanceVideo> list = etlBytedanceReportVideoDailyMapper.etlBytedanceVideoInfo(date);
|
|
|
|
+ if (null != list && !list.isEmpty()) {
|
|
|
|
+ countDownLatch = new CountDownLatch(list.size());
|
|
|
|
+ list.forEach(videoInfo -> executorService.submit(() -> {
|
|
try {
|
|
try {
|
|
- countDownLatch.await();
|
|
|
|
- } catch (InterruptedException e) {
|
|
|
|
- e.printStackTrace();
|
|
|
|
|
|
+ etlBytedanceReportVideoDailyMapper.replaceEtlVideoInfo(videoInfo);
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
+ } finally {
|
|
|
|
+ countDownLatch.countDown();
|
|
}
|
|
}
|
|
- return Result.ok("数据清洗完成。");
|
|
|
|
|
|
+ }));
|
|
|
|
+
|
|
|
|
+ try {
|
|
|
|
+ countDownLatch.await();
|
|
|
|
+ } catch (InterruptedException e) {
|
|
|
|
+ e.printStackTrace();
|
|
}
|
|
}
|
|
- });
|
|
|
|
- return Result.ok(future.get());
|
|
|
|
|
|
+ log.info("数据清洗完成");
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
-
|
|
|
|
-
|
|
|
|
}
|
|
}
|