|
@@ -49,14 +49,12 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
|
|
private IMaterialInfoService materialInfoService;
|
|
private IMaterialInfoService materialInfoService;
|
|
@Autowired
|
|
@Autowired
|
|
private IUserCompanyService companyService;
|
|
private IUserCompanyService companyService;
|
|
- ExecutorService executorService = Executors.newFixedThreadPool(5);
|
|
|
|
|
|
+ ExecutorService executorService = Executors.newFixedThreadPool(15);
|
|
ExecutorService videoExecutorService = Executors.newFixedThreadPool(10);
|
|
ExecutorService videoExecutorService = Executors.newFixedThreadPool(10);
|
|
|
|
|
|
@Override
|
|
@Override
|
|
public void etlKuaishouVideoInfo(String date) {
|
|
public void etlKuaishouVideoInfo(String date) {
|
|
- Date startDate = DateUtils.getStartDate(date);
|
|
|
|
- Date endDate = DateUtils.getEndDate(date);
|
|
|
|
- List<JSONObject> videoList = videoGetService.getVideoListByDate(startDate, endDate);
|
|
|
|
|
|
+ List<JSONObject> videoList = videoGetService.getVideoListByDate(date);
|
|
if (Check.isNull(videoList)) {
|
|
if (Check.isNull(videoList)) {
|
|
return;
|
|
return;
|
|
}
|
|
}
|
|
@@ -75,6 +73,11 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
|
|
if (Check.isNull(signature)) {
|
|
if (Check.isNull(signature)) {
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
+ Integer count = etlKuaishouVideoInfoMapper.getByMd5(signature);
|
|
|
|
+ if (count > 0) {
|
|
|
|
+ continue;
|
|
|
|
+ }
|
|
|
|
+
|
|
try {
|
|
try {
|
|
//1:查询素材名称
|
|
//1:查询素材名称
|
|
EtlKuaishouVideoInfo etlKuaishouVideoInfo = new EtlKuaishouVideoInfo();
|
|
EtlKuaishouVideoInfo etlKuaishouVideoInfo = new EtlKuaishouVideoInfo();
|
|
@@ -82,7 +85,9 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
|
|
if (null != materialInfo) {
|
|
if (null != materialInfo) {
|
|
etlKuaishouVideoInfo.setVideoName(materialInfo.getMaterialName());
|
|
etlKuaishouVideoInfo.setVideoName(materialInfo.getMaterialName());
|
|
} else {
|
|
} else {
|
|
- etlKuaishouVideoInfo.setVideoName(videoJson.getString("photoName"));
|
|
|
|
|
|
+ String photoName = etlKuaishouVideoInfoMapper.getPhotoName(signature);
|
|
|
|
+
|
|
|
|
+ etlKuaishouVideoInfo.setVideoName(photoName);
|
|
}
|
|
}
|
|
etlKuaishouVideoInfo.setVideoCode(signature);
|
|
etlKuaishouVideoInfo.setVideoCode(signature);
|
|
etlKuaishouVideoInfo.setUrl(videoJson.getString("url"));
|
|
etlKuaishouVideoInfo.setUrl(videoJson.getString("url"));
|
|
@@ -186,6 +191,7 @@ public class EtlKuaishouVideoInfoServiceImpl extends ServiceImpl<EtlKuaishouVide
|
|
}
|
|
}
|
|
if (!Check.isNull(addList)) {
|
|
if (!Check.isNull(addList)) {
|
|
etlKuaishouVideoInfoMapper.addBatch(addList);
|
|
etlKuaishouVideoInfoMapper.addBatch(addList);
|
|
|
|
+ log.info("视频数据清洗添加成功,日期:{}。条数:{}", date, addList.size());
|
|
|
|
|
|
}
|
|
}
|
|
});
|
|
});
|