|
@@ -47,6 +47,8 @@ import java.util.HashMap;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
import java.util.UUID;
|
|
|
+import java.util.concurrent.ExecutorService;
|
|
|
+import java.util.concurrent.Executors;
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
|
/**
|
|
@@ -851,24 +853,17 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
|
|
|
@Value("${pcm.local-path}")
|
|
|
private String downloadPath;
|
|
|
|
|
|
+
|
|
|
@Override
|
|
|
- public void importExcelMaterials(MultipartFile file) {
|
|
|
- List<FirstDeliveryValidMaterials> dataList = new ArrayList<>();
|
|
|
+ public void importExcelMaterials(MultipartFile file) throws InterruptedException {
|
|
|
+ ExecutorService executorService = Executors.newFixedThreadPool(8);
|
|
|
+ long t1 = System.currentTimeMillis();
|
|
|
+ List<FirstDeliveryValidMaterials> dataList = null;
|
|
|
StringBuffer path = new StringBuffer();
|
|
|
try {
|
|
|
path.append(downloadPath).append(DateUtils.getNowDateStr()).append("/").append(UUID.randomUUID().toString()).append("/");
|
|
|
log.info("-------下载到服务器地址:{}", path.toString());
|
|
|
- List<JSONObject> list = this.analysisBIGBIGFile(file, path.toString());
|
|
|
- for (JSONObject object : list) {
|
|
|
- FirstDeliveryValidMaterials material = new FirstDeliveryValidMaterials(object);
|
|
|
- if ("走量".equals(material.getOperationLabel())) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- if (!Check.isNull(material.getStatDate())) {
|
|
|
- material.setDeliveryYear(material.getStatDate().substring(0, 4));
|
|
|
- }
|
|
|
- dataList.add(material);
|
|
|
- }
|
|
|
+ dataList = this.analysisBIGBIGFile(file, path.toString());
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
} finally {
|
|
@@ -878,24 +873,33 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
|
|
|
//根据当前年份和月份,删除旧数据
|
|
|
FirstDeliveryValidMaterials material = dataList.get(0);
|
|
|
deliveryValidMapper.delMaterials(material.getDeliveryYear(), material.getDeliveryMonth());
|
|
|
- //插入 数据量大于5W,分批插入
|
|
|
- if (dataList.size() > 50000) {
|
|
|
- List<List<FirstDeliveryValidMaterials>> splitList = Lists.newArrayList(Lists.partition(dataList, 50000));
|
|
|
- for (List<FirstDeliveryValidMaterials> materials : splitList) {
|
|
|
+ //插入 数据量大于5Q,分批插入
|
|
|
+ if (dataList.size() > 5000) {
|
|
|
+ List<List<FirstDeliveryValidMaterials>> splitList = Lists.newArrayList(Lists.partition(dataList, 5000));
|
|
|
+ splitList.forEach(materials -> executorService.submit(() -> {
|
|
|
deliveryValidMapper.replaceBatchMaterials(materials);
|
|
|
- }
|
|
|
+ }));
|
|
|
} else {
|
|
|
deliveryValidMapper.replaceBatchMaterials(dataList);
|
|
|
}
|
|
|
}
|
|
|
- //执行宇鹏提供的sql
|
|
|
- deliveryValidMapper.replaceEtl();
|
|
|
+ executorService.shutdown();
|
|
|
+ while (true) {
|
|
|
+ if (executorService.isTerminated()) {
|
|
|
+ log.info("所有的子线程都结束了!");
|
|
|
+ //执行宇鹏提供的sql
|
|
|
+ deliveryValidMapper.replaceEtl();
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ Thread.sleep(1000L);//休眠1秒钟
|
|
|
+ }
|
|
|
+ log.info("------共用时" + (System.currentTimeMillis() - t1) / 1000 + "秒");
|
|
|
}
|
|
|
|
|
|
- public List<JSONObject> analysisBIGBIGFile(MultipartFile mfile, String path) throws Exception {
|
|
|
+ public List<FirstDeliveryValidMaterials> analysisBIGBIGFile(MultipartFile mfile, String path) throws Exception {
|
|
|
InputStream is = null;
|
|
|
String newFileUrl = FileUtil.approvalFile(mfile, path);
|
|
|
- List<JSONObject> list = null;
|
|
|
+ List<FirstDeliveryValidMaterials> dataList = null;
|
|
|
try {
|
|
|
File file = new File(newFileUrl);
|
|
|
is = new FileInputStream(file);
|
|
@@ -907,35 +911,38 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
|
|
|
//最后一列(列数)
|
|
|
int lastCellNum = 0;
|
|
|
// list存储数据集
|
|
|
- list = new ArrayList<>();
|
|
|
+ dataList = new ArrayList<>();
|
|
|
for (Sheet sheet : workbook) {
|
|
|
// sheet.getLastRowNum()获取最后一个非空行的行下标,比如总行数为n,则返回的为n-1
|
|
|
int sumPageNum = (sheet.getLastRowNum() + 1) / 20000 + 1; //计算分片数
|
|
|
for (int i = 1; i <= sumPageNum; i++) { //读取每片范围内的内容
|
|
|
- try {
|
|
|
- int minRowNum = 20000 * (i - 1) + 1; //计算每片读取的行数范围
|
|
|
- int maxRowNum = 20000 * i;
|
|
|
- for (Row row : sheet) {
|
|
|
- if (lastCellNum == 0) {
|
|
|
- lastCellNum = row.getLastCellNum();
|
|
|
- }
|
|
|
- if (row != null) {
|
|
|
- Map<String, Object> columnValueMap = new HashMap<>();
|
|
|
- if (row.getRowNum() >= minRowNum && row.getRowNum() <= maxRowNum) {//控制每片读取的行
|
|
|
- //每行数据对象,按照顺序从0到(lastCellNum-1)记录
|
|
|
- JSONObject cellEntity = new JSONObject();
|
|
|
- for (int k = 0; k < lastCellNum; k++) {
|
|
|
- cellEntity.put(String.valueOf(k), eeu.getCellValue(row.getCell(k)));
|
|
|
- }
|
|
|
- list.add(cellEntity);
|
|
|
- if (row.getRowNum() == maxRowNum) {//读取到每片最大行数限制,跳出,执行数据写入
|
|
|
- break;
|
|
|
- }
|
|
|
+ int minRowNum = 20000 * (i - 1) + 1; //计算每片读取的行数范围
|
|
|
+ int maxRowNum = 20000 * i;
|
|
|
+ for (Row row : sheet) {
|
|
|
+ if (lastCellNum == 0) {
|
|
|
+ lastCellNum = row.getLastCellNum();
|
|
|
+ }
|
|
|
+ if (row != null) {
|
|
|
+ Map<String, Object> columnValueMap = new HashMap<>();
|
|
|
+ if (row.getRowNum() >= minRowNum && row.getRowNum() <= maxRowNum) {//控制每片读取的行
|
|
|
+ //每行数据对象,按照顺序从0到(lastCellNum-1)记录
|
|
|
+ JSONObject cellEntity = new JSONObject();
|
|
|
+ for (int k = 0; k < lastCellNum; k++) {
|
|
|
+ cellEntity.put(String.valueOf(k), eeu.getCellValue(row.getCell(k)));
|
|
|
+ }
|
|
|
+ FirstDeliveryValidMaterials material = new FirstDeliveryValidMaterials(cellEntity);
|
|
|
+ if ("走量".equals(material.getOperationLabel())) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (!Check.isNull(material.getStatDate())) {
|
|
|
+ material.setDeliveryYear(material.getStatDate().substring(0, 4));
|
|
|
+ }
|
|
|
+ dataList.add(material);
|
|
|
+ if (row.getRowNum() == maxRowNum) {//读取到每片最大行数限制,跳出,执行数据写入
|
|
|
+ break;
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- } catch (Exception e) {
|
|
|
- e.printStackTrace();
|
|
|
}
|
|
|
}
|
|
|
}
|
|
@@ -945,7 +952,7 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
|
|
|
} finally {
|
|
|
IOUtils.closeQuietly(is);
|
|
|
}
|
|
|
- return list;
|
|
|
+ return dataList;
|
|
|
}
|
|
|
|
|
|
@Override
|