Browse Source

首投有效素材明细表 导入2

zhaoxian 1 year ago
parent
commit
79c919cf7b

+ 1 - 1
ruixuan-live/src/main/java/com/ruixuan/isc/service/IKuaishouPromoterService.java

@@ -99,5 +99,5 @@ public interface IKuaishouPromoterService {
 
     List<JSONObject> jyExpertBusiness();
 
-    void importExcelMaterials(MultipartFile file);
+    void importExcelMaterials(MultipartFile file) throws InterruptedException;
 }

+ 52 - 45
ruixuan-live/src/main/java/com/ruixuan/isc/service/impl/KuaishouPromoterServiceImpl.java

@@ -47,6 +47,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.regex.Pattern;
 
 /**
@@ -851,24 +853,17 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
     @Value("${pcm.local-path}")
     private String downloadPath;
 
+
     @Override
-    public void importExcelMaterials(MultipartFile file) {
-        List<FirstDeliveryValidMaterials> dataList = new ArrayList<>();
+    public void importExcelMaterials(MultipartFile file) throws InterruptedException {
+        ExecutorService executorService = Executors.newFixedThreadPool(8);
+        long t1 = System.currentTimeMillis();
+        List<FirstDeliveryValidMaterials> dataList = null;
         StringBuffer path = new StringBuffer();
         try {
             path.append(downloadPath).append(DateUtils.getNowDateStr()).append("/").append(UUID.randomUUID().toString()).append("/");
             log.info("-------下载到服务器地址:{}", path.toString());
-            List<JSONObject> list = this.analysisBIGBIGFile(file, path.toString());
-            for (JSONObject object : list) {
-                FirstDeliveryValidMaterials material = new FirstDeliveryValidMaterials(object);
-                if ("走量".equals(material.getOperationLabel())) {
-                    continue;
-                }
-                if (!Check.isNull(material.getStatDate())) {
-                    material.setDeliveryYear(material.getStatDate().substring(0, 4));
-                }
-                dataList.add(material);
-            }
+            dataList = this.analysisBIGBIGFile(file, path.toString());
         } catch (Exception e) {
             e.printStackTrace();
         } finally {
@@ -878,24 +873,33 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
             //根据当前年份和月份,删除旧数据
             FirstDeliveryValidMaterials material = dataList.get(0);
             deliveryValidMapper.delMaterials(material.getDeliveryYear(), material.getDeliveryMonth());
-            //插入 数据量大于5W,分批插入
-            if (dataList.size() > 50000) {
-                List<List<FirstDeliveryValidMaterials>> splitList = Lists.newArrayList(Lists.partition(dataList, 50000));
-                for (List<FirstDeliveryValidMaterials> materials : splitList) {
+            //插入 数据量大于5Q,分批插入
+            if (dataList.size() > 5000) {
+                List<List<FirstDeliveryValidMaterials>> splitList = Lists.newArrayList(Lists.partition(dataList, 5000));
+                splitList.forEach(materials -> executorService.submit(() -> {
                     deliveryValidMapper.replaceBatchMaterials(materials);
-                }
+                }));
             } else {
                 deliveryValidMapper.replaceBatchMaterials(dataList);
             }
         }
-        //执行宇鹏提供的sql
-        deliveryValidMapper.replaceEtl();
+        executorService.shutdown();
+        while (true) {
+            if (executorService.isTerminated()) {
+                log.info("所有的子线程都结束了!");
+                //执行宇鹏提供的sql
+                deliveryValidMapper.replaceEtl();
+                break;
+            }
+            Thread.sleep(1000L);//休眠1秒钟
+        }
+        log.info("------共用时" + (System.currentTimeMillis() - t1) / 1000 + "秒");
     }
 
-    public List<JSONObject> analysisBIGBIGFile(MultipartFile mfile, String path) throws Exception {
+    public List<FirstDeliveryValidMaterials> analysisBIGBIGFile(MultipartFile mfile, String path) throws Exception {
         InputStream is = null;
         String newFileUrl = FileUtil.approvalFile(mfile, path);
-        List<JSONObject> list = null;
+        List<FirstDeliveryValidMaterials> dataList = null;
         try {
             File file = new File(newFileUrl);
             is = new FileInputStream(file);
@@ -907,35 +911,38 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
             //最后一列(列数)
             int lastCellNum = 0;
             // list存储数据集
-            list = new ArrayList<>();
+            dataList = new ArrayList<>();
             for (Sheet sheet : workbook) {
                 // sheet.getLastRowNum()获取最后一个非空行的行下标,比如总行数为n,则返回的为n-1
                 int sumPageNum = (sheet.getLastRowNum() + 1) / 20000 + 1;       //计算分片数
                 for (int i = 1; i <= sumPageNum; i++) {     //读取每片范围内的内容
-                    try {
-                        int minRowNum = 20000 * (i - 1) + 1;    //计算每片读取的行数范围
-                        int maxRowNum = 20000 * i;
-                        for (Row row : sheet) {
-                            if (lastCellNum == 0) {
-                                lastCellNum = row.getLastCellNum();
-                            }
-                            if (row != null) {
-                                Map<String, Object> columnValueMap = new HashMap<>();
-                                if (row.getRowNum() >= minRowNum && row.getRowNum() <= maxRowNum) {//控制每片读取的行
-                                    //每行数据对象,按照顺序从0到(lastCellNum-1)记录
-                                    JSONObject cellEntity = new JSONObject();
-                                    for (int k = 0; k < lastCellNum; k++) {
-                                        cellEntity.put(String.valueOf(k), eeu.getCellValue(row.getCell(k)));
-                                    }
-                                    list.add(cellEntity);
-                                    if (row.getRowNum() == maxRowNum) {//读取到每片最大行数限制,跳出,执行数据写入
-                                        break;
-                                    }
+                    int minRowNum = 20000 * (i - 1) + 1;    //计算每片读取的行数范围
+                    int maxRowNum = 20000 * i;
+                    for (Row row : sheet) {
+                        if (lastCellNum == 0) {
+                            lastCellNum = row.getLastCellNum();
+                        }
+                        if (row != null) {
+                            Map<String, Object> columnValueMap = new HashMap<>();
+                            if (row.getRowNum() >= minRowNum && row.getRowNum() <= maxRowNum) {//控制每片读取的行
+                                //每行数据对象,按照顺序从0到(lastCellNum-1)记录
+                                JSONObject cellEntity = new JSONObject();
+                                for (int k = 0; k < lastCellNum; k++) {
+                                    cellEntity.put(String.valueOf(k), eeu.getCellValue(row.getCell(k)));
+                                }
+                                FirstDeliveryValidMaterials material = new FirstDeliveryValidMaterials(cellEntity);
+                                if ("走量".equals(material.getOperationLabel())) {
+                                    continue;
+                                }
+                                if (!Check.isNull(material.getStatDate())) {
+                                    material.setDeliveryYear(material.getStatDate().substring(0, 4));
+                                }
+                                dataList.add(material);
+                                if (row.getRowNum() == maxRowNum) {//读取到每片最大行数限制,跳出,执行数据写入
+                                    break;
                                 }
                             }
                         }
-                    } catch (Exception e) {
-                        e.printStackTrace();
                     }
                 }
             }
@@ -945,7 +952,7 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
         } finally {
             IOUtils.closeQuietly(is);
         }
-        return list;
+        return dataList;
     }
 
     @Override