|
@@ -2,17 +2,24 @@ package com.ruixuan.isc.service.impl;
|
|
|
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.google.common.collect.Lists;
|
|
|
+import com.monitorjbl.xlsx.StreamingReader;
|
|
|
import com.ruixuan.common.core.domain.AjaxResult;
|
|
|
import com.ruixuan.common.core.domain.entity.SysUser;
|
|
|
import com.ruixuan.common.utils.Check;
|
|
|
import com.ruixuan.common.utils.DateUtils;
|
|
|
+import com.ruixuan.common.utils.LoadFileUtil;
|
|
|
import com.ruixuan.common.utils.PageUtils;
|
|
|
import com.ruixuan.common.utils.RedisUtil;
|
|
|
import com.ruixuan.common.utils.Result;
|
|
|
+import com.ruixuan.common.utils.file.FileUtil;
|
|
|
import com.ruixuan.common.utils.http.HttpUtil;
|
|
|
+import com.ruixuan.data.utils.ExportExcelUtils;
|
|
|
+import com.ruixuan.isc.entity.FirstDeliveryValidMaterials;
|
|
|
import com.ruixuan.isc.entity.JYKuaishouPromoter;
|
|
|
import com.ruixuan.isc.entity.KuaishouItemCollectSamples;
|
|
|
import com.ruixuan.isc.entity.KuaishouPromoter;
|
|
|
+import com.ruixuan.isc.mapper.FirstDeliveryValidMapper;
|
|
|
import com.ruixuan.isc.mapper.KuaishouPromoterMapper;
|
|
|
import com.ruixuan.isc.service.IKuaishouItemCollectSamplesService;
|
|
|
import com.ruixuan.isc.service.IKuaishouPromoterService;
|
|
@@ -20,10 +27,18 @@ import com.ruixuan.system.service.ISysDeptService;
|
|
|
import com.ruixuan.system.service.ISysRoleService;
|
|
|
import com.ruixuan.system.service.ISysUserService;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.io.IOUtils;
|
|
|
+import org.apache.poi.ss.usermodel.Row;
|
|
|
+import org.apache.poi.ss.usermodel.Sheet;
|
|
|
+import org.apache.poi.ss.usermodel.Workbook;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
+import org.springframework.web.multipart.MultipartFile;
|
|
|
|
|
|
+import java.io.File;
|
|
|
+import java.io.FileInputStream;
|
|
|
+import java.io.InputStream;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.Arrays;
|
|
|
import java.util.Collections;
|
|
@@ -31,6 +46,7 @@ import java.util.Date;
|
|
|
import java.util.HashMap;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
+import java.util.UUID;
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
|
/**
|
|
@@ -55,6 +71,9 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
|
|
|
private KuaishouPromoterMapper kuaishouPromoterMapper;
|
|
|
|
|
|
@Autowired
|
|
|
+ private FirstDeliveryValidMapper deliveryValidMapper;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
private ISysDeptService deptService;
|
|
|
|
|
|
@Autowired
|
|
@@ -253,7 +272,6 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
|
|
|
|
|
|
/**
|
|
|
* 新增快手达人 信息
|
|
|
- *
|
|
|
*/
|
|
|
@Override
|
|
|
public JSONObject insertKuaishouPromoter(KuaishouPromoter promoter) {
|
|
@@ -830,7 +848,102 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
|
|
|
return kuaishouPromoterMapper.jyExpertBusiness();
|
|
|
}
|
|
|
|
|
|
+ @Value("${pcm.local-path}")
|
|
|
+ private String downloadPath;
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void importExcelMaterials(MultipartFile file) {
|
|
|
+ List<FirstDeliveryValidMaterials> dataList = new ArrayList<>();
|
|
|
+ StringBuffer path = new StringBuffer();
|
|
|
+ try {
|
|
|
+ path.append(downloadPath).append(DateUtils.getNowDateStr()).append("/").append(UUID.randomUUID().toString()).append("/");
|
|
|
+ log.info("-------下载到服务器地址:{}", path.toString());
|
|
|
+ List<JSONObject> list = this.analysisBIGBIGFile(file, path.toString());
|
|
|
+ for (JSONObject object : list) {
|
|
|
+ FirstDeliveryValidMaterials material = new FirstDeliveryValidMaterials(object);
|
|
|
+ if (!Check.isNull(material.getStatDate())) {
|
|
|
+ material.setDeliveryYear(material.getStatDate().substring(0, 4));
|
|
|
+ }
|
|
|
+ dataList.add(material);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ } finally {
|
|
|
+ LoadFileUtil.delFile(path.toString());
|
|
|
+ }
|
|
|
+ if (!Check.isNull(dataList)) {
|
|
|
+ //根据当前年份和月份,删除旧数据
|
|
|
+ FirstDeliveryValidMaterials material = dataList.get(0);
|
|
|
+ deliveryValidMapper.delMaterials(material.getDeliveryYear(), material.getDeliveryMonth());
|
|
|
+ //插入 数据量大于5W,分批插入
|
|
|
+ if (dataList.size() > 50000) {
|
|
|
+ List<List<FirstDeliveryValidMaterials>> splitList = Lists.newArrayList(Lists.partition(dataList, 50000));
|
|
|
+ for (List<FirstDeliveryValidMaterials> materials : splitList) {
|
|
|
+ deliveryValidMapper.replaceBatchMaterials(materials);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ deliveryValidMapper.replaceBatchMaterials(dataList);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //执行宇鹏提供的sql
|
|
|
+ deliveryValidMapper.replaceEtl();
|
|
|
+ }
|
|
|
|
|
|
+ public List<JSONObject> analysisBIGBIGFile(MultipartFile mfile, String path) throws Exception {
|
|
|
+ InputStream is = null;
|
|
|
+ String newFileUrl = FileUtil.approvalFile(mfile, path);
|
|
|
+ List<JSONObject> list = null;
|
|
|
+ try {
|
|
|
+ File file = new File(newFileUrl);
|
|
|
+ is = new FileInputStream(file);
|
|
|
+ Workbook workbook = StreamingReader.builder()
|
|
|
+ .rowCacheSize(20000)
|
|
|
+ .bufferSize(4096)
|
|
|
+ .open(is);
|
|
|
+ ExportExcelUtils eeu = new ExportExcelUtils();
|
|
|
+ //最后一列(列数)
|
|
|
+ int lastCellNum = 0;
|
|
|
+ // list存储数据集
|
|
|
+ list = new ArrayList<>();
|
|
|
+ for (Sheet sheet : workbook) {
|
|
|
+ // sheet.getLastRowNum()获取最后一个非空行的行下标,比如总行数为n,则返回的为n-1
|
|
|
+ int sumPageNum = (sheet.getLastRowNum() + 1) / 20000 + 1; //计算分片数
|
|
|
+ for (int i = 1; i <= sumPageNum; i++) { //读取每片范围内的内容
|
|
|
+ try {
|
|
|
+ int minRowNum = 20000 * (i - 1) + 1; //计算每片读取的行数范围
|
|
|
+ int maxRowNum = 20000 * i;
|
|
|
+ for (Row row : sheet) {
|
|
|
+ if (lastCellNum == 0) {
|
|
|
+ lastCellNum = row.getLastCellNum();
|
|
|
+ }
|
|
|
+ if (row != null) {
|
|
|
+ Map<String, Object> columnValueMap = new HashMap<>();
|
|
|
+ if (row.getRowNum() >= minRowNum && row.getRowNum() <= maxRowNum) {//控制每片读取的行
|
|
|
+ //每行数据对象,按照顺序从0到(lastCellNum-1)记录
|
|
|
+ JSONObject cellEntity = new JSONObject();
|
|
|
+ for (int k = 0; k < lastCellNum; k++) {
|
|
|
+ cellEntity.put(String.valueOf(k), eeu.getCellValue(row.getCell(k)));
|
|
|
+ }
|
|
|
+ list.add(cellEntity);
|
|
|
+ if (row.getRowNum() == maxRowNum) {//读取到每片最大行数限制,跳出,执行数据写入
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ workbook.close();
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ } finally {
|
|
|
+ IOUtils.closeQuietly(is);
|
|
|
+ }
|
|
|
+ return list;
|
|
|
+ }
|
|
|
|
|
|
@Override
|
|
|
public AjaxResult getPromoterInfo(String promoterId) {
|