Przeglądaj źródła

首投有效素材明细表 导入

zhaoxian 1 rok temu
rodzic
commit
c9e1bd5fa0

+ 2 - 2
ruixuan-admin/src/main/resources/application-dev.yml

@@ -107,9 +107,9 @@ spring:
   servlet:
     multipart:
       # 单个文件大小
-      max-file-size: 10MB
+      max-file-size: 40MB
       # 设置总上传的文件大小
-      max-request-size: 20MB
+      max-request-size: 60MB
   # 服务模块
   devtools:
     restart:

+ 2 - 2
ruixuan-admin/src/main/resources/application-prod.yml

@@ -104,9 +104,9 @@ spring:
   servlet:
      multipart:
        # 单个文件大小
-       max-file-size:  10MB
+       max-file-size:  40MB
        # 设置总上传的文件大小
-       max-request-size:  20MB
+       max-request-size:  60MB
   # 服务模块
   devtools:
     restart:

+ 10 - 0
ruixuan-live/pom.xml

@@ -118,6 +118,16 @@
             <artifactId>tencentcloud-speech-sdk-java</artifactId>
             <version>1.0.38</version>
         </dependency>
+        <dependency>
+            <groupId>com.monitorjbl</groupId>
+            <artifactId>xlsx-streamer</artifactId>
+            <version>2.2.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>29.0-jre</version>
+        </dependency>
 
     </dependencies>
 </project>

+ 26 - 1
ruixuan-live/src/main/java/com/ruixuan/isc/controller/KuaishouItemCollectSamplesController.java

@@ -10,14 +10,20 @@ import com.kuaidi100.sdk.request.SubscribeParam;
 import com.kuaidi100.sdk.request.SubscribeParameters;
 import com.kuaidi100.sdk.request.SubscribeReq;
 import com.kuaidi100.sdk.response.SubscribeResp;
+import com.monitorjbl.xlsx.StreamingReader;
 import com.ruixuan.common.core.controller.BaseController;
 import com.ruixuan.common.core.domain.AjaxResult;
 import com.ruixuan.common.core.domain.entity.SysUser;
 import com.ruixuan.common.core.page.TableDataInfo;
 import com.ruixuan.common.utils.Check;
 import com.ruixuan.common.utils.DateUtils;
+import com.ruixuan.common.utils.file.FileUtil;
 import com.ruixuan.data.utils.ExportExcelUtils;
-import com.ruixuan.isc.entity.*;
+import com.ruixuan.isc.entity.KuaishouItemCollectSampleExpress;
+import com.ruixuan.isc.entity.KuaishouItemCollectSampleLog;
+import com.ruixuan.isc.entity.KuaishouItemCollectSamples;
+import com.ruixuan.isc.entity.KuaishouItemList;
+import com.ruixuan.isc.entity.KuaishouPromoter;
 import com.ruixuan.isc.enums.CompanyEnum;
 import com.ruixuan.isc.service.IKuaishouItemCollectSamplesService;
 import com.ruixuan.isc.service.IKuaishouItemListService;
@@ -29,6 +35,10 @@ import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -41,10 +51,14 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
 import java.math.BigDecimal;
@@ -1033,6 +1047,17 @@ public class KuaishouItemCollectSamplesController extends BaseController {
         kuaishouItemCollectSamplesService.checkAndUpdateStatus();
     }
 
+
+    @PostMapping(value = "/importExcelMaterials")
+    @ResponseBody
+    public void importExcelMaterials(HttpServletResponse response, @RequestParam("file") MultipartFile file) {
+        response.setHeader("Access-Control-Allow-Origin", "*");
+        try {
+            promoterService.importExcelMaterials(file);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
 }
 
 

+ 101 - 0
ruixuan-live/src/main/java/com/ruixuan/isc/entity/FirstDeliveryValidMaterials.java

@@ -0,0 +1,101 @@
+package com.ruixuan.isc.entity;
+
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+
+import java.math.BigDecimal;
+
+/**
+ * 返点-首投有效素材明细
+ */
+@Data
+public class FirstDeliveryValidMaterials {
+
+    private Long id;
+    /**
+     * 年份
+     */
+    private String deliveryYear;
+    /**
+     * 首投月份
+     */
+    private String deliveryMonth;
+    /**
+     * 运营标签
+     */
+    private String operationLabel;
+    /**
+     * 素材ID
+     */
+    private Long materialId;
+    /**
+     * 广告主id
+     */
+    private Long accountId;
+    /**
+     * 首投日期
+     */
+    private String statDate;
+    /**
+     * 广告主名称
+     */
+    private String accountName;
+    /**
+     * 客户名称
+     */
+    private String customerName;
+    /**
+     * 广告ids
+     */
+    private String adIds;
+    /**
+     * 优化师名称
+     */
+    private String userName;
+    /**
+     * 返点核算政策类型
+     */
+    private String rebateType;
+    /**
+     * 结算行业一级行业
+     */
+    private String settlement1l;
+    /**
+     * 结算二级行业
+     */
+    private String settlement2l;
+    /**
+     * 是否首投有效素材
+     */
+    private String firstValid;
+    /**
+     * 首投7天累量消耗
+     */
+    private BigDecimal cost7day;
+    /**
+     * 消耗
+     */
+    private BigDecimal cost;
+
+    public FirstDeliveryValidMaterials() {
+    }
+
+    public FirstDeliveryValidMaterials(JSONObject obj) {
+        this.setDeliveryMonth(obj.getString("0"));//首投月份
+        this.setOperationLabel(obj.getString("1"));//运营标签
+        this.setMaterialId(Long.valueOf(obj.getString("2").replace("\"", "")));//素材id
+        this.setAccountId(Long.valueOf(obj.getString("4").replace("\"", "")));//广告主id
+        this.setStatDate(obj.getString("5"));//首投日期
+        this.setAccountName(obj.getString("7"));//广告主名称
+        this.setAdIds(obj.getString("8"));//广告ids
+        this.setCustomerName(obj.getString("9"));//客户名称
+        this.setUserName(obj.getString("10"));//优化师名称
+        this.setRebateType(obj.getString("11"));//返点核算政策类型
+        this.setSettlement1l(obj.getString("12"));//结算行业一级行业
+        this.setSettlement2l(obj.getString("13"));//结算二级行业
+        this.setFirstValid(obj.getString("14"));//是否首投有效素材
+        this.setCost7day(obj.getBigDecimal("15"));//首投7天累量消耗
+        this.setCost(obj.getBigDecimal("16"));//消耗
+    }
+
+}

+ 15 - 0
ruixuan-live/src/main/java/com/ruixuan/isc/mapper/FirstDeliveryValidMapper.java

@@ -0,0 +1,15 @@
+package com.ruixuan.isc.mapper;
+
+import com.ruixuan.isc.entity.FirstDeliveryValidMaterials;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+public interface FirstDeliveryValidMapper {
+
+    void delMaterials(@Param("deliveryYear") String deliveryYear, @Param("deliveryMonth") String deliveryMonth);
+
+    void replaceBatchMaterials(@Param("list") List<FirstDeliveryValidMaterials> list);
+
+    void replaceEtl();
+}

+ 2 - 0
ruixuan-live/src/main/java/com/ruixuan/isc/service/IKuaishouPromoterService.java

@@ -5,6 +5,7 @@ import com.ruixuan.common.core.domain.AjaxResult;
 import com.ruixuan.common.utils.Result;
 import com.ruixuan.isc.entity.JYKuaishouPromoter;
 import com.ruixuan.isc.entity.KuaishouPromoter;
+import org.springframework.web.multipart.MultipartFile;
 
 import java.util.List;
 
@@ -98,4 +99,5 @@ public interface IKuaishouPromoterService {
 
     List<JSONObject> jyExpertBusiness();
 
+    void importExcelMaterials(MultipartFile file);
 }

+ 114 - 1
ruixuan-live/src/main/java/com/ruixuan/isc/service/impl/KuaishouPromoterServiceImpl.java

@@ -2,17 +2,24 @@ package com.ruixuan.isc.service.impl;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Lists;
+import com.monitorjbl.xlsx.StreamingReader;
 import com.ruixuan.common.core.domain.AjaxResult;
 import com.ruixuan.common.core.domain.entity.SysUser;
 import com.ruixuan.common.utils.Check;
 import com.ruixuan.common.utils.DateUtils;
+import com.ruixuan.common.utils.LoadFileUtil;
 import com.ruixuan.common.utils.PageUtils;
 import com.ruixuan.common.utils.RedisUtil;
 import com.ruixuan.common.utils.Result;
+import com.ruixuan.common.utils.file.FileUtil;
 import com.ruixuan.common.utils.http.HttpUtil;
+import com.ruixuan.data.utils.ExportExcelUtils;
+import com.ruixuan.isc.entity.FirstDeliveryValidMaterials;
 import com.ruixuan.isc.entity.JYKuaishouPromoter;
 import com.ruixuan.isc.entity.KuaishouItemCollectSamples;
 import com.ruixuan.isc.entity.KuaishouPromoter;
+import com.ruixuan.isc.mapper.FirstDeliveryValidMapper;
 import com.ruixuan.isc.mapper.KuaishouPromoterMapper;
 import com.ruixuan.isc.service.IKuaishouItemCollectSamplesService;
 import com.ruixuan.isc.service.IKuaishouPromoterService;
@@ -20,10 +27,18 @@ import com.ruixuan.system.service.ISysDeptService;
 import com.ruixuan.system.service.ISysRoleService;
 import com.ruixuan.system.service.ISysUserService;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -31,6 +46,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.regex.Pattern;
 
 /**
@@ -55,6 +71,9 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
     private KuaishouPromoterMapper kuaishouPromoterMapper;
 
     @Autowired
+    private FirstDeliveryValidMapper deliveryValidMapper;
+
+    @Autowired
     private ISysDeptService deptService;
 
     @Autowired
@@ -253,7 +272,6 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
 
     /**
      * 新增快手达人 信息
-     *
      */
     @Override
     public JSONObject insertKuaishouPromoter(KuaishouPromoter promoter) {
@@ -830,7 +848,102 @@ public class KuaishouPromoterServiceImpl implements IKuaishouPromoterService {
         return kuaishouPromoterMapper.jyExpertBusiness();
     }
 
+    @Value("${pcm.local-path}")
+    private String downloadPath;
+
+    @Override
+    public void importExcelMaterials(MultipartFile file) {
+        List<FirstDeliveryValidMaterials> dataList = new ArrayList<>();
+        StringBuffer path = new StringBuffer();
+        try {
+            path.append(downloadPath).append(DateUtils.getNowDateStr()).append("/").append(UUID.randomUUID().toString()).append("/");
+            log.info("-------下载到服务器地址:{}", path.toString());
+            List<JSONObject> list = this.analysisBIGBIGFile(file, path.toString());
+            for (JSONObject object : list) {
+                FirstDeliveryValidMaterials material = new FirstDeliveryValidMaterials(object);
+                if (!Check.isNull(material.getStatDate())) {
+                    material.setDeliveryYear(material.getStatDate().substring(0, 4));
+                }
+                dataList.add(material);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            LoadFileUtil.delFile(path.toString());
+        }
+        if (!Check.isNull(dataList)) {
+            //根据当前年份和月份,删除旧数据
+            FirstDeliveryValidMaterials material = dataList.get(0);
+            deliveryValidMapper.delMaterials(material.getDeliveryYear(), material.getDeliveryMonth());
+            //插入 数据量大于5W,分批插入
+            if (dataList.size() > 50000) {
+                List<List<FirstDeliveryValidMaterials>> splitList = Lists.newArrayList(Lists.partition(dataList, 50000));
+                for (List<FirstDeliveryValidMaterials> materials : splitList) {
+                    deliveryValidMapper.replaceBatchMaterials(materials);
+                }
+            } else {
+                deliveryValidMapper.replaceBatchMaterials(dataList);
+            }
+        }
+        //执行宇鹏提供的sql
+        deliveryValidMapper.replaceEtl();
+    }
 
+    public List<JSONObject> analysisBIGBIGFile(MultipartFile mfile, String path) throws Exception {
+        InputStream is = null;
+        String newFileUrl = FileUtil.approvalFile(mfile, path);
+        List<JSONObject> list = null;
+        try {
+            File file = new File(newFileUrl);
+            is = new FileInputStream(file);
+            Workbook workbook = StreamingReader.builder()
+                    .rowCacheSize(20000)
+                    .bufferSize(4096)
+                    .open(is);
+            ExportExcelUtils eeu = new ExportExcelUtils();
+            //最后一列(列数)
+            int lastCellNum = 0;
+            // list存储数据集
+            list = new ArrayList<>();
+            for (Sheet sheet : workbook) {
+                // sheet.getLastRowNum()获取最后一个非空行的行下标,比如总行数为n,则返回的为n-1
+                int sumPageNum = (sheet.getLastRowNum() + 1) / 20000 + 1;       //计算分片数
+                for (int i = 1; i <= sumPageNum; i++) {     //读取每片范围内的内容
+                    try {
+                        int minRowNum = 20000 * (i - 1) + 1;    //计算每片读取的行数范围
+                        int maxRowNum = 20000 * i;
+                        for (Row row : sheet) {
+                            if (lastCellNum == 0) {
+                                lastCellNum = row.getLastCellNum();
+                            }
+                            if (row != null) {
+                                Map<String, Object> columnValueMap = new HashMap<>();
+                                if (row.getRowNum() >= minRowNum && row.getRowNum() <= maxRowNum) {//控制每片读取的行
+                                    //每行数据对象,按照顺序从0到(lastCellNum-1)记录
+                                    JSONObject cellEntity = new JSONObject();
+                                    for (int k = 0; k < lastCellNum; k++) {
+                                        cellEntity.put(String.valueOf(k), eeu.getCellValue(row.getCell(k)));
+                                    }
+                                    list.add(cellEntity);
+                                    if (row.getRowNum() == maxRowNum) {//读取到每片最大行数限制,跳出,执行数据写入
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+            }
+            workbook.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            IOUtils.closeQuietly(is);
+        }
+        return list;
+    }
 
     @Override
     public AjaxResult getPromoterInfo(String promoterId) {

+ 105 - 0
ruixuan-live/src/main/resources/mapper/isc/FirstDeliveryValidMapper.xml

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.ruixuan.isc.mapper.FirstDeliveryValidMapper">
+
+    <delete id="delMaterials">
+        delete
+        from `jeecg-boot`.first_delivery_valid_materials
+        where delivery_year = #{deliveryYear}
+          AND delivery_month = #{deliveryMonth}
+    </delete>
+
+    <insert id="replaceBatchMaterials">
+        insert into `jeecg-boot`.first_delivery_valid_materials(
+        delivery_year,
+        delivery_month,
+        operation_label,
+        material_id,
+        account_id,
+        stat_date,
+        account_name,
+        ad_ids,
+        customer_name,
+        user_name,
+        rebate_type,
+        settlement_1l,
+        settlement_2l,
+        first_valid,
+        cost_7day,
+        cost
+        )
+        values
+        <foreach collection="list" item="li" separator=",">
+            (
+            #{li.deliveryYear},
+            #{li.deliveryMonth},
+            #{li.operationLabel},
+            #{li.materialId},
+            #{li.accountId},
+            #{li.statDate},
+            #{li.accountName},
+            #{li.adIds},
+            #{li.customerName},
+            #{li.userName},
+            #{li.rebateType},
+            #{li.settlement1l},
+            #{li.settlement2l},
+            #{li.firstValid},
+            #{li.cost7day},
+            #{li.cost}
+            )
+        </foreach>
+    </insert>
+
+    <insert id="replaceEtl">
+        replace
+        into `jeecg-boot`.`etl_first_delivery_valid_materials`
+        select `delivery_month`,
+               `operation_label`,
+               a.`material_id`,
+               e.signature,
+               `shot_leader_id`,
+               `shot_leader_name`,
+               f.`shot_id`,
+               f.`plan_id`,
+               f.`leader_id`,
+               f.`clip_name`,
+               f.`shot_name`,
+               f.`plan_name`,
+               f.`leader_name`,
+               c.id                    as `project_id`,
+               c.project_name,
+               c.responsible_id           project_leader_id,
+               d.realname              as project_leader_name,
+               `stat_date`,
+               `customer_name`,
+               `rebate_type`,
+               `settlement_1l`,
+               `settlement_2l`,
+               `first_valid`,
+               `cost_7day`,
+               `cost`,
+               `delivery_year`,
+               c.design_responsible_id as project_design_leader_id,
+               h.realname              as project_design_leader_name,
+               f.clip_id               as clip_id
+        from `jeecg-boot`.first_delivery_valid_materials a
+                 left join `jeecg-boot`.ctop_user_allocation b
+                           on a.account_id = b.account_id
+                 left join `jeecg-boot`.ctop_project c
+                           on b.project_id = c.id
+                 left join `jeecg-boot`.sys_user d
+                           on c.responsible_id = d.id
+                 left join application.bytedance_video_get e
+                           on a.material_id = e.material_id
+                 left join application.app_bytedance_material_info f
+                           on e.signature = f.material_id
+                 left join `jeecg-boot`.ctop_material_ascription g
+                           on e.signature = g.material_id
+                 left join `jeecg-boot`.sys_user h
+                           on c.design_responsible_id = h.id
+        where c.id is not null
+        group by a.delivery_month, a.material_id, signature, b.project_id, settlement_2l, a.delivery_year
+    </insert>
+
+</mapper>