Selaa lähdekoodia

增加代理商账户爬虫

yumeng 5 vuotta sitten
vanhempi
commit
90e561f96b
18 muutettua tiedostoa jossa 867 lisäystä ja 21 poistoa
  1. 244 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/KuaishouAgentAccountController.java
  2. 20 0
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/DataeyeJob.java
  3. 43 2
      module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java
  4. 149 0
      module-common/src/main/java/cn/com/ctop/common/module/utils/JavacvUtil.java
  5. 17 1
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompany.java
  6. 4 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMedia.java
  7. 7 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeCompanyMapper.java
  8. 8 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeCompanyService.java
  9. 71 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyServiceImpl.java
  10. 108 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouAgentAccount.java
  11. 19 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/job/KuaishouAgentAccountJob.java
  12. 18 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouAgentAccountMapper.java
  13. 49 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouAgentAccountMapper.xml
  14. 14 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouAgentAccountService.java
  15. 1 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouReportDailyAgentService.java
  16. 19 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouAgentAccountServiceImpl.java
  17. 75 17
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouReportDailyAgentServiceImpl.java
  18. 1 1
      pom.xml

+ 244 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/KuaishouAgentAccountController.java

@@ -0,0 +1,244 @@
+package org.jeecg.modules.ctop.controller;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.jeecg.common.api.vo.Result;
+import org.jeecg.common.system.query.QueryGenerator;
+import org.jeecg.common.aspect.annotation.AutoLog;
+import org.jeecg.common.util.oConvertUtils;
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAgentAccount;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouAgentAccountService;
+import java.util.Date;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import lombok.extern.slf4j.Slf4j;
+
+import org.jeecgframework.poi.excel.ExcelImportUtil;
+import org.jeecgframework.poi.excel.def.NormalExcelConstants;
+import org.jeecgframework.poi.excel.entity.ExportParams;
+import org.jeecgframework.poi.excel.entity.ImportParams;
+import org.jeecgframework.poi.excel.view.JeecgEntityExcelView;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.multipart.MultipartHttpServletRequest;
+import org.springframework.web.servlet.ModelAndView;
+import com.alibaba.fastjson.JSON;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+
+ /**
+ * 快手代理商账户表
+ * @author jeecg-boot
+ * @date   2020-03-03
+ * @version V1.0
+ */
+@Slf4j
+@Api(tags="快手代理商账户表")
+@RestController
+@RequestMapping("/kuaishou/kuaishouAgentAccount")
+public class KuaishouAgentAccountController {
+	@Autowired
+	private IKuaishouAgentAccountService kuaishouAgentAccountService;
+	
+	/**
+	  * 分页列表查询
+	 * @param kuaishouAgentAccount
+	 * @param pageNo
+	 * @param pageSize
+	 * @param req
+	 * @return
+	 */
+	@AutoLog(value = "快手代理商账户表-分页列表查询")
+	@ApiOperation(value="快手代理商账户表-分页列表查询", notes="快手代理商账户表-分页列表查询")
+	@GetMapping(value = "/list")
+	public Result<IPage<KuaishouAgentAccount>> queryPageList(KuaishouAgentAccount kuaishouAgentAccount,
+									  @RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
+									  @RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
+									  HttpServletRequest req) {
+		Result<IPage<KuaishouAgentAccount>> result = new Result<IPage<KuaishouAgentAccount>>();
+		QueryWrapper<KuaishouAgentAccount> queryWrapper = QueryGenerator.initQueryWrapper(kuaishouAgentAccount, req.getParameterMap());
+		Page<KuaishouAgentAccount> page = new Page<KuaishouAgentAccount>(pageNo, pageSize);
+		IPage<KuaishouAgentAccount> pageList = kuaishouAgentAccountService.page(page, queryWrapper);
+		result.setSuccess(true);
+		result.setResult(pageList);
+		return result;
+	}
+	
+	/**
+	  *   添加
+	 * @param kuaishouAgentAccount
+	 * @return
+	 */
+	@AutoLog(value = "快手代理商账户表-添加")
+	@ApiOperation(value="快手代理商账户表-添加", notes="快手代理商账户表-添加")
+	@PostMapping(value = "/add")
+	public Result<KuaishouAgentAccount> add(@RequestBody KuaishouAgentAccount kuaishouAgentAccount) {
+		Result<KuaishouAgentAccount> result = new Result<KuaishouAgentAccount>();
+		try {
+			kuaishouAgentAccountService.save(kuaishouAgentAccount);
+			result.success("添加成功!");
+		} catch (Exception e) {
+			log.error(e.getMessage(),e);
+			result.error500("操作失败");
+		}
+		return result;
+	}
+	
+	/**
+	  *  编辑
+	 * @param kuaishouAgentAccount
+	 * @return
+	 */
+	@AutoLog(value = "快手代理商账户表-编辑")
+	@ApiOperation(value="快手代理商账户表-编辑", notes="快手代理商账户表-编辑")
+	@PutMapping(value = "/edit")
+	public Result<KuaishouAgentAccount> edit(@RequestBody KuaishouAgentAccount kuaishouAgentAccount) {
+		Result<KuaishouAgentAccount> result = new Result<KuaishouAgentAccount>();
+		KuaishouAgentAccount kuaishouAgentAccountEntity = kuaishouAgentAccountService.getById(kuaishouAgentAccount.getId());
+		if(kuaishouAgentAccountEntity==null) {
+			result.error500("未找到对应实体");
+		}else {
+			boolean ok = kuaishouAgentAccountService.updateById(kuaishouAgentAccount);
+			if(ok) {
+				result.success("修改成功!");
+			}
+		}
+		
+		return result;
+	}
+	
+	/**
+	  *   通过id删除
+	 * @param id
+	 * @return
+	 */
+	@AutoLog(value = "快手代理商账户表-通过id删除")
+	@ApiOperation(value="快手代理商账户表-通过id删除", notes="快手代理商账户表-通过id删除")
+	@DeleteMapping(value = "/delete")
+	public Result<?> delete(@RequestParam(name="id",required=true) String id) {
+		try {
+			kuaishouAgentAccountService.removeById(id);
+		} catch (Exception e) {
+			log.error("删除失败",e.getMessage());
+			return Result.error("删除失败!");
+		}
+		return Result.ok("删除成功!");
+	}
+	
+	/**
+	  *  批量删除
+	 * @param ids
+	 * @return
+	 */
+	@AutoLog(value = "快手代理商账户表-批量删除")
+	@ApiOperation(value="快手代理商账户表-批量删除", notes="快手代理商账户表-批量删除")
+	@DeleteMapping(value = "/deleteBatch")
+	public Result<KuaishouAgentAccount> deleteBatch(@RequestParam(name="ids",required=true) String ids) {
+		Result<KuaishouAgentAccount> result = new Result<KuaishouAgentAccount>();
+		if(ids==null || "".equals(ids.trim())) {
+			result.error500("参数不识别!");
+		}else {
+			this.kuaishouAgentAccountService.removeByIds(Arrays.asList(ids.split(",")));
+			result.success("删除成功!");
+		}
+		return result;
+	}
+	
+	/**
+	  * 通过id查询
+	 * @param id
+	 * @return
+	 */
+	@AutoLog(value = "快手代理商账户表-通过id查询")
+	@ApiOperation(value="快手代理商账户表-通过id查询", notes="快手代理商账户表-通过id查询")
+	@GetMapping(value = "/queryById")
+	public Result<KuaishouAgentAccount> queryById(@RequestParam(name="id",required=true) String id) {
+		Result<KuaishouAgentAccount> result = new Result<KuaishouAgentAccount>();
+		KuaishouAgentAccount kuaishouAgentAccount = kuaishouAgentAccountService.getById(id);
+		if(kuaishouAgentAccount==null) {
+			result.error500("未找到对应实体");
+		}else {
+			result.setResult(kuaishouAgentAccount);
+			result.setSuccess(true);
+		}
+		return result;
+	}
+
+  /**
+      * 导出excel
+   *
+   * @param request
+   * @param response
+   */
+  @RequestMapping(value = "/exportXls")
+  public ModelAndView exportXls(HttpServletRequest request, HttpServletResponse response) {
+      // Step.1 组装查询条件
+      QueryWrapper<KuaishouAgentAccount> queryWrapper = null;
+      try {
+          String paramsStr = request.getParameter("paramsStr");
+          if (oConvertUtils.isNotEmpty(paramsStr)) {
+              String deString = URLDecoder.decode(paramsStr, "UTF-8");
+              KuaishouAgentAccount kuaishouAgentAccount = JSON.parseObject(deString, KuaishouAgentAccount.class);
+              queryWrapper = QueryGenerator.initQueryWrapper(kuaishouAgentAccount, request.getParameterMap());
+          }
+      } catch (UnsupportedEncodingException e) {
+          e.printStackTrace();
+      }
+
+      //Step.2 AutoPoi 导出Excel
+      ModelAndView mv = new ModelAndView(new JeecgEntityExcelView());
+      List<KuaishouAgentAccount> pageList = kuaishouAgentAccountService.list(queryWrapper);
+      //导出文件名称
+      mv.addObject(NormalExcelConstants.FILE_NAME, "快手代理商账户表列表");
+      mv.addObject(NormalExcelConstants.CLASS, KuaishouAgentAccount.class);
+      mv.addObject(NormalExcelConstants.PARAMS, new ExportParams("快手代理商账户表列表数据", "导出人:Jeecg", "导出信息"));
+      mv.addObject(NormalExcelConstants.DATA_LIST, pageList);
+      return mv;
+  }
+
+  /**
+      * 通过excel导入数据
+   *
+   * @param request
+   * @param response
+   * @return
+   */
+  @RequestMapping(value = "/importExcel", method = RequestMethod.POST)
+  public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
+      MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
+      Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
+      for (Map.Entry<String, MultipartFile> entity : fileMap.entrySet()) {
+          MultipartFile file = entity.getValue();
+          ImportParams params = new ImportParams();
+          params.setTitleRows(2);
+          params.setHeadRows(1);
+          params.setNeedSave(true);
+          try {
+              List<KuaishouAgentAccount> listKuaishouAgentAccounts = ExcelImportUtil.importExcel(file.getInputStream(), KuaishouAgentAccount.class, params);
+              kuaishouAgentAccountService.saveBatch(listKuaishouAgentAccounts);
+              return Result.ok("文件导入成功!数据行数:" + listKuaishouAgentAccounts.size());
+          } catch (Exception e) {
+              log.error(e.getMessage(),e);
+              return Result.error("文件导入失败:"+e.getMessage());
+          } finally {
+              try {
+                  file.getInputStream().close();
+              } catch (IOException e) {
+                  e.printStackTrace();
+              }
+          }
+      }
+      return Result.ok("文件导入失败!");
+  }
+
+
+}

+ 20 - 0
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/job/DataeyeJob.java

@@ -0,0 +1,20 @@
+package org.jeecg.modules.ctop.job;
+
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import lombok.extern.slf4j.Slf4j;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+@Slf4j
+public class DataeyeJob implements Job {
+    @Autowired
+    private DataeyeCompanyService dataeyeCompanyService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        for (int i = 1; i < 1000; i++){
+            dataeyeCompanyService.getCompanyInfo(i);
+        }
+    }
+}

+ 43 - 2
module-common/src/main/java/cn/com/ctop/common/module/utils/HttpUtils.java

@@ -158,14 +158,55 @@ public class HttpUtils {
         return HttpClients.createDefault();
     }
 
+    public static String httpPostFormRequest(String url, Map<String, Object> param, Map<String, String> headers) {
+        HttpClient httpClient = createSslClientDefault();
+        String strReturn = "";
+        try {
+            HttpPost httppost = new HttpPost(url);
+            if(headers != null){
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
+            }
+            List<NameValuePair> paramList = new ArrayList<NameValuePair>();
+            for (String key : param.keySet()) {
+                BasicNameValuePair basicNameValuePair = new BasicNameValuePair(key, String.valueOf(param.get(key)));
+                paramList.add(basicNameValuePair);
+            }
+
+            // 第二步:我们发现Entity是一个接口,所以只能找实现类,发现实现类又需要一个集合,集合的泛型是NameValuePair类型
+            UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(paramList);
+            // 第一步:通过setEntity 将我们的entity对象传递过去
+            httppost.setEntity(formEntity);
+            HttpEntity respentity;
+
+            HttpResponse response = httpClient.execute(httppost);
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
+                String newUrl = response.getFirstHeader("Location").getValue();
+                return httpPostFormRequest(newUrl, param, headers);
+            } else if (statusCode == HttpStatus.SC_OK) {
+                respentity = response.getEntity();
+                strReturn = EntityUtils.toString(respentity);
+                return strReturn;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error(e.getMessage());
+        }
+        return strReturn;
+    }
+
     public static String httpPostParamRequest(String url, Map<String, Object> param, Map<String, String> headers) {
         HttpClient httpClient = createSslClientDefault();
         String strReturn = "";
         try {
             HttpPost httppost = new HttpPost(url);
             httppost.setHeader("User-Agent", USER_AGENT);
-            for (String key : headers.keySet()) {
-                httppost.setHeader(key, headers.get(key));
+            if(headers != null){
+                for (String key : headers.keySet()) {
+                    httppost.setHeader(key, headers.get(key));
+                }
             }
             List<NameValuePair> paramList = new ArrayList<NameValuePair>();
             for (String key : param.keySet()) {

+ 149 - 0
module-common/src/main/java/cn/com/ctop/common/module/utils/JavacvUtil.java

@@ -0,0 +1,149 @@
+package cn.com.ctop.common.module.utils;
+
+import com.google.common.collect.Lists;
+import org.bytedeco.javacpp.opencv_core;
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.Java2DFrameConverter;
+import org.opencv.core.CvType;
+import org.opencv.core.Mat;
+import org.opencv.core.MatOfByte;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.awt.image.DataBufferByte;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+public class JavacvUtil {
+    /**
+     * 视频文件指定时间段的帧截取
+     * @param file
+     * @param start
+     * @param end
+     */
+    public static List<File> videoIntercept(File file, Integer start, Integer end) {
+        Frame frame = null;
+        List<File> files = Lists.newArrayList();
+        FFmpegFrameGrabber fFmpegFrameGrabber = new FFmpegFrameGrabber(file);
+        String filePath = "D://video//images//";
+        String fileTargetName = "movie";
+        try {
+            fFmpegFrameGrabber.start();
+            int ftp = fFmpegFrameGrabber.getLengthInFrames();
+            System.out.println("开始视频提取帧");
+            for (int i=0 ; i < ftp ; i++){
+                if( i >= start && i <= end){
+                    frame = fFmpegFrameGrabber.grabImage();
+                    doExecuteFrame(frame, filePath, fileTargetName, i ,files);
+                }
+            }
+            System.out.println("============运行结束============");
+            fFmpegFrameGrabber.stop();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return files;
+    }
+
+    public static void doExecuteFrame(Frame frame, String targetFilePath, String targetFileName, int index ,List<File> files) {
+        if ( frame == null || frame.image == null) {
+            return;
+        }
+        Java2DFrameConverter converter = new Java2DFrameConverter();
+        String imageMat = "jpg";
+        String fileName = targetFilePath + targetFileName + "_" + index + "." + imageMat;
+        BufferedImage bi = converter.getBufferedImage(frame);
+
+//        Mat img = Imgcodecs.imread("");
+//        Mat imgHSV = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
+//        Mat img2 = new Mat(img.rows(), img.cols(), CvType.CV_8UC3);
+
+        //转成HSV空间
+//        Imgproc.cvtColor(img, imgHSV, Imgproc.COLOR_BGR2HSV);
+//        //转回BGR空间
+//        Imgproc.cvtColor(imgHSV, img2, Imgproc.COLOR_HSV2BGR);
+
+        File output = new File(fileName);
+        files.add(output);
+        try{
+            ImageIO.write(bi, imageMat, output);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void main(String[] args) {
+        List<File> files = videoIntercept(new File("C:\\Users\\宋英豪\\Desktop\\nfpHmswW.mp4"), 10, 100);
+        System.out.println(files);
+    }
+
+    /**
+     * Mat转换成BufferedImage
+     *
+     * @param matrix
+     *            要转换的Mat
+     * @param fileExtension
+     *            格式为 ".jpg", ".png", etc
+     * @return
+     */
+    public static BufferedImage Mat2BufImg (Mat matrix, String fileExtension) {
+        // convert the matrix into a matrix of bytes appropriate for
+        // this file extension
+        MatOfByte mob = new MatOfByte();
+        Imgcodecs.imencode(fileExtension, matrix, mob);
+        // convert the "matrix of bytes" into a byte array
+        byte[] byteArray = mob.toArray();
+        BufferedImage bufImage = null;
+        try {
+            InputStream in = new ByteArrayInputStream(byteArray);
+            bufImage = ImageIO.read(in);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return bufImage;
+    }
+
+    /**
+     * BufferedImage转换成Mat
+     *
+     * @param original
+     *            要转换的BufferedImage
+     * @param imgType
+     *            bufferedImage的类型 如 BufferedImage.TYPE_3BYTE_BGR
+     * @param matType
+     *            转换成mat的type 如 CvType.CV_8UC3
+     */
+    public static Mat BufImg2Mat (BufferedImage original, int imgType, int matType) {
+        if (original == null) {
+            throw new IllegalArgumentException("original == null");
+        }
+
+        // Don't convert if it already has correct type
+        if (original.getType() != imgType) {
+
+            // Create a buffered image
+            BufferedImage image = new BufferedImage(original.getWidth(), original.getHeight(), imgType);
+
+            // Draw the image onto the new buffer
+            Graphics2D g = image.createGraphics();
+            try {
+                g.setComposite(AlphaComposite.Src);
+                g.drawImage(original, 0, 0, null);
+            } finally {
+                g.dispose();
+            }
+        }
+
+        byte[] pixels = ((DataBufferByte) original.getRaster().getDataBuffer()).getData();
+        Mat mat = Mat.eye(original.getHeight(), original.getWidth(), matType);
+        mat.put(0, 0, pixels);
+        return mat;
+    }
+}

+ 17 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeCompany.java

@@ -1,7 +1,23 @@
 package cn.com.ctop.crawler.modules.dataeye.entity;
 
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import java.util.Date;
+@Data
+@TableName("ctop_dataeye_company")
 public class DataeyeCompany {
-    private Integer id;
+    @TableId(value = "id", type = IdType.INPUT)
+    private Integer companyId;
     private String companyName;
+    private Date firstSeen;
+    private Date lastSeen;
+    private Integer days;
+    private Integer materialNum;
+    private Integer productNum;
+    private Integer mediaNum;
 
 }

+ 4 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/entity/DataeyeMedia.java

@@ -8,6 +8,10 @@ public class DataeyeMedia {
     private Integer id;
     private String mediaName;
     private String logoUrl;
+    private String supportIos;
+    private String supportAndroid;
+    private String positionList;
+    private Integer isUnion;
     private Integer isShow;
     private Date createTime;
     private Date updateTime;

+ 7 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/mapper/DataeyeCompanyMapper.java

@@ -0,0 +1,7 @@
+package cn.com.ctop.crawler.modules.dataeye.mapper;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+public interface DataeyeCompanyMapper extends BaseMapper<DataeyeCompany> {
+}

+ 8 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/DataeyeCompanyService.java

@@ -0,0 +1,8 @@
+package cn.com.ctop.crawler.modules.dataeye.service;
+
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+public interface DataeyeCompanyService extends IService<DataeyeCompany> {
+    public void getCompanyInfo(Integer companyId);
+}

+ 71 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/dataeye/service/impl/DataeyeCompanyServiceImpl.java

@@ -0,0 +1,71 @@
+package cn.com.ctop.crawler.modules.dataeye.service.impl;
+
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.core.util.SignUtil;
+import cn.com.ctop.crawler.modules.dataeye.entity.DataeyeCompany;
+import cn.com.ctop.crawler.modules.dataeye.mapper.DataeyeCompanyMapper;
+import cn.com.ctop.crawler.modules.dataeye.service.DataeyeCompanyService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.jeecg.common.util.DateUtils;
+import org.springframework.stereotype.Service;
+import sun.misc.BASE64Encoder;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+@Service
+public class DataeyeCompanyServiceImpl extends ServiceImpl<DataeyeCompanyMapper, DataeyeCompany> implements DataeyeCompanyService {
+
+    public void getCompanyInfo(Integer companyId){
+        Map<String,String> headerMap = new HashMap<>();
+        headerMap.put("Host","adx.dataeye.com");
+        headerMap.put("Origin","https://adx.dataeye.com");
+        headerMap.put("Referer","https://adx.dataeye.com/");
+        BasicClientCookie ck = new BasicClientCookie("tokenglobal", "721501ec4ec04a50bbffc0c0240cbbdd63a0d5456205b9a5ea1581ce6c6554ca");
+        ck.setDomain("adx.dataeye.com");
+        ck.setExpiryDate(DateUtils.addDay(new Date(),365));
+        ck.setPath("/");
+        HttpUtils.COOKIESTORE.addCookie(ck);
+        Map<String,Object> paramMap = new HashMap<>();
+        paramMap.put("companyId",companyId);
+        paramMap.put("thisTimes",(long)(new Date().getTime()/100));
+        String sign = SignUtil.getDataeyeSign(paramMap);
+        paramMap.put("sign",sign);
+        System.out.println(new Gson().toJson(paramMap));
+        String response = HttpUtils.httpPostFormRequest("https://adx.dataeye.com/company/getCompanyInfo",paramMap,headerMap);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        try {
+            JsonNode rootNode = mapper.readTree(response);
+            int statusCode = rootNode.get("statusCode").asInt();
+            if (statusCode == 200){
+                String content = rootNode.get("content").toString();
+                DataeyeCompany company = mapper.readValue(content,DataeyeCompany.class);
+                this.save(company);
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        System.out.println(response);
+    }
+
+
+    public static void main(String[] args){
+        DataeyeCompanyServiceImpl i = new DataeyeCompanyServiceImpl();
+        i.getCompanyInfo(1);
+//        Date date = new Date(1582598505897L);
+//        System.out.println(DateUtils.formatDate(1582598505897L));
+
+
+    }
+}

+ 108 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/entity/KuaishouAgentAccount.java

@@ -0,0 +1,108 @@
+package cn.com.ctop.kuaishou.modules.report.entity;
+
+import java.io.Serializable;
+import java.util.Date;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.jeecgframework.poi.excel.annotation.Excel;
+
+/**
+ * 快手代理商账户表
+ * @author jeecg-boot
+ * @date   2020-03-03
+ * @version V1.0
+ */
+@Data
+@TableName("ctop_kuaishou_agent_account")
+@EqualsAndHashCode(callSuper = false)
+@Accessors(chain = true)
+@ApiModel(value="ctop_kuaishou_agent_account对象", description="快手代理商账户表")
+public class KuaishouAgentAccount {
+    
+	/**id*/
+	@TableId(type = IdType.INPUT)
+    @ApiModelProperty(value = "id")
+	@JsonProperty("userId")
+	private Integer id;
+	/**accountId*/
+	@Excel(name = "accountId", width = 15)
+    @ApiModelProperty(value = "accountId")
+	private Integer accountId;
+	/**accountName*/
+	@Excel(name = "accountName", width = 15)
+    @ApiModelProperty(value = "accountName")
+	private String accountName;
+	/**balance*/
+	@Excel(name = "balance", width = 15)
+    @ApiModelProperty(value = "balance")
+	private java.math.BigDecimal balance;
+	/**creditBalance*/
+	@Excel(name = "creditBalance", width = 15)
+    @ApiModelProperty(value = "creditBalance")
+	private java.math.BigDecimal creditBalance;
+	/**extendedBalance*/
+	@Excel(name = "extendedBalance", width = 15)
+    @ApiModelProperty(value = "extendedBalance")
+	private java.math.BigDecimal extendedBalance;
+	/**rebate*/
+	@Excel(name = "rebate", width = 15)
+    @ApiModelProperty(value = "rebate")
+	private java.math.BigDecimal rebate;
+	/**preRebate*/
+	@Excel(name = "preRebate", width = 15)
+    @ApiModelProperty(value = "preRebate")
+	private java.math.BigDecimal preRebate;
+	/**contractRebate*/
+	@Excel(name = "contractRebate", width = 15)
+    @ApiModelProperty(value = "contractRebate")
+	private java.math.BigDecimal contractRebate;
+	/**directRebate*/
+	@Excel(name = "directRebate", width = 15)
+    @ApiModelProperty(value = "directRebate")
+	private java.math.BigDecimal directRebate;
+	/**totalBalance*/
+	@Excel(name = "totalBalance", width = 15)
+    @ApiModelProperty(value = "totalBalance")
+	private java.math.BigDecimal totalBalance;
+	/**kuaibiBalance*/
+	@Excel(name = "kuaibiBalance", width = 15)
+    @ApiModelProperty(value = "kuaibiBalance")
+	private java.math.BigDecimal kuaibiBalance;
+	/**pushBalance*/
+	@Excel(name = "pushBalance", width = 15)
+    @ApiModelProperty(value = "pushBalance")
+	private java.math.BigDecimal pushBalance;
+	/**recharged*/
+	@Excel(name = "recharged", width = 15)
+    @ApiModelProperty(value = "recharged")
+	private Boolean recharged;
+	/**corporationName*/
+	@Excel(name = "corporationName", width = 15)
+    @ApiModelProperty(value = "corporationName")
+	private String corporationName;
+	/**reviewStatus*/
+	@Excel(name = "reviewStatus", width = 15)
+    @ApiModelProperty(value = "reviewStatus")
+	private Integer reviewStatus;
+	/**frozenStatus*/
+	@Excel(name = "frozenStatus", width = 15)
+    @ApiModelProperty(value = "frozenStatus")
+	private Integer frozenStatus;
+	/**transferAccountStatus*/
+	@Excel(name = "transferAccountStatus", width = 15)
+    @ApiModelProperty(value = "transferAccountStatus")
+	private Boolean transferAccountStatus;
+	/**createTime*/
+    @ApiModelProperty(value = "createTime")
+	private Date createTime;
+}

+ 19 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/job/KuaishouAgentAccountJob.java

@@ -0,0 +1,19 @@
+package cn.com.ctop.kuaishou.modules.report.job;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyAgentService;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class KuaishouAgentAccountJob implements Job {
+
+    @Autowired
+    private IKuaishouReportDailyAgentService kuaishouReportDailyAgentService;
+    @Override
+    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
+        int currentPage = 1;
+        while (kuaishouReportDailyAgentService.getAccount(currentPage)){
+            currentPage++;
+        }
+    }
+}

+ 18 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/KuaishouAgentAccountMapper.java

@@ -0,0 +1,18 @@
+package cn.com.ctop.kuaishou.modules.report.mapper;
+
+import java.util.List;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouReportDailyAgent;
+import org.apache.ibatis.annotations.Param;
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAgentAccount;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * 快手代理商账户表
+ * @author: jeecg-boot
+ * @date:   2020-03-03
+ * @cersion: V1.0
+ */
+public interface KuaishouAgentAccountMapper extends BaseMapper<KuaishouAgentAccount> {
+    public void replaceBatch(List<KuaishouAgentAccount> list);
+}

+ 49 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouAgentAccountMapper.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAgentAccountMapper">
+    <insert id="replaceBatch">
+        replace into ctop_kuaishou_agent_account(
+        `id`,
+        `account_id`,
+        `account_name`,
+        `balance`,
+        `credit_balance`,
+        `extended_balance`,
+        `rebate`,
+        `pre_rebate`,
+        `contract_rebate`,
+        `direct_rebate`,
+        `total_balance`,
+        `kuaibi_balance`,
+        `push_balance`,
+        `recharged`,
+        `corporation_name`,
+        `review_status`,
+        `frozen_status`,
+        `transfer_account_status`,
+        `create_time`
+        )
+        values
+        <foreach collection="list" item="agent" separator=",">
+            (#{agent.id},
+            #{agent.accountId},
+            #{agent.accountName},
+            #{agent.balance},
+            #{agent.creditBalance},
+            #{agent.extendedBalance},
+            #{agent.rebate},
+            #{agent.preRebate},
+            #{agent.contractRebate},
+            #{agent.directRebate},
+            #{agent.totalBalance},
+            #{agent.kuaibiBalance},
+            #{agent.pushBalance},
+            #{agent.recharged},
+            #{agent.corporationName},
+            #{agent.reviewStatus},
+            #{agent.frozenStatus},
+            #{agent.transferAccountStatus},
+            #{agent.createTime})
+        </foreach>
+    </insert>
+</mapper>

+ 14 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouAgentAccountService.java

@@ -0,0 +1,14 @@
+package cn.com.ctop.kuaishou.modules.report.service;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAgentAccount;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * 快手代理商账户表
+ * @author jeecg-boot
+ * @date   2020-03-03
+ * @version V1.0
+ */
+public interface IKuaishouAgentAccountService extends IService<KuaishouAgentAccount> {
+
+}

+ 1 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/IKuaishouReportDailyAgentService.java

@@ -14,4 +14,5 @@ import java.util.List;
 public interface IKuaishouReportDailyAgentService extends IService<KuaishouReportDailyAgent> {
     public void replaceBatch(List<KuaishouReportDailyAgent> list);
     public void getReport(String startDate,String endDate);
+    public boolean getAccount(int currentPage);
 }

+ 19 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouAgentAccountServiceImpl.java

@@ -0,0 +1,19 @@
+package cn.com.ctop.kuaishou.modules.report.service.impl;
+
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAgentAccount;
+import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAgentAccountMapper;
+import cn.com.ctop.kuaishou.modules.report.service.IKuaishouAgentAccountService;
+import org.springframework.stereotype.Service;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+
+/**
+ * 快手代理商账户表
+ * @author jeecg-boot
+ * @date   2020-03-03
+ * @version V1.0
+ */
+@Service
+public class KuaishouAgentAccountServiceImpl extends ServiceImpl<KuaishouAgentAccountMapper, KuaishouAgentAccount> implements IKuaishouAgentAccountService {
+
+}

+ 75 - 17
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/service/impl/KuaishouReportDailyAgentServiceImpl.java

@@ -1,8 +1,10 @@
 package cn.com.ctop.kuaishou.modules.report.service.impl;
 
 import cn.com.ctop.common.module.utils.HttpUtils2;
+import cn.com.ctop.kuaishou.modules.report.entity.KuaishouAgentAccount;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouReportDailyAgent;
 import cn.com.ctop.kuaishou.modules.report.entity.KuaishouReportDailyAgentSum;
+import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouAgentAccountMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouReportDailyAgentMapper;
 import cn.com.ctop.kuaishou.modules.report.mapper.KuaishouReportDailyAgentSumMapper;
 import cn.com.ctop.kuaishou.modules.report.service.IKuaishouReportDailyAgentService;
@@ -40,6 +42,8 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
     private KuaishouReportDailyAgentMapper kuaishouReportDailyAgentMapper;
     @Autowired
     private KuaishouReportDailyAgentSumMapper kuaishouReportDailyAgentSumMapper;
+    @Autowired
+    private KuaishouAgentAccountMapper kuaishouAgentAccountMapper;
     @Value("${jeecg.path.chrome-driver}")
     private String chromeDriver;
     @Override
@@ -47,7 +51,8 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
         kuaishouReportDailyAgentMapper.replaceBatch(list);
     }
 
-    public void getReport(String startDate,String endDate){
+    public boolean loginAgent(){
+        boolean loginStatus = false;
         System.getProperties().setProperty("webdriver.chrome.driver", "D:/chromedriver.exe");
         String url = "https://agent.e.kuaishou.com";
         ChromeOptions chromeOptions = new ChromeOptions();
@@ -75,15 +80,7 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
             //点击登录
             loginElement.click();
             Thread.sleep(3000L);
-            Map<String,String> headerMap = new HashMap<String,String>();
-            headerMap.put("Accept","application/json");
-            headerMap.put("Accept-Encoding","gzip, deflate, br");
-            headerMap.put("Accept-Language","zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
-            headerMap.put("Connection","keep-alive");
-            headerMap.put("Host","agent.e.kuaishou.com");
-            headerMap.put("Referer","https://agent.e.kuaishou.com/");
-            headerMap.put("Content-Type","application/json;charset=utf-8");
-            headerMap.put("User-Agent",HttpUtils2.USER_AGENT);
+
             Set<Cookie> cookies = webDriver.manage().getCookies();
             Iterator iterator = cookies.iterator();
             CookieStore cookieStore = new BasicCookieStore();
@@ -103,8 +100,75 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
                 clientCookie.setPath(cookieMap.get("path"));
                 HttpUtils2.cookieStore.addCookie(clientCookie);
             }
-            String currentDate = startDate;
+            loginStatus = true;
+        }catch (Exception e) {
+            e.printStackTrace();
+            loginStatus = false;
+        }finally {
+            webDriver.manage().deleteAllCookies();
+//            webDriver.close();
+            webDriver.quit();
+        }
+        return loginStatus;
+    }
 
+    public boolean getAccount(int currentPage){
+        boolean hasNextPage = false;
+        try {
+            if (!loginAgent()){
+                return hasNextPage;
+            }
+            Map<String,String> headerMap = new HashMap<String,String>();
+            headerMap.put("Accept","application/json");
+            headerMap.put("Accept-Encoding","gzip, deflate, br");
+            headerMap.put("Accept-Language","zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
+            headerMap.put("Connection","keep-alive");
+            headerMap.put("Host","agent.e.kuaishou.com");
+            headerMap.put("Referer","https://agent.e.kuaishou.com/");
+            headerMap.put("Content-Type","application/json;charset=utf-8");
+            headerMap.put("User-Agent",HttpUtils2.USER_AGENT);
+            String postData = "{\"pageInfo\":{\"currentPage\":"+currentPage+",\"pageSize\":20,\"total\":1000},\"reviewStatus\":\"-1\",\"selectType\":1,\"selectValue\":\"\",\"createBeginTime\":0,\"createEndTime\":0}";
+            String listUrl = "https://agent.e.kuaishou.com/rest/dsp/agent/account/list";
+            String result = HttpUtils2.httpPostRequestTest(listUrl,postData,headerMap);
+            ObjectMapper mapper = new ObjectMapper();
+            JsonNode resultNode = mapper.readTree(result);
+            Integer code = resultNode.get("result").intValue();
+            if(code == 1) {
+                Iterator<JsonNode> iterator1 = resultNode.get("adDspAccountList").elements();
+                int i = 0;
+                List<KuaishouAgentAccount> accountList = new ArrayList<>();
+                while (iterator1.hasNext()) {
+                    hasNextPage = true;
+                    JsonNode dataNode = iterator1.next();
+                    KuaishouAgentAccount agentAccount = mapper.readValue(dataNode.toString(),KuaishouAgentAccount.class);
+                    accountList.add(agentAccount);
+                }
+                if (accountList != null && accountList.size() > 0){
+                    kuaishouAgentAccountMapper.replaceBatch(accountList);
+                }
+            }
+
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+        return hasNextPage;
+    }
+
+    public void getReport(String startDate,String endDate){
+        try{
+            if (!loginAgent()){
+                return;
+            }
+            String currentDate = startDate;
+            Map<String,String> headerMap = new HashMap<String,String>();
+            headerMap.put("Accept","application/json");
+            headerMap.put("Accept-Encoding","gzip, deflate, br");
+            headerMap.put("Accept-Language","zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
+            headerMap.put("Connection","keep-alive");
+            headerMap.put("Host","agent.e.kuaishou.com");
+            headerMap.put("Referer","https://agent.e.kuaishou.com/");
+            headerMap.put("Content-Type","application/json;charset=utf-8");
+            headerMap.put("User-Agent",HttpUtils2.USER_AGENT);
             while(DateUtils.compareDate(currentDate,endDate)<=0){
                 String listUrl = "https://agent.e.kuaishou.com/rest/dsp/agent/account/data/list";
                 String postData = "{\"agentId\":403756224,\"date\":\""+currentDate+"\",\"sortKey\":\"totalCharge\",\"isAscending\":false,\"pageInfo\":{\"totalCount\":1616,\"currentPage\":1,\"pageSize\":999999}}";
@@ -172,14 +236,8 @@ public class KuaishouReportDailyAgentServiceImpl extends ServiceImpl<KuaishouRep
                 System.out.println("代理商日报"+currentDate+"入库完成");
                 currentDate = DateUtils.addDay(currentDate,1);
             }
-
-
         } catch (Exception e) {
             e.printStackTrace();
-        } finally {
-            webDriver.manage().deleteAllCookies();
-//            webDriver.close();
-            webDriver.quit();
         }
     }
 }

+ 1 - 1
pom.xml

@@ -25,7 +25,7 @@
         <module>module-ctop</module>
         <module>performance-appraisal</module>
         <module>jeecg-boot-module-system</module>
-        <module>moudule-eureca</module>
+<!--        <module>moudule-eureca</module>-->
 
     </modules>