瀏覽代碼

Merge branch 'master' of http://git.tjyourong.com.cn/ctop/adsp-boot

 Conflicts:
	module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java
xuzuoyun 4 年之前
父節點
當前提交
99a2516e48
共有 16 個文件被更改,包括 1136 次插入48 次删除
  1. 53 35
      jeecg-boot-base-common/src/main/java/org/jeecg/common/util/MD5Util.java
  2. 92 0
      jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/DataSourceCachePool.java
  3. 31 0
      jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/DbValidationQueryEnum.java
  4. 298 0
      jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/DynamicDBUtil.java
  5. 171 0
      jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/FreemarkerParseFactory.java
  6. 212 0
      jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/SqlUtils.java
  7. 6 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/ProjectMemberController.java
  8. 68 3
      jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java
  9. 4 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/TaobaoTttmService.java
  10. 158 0
      module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoTttmServiceImpl.java
  11. 2 0
      module-job-bytedance/src/main/resources/application-prod.yml
  12. 2 0
      module-job-kuaishou/src/main/resources/application-prod.yml
  13. 1 2
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouHistoryReportTaskServiceImpl.java
  14. 8 1
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyAccountMapper.xml
  15. 2 0
      module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCreativeMapper.xml
  16. 28 4
      pom.xml

File diff suppressed because it is too large
+ 53 - 35
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/MD5Util.java


+ 92 - 0
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/DataSourceCachePool.java

@@ -0,0 +1,92 @@
+package org.jeecg.common.util.db;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import org.jeecg.common.constant.CacheConstant;
+import org.jeecg.common.system.api.ISysBaseApi;
+import org.jeecg.common.system.vo.DynamicDataSourceModel;
+import org.jeecg.common.util.SpringContextUtils;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * 数据源缓存池
+ */
+public class DataSourceCachePool {
+    /** 数据源连接池缓存【本地 class缓存 - 不支持分布式】 */
+    private static Map<String, DruidDataSource> dbSources = new HashMap<>();
+    private static RedisTemplate<String, Object> redisTemplate;
+
+    private static RedisTemplate<String, Object> getRedisTemplate() {
+        if (redisTemplate == null) {
+            redisTemplate = (RedisTemplate<String, Object>) SpringContextUtils.getBean("redisTemplate");
+        }
+        return redisTemplate;
+    }
+
+    /**
+     * 获取多数据源缓存
+     *
+     * @param dbKey
+     * @return
+     */
+    public static DynamicDataSourceModel getCacheDynamicDataSourceModel(String dbKey) {
+        String redisCacheKey = CacheConstant.SYS_DYNAMICDB_CACHE + dbKey;
+        if (getRedisTemplate().hasKey(redisCacheKey)) {
+            return (DynamicDataSourceModel) getRedisTemplate().opsForValue().get(redisCacheKey);
+        }
+        ISysBaseApi sysBaseAPI = SpringContextUtils.getBean(ISysBaseApi.class);
+        DynamicDataSourceModel dbSource = sysBaseAPI.getDynamicDbSourceByCode(dbKey);
+        if (dbSource != null) {
+            getRedisTemplate().opsForValue().set(redisCacheKey, dbSource);
+        }
+        return dbSource;
+    }
+
+    public static DruidDataSource getCacheBasicDataSource(String dbKey) {
+        return dbSources.get(dbKey);
+    }
+
+    /**
+     * put 数据源缓存
+     *
+     * @param dbKey
+     * @param db
+     */
+    public static void putCacheBasicDataSource(String dbKey, DruidDataSource db) {
+        dbSources.put(dbKey, db);
+    }
+
+    /**
+     * 清空数据源缓存
+     */
+    public static void cleanAllCache() {
+        //关闭数据源连接
+        for(Map.Entry<String, DruidDataSource> entry : dbSources.entrySet()){
+            String dbkey = entry.getKey();
+            DruidDataSource druidDataSource = entry.getValue();
+            if(druidDataSource!=null && druidDataSource.isEnable()){
+                druidDataSource.close();
+            }
+            //清空redis缓存
+            getRedisTemplate().delete(CacheConstant.SYS_DYNAMICDB_CACHE + dbkey);
+        }
+        //清空缓存
+        dbSources.clear();
+    }
+
+    public static void removeCache(String dbKey) {
+        //关闭数据源连接
+        DruidDataSource druidDataSource = dbSources.get(dbKey);
+        if(druidDataSource!=null && druidDataSource.isEnable()){
+            druidDataSource.close();
+        }
+        //清空redis缓存
+        getRedisTemplate().delete(CacheConstant.SYS_DYNAMICDB_CACHE + dbKey);
+        //清空缓存
+        dbSources.remove(dbKey);
+    }
+
+}

+ 31 - 0
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/DbValidationQueryEnum.java

@@ -0,0 +1,31 @@
+package org.jeecg.common.util.db;
+
+public enum DbValidationQueryEnum {
+    ORACLE("oracle", "SELECT 1 FROM DUAL"),
+    MYSQL("mysql", "select 1"),
+    SQLSERVER("sqlserver", "SELECT 1 FROM DUAL"),;
+
+    DbValidationQueryEnum(String dbType, String validationQuerySql) {
+        this.dbType = dbType;
+        this.validationQuerySql = validationQuerySql;
+    }
+
+    private String dbType;
+    private String validationQuerySql;
+
+    public String getDbType() {
+        return dbType;
+    }
+
+    public void setDbType(String dbType) {
+        this.dbType = dbType;
+    }
+
+    public String getValidationQuerySql() {
+        return validationQuerySql;
+    }
+
+    public void setValidationQuerySql(String validationQuerySql) {
+        this.validationQuerySql = validationQuerySql;
+    }
+}

+ 298 - 0
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/DynamicDBUtil.java

@@ -0,0 +1,298 @@
+package org.jeecg.common.util.db;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.ArrayUtils;
+import org.jeecg.common.exception.JeecgBootException;
+import org.jeecg.common.system.vo.DynamicDataSourceModel;
+import org.jeecg.common.util.ReflectHelper;
+import org.jeecg.common.util.oConvertUtils;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Spring JDBC 实时数据库访问
+ *
+ * @author chenguobin
+ * @version 1.0
+ * @date 2014-09-05
+ */
+@Slf4j
+public class DynamicDBUtil {
+
+    /**
+     * 获取数据源【最底层方法,不要随便调用】
+     *
+     * @param dbSource
+     * @return
+     */
+    private static DruidDataSource getJdbcDataSource(final DynamicDataSourceModel dbSource) {
+        DruidDataSource dataSource = new DruidDataSource();
+
+        String driverClassName = dbSource.getDbDriver();
+        String url = dbSource.getDbUrl();
+        String dbUser = dbSource.getDbUsername();
+        String dbPassword = dbSource.getDbPassword();
+        dataSource.setDriverClassName(driverClassName);
+        dataSource.setUrl(url);
+        //dataSource.setValidationQuery("SELECT 1 FROM DUAL");
+        dataSource.setTestWhileIdle(true);
+        dataSource.setTestOnBorrow(false);
+        dataSource.setTestOnReturn(false);
+        dataSource.setBreakAfterAcquireFailure(true);
+        dataSource.setConnectionErrorRetryAttempts(0);
+        dataSource.setUsername(dbUser);
+        dataSource.setMaxWait(60000);
+        dataSource.setPassword(dbPassword);
+
+        log.info("******************************************");
+        log.info("*                                        *");
+        log.info("*====【"+dbSource.getCode()+"】=====Druid连接池已启用 ====*");
+        log.info("*                                        *");
+        log.info("******************************************");
+        return dataSource;
+    }
+
+    /**
+     * 通过 dbKey ,获取数据源
+     *
+     * @param dbKey
+     * @return
+     */
+    public static DruidDataSource getDbSourceByDbKey(final String dbKey) {
+        //获取多数据源配置
+        DynamicDataSourceModel dbSource = DataSourceCachePool.getCacheDynamicDataSourceModel(dbKey);
+        //先判断缓存中是否存在数据库链接
+        DruidDataSource cacheDbSource = DataSourceCachePool.getCacheBasicDataSource(dbKey);
+        if (cacheDbSource != null && !cacheDbSource.isClosed()) {
+            log.debug("--------getDbSourceBydbKey------------------从缓存中获取DB连接-------------------");
+            return cacheDbSource;
+        } else {
+            DruidDataSource dataSource = getJdbcDataSource(dbSource);
+            if(dataSource!=null && dataSource.isEnable()){
+                DataSourceCachePool.putCacheBasicDataSource(dbKey, dataSource);
+            }else{
+                throw new JeecgBootException("动态数据源连接失败,dbKey:"+dbKey);
+            }
+            log.info("--------getDbSourceBydbKey------------------创建DB数据库连接-------------------");
+            return dataSource;
+        }
+    }
+
+    /**
+     * 关闭数据库连接池
+     *
+     * @param dbKey
+     * @return
+     */
+    public static void closeDbKey(final String dbKey) {
+        DruidDataSource dataSource = getDbSourceByDbKey(dbKey);
+        try {
+            if (dataSource != null && !dataSource.isClosed()) {
+                dataSource.getConnection().commit();
+                dataSource.getConnection().close();
+                dataSource.close();
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+
+    private static JdbcTemplate getJdbcTemplate(String dbKey) {
+        DruidDataSource dataSource = getDbSourceByDbKey(dbKey);
+        return new JdbcTemplate(dataSource);
+    }
+
+    /**
+     * Executes the SQL statement in this <code>PreparedStatement</code> object,
+     * which must be an SQL Data Manipulation Language (DML) statement, such as <code>INSERT</code>, <code>UPDATE</code> or
+     * <code>DELETE</code>; or an SQL statement that returns nothing,
+     * such as a DDL statement.
+     */
+    public static int update(final String dbKey, String sql, Object... param) {
+        int effectCount;
+        JdbcTemplate jdbcTemplate = getJdbcTemplate(dbKey);
+        if (ArrayUtils.isEmpty(param)) {
+            effectCount = jdbcTemplate.update(sql);
+        } else {
+            effectCount = jdbcTemplate.update(sql, param);
+        }
+        return effectCount;
+    }
+
+    /**
+     * 支持miniDao语法操作的Update
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句,sql支持minidao语法逻辑
+     * @param data  sql语法中需要判断的数据及sql拼接注入中需要的数据
+     * @return
+     */
+    public static int updateByHash(final String dbKey, String sql, HashMap<String, Object> data) {
+        int effectCount;
+        JdbcTemplate jdbcTemplate = getJdbcTemplate(dbKey);
+        //根据模板获取sql
+        sql = FreemarkerParseFactory.parseTemplateContent(sql, data);
+        NamedParameterJdbcTemplate namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(jdbcTemplate.getDataSource());
+        effectCount = namedParameterJdbcTemplate.update(sql, data);
+        return effectCount;
+    }
+
+    public static Object findOne(final String dbKey, String sql, Object... param) {
+        List<Map<String, Object>> list;
+        list = findList(dbKey, sql, param);
+        if (oConvertUtils.listIsEmpty(list)) {
+            log.error("Except one, but not find actually");
+        }
+        if (list.size() > 1) {
+            log.error("Except one, but more than one actually");
+        }
+        return list.get(0);
+    }
+
+    /**
+     * 支持miniDao语法操作的查询 返回HashMap
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句,sql支持minidao语法逻辑
+     * @param data  sql语法中需要判断的数据及sql拼接注入中需要的数据
+     * @return
+     */
+    public static Object findOneByHash(final String dbKey, String sql, HashMap<String, Object> data) {
+        List<Map<String, Object>> list;
+        list = findListByHash(dbKey, sql, data);
+        if (oConvertUtils.listIsEmpty(list)) {
+            log.error("Except one, but not find actually");
+        }
+        if (list.size() > 1) {
+            log.error("Except one, but more than one actually");
+        }
+        return list.get(0);
+    }
+
+    /**
+     * 直接sql查询 根据clazz返回单个实例
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句
+     * @param clazz 返回实例的Class
+     * @param param
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    public static <T> Object findOne(final String dbKey, String sql, Class<T> clazz, Object... param) {
+        Map<String, Object> map = (Map<String, Object>) findOne(dbKey, sql, param);
+        return ReflectHelper.setAll(clazz, map);
+    }
+
+    /**
+     * 支持miniDao语法操作的查询 返回单个实例
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句,sql支持minidao语法逻辑
+     * @param clazz 返回实例的Class
+     * @param data  sql语法中需要判断的数据及sql拼接注入中需要的数据
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    public static <T> Object findOneByHash(final String dbKey, String sql, Class<T> clazz, HashMap<String, Object> data) {
+        Map<String, Object> map = (Map<String, Object>) findOneByHash(dbKey, sql, data);
+        return ReflectHelper.setAll(clazz, map);
+    }
+
+    public static List<Map<String, Object>> findList(final String dbKey, String sql, Object... param) {
+        List<Map<String, Object>> list;
+        JdbcTemplate jdbcTemplate = getJdbcTemplate(dbKey);
+
+        if (ArrayUtils.isEmpty(param)) {
+            list = jdbcTemplate.queryForList(sql);
+        } else {
+            list = jdbcTemplate.queryForList(sql, param);
+        }
+        return list;
+    }
+
+    /**
+     * 支持miniDao语法操作的查询
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句,sql支持minidao语法逻辑
+     * @param data  sql语法中需要判断的数据及sql拼接注入中需要的数据
+     * @return
+     */
+    public static List<Map<String, Object>> findListByHash(final String dbKey, String sql, HashMap<String, Object> data) {
+        List<Map<String, Object>> list;
+        JdbcTemplate jdbcTemplate = getJdbcTemplate(dbKey);
+        //根据模板获取sql
+        sql = FreemarkerParseFactory.parseTemplateContent(sql, data);
+        NamedParameterJdbcTemplate namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(jdbcTemplate.getDataSource());
+        list = namedParameterJdbcTemplate.queryForList(sql, data);
+        return list;
+    }
+
+    //此方法只能返回单列,不能返回实体类
+    public static <T> List<T> findList(final String dbKey, String sql, Class<T> clazz, Object... param) {
+        List<T> list;
+        JdbcTemplate jdbcTemplate = getJdbcTemplate(dbKey);
+
+        if (ArrayUtils.isEmpty(param)) {
+            list = jdbcTemplate.queryForList(sql, clazz);
+        } else {
+            list = jdbcTemplate.queryForList(sql, clazz, param);
+        }
+        return list;
+    }
+
+    /**
+     * 支持miniDao语法操作的查询 返回单列数据list
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句,sql支持minidao语法逻辑
+     * @param clazz 类型Long、String等
+     * @param data  sql语法中需要判断的数据及sql拼接注入中需要的数据
+     * @return
+     */
+    public static <T> List<T> findListByHash(final String dbKey, String sql, Class<T> clazz, HashMap<String, Object> data) {
+        List<T> list;
+        JdbcTemplate jdbcTemplate = getJdbcTemplate(dbKey);
+        //根据模板获取sql
+        sql = FreemarkerParseFactory.parseTemplateContent(sql, data);
+        NamedParameterJdbcTemplate namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(jdbcTemplate.getDataSource());
+        list = namedParameterJdbcTemplate.queryForList(sql, data, clazz);
+        return list;
+    }
+
+    /**
+     * 直接sql查询 返回实体类列表
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句,sql支持 minidao 语法逻辑
+     * @param clazz 返回实体类列表的class
+     * @param param sql拼接注入中需要的数据
+     * @return
+     */
+    public static <T> List<T> findListEntities(final String dbKey, String sql, Class<T> clazz, Object... param) {
+        List<Map<String, Object>> queryList = findList(dbKey, sql, param);
+        return ReflectHelper.transList2Entrys(queryList, clazz);
+    }
+
+    /**
+     * 支持miniDao语法操作的查询 返回实体类列表
+     *
+     * @param dbKey 数据源标识
+     * @param sql   执行sql语句,sql支持minidao语法逻辑
+     * @param clazz 返回实体类列表的class
+     * @param data  sql语法中需要判断的数据及sql拼接注入中需要的数据
+     * @return
+     */
+    public static <T> List<T> findListEntitiesByHash(final String dbKey, String sql, Class<T> clazz, HashMap<String, Object> data) {
+        List<Map<String, Object>> queryList = findListByHash(dbKey, sql, data);
+        return ReflectHelper.transList2Entrys(queryList, clazz);
+    }
+}

+ 171 - 0
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/FreemarkerParseFactory.java

@@ -0,0 +1,171 @@
+package org.jeecg.common.util.db;
+
+import freemarker.cache.StringTemplateLoader;
+import freemarker.core.ParseException;
+import freemarker.template.Configuration;
+import freemarker.template.Template;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
+import org.jeecgframework.codegenerate.generate.util.SimpleFormat;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/**
+ * @author 赵俊夫
+ * @version V1.0
+ * @Title:FreemarkerHelper
+ * @description:Freemarker引擎协助类
+ * @date Jul 5, 2013 2:58:29 PM
+ */
+@Slf4j
+public class FreemarkerParseFactory {
+
+    private static final String ENCODE = "utf-8";
+    /**
+     * 参数格式化工具类
+     */
+    private static final String MINI_DAO_FORMAT = "DaoFormat";
+
+    /**
+     * 文件缓存
+     */
+    private static final Configuration _tplConfig = new Configuration();
+    /**
+     * SQL 缓存
+     */
+    private static final Configuration _sqlConfig = new Configuration();
+
+    private static StringTemplateLoader stringTemplateLoader = new StringTemplateLoader();
+
+    // 使用内嵌的(?ms)打开单行和多行模式
+    private final static Pattern p = Pattern
+            .compile("(?ms)/\\*.*?\\*/|^\\s*//.*?$");
+
+    static {
+        _tplConfig.setClassForTemplateLoading(
+                new FreemarkerParseFactory().getClass(), "/");
+        _tplConfig.setNumberFormat("0.#####################");
+        _sqlConfig.setTemplateLoader(stringTemplateLoader);
+        _sqlConfig.setNumberFormat("0.#####################");
+        //classic_compatible设置,解决报空指针错误
+        _sqlConfig.setClassicCompatible(true);
+    }
+
+    /**
+     * 判断模板是否存在
+     *
+     * @throws Exception
+     */
+    public static boolean isExistTemplate(String tplName) throws Exception {
+        try {
+            Template mytpl = _tplConfig.getTemplate(tplName, "UTF-8");
+            if (mytpl == null) {
+                return false;
+            }
+        } catch (Exception e) {
+            //update-begin--Author:scott  Date:20180320 for:解决问题 - 错误提示sql文件不存在,实际问题是sql freemarker用法错误-----
+            if (e instanceof ParseException) {
+                log.error(e.getMessage(), e.fillInStackTrace());
+                throw new Exception(e);
+            }
+            log.debug("----isExistTemplate----" + e.toString());
+            //update-end--Author:scott  Date:20180320 for:解决问题 - 错误提示sql文件不存在,实际问题是sql freemarker用法错误------
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * 解析ftl模板
+     *
+     * @param tplName 模板名
+     * @param paras   参数
+     * @return
+     */
+    public static String parseTemplate(String tplName, Map<String, Object> paras) {
+        try {
+            log.debug(" minidao sql templdate : " + tplName);
+            StringWriter swriter = new StringWriter();
+            Template mytpl = _tplConfig.getTemplate(tplName, ENCODE);
+            if (paras.containsKey(MINI_DAO_FORMAT)) {
+                throw new RuntimeException("DaoFormat 是 minidao 保留关键字,不允许使用 ,请更改参数定义!");
+            }
+            paras.put(MINI_DAO_FORMAT, new SimpleFormat());
+            mytpl.process(paras, swriter);
+            String sql = getSqlText(swriter.toString());
+            paras.remove(MINI_DAO_FORMAT);
+            return sql;
+        } catch (Exception e) {
+            log.error(e.getMessage(), e.fillInStackTrace());
+            log.error("发送一次的模板key:{ " + tplName + " }");
+            //System.err.println(e.getMessage());
+            //System.err.println("模板名:{ "+ tplName +" }");
+            throw new RuntimeException("解析SQL模板异常");
+        }
+    }
+
+    /**
+     * 解析ftl
+     *
+     * @param tplContent 模板内容
+     * @param paras      参数
+     * @return String 模板解析后内容
+     */
+    public static String parseTemplateContent(String tplContent,
+                                              Map<String, Object> paras) {
+        try {
+            StringWriter swriter = new StringWriter();
+            if (stringTemplateLoader.findTemplateSource("sql_" + tplContent.hashCode()) == null) {
+                stringTemplateLoader.putTemplate("sql_" + tplContent.hashCode(), tplContent);
+            }
+            Template mytpl = _sqlConfig.getTemplate("sql_" + tplContent.hashCode(), ENCODE);
+            if (paras.containsKey(MINI_DAO_FORMAT)) {
+                throw new RuntimeException("DaoFormat 是 minidao 保留关键字,不允许使用 ,请更改参数定义!");
+            }
+            paras.put(MINI_DAO_FORMAT, new SimpleFormat());
+            mytpl.process(paras, swriter);
+            String sql = getSqlText(swriter.toString());
+            paras.remove(MINI_DAO_FORMAT);
+            return sql;
+        } catch (Exception e) {
+            log.error(e.getMessage(), e.fillInStackTrace());
+            log.error("发送一次的模板key:{ " + tplContent + " }");
+            //System.err.println(e.getMessage());
+            //System.err.println("模板内容:{ "+ tplContent +" }");
+            throw new RuntimeException("解析SQL模板异常");
+        }
+    }
+
+    /**
+     * 除去无效字段,去掉注释 不然批量处理可能报错 去除无效的等于
+     */
+    private static String getSqlText(String sql) {
+        // 将注释替换成""
+        sql = p.matcher(sql).replaceAll("");
+        sql = sql.replaceAll("\\n", " ").replaceAll("\\t", " ")
+                .replaceAll("\\s{1,}", " ").trim();
+        // 去掉 最后是 where这样的问题
+        if (sql.endsWith("where") || sql.endsWith("where ")) {
+            sql = sql.substring(0, sql.lastIndexOf("where"));
+        }
+        // 去掉where and 这样的问题
+        int index = 0;
+        while ((index = StringUtils.indexOfIgnoreCase(sql, "where and", index)) != -1) {
+            sql = sql.substring(0, index + 5)
+                    + sql.substring(index + 9, sql.length());
+        }
+        // 去掉 , where 这样的问题
+        index = 0;
+        while ((index = StringUtils.indexOfIgnoreCase(sql, ", where", index)) != -1) {
+            sql = sql.substring(0, index)
+                    + sql.substring(index + 1, sql.length());
+        }
+        // 去掉 最后是 ,这样的问题
+        if (sql.endsWith(",") || sql.endsWith(", ")) {
+            sql = sql.substring(0, sql.lastIndexOf(","));
+        }
+        return sql;
+    }
+}

+ 212 - 0
jeecg-boot-base-common/src/main/java/org/jeecg/common/util/db/SqlUtils.java

@@ -0,0 +1,212 @@
+package org.jeecg.common.util.db;
+
+import org.apache.commons.lang.StringUtils;
+import org.jeecg.common.constant.DataBaseConstant;
+import org.jeecg.common.system.vo.DynamicDataSourceModel;
+
+import java.text.MessageFormat;
+import java.util.Map;
+
+/**
+ * 根据不同的数据库,动态生成SQL,例如分页
+ */
+public class SqlUtils {
+
+    public static final String DATABSE_TYPE_MYSQL = "mysql";
+    public static final String DATABSE_TYPE_POSTGRE = "postgresql";
+    public static final String DATABSE_TYPE_ORACLE = "oracle";
+    public static final String DATABSE_TYPE_SQLSERVER = "sqlserver";
+
+
+    /**
+     * 分页SQL
+     */
+    public static final String MYSQL_SQL = "select * from ( {0}) sel_tab00 limit {1},{2}";
+    public static final String POSTGRE_SQL = "select * from ( {0}) sel_tab00 limit {2} offset {1}";
+    public static final String ORACLE_SQL = "select * from (select row_.*,rownum rownum_ from ({0}) row_ where rownum <= {1}) where rownum_>{2}";
+    public static final String SQLSERVER_SQL = "select * from ( select row_number() over(order by tempColumn) tempRowNumber, * from (select top {1} tempColumn = 0, {0}) t ) tt where tempRowNumber > {2}";
+
+    /**
+     * 获取所有表的SQL
+     */
+    public static final String MYSQL_ALLTABLES_SQL = "select distinct table_name from information_schema.columns where table_schema = {0}";
+    public static final String POSTGRE__ALLTABLES_SQL = "SELECT distinct c.relname AS  table_name FROM pg_class c";
+    public static final String ORACLE__ALLTABLES_SQL = "select distinct colstable.table_name as  table_name from user_tab_cols colstable";
+    public static final String SQLSERVER__ALLTABLES_SQL = "select distinct c.name as  table_name from sys.objects c";
+
+    /**
+     * 获取指定表的所有列名
+     */
+    public static final String MYSQL_ALLCOLUMNS_SQL = "select column_name from information_schema.columns where table_name = {0} and table_schema = {1}";
+    public static final String POSTGRE_ALLCOLUMNS_SQL = "select table_name from information_schema.columns where table_name = {0}";
+    public static final String ORACLE_ALLCOLUMNS_SQL = "select column_name from all_tab_columns where table_name ={0}";
+    public static final String SQLSERVER_ALLCOLUMNS_SQL = "select name from syscolumns where id={0}";
+
+    /*
+     * 判断数据库类型
+     */
+
+    public static boolean dbTypeIsMySQL(String dbType) {
+        return dbTypeIf(dbType, DATABSE_TYPE_MYSQL, DataBaseConstant.DB_TYPE_MYSQL_NUM);
+    }
+
+    public static boolean dbTypeIsOracle(String dbType) {
+        return dbTypeIf(dbType, DATABSE_TYPE_ORACLE, DataBaseConstant.DB_TYPE_ORACLE_NUM);
+    }
+
+    public static boolean dbTypeIsSQLServer(String dbType) {
+        return dbTypeIf(dbType, DATABSE_TYPE_SQLSERVER, DataBaseConstant.DB_TYPE_SQLSERVER_NUM);
+    }
+
+    public static boolean dbTypeIsPostgre(String dbType) {
+        return dbTypeIf(dbType, DATABSE_TYPE_POSTGRE, DataBaseConstant.DB_TYPE_POSTGRESQL_NUM);
+    }
+
+    /**
+     * 判断数据库类型
+     */
+    public static boolean dbTypeIf(String dbType, String... correctTypes) {
+        for (String type : correctTypes) {
+            if (type.equalsIgnoreCase(dbType)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * 获取全 SQL
+     * 拼接 where 条件
+     *
+     * @param sql
+     * @param params
+     * @return
+     */
+    public static String getFullSql(String sql, Map params) {
+        return getFullSql(sql, params, null, null);
+    }
+
+    /**
+     * 获取全 SQL
+     * 拼接 where 条件
+     * 拼接 order 排序
+     *
+     * @param sql
+     * @param params
+     * @param orderColumn 排序字段
+     * @param orderBy     排序方式,只能是 DESC 或 ASC
+     * @return
+     */
+    public static String getFullSql(String sql, Map params, String orderColumn, String orderBy) {
+        StringBuilder sqlBuilder = new StringBuilder();
+        sqlBuilder.append("SELECT t.* FROM ( ").append(sql).append(" ) t ");
+        if (params != null && params.size() >= 1) {
+            sqlBuilder.append("WHERE 1=1 ");
+            for (Object key : params.keySet()) {
+                String value = String.valueOf(params.get(key));
+                if (StringUtils.isNotBlank(value)) {
+                    sqlBuilder.append(" AND (").append(key).append(" = N'").append(value).append("')");
+                }
+            }
+            if (StringUtils.isNotBlank(orderColumn) && StringUtils.isNotBlank(orderBy)) {
+                sqlBuilder.append("ORDER BY ").append(orderColumn).append(" ").append("DESC".equalsIgnoreCase(orderBy) ? "DESC" : "ASC");
+            }
+        }
+        return sqlBuilder.toString();
+    }
+
+    /**
+     * 获取求数量 SQL
+     *
+     * @param sql
+     * @return
+     */
+    public static String getCountSql(String sql) {
+        return String.format("SELECT COUNT(1) \"total\" FROM ( %s ) temp_count", sql);
+    }
+
+    /**
+     * 生成分页查询 SQL
+     *
+     * @param dbType 数据库类型
+     * @param sql
+     * @param page
+     * @param rows
+     * @return
+     */
+    public static String createPageSqlByDBType(String dbType, String sql, int page, int rows) {
+        int beginNum = (page - 1) * rows;
+        Object[] sqlParam = new Object[3];
+        sqlParam[0] = sql;
+        sqlParam[1] = String.valueOf(beginNum);
+        sqlParam[2] = String.valueOf(rows);
+        if (dbTypeIsMySQL(dbType)) {
+            sql = MessageFormat.format(MYSQL_SQL, sqlParam);
+        } else if (dbTypeIsPostgre(dbType)) {
+            sql = MessageFormat.format(POSTGRE_SQL, sqlParam);
+        } else {
+            int beginIndex = (page - 1) * rows;
+            int endIndex = beginIndex + rows;
+            sqlParam[2] = Integer.toString(beginIndex);
+            sqlParam[1] = Integer.toString(endIndex);
+            if (dbTypeIsOracle(dbType)) {
+                sql = MessageFormat.format(ORACLE_SQL, sqlParam);
+            } else if (dbTypeIsSQLServer(dbType)) {
+                sqlParam[0] = sql.substring(getAfterSelectInsertPoint(sql));
+                sql = MessageFormat.format(SQLSERVER_SQL, sqlParam);
+            }
+        }
+        return sql;
+    }
+
+    /**
+     * 生成分页查询 SQL
+     *
+     * @param sql
+     * @param page
+     * @param rows
+     * @return
+     */
+    public static String createPageSqlByDBKey(String dbKey, String sql, int page, int rows) {
+        DynamicDataSourceModel dynamicSourceEntity = DataSourceCachePool.getCacheDynamicDataSourceModel(dbKey);
+        String dbType = dynamicSourceEntity.getDbType();
+        return createPageSqlByDBType(dbType, sql, page, rows);
+    }
+
+    private static int getAfterSelectInsertPoint(String sql) {
+        int selectIndex = sql.toLowerCase().indexOf("select");
+        int selectDistinctIndex = sql.toLowerCase().indexOf("select distinct");
+        return selectIndex + (selectDistinctIndex == selectIndex ? 15 : 6);
+    }
+
+    public static String getAllTableSql(String dbType, Object... params) {
+        if (StringUtils.isNotEmpty(dbType)) {
+            if (dbTypeIsMySQL(dbType)) {
+                return MessageFormat.format(MYSQL_ALLTABLES_SQL, params);
+            } else if (dbTypeIsOracle(dbType)) {
+                return ORACLE__ALLTABLES_SQL;
+            } else if (dbTypeIsPostgre(dbType)) {
+                return POSTGRE__ALLTABLES_SQL;
+            } else if (dbTypeIsSQLServer(dbType)) {
+                return SQLSERVER__ALLTABLES_SQL;
+            }
+        }
+        return null;
+    }
+
+    public static String getAllColumnSQL(String dbType, Object... params) {
+        if (StringUtils.isNotEmpty(dbType)) {
+            if (dbTypeIsMySQL(dbType)) {
+                return MessageFormat.format(MYSQL_ALLCOLUMNS_SQL, params);
+            } else if (dbTypeIsOracle(dbType)) {
+                return MessageFormat.format(ORACLE_ALLCOLUMNS_SQL, params);
+            } else if (dbTypeIsPostgre(dbType)) {
+                return MessageFormat.format(POSTGRE_ALLCOLUMNS_SQL, params);
+            } else if (dbTypeIsSQLServer(dbType)) {
+                return MessageFormat.format(SQLSERVER_ALLCOLUMNS_SQL, params);
+            }
+        }
+        return null;
+    }
+
+}

+ 6 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/ProjectMemberController.java

@@ -222,6 +222,9 @@ public class ProjectMemberController {
                 throw new Exception("入参不能为空");
             }
             Long projectId = jsonObject.getLong("projectId");
+            if (Check.isNull(projectId)) {
+                throw new Exception("请选择项目");
+            }
             String projectName = jsonObject.getString("projectName");
             JSONArray memberArr = jsonObject.getJSONArray("memberArr");
             if (Check.isNull(memberArr)) {
@@ -268,7 +271,7 @@ public class ProjectMemberController {
      * @param projectMember
      * @return
      */
-    @PostMapping(value = "/add")
+   /* @PostMapping(value = "/add")
     public Result<ProjectMember> add(@RequestBody ProjectMember projectMember) {
         Result<ProjectMember> result = new Result<>();
         try {
@@ -279,7 +282,7 @@ public class ProjectMemberController {
             result.error500("操作失败");
         }
         return result;
-    }
+    }*/
 
     /**
      * 编辑
@@ -389,7 +392,7 @@ public class ProjectMemberController {
         Result<List<Map<String, Object>>> resultBody = new Result<>();
         try {
             String userId = ((LoginUser) SecurityUtils.getSubject().getPrincipal()).getId();
-            List<Map<String, Object>> result = projectMemberService.getProjectAccountByUserId(userId,mediaType);
+            List<Map<String, Object>> result = projectMemberService.getProjectAccountByUserId(userId, mediaType);
             resultBody.setSuccess(true);
             resultBody.setResult(result);
         } catch (Exception e) {

+ 68 - 3
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -69,12 +69,80 @@ public class TestController {
     private IBytedanceReportService bytedanceReportService;
     @Autowired
     private IKuaishouInterfaceService2 kuaishouInterfaceService2;
+    @Autowired
+    private IUserAllocationService userAllocationService;
 
     static ExecutorService executorService = Executors.newFixedThreadPool(15);
     static ExecutorService videoService = Executors.newFixedThreadPool(5);
     static ExecutorService suzhaoService = Executors.newFixedThreadPool(5);
 
 
+    @GetMapping(value = "/getDailyDataByProjectId")
+    public String getDailyDataByProjectId(Long projectId, String startDate, String endDate) {
+        Result<String> result = new Result<>();
+        try {
+
+            QueryWrapper<UserAllocation> userAllocationQueryWrapper = new QueryWrapper<>();
+            userAllocationQueryWrapper.eq("project_id", projectId);
+           // userAllocationQueryWrapper.eq("account_status", 0);
+            List<UserAllocation> list = userAllocationService.list(userAllocationQueryWrapper);
+            if (!Check.isNull(list)) {
+                for (UserAllocation allocation : list) {
+                    Long accountId = allocation.getAccountId();
+                    QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
+                    tokenQueryWrapper.eq("media_id", 2);
+                    tokenQueryWrapper.eq("account_id", accountId);
+                    CtopOauthToken ctopOauthTokens = oauthTokenMapper.selectOne(tokenQueryWrapper);
+                    if (!Check.isNull(ctopOauthTokens)) {
+                        historyReportTaskService.createTask(ctopOauthTokens.getAccountId(), ctopOauthTokens.getAccessToken(), startDate, endDate, CtopAdConstant.KUAISHOU_LOAD_JOB_TYPE_DAILY);
+
+                    }
+                }
+
+            }
+
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            result.setSuccess(false);
+        }
+        return "success";
+    }
+
+
+    @GetMapping(value = "/getHistoryDataByProjectId")
+    public String getHistoryDataByProjectId(Long projectId, String startDate, String endDate) {
+        Result<String> result = new Result<>();
+        try {
+
+            QueryWrapper<UserAllocation> userAllocationQueryWrapper = new QueryWrapper<>();
+            userAllocationQueryWrapper.eq("project_id", projectId);
+           // userAllocationQueryWrapper.eq("account_status", 0);
+            List<UserAllocation> list = userAllocationService.list(userAllocationQueryWrapper);
+            if (!Check.isNull(list)) {
+                for (UserAllocation allocation : list) {
+                    Long accountId = allocation.getAccountId();
+                    QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
+                    tokenQueryWrapper.eq("media_id", 2);
+                    tokenQueryWrapper.eq("account_id", accountId);
+                    CtopOauthToken ctopOauthTokens = oauthTokenMapper.selectOne(tokenQueryWrapper);
+                    if (!Check.isNull(ctopOauthTokens)) {
+                        historyReportTaskService.createTask(ctopOauthTokens.getAccountId(), ctopOauthTokens.getAccessToken(), startDate, endDate, CtopAdConstant.KUAISHOU_LOAD_JOB_TYPE_HISTORY);
+
+                    }
+                }
+
+            }
+
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            result.setSuccess(false);
+        }
+        return "success";
+    }
+
+
     @GetMapping(value = "/getDailyDataByAccountId")
     public String getDailyDataByAccountId(Long accountId) {
         Result<String> result = new Result<>();
@@ -617,9 +685,6 @@ public class TestController {
     }
 
 
-    @Autowired
-    private IUserAllocationService userAllocationService;
-
     @GetMapping(value = "/getPlanAndGroupByAccount")
     public void getPlanAndGroupByAccount(Long accountId) {
         QueryWrapper<CtopOauthToken> oauthTokenQueryWrapper = new QueryWrapper<>();

+ 4 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/TaobaoTttmService.java

@@ -0,0 +1,4 @@
+package cn.com.ctop.crawler.modules.taobao.service;
+
+public interface TaobaoTttmService {
+}

+ 158 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoTttmServiceImpl.java

@@ -0,0 +1,158 @@
+package cn.com.ctop.crawler.modules.taobao.service.impl;
+
+import cn.com.ctop.common.module.utils.HttpUtils;
+import cn.com.ctop.crawler.modules.taobao.service.TaobaoJhsbybtService;
+import cn.com.ctop.crawler.modules.taobao.service.TaobaoTttmService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.http.cookie.Cookie;
+import org.springframework.stereotype.Service;
+
+import java.io.FileNotFoundException;
+import java.io.PrintStream;
+import java.util.*;
+
+@Service
+public class TaobaoTttmServiceImpl implements TaobaoTttmService {
+
+    public String getGoodsList(int page){
+        String url = "https://h5api.m.taobao.com/h5/mtop.panama.fiona.commonbizservice/1.1/";
+        Map<String,Object> paramMap = new HashMap<String,Object>();
+        paramMap.put("jsv","2.6.0");
+        paramMap.put("appKey","12574478");
+        paramMap.put("t",new Date().getTime());
+        paramMap.put("sign","56b679fe3a6de0cd4ffa8ccb8680ff83");
+        paramMap.put("timeout",3000);
+        paramMap.put("api","mtop.panama.fiona.commonbizservice");
+        paramMap.put("params","[object Object]");
+        paramMap.put("dataType","jsonp");
+        paramMap.put("v","1.0");
+        paramMap.put("preventFallback",true);
+        paramMap.put("type","jsonp");
+        paramMap.put("callback","mtopjsonp2");
+        paramMap.put("data","{\"bizCode\":\"home_feeds\",\"optStr\":\"preview=false;outerBackupKey=prod-home_feeds-currentPage_"+page+"\",\"currentPage\":"+page+",\"outerBackupKey\":\"prod-home_feeds-currentPage_"+page+"\"}");
+        Map<String,String> headerMap = new HashMap<String,String>();
+        headerMap.put("Accept-Encoding","gzip, deflate, br");
+        headerMap.put("Connection","keep-alive");
+        headerMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36");
+        headerMap.put("Accept","*/*");
+        headerMap.put("Content-Type","application/xml");
+        headerMap.put("Referer","https://market.m.taobao.com/app/tmall-def/jhsbybt/web/index.html?disableNav=YES&itemIds=617136337387&spm=608.14027681/8.bubbles.d0");
+        String result = HttpUtils.httpGet(url,paramMap,headerMap);
+        List<Cookie> cookieList = HttpUtils.COOKIESTORE.getCookies();
+        String token = "";
+        String cookie = "";
+        for (Cookie ck : cookieList){
+            if(ck.getName().equals("_m_h5_tk")){
+                token = ck.getValue().split("_")[0];
+            }
+            cookie = cookie + ck.getName()+"="+ck.getValue()+";";
+        }
+        String sign = DigestUtils.md5Hex(token+"&"+paramMap.get("t")+"&"+paramMap.get("appKey")+"&"+paramMap.get("data"));
+        paramMap.put("sign",sign);
+        headerMap.put("Cookie",cookie);
+        result = HttpUtils.httpGet(url,paramMap,headerMap);
+        return result;
+    }
+
+    public static void main(String[] args){
+        try {
+            PrintStream out = new PrintStream("D:/tttm0831.csv");
+            System.setOut(out);
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        }
+        System.out.println("商品id,商品名,销量,商品图,商品链接,优惠价,原价");
+        TaobaoTttmServiceImpl tb = new TaobaoTttmServiceImpl();
+        int i = 2;
+        Boolean hasMore = true;
+        while (hasMore){
+            try {
+                String result = tb.getGoodsList(i);
+                result = result.replace("mtopjsonp2(","");
+                result = result.substring(0,result.length()-1);
+//                System.out.println(result);
+                ObjectMapper mapper = new ObjectMapper();
+                JsonNode rootNode = mapper.readTree(result);
+                Boolean success = rootNode.get("data").get("data").get("homeFeeds").isNull();
+                int size = rootNode.get("data").get("data").get("homeFeeds").size();
+                if(size == 0){
+                    hasMore = false;
+                }
+//                while (!success){
+//                    Thread.sleep(3000L);
+//                    result = tb.getGoodsList(i);
+//                    result = result.replace("mtopjsonp2(","");
+//                    result = result.substring(0,result.length()-1);
+//                    mapper = new ObjectMapper();
+//                    rootNode = mapper.readTree(result);
+//                    success = rootNode.get("data").get("resultValue").get("8613423").get("success").asBoolean();
+//                }
+//                hasMore = rootNode.get("data").get("resultValue").get("8613423").get("data").get("hasMore").asBoolean();
+                Iterator<JsonNode> items = rootNode.get("data").get("data").get("homeFeeds").iterator();
+
+                while (items.hasNext()){
+
+                    JsonNode itemNode = items.next();
+                    String itemType = itemNode.get("itemType").asText();
+                    if (itemType.equals("rankList")){
+                        Iterator<JsonNode> subitems = itemNode.get("items").iterator();
+                        while (subitems.hasNext()){
+                            JsonNode subNode = subitems.next();
+                            String cjmsItemName = subNode.get("shortName").asText();
+                            Integer itemSoldInfo = 0;
+                            String picUrl = subNode.get("preview").asText();
+//                    String cjmsTaobaoEntrance = itemNode.get("cjmsTaobaoEntrance").asText();
+//                    String couponAmount = itemNode.get("couponAmount").asText();
+                            String orgPrice = subNode.get("originPrice").asText();
+//                    String brandLogo = itemNode.get("brandLogo").asText();
+//                    String couponPrice = itemNode.get("couponPrice").asText();
+                            String actPrice = subNode.get("actPrice").asText();
+//                    String cutPrice = itemNode.get("cutPrice").asText();
+                            String itemUrl = subNode.get("itemHref").asText();
+                            String itemId = subNode.get("itemId").asText();
+                            System.out.println(itemId+","+cjmsItemName+","+
+                                    itemSoldInfo+","+
+//                            brandLogo+","+
+                                    picUrl+","+
+//                            cjmsTaobaoEntrance+","+
+                                    itemUrl+","+
+                                    actPrice+","+
+//                            cutPrice+","+
+                                    orgPrice+",");
+                        }
+                    }else if(itemType.equals("item")){
+                        String cjmsItemName = itemNode.get("shortName").asText();
+                        Integer itemSoldInfo = itemNode.get("soldCount").asInt();
+                        String picUrl = itemNode.get("preview").asText();
+//                    String cjmsTaobaoEntrance = itemNode.get("cjmsTaobaoEntrance").asText();
+//                    String couponAmount = itemNode.get("couponAmount").asText();
+                        String orgPrice = itemNode.get("originPrice").asText();
+//                    String brandLogo = itemNode.get("brandLogo").asText();
+//                    String couponPrice = itemNode.get("couponPrice").asText();
+                        String actPrice = itemNode.get("actPrice").asText();
+//                    String cutPrice = itemNode.get("cutPrice").asText();
+                        String itemUrl = itemNode.get("itemHref").asText();
+                        String itemId = itemNode.get("itemId").asText();
+                        System.out.println(itemId+","+cjmsItemName+","+
+                                itemSoldInfo+","+
+//                            brandLogo+","+
+                                picUrl+","+
+//                            cjmsTaobaoEntrance+","+
+                                itemUrl+","+
+                                actPrice+","+
+//                            cutPrice+","+
+                                orgPrice+",");
+                    }
+
+                }
+                i++;
+                Thread.sleep(3000L);
+            }catch (Exception e){
+                e.printStackTrace();
+            }
+        }
+
+    }
+}

+ 2 - 0
module-job-bytedance/src/main/resources/application-prod.yml

@@ -148,6 +148,8 @@ jeecg:
     bak-database-file: /data/data/bak/
     report-history: /data/upload/report/history/
     report-daily: /data/upload/report/daily/
+    kuaishou-agent-image: /data/upload/image/temp.png
+    kuaishou-account-captcha-image: /data/upload/image/captcha.png
 kuaishou:
   group-control:
     path: http://39.106.184.70

+ 2 - 0
module-job-kuaishou/src/main/resources/application-prod.yml

@@ -153,6 +153,8 @@ jeecg:
     bak-database-file: /data/data/bak/
     report-history: /data/upload/report/history/
     report-daily: /data/upload/report/daily/
+    kuaishou-agent-image: /data/upload/image/temp.png
+    kuaishou-account-captcha-image: /data/upload/image/captcha.png
 kuaishou:
   group-control:
     path: http://39.106.184.70

+ 1 - 2
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaiShouHistoryReportTaskServiceImpl.java

@@ -16,7 +16,6 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
-import com.xxl.job.core.handler.annotation.XxlJob;
 import com.xxl.job.core.log.XxlJobLogger;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -93,7 +92,7 @@ public class KuaiShouHistoryReportTaskServiceImpl extends ServiceImpl<KuaiShouHi
                             JSONObject dataJson = resultJson.getJSONObject("data");
                             KuaiShouDailyReportTask dailyReportTask = new KuaiShouDailyReportTask();
                             dailyReportTask.setAccountId(accountId);
-                            dailyReportTask.setStatDate(startDate);
+                            dailyReportTask.setStatDate(endDate);
                             dailyReportTask.setTaskId(dataJson.getString("task_id"));
                             dailyReportTask.setViewType(type);
                             dailyReportTask.setTaskStatus(0);

+ 8 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyAccountMapper.xml

@@ -456,6 +456,7 @@
         action_ratio,
         impression_1k_cost,
         photo_click_cost,
+        @click_1k_cost,
         action_cost,
         event_pay_first_day,
         event_pay_purchase_amount_first_day,
@@ -514,7 +515,13 @@
         @event_new_user_jinjian_page_roi,
         @event_new_user_credit_grant_page,
         @event_new_user_credit_grant_page_cost,
-        @event_new_user_credit_grant_page_roi
+        @event_new_user_credit_grant_page_roi,
+        @event_appoint_form,
+        @event_appoint_form_cost,
+        @event_appoint_form_ratio,
+        @event_appoint_jump_click,
+        @event_appoint_jump_click_cost,
+        @event_appoint_jump_click_ratio
         )
           set account_id = #{accountId}
     </insert>

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyCreativeMapper.xml

@@ -14,7 +14,9 @@
 		campaign_id,
 		unit_name,
 		campaign_name,
+		photo_id,
 		@photo_url,
+		@image_token,
 		@cover_url,
 		@description,
 		@ad_scene,

+ 28 - 4
pom.xml

@@ -16,8 +16,8 @@
 
     <modules>
         <module>jeecg-boot-base-common</module>
-        <module>jeecg-boot-module-activiti</module>
         <module>module-common</module>
+        <module>jeecg-boot-module-activiti</module>
         <module>module-crawler</module>
         <module>module-kuaishou</module>
         <module>module-toutiao</module>
@@ -73,7 +73,7 @@
 
         <repository>
             <id>ctop</id>
-            <url>http://39.106.184.70:8081/repository/3rd_part</url>
+            <url>http://118.24.244.213:8081/repository/3rd_part</url>
         </repository>
     </repositories>
 
@@ -83,7 +83,7 @@
                 <enabled>true</enabled>
             </snapshots>
             <id>ctop</id>
-            <url>http://39.106.184.70:8081/repository/3rd_part</url>
+            <url>http://118.24.244.213:8081/repository/3rd_part</url>
         </pluginRepository>
     </pluginRepositories>
 
@@ -229,7 +229,22 @@
             <version>7.2.1.jre8</version>
             <scope>runtime</scope>
         </dependency>
-
+        <!-- mini文件存储服务 -->
+        <dependency>
+            <groupId>io.minio</groupId>
+            <artifactId>minio</artifactId>
+            <version>4.0.0</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>guava</artifactId>
+                    <groupId>com.google.guava</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jsr305</artifactId>
+                    <groupId>com.google.code.findbugs</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <!-- Quartz定时任务 -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -310,6 +325,15 @@
             <version>4.5.9</version>
         </dependency>
         <dependency>
+            <groupId>javax.persistence</groupId>
+            <artifactId>javax.persistence-api</artifactId>
+            <version>2.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jpa</artifactId>
+        </dependency>
+        <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpcore</artifactId>
             <version>4.4</version>