瀏覽代碼

Merge remote-tracking branch 'origin/master'

syh 4 年之前
父節點
當前提交
b61967b197

+ 34 - 1
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -84,7 +84,7 @@ public class TestController {
 
             QueryWrapper<UserAllocation> userAllocationQueryWrapper = new QueryWrapper<>();
             userAllocationQueryWrapper.eq("project_id", projectId);
-            userAllocationQueryWrapper.eq("account_status", 0);
+           // userAllocationQueryWrapper.eq("account_status", 0);
             List<UserAllocation> list = userAllocationService.list(userAllocationQueryWrapper);
             if (!Check.isNull(list)) {
                 for (UserAllocation allocation : list) {
@@ -110,6 +110,39 @@ public class TestController {
     }
 
 
+    @GetMapping(value = "/getHistoryDataByProjectId")
+    public String getHistoryDataByProjectId(Long projectId, String startDate, String endDate) {
+        Result<String> result = new Result<>();
+        try {
+
+            QueryWrapper<UserAllocation> userAllocationQueryWrapper = new QueryWrapper<>();
+            userAllocationQueryWrapper.eq("project_id", projectId);
+           // userAllocationQueryWrapper.eq("account_status", 0);
+            List<UserAllocation> list = userAllocationService.list(userAllocationQueryWrapper);
+            if (!Check.isNull(list)) {
+                for (UserAllocation allocation : list) {
+                    Long accountId = allocation.getAccountId();
+                    QueryWrapper<CtopOauthToken> tokenQueryWrapper = new QueryWrapper<>();
+                    tokenQueryWrapper.eq("media_id", 2);
+                    tokenQueryWrapper.eq("account_id", accountId);
+                    CtopOauthToken ctopOauthTokens = oauthTokenMapper.selectOne(tokenQueryWrapper);
+                    if (!Check.isNull(ctopOauthTokens)) {
+                        historyReportTaskService.createTask(ctopOauthTokens.getAccountId(), ctopOauthTokens.getAccessToken(), startDate, endDate, CtopAdConstant.KUAISHOU_LOAD_JOB_TYPE_HISTORY);
+
+                    }
+                }
+
+            }
+
+
+        } catch (Exception e) {
+            e.printStackTrace();
+            result.setSuccess(false);
+        }
+        return "success";
+    }
+
+
     @GetMapping(value = "/getDailyDataByAccountId")
     public String getDailyDataByAccountId(Long accountId) {
         Result<String> result = new Result<>();

文件差異過大導致無法顯示
+ 59 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/service/impl/KuaishouCrawlerServiceImpl.java


+ 18 - 0
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/HttpParamUtil.java

@@ -42,6 +42,24 @@ public class HttpParamUtil {
         return sb.toString().substring(1);
     }
 
+    public static <K, V> String mapToUrlStringNoEncoder(Map<K, V> map, List<String> ignoreKeyList){
+        if (map == null) {
+            return null;
+        }
+        StringBuffer sb = new StringBuffer();
+        try{
+            for(K key :map.keySet()){
+                if(ignoreKeyList!=null && ignoreKeyList.contains(key)){
+                    continue;
+                }
+                sb.append("&"+key + "=" + String.valueOf(map.get(key)==null?"":map.get(key)));
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+        return sb.toString().substring(1);
+    }
+
     public static Map<String,String> urlParamToMap(String urlParam){
         if (urlParam == null || "".equalsIgnoreCase(urlParam)) {
             return null;

+ 8 - 1
module-crawler/src/main/java/cn/com/ctop/crawler/modules/core/util/KuaishouUtil.java

@@ -36,17 +36,21 @@ public class KuaishouUtil {
         Map<String,String> dynamicParams = httpEntity.getHttpBody().getDynamicParams();
         List<String> ignoreKeyList = new ArrayList<String>();
         ignoreKeyList.add("sig");
+        ignoreKeyList.add("__NS_sig3");
         ignoreKeyList.add("__NStokensig");
         String staticQueryString = HttpParamUtil.mapToUrlString(staticParams,null);
         Map<String,String> allMap = new HashMap<String,String>();
         allMap.putAll(staticParams);
         allMap.putAll(dynamicParams);
         if (staticParams.get("kpf") != null && "ANDROID_PHONE".equals(staticParams.get("kpf"))) {
+            System.out.println(HttpParamUtil.mapToString(SortUtil.sortMapByKey(allMap), ignoreKeyList));
             String sig = SignUtil.getMd5Sign(HttpParamUtil.mapToString(SortUtil.sortMapByKey(allMap), ignoreKeyList), salt);
             dynamicParams.put("sig",sig);
             allMap.put("sig",sig);
+            System.out.println(sig);
         }
-        String dynamicQueryString = HttpParamUtil.mapToUrlString(dynamicParams,null);
+
+        String dynamicQueryString = HttpParamUtil.mapToUrlStringNoEncoder(dynamicParams,null);
         BasicCookieStore cookieStore = new BasicCookieStore();
         for(HttpCookie cookie : httpEntity.getCookies()){
             BasicClientCookie clientCookie = new BasicClientCookie(cookie.getName(),cookie.getValue());
@@ -72,6 +76,7 @@ public class KuaishouUtil {
             }
             CloseableHttpClient httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(globalConfig).setConnectionReuseStrategy((response, context) -> false ).setSSLSocketFactory(sslFactory).build();
             HttpPost httppost = new HttpPost(httpEntity.getUrl()+httpEntity.getPath()+"?"+staticQueryString);
+            System.out.println(httpEntity.getUrl()+httpEntity.getPath()+"?"+staticQueryString);
             HttpHeader header = httpEntity.getHttpHeader();
             Map<String,String> headers = header.getHeaders();
             if(headers != null){
@@ -80,10 +85,12 @@ public class KuaishouUtil {
                 }
             }
             httppost.setEntity(new StringEntity(dynamicQueryString));
+            System.out.println(dynamicQueryString);
             HttpResponse response = httpClient.execute(httppost);
             int code = response.getStatusLine().getStatusCode();
             org.apache.http.HttpEntity respentity = response.getEntity();
             String strReturn = EntityUtils.toString(respentity);
+            System.out.println(strReturn);
             long endTime = System.currentTimeMillis();
             crawlerLog.setUrl(httpEntity.getUrl());
             crawlerLog.setPath(httpEntity.getPath());

+ 1 - 2
module-crawler/src/main/java/cn/com/ctop/crawler/modules/taobao/service/impl/TaobaoJhsbybtServiceImpl.java

@@ -57,7 +57,7 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
 
     public static void main(String[] args){
         try {
-            PrintStream out = new PrintStream("D:/bybt0831.csv");
+            PrintStream out = new PrintStream("D:/bybt0901.csv");
             System.setOut(out);
         } catch (FileNotFoundException e) {
             e.printStackTrace();
@@ -71,7 +71,6 @@ public class TaobaoJhsbybtServiceImpl implements TaobaoJhsbybtService {
                 String result = tb.getGoodsList(i);
                 result = result.replace("mtopjsonp2(","");
                 result = result.substring(0,result.length()-1);
-//                System.out.println(result);
                 ObjectMapper mapper = new ObjectMapper();
                 JsonNode rootNode = mapper.readTree(result);
                 Boolean success = rootNode.get("data").get("resultValue").get("8613423").get("success").asBoolean();

+ 8 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/report/mapper/xml/KuaishouReportDailyAccountMapper.xml

@@ -456,6 +456,7 @@
         action_ratio,
         impression_1k_cost,
         photo_click_cost,
+        @click_1k_cost,
         action_cost,
         event_pay_first_day,
         event_pay_purchase_amount_first_day,
@@ -514,7 +515,13 @@
         @event_new_user_jinjian_page_roi,
         @event_new_user_credit_grant_page,
         @event_new_user_credit_grant_page_cost,
-        @event_new_user_credit_grant_page_roi
+        @event_new_user_credit_grant_page_roi,
+        @event_appoint_form,
+        @event_appoint_form_cost,
+        @event_appoint_form_ratio,
+        @event_appoint_jump_click,
+        @event_appoint_jump_click_cost,
+        @event_appoint_jump_click_ratio
         )
           set account_id = #{accountId}
     </insert>

+ 0 - 1
pom.xml

@@ -16,7 +16,6 @@
 
     <modules>
         <module>jeecg-boot-base-common</module>
-        <module>jeecg-boot-module-activiti</module>
         <module>module-common</module>
         <module>jeecg-boot-module-activiti</module>
         <module>module-crawler</module>