Parcourir la source

店铺名称数据抓取

yumeng il y a 10 mois
Parent
commit
bef7082bc3

+ 22 - 13
jeecg-boot-module-system/src/main/java/org/jeecg/modules/ctop/controller/TestController.java

@@ -181,24 +181,33 @@ public class TestController {
             return;
         }
 
-
         String date = DateUtils.getNowDate("yyyy-MM-dd");
         String yesDate = DateUtils.getAnotherDay("yyyy-MM-dd", date, -1);
-
-
         for (int i = 0; i < cookies.size(); i++) {
             JSONObject jsonObject = cookies.get(i);
             String cookie = jsonObject.getString("cookie");
-            List<String> activityKeyWords = ikuaishouSupplyChainService.getActivityKeyWord(yesDate, "ruixuan");
-            if (!Check.isNull(activityKeyWords)) {
-                for (int j = 0; j < activityKeyWords.size(); j++) {
-                    String activityKeyWord = activityKeyWords.get(i);
-                    ikuaishouSupplyChainService.getCcrInfo(cookie, 0, "ruixuan", activityKeyWord);
-                }
-
-            }
-
-
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "ruixuan", yesDate);
+        }
+        Thread.sleep(5 * 1000);
+        List<JSONObject> mgsCookies = ikuaishouSupplyChainService.getMgsCookie();
+        for (int i = 0; i < mgsCookies.size(); i++) {
+            JSONObject jsonObject = mgsCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "miaogousi", yesDate);
+        }
+        Thread.sleep(5 * 1000);
+        List<JSONObject> rocketCookies = ikuaishouSupplyChainService.getRocketCookie();
+        for (int i = 0; i < rocketCookies.size(); i++) {
+            JSONObject jsonObject = rocketCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "rocket", yesDate);
+        }
+        Thread.sleep(5 * 1000);
+        List<JSONObject> yufuCookies = ikuaishouSupplyChainService.getYuFuCiikie();
+        for (int i = 0; i < yufuCookies.size(); i++) {
+            JSONObject jsonObject = yufuCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "yufu", yesDate);
         }
     }
 

+ 78 - 11
module-job-kuaishou/src/main/java/cn/com/ctop/job/kuaishou/handler/settlementJob.java

@@ -211,10 +211,7 @@ public class settlementJob {
                     String activityKeyWord = activityKeyWords.get(i);
                     ikuaishouSupplyChainService.getCcrInfo(cookie, 0, "ruixuan", activityKeyWord);
                 }
-
             }
-
-
         }
 
         Thread.sleep(5 * 1000);
@@ -229,11 +226,8 @@ public class settlementJob {
                     String activityKeyWord = activityKeyWords.get(i);
                     ikuaishouSupplyChainService.getCcrInfo(cookie, 0, "miaogousi", activityKeyWord);
                 }
-
             }
-
         }
-
         Thread.sleep(5 * 1000);
         List<JSONObject> rocketCookies = ikuaishouSupplyChainService.getRocketCookie();
         for (int i = 0; i < rocketCookies.size(); i++) {
@@ -244,13 +238,9 @@ public class settlementJob {
                 for (int j = 0; j < activityKeyWords.size(); j++) {
                     String activityKeyWord = activityKeyWords.get(i);
                     ikuaishouSupplyChainService.getCcrInfo(cookie, 0, "rocket", activityKeyWord);
-
                 }
-
             }
-
         }
-
         Thread.sleep(5 * 1000);
         List<JSONObject> yufuCookies = ikuaishouSupplyChainService.getYuFuCiikie();
         for (int i = 0; i < yufuCookies.size(); i++) {
@@ -262,14 +252,91 @@ public class settlementJob {
                     String activityKeyWord = activityKeyWords.get(i);
                     ikuaishouSupplyChainService.getCcrInfo(cookie, 0, "yufu", activityKeyWord);
                 }
-
             }
+        }
+    }
+
+
 
 
+
+    @XxlJob("getSellerInfoNowDate")
+    public void getSellerInfoNowDate() throws InterruptedException {
+        List<JSONObject> cookies = ikuaishouSupplyChainService.getCookie();
+        if (Check.isNull(cookies)) {
+            return;
         }
 
+        String date = DateUtils.getNowDate("yyyy-MM-dd");
+        for (int i = 0; i < cookies.size(); i++) {
+            JSONObject jsonObject = cookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "ruixuan", date);
+        }
 
+        Thread.sleep(5 * 1000);
+        List<JSONObject> mgsCookies = ikuaishouSupplyChainService.getMgsCookie();
+        for (int i = 0; i < mgsCookies.size(); i++) {
+            JSONObject jsonObject = mgsCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "miaogousi", date);
+        }
+
+        Thread.sleep(5 * 1000);
+        List<JSONObject> rocketCookies = ikuaishouSupplyChainService.getRocketCookie();
+        for (int i = 0; i < rocketCookies.size(); i++) {
+            JSONObject jsonObject = rocketCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "rocket", date);
+        }
+
+        Thread.sleep(5 * 1000);
+        List<JSONObject> yufuCookies = ikuaishouSupplyChainService.getYuFuCiikie();
+        for (int i = 0; i < yufuCookies.size(); i++) {
+            JSONObject jsonObject = yufuCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "yufu", date);
+        }
+    }
+
+
+    @XxlJob("getSellerInfoYesDate")
+    public void getSellerInfoYesDate() throws InterruptedException {
+        List<JSONObject> cookies = ikuaishouSupplyChainService.getCookie();
+        if (Check.isNull(cookies)) {
+            return;
+        }
+
+        String date = DateUtils.getNowDate("yyyy-MM-dd");
+        String yesDate = DateUtils.getAnotherDay("yyyy-MM-dd", date, -1);
+        for (int i = 0; i < cookies.size(); i++) {
+            JSONObject jsonObject = cookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "ruixuan", yesDate);
+        }
+        Thread.sleep(5 * 1000);
+        List<JSONObject> mgsCookies = ikuaishouSupplyChainService.getMgsCookie();
+        for (int i = 0; i < mgsCookies.size(); i++) {
+            JSONObject jsonObject = mgsCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "miaogousi", yesDate);
+        }
+        Thread.sleep(5 * 1000);
+        List<JSONObject> rocketCookies = ikuaishouSupplyChainService.getRocketCookie();
+        for (int i = 0; i < rocketCookies.size(); i++) {
+            JSONObject jsonObject = rocketCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "rocket", yesDate);
+        }
+        Thread.sleep(5 * 1000);
+        List<JSONObject> yufuCookies = ikuaishouSupplyChainService.getYuFuCiikie();
+        for (int i = 0; i < yufuCookies.size(); i++) {
+            JSONObject jsonObject = yufuCookies.get(i);
+            String cookie = jsonObject.getString("cookie");
+            ikuaishouSupplyChainService.getSellerInfo(cookie, 0, "yufu", yesDate);
+        }
     }
 
 
 }
+

+ 8 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/KuaishouSupplyChainMapper.java

@@ -67,4 +67,12 @@ public interface KuaishouSupplyChainMapper extends BaseMapper<KuaishouSupplyChai
     void replaceYUfuCcr(@Param("adds") List<JSONObject> adds);
 
     List<String> getActivityKeyWord(@Param("date") String date, @Param("db") String db);
+
+    void replaceRuiXuanSellerInfo(@Param("adds") List<JSONObject> adds);
+
+    void replaceMiaogousiSellerInfo(@Param("adds") List<JSONObject> adds);
+
+    void replaceRocketSellerInfo(@Param("adds") List<JSONObject> adds);
+
+    void replaceYUfuSellerInfo(@Param("adds") List<JSONObject> adds);
 }

+ 55 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/mapper/xml/KuaishouSupplyChainMapper.xml

@@ -167,6 +167,61 @@
         </foreach>
 
     </insert>
+    <insert id="replaceRuiXuanSellerInfo">
+
+        replace into ruixuan.kuaishou_seller_info(
+        seller_id,
+        seller_name
+        )
+        values
+        <foreach collection="adds" item="add" separator=",">
+            (
+            #{add.sellerId},
+            #{add.sellerName}
+            )
+        </foreach>
+    </insert>
+    <insert id="replaceMiaogousiSellerInfo">
+        replace into miaogousi.kuaishou_seller_info(
+        seller_id,
+        seller_name
+        )
+        values
+        <foreach collection="adds" item="add" separator=",">
+            (
+            #{add.sellerId},
+            #{add.sellerName}
+            )
+        </foreach>
+    </insert>
+    <insert id="replaceRocketSellerInfo" >
+        replace into rocket.kuaishou_seller_info(
+        seller_id,
+        seller_name
+        )
+        values
+        <foreach collection="adds" item="add" separator=",">
+            (
+            #{add.sellerId},
+            #{add.sellerName}
+            )
+        </foreach>
+
+    </insert>
+    <insert id="replaceYUfuSellerInfo" >
+        replace into yufu.kuaishou_seller_info(
+        seller_id,
+        seller_name
+        )
+        values
+        <foreach collection="adds" item="add" separator=",">
+            (
+            #{add.sellerId},
+            #{add.sellerName}
+            )
+        </foreach>
+
+    </insert>
     <delete id="deletePromoterBindChannelByDate" parameterType="java.lang.Long">
         delete from ruixuan.promoter_bind_channel_date where stat_date = #{date}
     </delete>

+ 2 - 0
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/IKuaishouSupplyChainService.java

@@ -60,4 +60,6 @@ public interface IKuaishouSupplyChainService extends IService<KuaishouSupplyChai
     void cleanYufuKuaiShouPromoterBindChannel(Long date);
 
     List<String> getActivityKeyWord(String yesDate, String ruixuan);
+
+    void getSellerInfo(String cookie, Integer offset, String db,String date) throws InterruptedException;
 }

+ 63 - 1
module-kuaishou/src/main/java/cn/com/ctop/kuaishou/modules/batch/service/impl/KuaishouSupplyChainServiceImpl.java

@@ -190,7 +190,7 @@ public class KuaishouSupplyChainServiceImpl extends ServiceImpl<KuaishouSupplyCh
 
     @Override
     public void getCcrInfo(String cookie, Integer offset, String type, String activityKeyWord) throws InterruptedException {
-        Thread.sleep(1000);
+        Thread.sleep(3000);
         Map<String, Object> params = new HashMap<>();
         params.put("filters", "{}");
         params.put("sorter", "{}");
@@ -293,6 +293,68 @@ public class KuaishouSupplyChainServiceImpl extends ServiceImpl<KuaishouSupplyCh
         return supplyChainMapper.getActivityKeyWord(yesDate, db);
     }
 
+    @Override
+    public void getSellerInfo(String cookie, Integer offset, String db, String date) throws InterruptedException {
+        Thread.sleep(2000);
+        Map<String, Object> params = new HashMap<>();
+        params.put("startDate", "2024-07-01");
+        params.put("endDate", "2024-08-20");
+        params.put("limit", 20);
+        params.put("offset", offset);
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Cookie", cookie);
+
+        String resultStr = HttpUtils.httpGet("https://cps.kwaixiaodian.com/distribute/pc/investment/dashboard/seller/detail", params, headers);
+        JSONObject result = JSONObject.parseObject(resultStr);
+        Integer result1 = result.getInteger("result");
+        if (result1 == 1) {
+            List<JSONObject> adds = new ArrayList<>();
+            if (!Check.isNull(result)) {
+                JSONObject data = result.getJSONObject("data");
+                if (!Check.isNull(data)) {
+                    JSONArray dataArray = data.getJSONArray("data");
+                    if (!Check.isNull(dataArray)) {
+                        for (int i = 0; i < dataArray.size(); i++) {
+                            JSONObject dateJson = dataArray.getJSONObject(i);
+                            Long sellerId = dateJson.getLong("id");
+                            String sellerName = dateJson.getString("name");
+                            JSONObject addJson = new JSONObject();
+                            addJson.put("sellerId", sellerId);
+                            addJson.put("sellerName", sellerName);
+                            adds.add(addJson);
+                        }
+
+                    }
+
+
+                }
+
+            }
+            if (!Check.isNull(adds)) {
+                if ("ruixuan".equals(db)) {
+                    supplyChainMapper.replaceRuiXuanSellerInfo(adds);
+                }
+                if ("miaogousi".equals(db)) {
+                    supplyChainMapper.replaceMiaogousiSellerInfo(adds);
+                }
+                if ("rocket".equals(db)) {
+                    supplyChainMapper.replaceRocketSellerInfo(adds);
+                }
+
+                if ("yufu".equals(db)) {
+                    supplyChainMapper.replaceYUfuSellerInfo(adds);
+                }
+
+                if (adds.size() >= 20) {
+                    getSellerInfo(cookie, offset + 20, db, date);
+                }
+            }
+        } else {
+            log.error("{},获取商铺数据异常,返回数据: {}", db, result);
+            sendMessageService.sendMessage("113dee46c7df464da78c07a985e92cd1", db + "获取ccr数据失败,返回数据:" + resultStr);
+        }
+    }
+
 
     @Override
     public List<JSONObject> getCookie() {