pull

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/controller/FeishuController.java

@@ -43,8 +43,8 @@ public class FeishuController {
 
     @Autowired
     IUserLoginRecordService userLoginRecordService;
-    @GetMapping("/test")
-    public void test() throws Exception {
+    @PostMapping("/test")
+    public void test(@RequestBody String str) throws Exception {
 //        bytedanceProductUpdateService.initNew();
 //        bytedanceProductUpdateService.run();
 //        reportsAdPushService.test();
@@ -57,7 +57,9 @@ public class FeishuController {
 
 //        csjResoureJob.csjResourceOssJob();
 //    dianzhongConfigService.pullMovieInfo("2025-08-08", 78012L);
-        dianzhongConfigService.pullChannelInfo();
+//        dianzhongConfigService.pullChannelInfo();
+//        dianzhongConfigService.pullMovieInfo(str);
+        dianzhongConfigService.pullMovie();
     }
 
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/entity/DianzhongMovieConfig.java

@@ -7,18 +7,18 @@ public class DianzhongMovieConfig {
 
     private String movieId; //剧目ID
     private String movieName; //剧目名称
-    private String introduction; //简介
-    private Integer wordCount; //字数
     private String coverImgUrl; //封面图链接
+    private Integer chapterCount;
+    private Integer freeChapterCount;
     private String type1; //男频/女频
     private String type2; //二级分类
     private String type3; //三级分类
     private String firstShelfTime; //上架时间
-    private String showTitle; //显示标题
     private String remark; //备注
     private String createBy; //创建者
     private String createTime; //创建时间
     private String updateBy; //更新者
     private String updateTime; //修改时间
+    private String infoJson;//原始json
 
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/mapper/xml/DianzhongConfigMapper.xml

@@ -26,33 +26,32 @@
         <trim prefix="(" suffix=")" suffixOverrides=",">
             <if test="movieId != null">movie_id,</if>
             <if test="movieName != null">movie_name,</if>
-            <if test="introduction != null">introduction,</if>
-            <if test="wordCount != null">word_count,</if>
             <if test="coverImgUrl != null">cover_img_url,</if>
+            <if test="chapterCount != null">chapter_count,</if>
+            <if test="freeChapterCount != null">free_chapter_count,</if>
             <if test="type1 != null">type_1,</if>
             <if test="type2 != null">type_2,</if>
             <if test="type3 != null">type_3,</if>
             <if test="firstShelfTime != null">first_shelf_time,</if>
-            <if test="showTitle != null">show_title,</if>
             <if test="remark != null">remark,</if>
             <if test="createBy != null">create_by,</if>
             <if test="updateBy != null">update_by,</if>
+            <if test="infoJson != null">info_json,</if>
         </trim>
         VALUES
         <trim prefix="(" suffix=")" suffixOverrides=",">
             <if test="movieId != null">#{movieId},</if>
             <if test="movieName != null">#{movieName},</if>
-            <if test="introduction != null">#{introduction},</if>
-            <if test="wordCount != null">#{wordCount},</if>
             <if test="coverImgUrl != null">#{coverImgUrl},</if>
+            <if test="chapterCount != null">#{chapterCount},</if>
+            <if test="freeChapterCount != null">#{freeChapterCount},</if>
             <if test="type1 != null">#{type1},</if>
             <if test="type2 != null">#{type2},</if>
             <if test="type3 != null">#{type3},</if>
             <if test="firstShelfTime != null">#{firstShelfTime},</if>
-            <if test="showTitle != null">#{showTitle},</if>
             <if test="remark != null">#{remark},</if>
             <if test="createBy != null">#{createBy},</if>
-            <if test="updateBy != null">#{updateBy},</if>
+            <if test="infoJson != null">#{infoJson},</if>
         </trim>
     </insert>
 

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/IDianzhongConfigService.java

@@ -1,11 +1,15 @@
 package cn.com.ctop.job.bytedance.data.service;
 
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+
 public interface IDianzhongConfigService {
     /**
      * 同步剧目信息
-     * @param date 上架时间
      */
-    void pullMovieInfo(String date, Long channelId);
+    void pullMovieInfo(String str) throws IOException;
 
     void pullChannelInfo();
+
+    void pullMovie() throws IOException;
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/data/service/impl/DianzhongConfigServiceImpl.java

@@ -11,10 +11,25 @@ import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.message.BasicNameValuePair;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 @Service
 @Slf4j
@@ -69,6 +84,94 @@ public class DianzhongConfigServiceImpl implements IDianzhongConfigService {
         }
     }
 
+    @Override
+    public void pullMovie() throws IOException {
+        pullMoviePage(0, 100);
+    }
+
+    private void pullMoviePage(int offset, int limit) throws IOException {
+        log.info("拉取： 点众剧目，offset: {}, limit: {}", offset, limit);
+        String url = "https://vip.wqxsw.com/admin/referral/book/index?sort=idx&order=desc&offset="+offset+"&limit="+limit+"&filter=%7B%7D&op=%7B%7D&_="+System.currentTimeMillis();
+        //免费付费剧目ID相同，拉一次就可以
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Cookie", "PHPSESSID=a2t3dp59hn92sjm0a7spe82b2s; keeplogin=29581%7C604800%7C1755853777%7Cea024be2fbc5c0604c25514f216e178f");
+        headers.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
+        headers.put("Accept", "application/json, text/javascript, */*; q=0.01");
+        headers.put("x-requested-with", "XMLHttpRequest");
+
+            String res = HttpUtils.httpGet(url, null, headers);
+            if (StringUtils.isNotEmpty(res)){
+                pullMovieInfo(res);
+                if (offset <=100){ //每次只拉最近300个
+                    pullMoviePage(offset+limit, limit);
+                }
+            }
+    }
+
+
+    @Override
+    public void pullMovieInfo(String str) throws IOException {
+        JSONObject res = JSONArray.parseObject(str);
+        JSONArray rows = res.getJSONArray("rows");
+        if (rows==null || rows.isEmpty()){
+            return;
+        }
+        for (int i = 0; i < rows.size(); i++) {
+            JSONObject minfo = rows.getJSONObject(i);
+            DianzhongMovieConfig config = new DianzhongMovieConfig();
+            config.setMovieId(minfo.getString("bookId"));
+            config.setMovieName(minfo.getString("bookName"));
+            config.setCoverImgUrl(minfo.getString("coverCutWap"));
+            config.setChapterCount(minfo.getInteger("totalChapterNum"));
+            config.setFreeChapterCount(minfo.getInteger("freeChapterNum"));
+            config.setType1(minfo.getString("bookTypeOne"));
+            config.setType2(minfo.getString("bookTypeTwo"));
+            config.setType3(minfo.getString("bookTypeThree"));
+            config.setFirstShelfTime(minfo.getString("firstShelfTime"));
+            config.setInfoJson(minfo.toJSONString());
+
+            dianzhongConfigMapper.insertConfigSelective(config);
+        }
+
+    }
+
+
+
+    private void tt() throws Exception {
+        String loginUrl = "https://vip.wqxsw.com/admin/index/login?url=%2Fadmin%2Findex%2Findex";
+        //免费付费剧目ID相同，拉一次就可以
+        HttpPost httpPost = new HttpPost(loginUrl);
+        //PHPSESSID=a2t3dp59hn92sjm0a7spe82b2s; keeplogin=29581%7C604800%7C1755853777%7Cea024be2fbc5c0604c25514f216e178f
+        httpPost.setHeader("Cookie", "PHPSESSID=n3620u5ck12dplrbqfmvslegt2");
+        httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
+        httpPost.setHeader("Accept", "application/json, text/javascript, */*; q=0.01");
+        httpPost.setHeader("x-requested-with", "XMLHttpRequest");
+        List<NameValuePair> params = new ArrayList<>();
+        params.add(new BasicNameValuePair("username", "tjxldyvip1"));
+        params.add(new BasicNameValuePair("password", "Aa112233"));
+        params.add(new BasicNameValuePair("keeplogin", "1"));
+        HttpEntity httpEntity = new UrlEncodedFormEntity(params);
+        httpPost.setEntity(httpEntity);
+
+        HttpClient httpclient = HttpUtils.getHttpclient();
+        HttpResponse response = httpclient.execute(httpPost);
+        int statusCode = response.getStatusLine().getStatusCode();
+        if (statusCode != 200) {
+            log.error("点众登录失败，状态码: {}", statusCode);
+            return;
+        }
+        Header[] headers = response.getHeaders("Set-Cookie");
+        //获取其中的keeplogin
+        String cookie = null;
+        for (Header header : headers) {
+            String value = header.getValue();
+            if (value.contains("keeplogin")) {
+                cookie = value;
+                break;
+            }
+        }
+    }
+
 
 
 
@@ -103,54 +206,7 @@ public class DianzhongConfigServiceImpl implements IDianzhongConfigService {
         return object;
     }
 
-//    private DianZhongConfig.DZConfig getAuthConfig(){
-//        return dianZhongConfig.getDzConfig().get(0);
-//    }
-    //废弃
-@Override
-public void pullMovieInfo(String date, Long channelId) {
-    String url = "https://video.wqxsw.com/api/channelapi/books";
-    JSONObject requestJson = initRequestJson(dianZhongConfig.getDzConfig().get(0));
-    requestJson.put("date", date);
-    if (channelId != null){
-        requestJson.put("channelId", channelId);
-    }
-    System.out.println("请求点众书籍列表接口，参数：" + requestJson.toJSONString());
-    String res = HttpUtils.httpPostRequest(url, requestJson, null);
-    if (StringUtils.isEmpty(res)){
-        log.error("点众书籍列表接口返回结果为空");
-        return;
-    }
-    JSONObject resObj = JSONObject.parseObject(res);
-    int code = resObj.getIntValue("code");
-    if (code >1){
-        log.error("点众书籍列表接口返回错误，code: {}, msg: {}", code, resObj.getString("msg"));
-        return;
-    }
-    JSONArray data = resObj.getJSONArray("data");
-    if (data == null || data.isEmpty()) {
-        System.out.println(res);
-        log.info("点众书籍列表接口没有返回数据");
-        return;
-    }
-    for (int i = 0; i < data.size(); i++) {
-        JSONObject m = data.getJSONObject(i);
-        DianzhongMovieConfig config = new DianzhongMovieConfig();
-        config.setMovieId(m.getString("bookId"));
-        config.setMovieName(m.getString("bookName"));
-        config.setIntroduction(m.getString("introduction"));
-        config.setWordCount(m.getInteger("totalWordSize"));
-        config.setCoverImgUrl(m.getString("coverWap"));
-        config.setType1(m.getString("bookTypeOneText"));
-        config.setType2(m.getString("bookTypeTwoText"));
-        config.setType3(m.getString("bookTypeThreeText"));
-        config.setFirstShelfTime(m.getString("firstShelfTime"));
-        config.setShowTitle(m.getString("showTitle"));
-        config.setUpdateBy("system");
-        config.setCreateBy("system");
-
-        System.out.println(JSONObject.toJSONString(config));
-        dianzhongConfigMapper.insertConfigSelective(config);
-    }
-}
+
+
+
 }

job-bytedance/src/main/java/cn/com/ctop/job/bytedance/handler/DianzhongConfigJob.java

@@ -0,0 +1,34 @@
+package cn.com.ctop.job.bytedance.handler;
+
+import cn.com.ctop.job.bytedance.data.service.IDianzhongConfigService;
+import cn.com.ctop.job.bytedance.utils.FeishuRobotUtil;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+
+@Slf4j
+@Component
+public class DianzhongConfigJob {
+    @Autowired
+    IDianzhongConfigService dianzhongConfigService;
+
+    public static final String hook_url = "https://open.feishu.cn/open-apis/bot/v2/hook/a0fcfb57-c053-47bd-b1e7-b826f44841e4";
+
+
+    @XxlJob("pullDianzhongChannel")
+    public void pullDianzhongChannel(){
+        dianzhongConfigService.pullChannelInfo();
+    }
+
+    @XxlJob("pullDianzhongMovie")
+    public void pullDianzhongMovie() throws IOException {
+        try {
+            dianzhongConfigService.pullMovie();
+        }catch (Exception e){
+            FeishuRobotUtil.sendTextToWebhook("点众短剧同步失败", hook_url);
+        }
+    }
+}