|
|
@@ -0,0 +1,369 @@
|
|
|
+package cn.com.ctop.job.bytedance.data.service.impl;
|
|
|
+
|
|
|
+import cn.com.ctop.job.bytedance.data.entity.BytedanceAdvertiserList;
|
|
|
+import cn.com.ctop.job.bytedance.data.mapper.EbpAdvertiserListMapper;
|
|
|
+import cn.com.ctop.job.bytedance.data.service.IEbpAdvertiserListService;
|
|
|
+import cn.com.ctop.job.bytedance.data.utils.Check;
|
|
|
+import cn.com.ctop.job.bytedance.data.utils.HttpUtils;
|
|
|
+import com.alibaba.fastjson.JSONArray;
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+
|
|
|
+import java.util.*;
|
|
|
+import java.util.concurrent.*;
|
|
|
+
|
|
|
+@Service
|
|
|
+@Slf4j
|
|
|
+public class EbpAdvertiserListServiceImpl implements IEbpAdvertiserListService {
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private EbpAdvertiserListMapper ebpAdvertiserListMapper;
|
|
|
+
|
|
|
+ private static final String EBP_ADVERTISER_LIST_URL = "https://api.oceanengine.com/open_api/2/ebp/advertiser/list/";
|
|
|
+ /** 单页失败重试上限(与 nuojing-data-service 授权回调同口径) */
|
|
|
+ private static final int EBP_PAGE_RETRY_LIMIT = 5;
|
|
|
+ /** 平台限制单组织仅返回前 10000 条,超出部分请求必然报 40000,按 100 条/页截断到 100 页 */
|
|
|
+ private static final int EBP_MAX_PAGE = 100;
|
|
|
+ /** 触发开发者限流(40110)后的惩罚等待 */
|
|
|
+ private static final long EBP_RATE_LIMITED_WAIT_MS = 65_000L;
|
|
|
+ /** 普通失败重试间隔 */
|
|
|
+ private static final long EBP_RETRY_WAIT_MS = 1_000L;
|
|
|
+ /** EBP 接口全局节流:8 QPS(单次最小间隔 125ms),并行线程共享同一时间轴排队;
|
|
|
+ * 接口 60s 内请求数超配额*100 即触发平台限流 */
|
|
|
+ private static final long EBP_MIN_REQUEST_INTERVAL_MS = 125L;
|
|
|
+ private static final Object EBP_RATE_LOCK = new Object();
|
|
|
+ private static long ebpNextRequestAt = 0L;
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public JSONObject ebpAdvertiserSync() {
|
|
|
+ long syncStart = System.currentTimeMillis();
|
|
|
+ JSONObject returnJson = new JSONObject();
|
|
|
+ // 1. 查库取 EBP 工作台组织与 token:token 取自 media_auth_account,
|
|
|
+ // enterprise_organization_id(bp_admin_id)取自 media_bytedance_ebp_admin_relation
|
|
|
+ List<JSONObject> ebpAdminList = ebpAdvertiserListMapper.getEbpAdminList();
|
|
|
+ if (Check.isNull(ebpAdminList) || ebpAdminList.isEmpty()) {
|
|
|
+ returnJson.put("code", 500);
|
|
|
+ returnJson.put("message", "未取到有效 EBP 工作台组织授权,跳过本次同步");
|
|
|
+ return returnJson;
|
|
|
+ }
|
|
|
+ log.info("字节EBP任务-待同步工作台组织数:{}", ebpAdminList.size());
|
|
|
+
|
|
|
+ // 2. 串行拉各组织第 1 页确定总页数(每组织仅 1 次请求,代价可忽略)
|
|
|
+ int pageSize = 100;
|
|
|
+ long phaseStart = System.currentTimeMillis();
|
|
|
+ List<EbpOrgFetchPlan> plans = new ArrayList<>();
|
|
|
+ boolean allSuccess = true;
|
|
|
+ for (JSONObject ebpAdmin : ebpAdminList) {
|
|
|
+ Long adminAdvertiserId = ebpAdmin.getLong("adminAdvertiserId");
|
|
|
+ Long bpAdminId = ebpAdmin.getLong("bpAdminId");
|
|
|
+ String accessToken = ebpAdmin.getString("accessToken");
|
|
|
+ if (Check.isNull(accessToken)) {
|
|
|
+ // token 缺失时账户集合不完整,本次不做失效标记,避免误伤
|
|
|
+ log.error("字节EBP任务-管家{}未取到有效token,组织{}本次跳过", adminAdvertiserId, bpAdminId);
|
|
|
+ allSuccess = false;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ long orgFirstPageStart = System.currentTimeMillis();
|
|
|
+ EbpPageResult firstPage = fetchEbpAdvertiserPageWithRetry(adminAdvertiserId, bpAdminId, accessToken, 1, pageSize);
|
|
|
+ if (firstPage == null) {
|
|
|
+ // 组织首页拉取失败时账户集合不完整,本次不做失效标记,避免误伤
|
|
|
+ log.warn("字节EBP任务-工作台组织首页拉取失败,该组织本次跳过:bpAdminId={}", bpAdminId);
|
|
|
+ allSuccess = false;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ int orgTotalPage = Math.max(firstPage.getTotalPage(), 1);
|
|
|
+ if (orgTotalPage > EBP_MAX_PAGE) {
|
|
|
+ // 平台限制单组织仅返回前 10000 条,超出部分请求必然报 40000;截断后数据不完整,本次不做失效标记
|
|
|
+ log.warn("字节EBP任务-组织账户超出平台{}条上限,仅拉取前{}页且本次不做失效标记:bpAdminId={}, totalPage={}",
|
|
|
+ EBP_MAX_PAGE * pageSize, EBP_MAX_PAGE, bpAdminId, orgTotalPage);
|
|
|
+ orgTotalPage = EBP_MAX_PAGE;
|
|
|
+ allSuccess = false;
|
|
|
+ }
|
|
|
+ plans.add(new EbpOrgFetchPlan(adminAdvertiserId, bpAdminId, accessToken, orgTotalPage, firstPage.getIds()));
|
|
|
+ log.info("字节EBP任务-组织首页拉取耗时={}ms, bpAdminId={}, totalPage={}, 账户数={}",
|
|
|
+ System.currentTimeMillis() - orgFirstPageStart, bpAdminId, firstPage.getTotalPage(), firstPage.getIds().size());
|
|
|
+ }
|
|
|
+ log.info("字节EBP任务-组织首页阶段总耗时={}ms, 组织数={}", System.currentTimeMillis() - phaseStart, plans.size());
|
|
|
+
|
|
|
+ // 3. 全部组织的其余分页提交线程池并行拉取
|
|
|
+ phaseStart = System.currentTimeMillis();
|
|
|
+ ExecutorService executor = Executors.newFixedThreadPool(10);
|
|
|
+ List<Future<EbpPageResult>> futures = new ArrayList<>();
|
|
|
+ for (EbpOrgFetchPlan plan : plans) {
|
|
|
+ for (int page = 2; page <= plan.getTotalPage(); page++) {
|
|
|
+ final int currentPage = page;
|
|
|
+ futures.add(executor.submit(() -> fetchEbpAdvertiserPageWithRetry(plan.getAdminAdvertiserId(), plan.getBpAdminId(), plan.getAccessToken(), currentPage, pageSize)));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Map<Long, Map<Integer, List<Long>>> idsByOrgPage = new HashMap<>();
|
|
|
+ int successPages = 0;
|
|
|
+ int failedPages = 0;
|
|
|
+ for (Future<EbpPageResult> future : futures) {
|
|
|
+ try {
|
|
|
+ EbpPageResult result = future.get();
|
|
|
+ if (result == null || result.getIds() == null) {
|
|
|
+ allSuccess = false;
|
|
|
+ failedPages++;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ successPages++;
|
|
|
+ idsByOrgPage.computeIfAbsent(result.getBpAdminId(), key -> new HashMap<>()).put(result.getPageNumber(), result.getIds());
|
|
|
+ } catch (InterruptedException | ExecutionException e) {
|
|
|
+ allSuccess = false;
|
|
|
+ failedPages++;
|
|
|
+ log.error("字节EBP任务-多线程请求工作台组织账户失败: {}", e.getMessage());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ executor.shutdown();
|
|
|
+ log.info("字节EBP任务-并行分页阶段耗时={}ms, 提交页数={}, 成功页数={}, 失败页数={}",
|
|
|
+ System.currentTimeMillis() - phaseStart, futures.size(), successPages, failedPages);
|
|
|
+
|
|
|
+ // 4. 按组织顺序、页序合并各组织账户(失效按组织域执行,不跨组织合并)
|
|
|
+ long mergeStart = System.currentTimeMillis();
|
|
|
+ Map<Long, Set<Long>> mergedIdsByOrg = new LinkedHashMap<>();
|
|
|
+ int totalCount = 0;
|
|
|
+ for (EbpOrgFetchPlan plan : plans) {
|
|
|
+ Set<Long> mergedIds = new LinkedHashSet<>();
|
|
|
+ mergedIds.addAll(plan.getFirstPageIds());
|
|
|
+ Map<Integer, List<Long>> orgPages = idsByOrgPage.getOrDefault(plan.getBpAdminId(), Collections.emptyMap());
|
|
|
+ int orgAccountCount = plan.getFirstPageIds().size();
|
|
|
+ for (int page = 2; page <= plan.getTotalPage(); page++) {
|
|
|
+ List<Long> ids = orgPages.get(page);
|
|
|
+ if (ids != null) {
|
|
|
+ orgAccountCount += ids.size();
|
|
|
+ mergedIds.addAll(ids);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ mergedIdsByOrg.put(plan.getBpAdminId(), mergedIds);
|
|
|
+ totalCount += mergedIds.size();
|
|
|
+ log.info("字节EBP任务-组织账户合并:bpAdminId={}, 账户数={}", plan.getBpAdminId(), orgAccountCount);
|
|
|
+ }
|
|
|
+ log.info("字节EBP任务-分页结果合并耗时={}ms, 组织数={}, 账户数={}",
|
|
|
+ System.currentTimeMillis() - mergeStart, mergedIdsByOrg.size(), totalCount);
|
|
|
+
|
|
|
+ // 5. 全量同步:所有组织分页均拉取成功才软失效该组织域内已消失账户(置 is_valid=0 并清 bp_admin_id,
|
|
|
+ // 与老工作台账户互不干扰);任一分页失败则跳过,防止误伤
|
|
|
+ int invalidatedCount = 0;
|
|
|
+ if (allSuccess) {
|
|
|
+ long invalidateStart = System.currentTimeMillis();
|
|
|
+ for (EbpOrgFetchPlan plan : plans) {
|
|
|
+ Set<Long> mergedIds = mergedIdsByOrg.getOrDefault(plan.getBpAdminId(), Collections.emptySet());
|
|
|
+ // 该组织已确认无账户时按全量同步语义软失效该组织全部账户;否则软失效不在列表中的账户
|
|
|
+ List<Long> protectIds = mergedIds.isEmpty() ? null : new ArrayList<>(mergedIds);
|
|
|
+ int count = ebpAdvertiserListMapper.ebpInvalidateAdvertiserAccounts(plan.getAdminAdvertiserId(), plan.getBpAdminId(), protectIds);
|
|
|
+ invalidatedCount += count;
|
|
|
+ log.info("字节EBP任务-组织软失效:adminAdvertiserId={}, bpAdminId={}, 失效账户数={}",
|
|
|
+ plan.getAdminAdvertiserId(), plan.getBpAdminId(), count);
|
|
|
+ }
|
|
|
+ log.info("字节EBP任务-存量失效账户标记:{}条, 耗时={}ms", invalidatedCount, System.currentTimeMillis() - invalidateStart);
|
|
|
+ }
|
|
|
+ log.info("字节EBP任务-全量同步总耗时={}ms, 组织数={}, 账户数={}, 失效账户数={}",
|
|
|
+ System.currentTimeMillis() - syncStart, mergedIdsByOrg.size(), totalCount, invalidatedCount);
|
|
|
+ returnJson.put("code", allSuccess ? 200 : 500);
|
|
|
+ returnJson.put("total", totalCount);
|
|
|
+ returnJson.put("message", allSuccess ? "全量同步成功,已同步标记失效账户" : "存在拉取失败分页,跳过本次失效标记");
|
|
|
+ return returnJson;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 拉取工作台组织单页(最多重试 EBP_PAGE_RETRY_LIMIT 次,每次请求前经全局节流):
|
|
|
+ * 40000(超出平台 10000 条上限)放弃本页不重试;40110(开发者限流)等惩罚窗口后重试;
|
|
|
+ * 其余失败等 1s 后重试;成功返回分页结果,失败返回 null
|
|
|
+ */
|
|
|
+ private EbpPageResult fetchEbpAdvertiserPageWithRetry(Long adminAdvertiserId, Long bpAdminId, String accessToken, int page, int pageSize) {
|
|
|
+ for (int attempt = 1; attempt <= EBP_PAGE_RETRY_LIMIT; attempt++) {
|
|
|
+ acquireEbpRateToken();
|
|
|
+ long requestStart = System.currentTimeMillis();
|
|
|
+ String s;
|
|
|
+ try {
|
|
|
+ s = requestEbpAdvertiserList(bpAdminId, accessToken, page, pageSize);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("请求字节EBP工作台组织账户第{}页异常,bpAdminId:{},第{}次尝试", page, bpAdminId, attempt, e);
|
|
|
+ sleepQuietly(EBP_RETRY_WAIT_MS);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ JSONObject jsonObject = JSONObject.parseObject(s);
|
|
|
+ Integer code = Check.isNull(jsonObject) ? null : jsonObject.getInteger("code");
|
|
|
+ if (code == null || code != 0) {
|
|
|
+ if (code != null && code == 40000) {
|
|
|
+ log.warn("字节EBP任务-超出平台10000条拉取上限(40000),放弃本页且不重试:bpAdminId={}, page={}", bpAdminId, page);
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ if (code != null && code == 40110) {
|
|
|
+ log.warn("字节EBP任务-触发开发者限流(40110),等待{}s后重试:bpAdminId={}, page={}, 第{}次尝试",
|
|
|
+ EBP_RATE_LIMITED_WAIT_MS / 1000, bpAdminId, page, attempt);
|
|
|
+ sleepQuietly(EBP_RATE_LIMITED_WAIT_MS);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ log.error("请求字节EBP工作台组织账户列表失败,bpAdminId:{},page:{},code:{},第{}次尝试,返回信息:{}",
|
|
|
+ bpAdminId, page, code, attempt, jsonObject);
|
|
|
+ sleepQuietly(EBP_RETRY_WAIT_MS);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ JSONObject data = jsonObject.getJSONObject("data");
|
|
|
+ // data 缺失时按失败处理:否则会被当成"组织下没有账户",进而误把历史账户全部置失效
|
|
|
+ if (Check.isNull(data)) {
|
|
|
+ log.warn("字节EBP任务-返回data为空,准备重试:bpAdminId={}, page={}, 第{}次尝试", bpAdminId, page, attempt);
|
|
|
+ sleepQuietly(EBP_RETRY_WAIT_MS);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ EbpPageResult result = parseEbpAdvertiserPage(adminAdvertiserId, bpAdminId, page, data);
|
|
|
+ log.info("字节EBP任务-分页请求耗时={}ms, bpAdminId={}, page={}, 账户数={}",
|
|
|
+ System.currentTimeMillis() - requestStart, bpAdminId, page, result.getIds().size());
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 请求工作台组织账户单页接口(account_source=AD)
|
|
|
+ */
|
|
|
+ private String requestEbpAdvertiserList(Long bpAdminId, String accessToken, int page, int pageSize) {
|
|
|
+ Map<String, Object> paramsMap = new HashMap<>();
|
|
|
+ paramsMap.put("enterprise_organization_id", bpAdminId);
|
|
|
+ paramsMap.put("account_source", "AD");
|
|
|
+ paramsMap.put("page", page);
|
|
|
+ paramsMap.put("page_size", pageSize);
|
|
|
+ Map<String, String> headers = new HashMap<>();
|
|
|
+ headers.put("Access-Token", accessToken);
|
|
|
+ return HttpUtils.httpGet(EBP_ADVERTISER_LIST_URL, paramsMap, headers);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 解析工作台组织单页账户并入库(带 bp_admin_id/account_type/raw_response,
|
|
|
+ * account_role 留空保留既有值,与 nuojing-data-service 同口径)
|
|
|
+ */
|
|
|
+ private EbpPageResult parseEbpAdvertiserPage(Long adminAdvertiserId, Long bpAdminId, int page, JSONObject data) {
|
|
|
+ int totalPage = 0;
|
|
|
+ JSONObject pageInfo = data.getJSONObject("page_info");
|
|
|
+ if (!Check.isNull(pageInfo) && pageInfo.getInteger("total_page") != null) {
|
|
|
+ totalPage = pageInfo.getInteger("total_page");
|
|
|
+ }
|
|
|
+ List<BytedanceAdvertiserList> advertiserLists = new ArrayList<>();
|
|
|
+ JSONArray accountList = data.getJSONArray("account_list");
|
|
|
+ if (!Check.isNull(accountList)) {
|
|
|
+ for (int i = 0; i < accountList.size(); i++) {
|
|
|
+ JSONObject accountInfo = accountList.getJSONObject(i);
|
|
|
+ if (accountInfo.getLong("account_id") == null) {
|
|
|
+ // 条目缺 account_id 时静默跳过会让账户"凭空消失",留痕便于对照平台报文
|
|
|
+ log.warn("字节EBP任务-账户条目缺少account_id,跳过:bpAdminId={}, entry={}", bpAdminId, accountInfo);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ BytedanceAdvertiserList advertiserList = new BytedanceAdvertiserList();
|
|
|
+ advertiserList.setAdminAdvertiserId(adminAdvertiserId);
|
|
|
+ advertiserList.setBpAdminId(bpAdminId);
|
|
|
+ advertiserList.setAdvertiserId(accountInfo.getLong("account_id"));
|
|
|
+ advertiserList.setAdvertiserName(accountInfo.getString("account_name"));
|
|
|
+ advertiserList.setAccountType(accountInfo.getString("account_type"));
|
|
|
+ advertiserList.setRawResponse(accountInfo.toJSONString());
|
|
|
+ advertiserLists.add(advertiserList);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (!advertiserLists.isEmpty()) {
|
|
|
+ ebpAdvertiserListMapper.ebpReplaceAdvertiserLists(advertiserLists);
|
|
|
+ }
|
|
|
+ List<Long> ids = new ArrayList<>();
|
|
|
+ for (BytedanceAdvertiserList advertiserList : advertiserLists) {
|
|
|
+ ids.add(advertiserList.getAdvertiserId());
|
|
|
+ }
|
|
|
+ return new EbpPageResult(adminAdvertiserId, bpAdminId, page, totalPage, ids);
|
|
|
+ }
|
|
|
+
|
|
|
+ /** 进程内节流:按 EBP_MIN_REQUEST_INTERVAL_MS 预约下一个请求时隙,拿不到时隙的线程睡到自己的时隙再发请求 */
|
|
|
+ private void acquireEbpRateToken() {
|
|
|
+ long waitMillis;
|
|
|
+ synchronized (EBP_RATE_LOCK) {
|
|
|
+ long now = System.currentTimeMillis();
|
|
|
+ long slot = Math.max(now, ebpNextRequestAt);
|
|
|
+ ebpNextRequestAt = slot + EBP_MIN_REQUEST_INTERVAL_MS;
|
|
|
+ waitMillis = slot - now;
|
|
|
+ }
|
|
|
+ if (waitMillis > 0) {
|
|
|
+ sleepQuietly(waitMillis);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void sleepQuietly(long millis) {
|
|
|
+ try {
|
|
|
+ Thread.sleep(millis);
|
|
|
+ } catch (InterruptedException e) {
|
|
|
+ Thread.currentThread().interrupt();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 组织拉取计划:第 1 页已解析账户ID + 总页数
|
|
|
+ static class EbpOrgFetchPlan {
|
|
|
+ private final Long adminAdvertiserId;
|
|
|
+ private final Long bpAdminId;
|
|
|
+ private final String accessToken;
|
|
|
+ private final int totalPage;
|
|
|
+ private final List<Long> firstPageIds;
|
|
|
+
|
|
|
+ EbpOrgFetchPlan(Long adminAdvertiserId, Long bpAdminId, String accessToken, int totalPage, List<Long> firstPageIds) {
|
|
|
+ this.adminAdvertiserId = adminAdvertiserId;
|
|
|
+ this.bpAdminId = bpAdminId;
|
|
|
+ this.accessToken = accessToken;
|
|
|
+ this.totalPage = totalPage;
|
|
|
+ this.firstPageIds = firstPageIds;
|
|
|
+ }
|
|
|
+
|
|
|
+ public Long getAdminAdvertiserId() {
|
|
|
+ return adminAdvertiserId;
|
|
|
+ }
|
|
|
+
|
|
|
+ public Long getBpAdminId() {
|
|
|
+ return bpAdminId;
|
|
|
+ }
|
|
|
+
|
|
|
+ public String getAccessToken() {
|
|
|
+ return accessToken;
|
|
|
+ }
|
|
|
+
|
|
|
+ public int getTotalPage() {
|
|
|
+ return totalPage;
|
|
|
+ }
|
|
|
+
|
|
|
+ public List<Long> getFirstPageIds() {
|
|
|
+ return firstPageIds;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 单页拉取结果:ids 为 null 表示重试后仍失败
|
|
|
+ static class EbpPageResult {
|
|
|
+ private final Long adminAdvertiserId;
|
|
|
+ private final Long bpAdminId;
|
|
|
+ private final int pageNumber;
|
|
|
+ private final int totalPage;
|
|
|
+ private final List<Long> ids;
|
|
|
+
|
|
|
+ EbpPageResult(Long adminAdvertiserId, Long bpAdminId, int pageNumber, int totalPage, List<Long> ids) {
|
|
|
+ this.adminAdvertiserId = adminAdvertiserId;
|
|
|
+ this.bpAdminId = bpAdminId;
|
|
|
+ this.pageNumber = pageNumber;
|
|
|
+ this.totalPage = totalPage;
|
|
|
+ this.ids = ids;
|
|
|
+ }
|
|
|
+
|
|
|
+ public Long getAdminAdvertiserId() {
|
|
|
+ return adminAdvertiserId;
|
|
|
+ }
|
|
|
+
|
|
|
+ public Long getBpAdminId() {
|
|
|
+ return bpAdminId;
|
|
|
+ }
|
|
|
+
|
|
|
+ public int getPageNumber() {
|
|
|
+ return pageNumber;
|
|
|
+ }
|
|
|
+
|
|
|
+ public int getTotalPage() {
|
|
|
+ return totalPage;
|
|
|
+ }
|
|
|
+
|
|
|
+ public List<Long> getIds() {
|
|
|
+ return ids;
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|