KuaishouColdWorker.java 2.7 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970
  1. package com.adx.tencent.kuaishou.service;
  2. import com.adx.tencent.kuaishou.model.KuaishouBidRecord;
  3. import com.adx.tencent.kuaishou.model.KuaishouTrackingRecord;
  4. import com.adx.tencent.kuaishou.store.KuaishouColdStore;
  5. import com.adx.tencent.kuaishou.store.KuaishouHotStore;
  6. import com.adx.tencent.storage.model.QueuedEvent;
  7. import com.fasterxml.jackson.databind.ObjectMapper;
  8. import java.time.Duration;
  9. import java.util.ArrayList;
  10. import java.util.List;
  11. public class KuaishouColdWorker {
  12. private final KuaishouHotStore hotStore;
  13. private final KuaishouColdStore coldStore;
  14. private final String group;
  15. private final String consumer;
  16. private final int batch;
  17. private final Duration pendingIdle;
  18. private final long streamMaxLen;
  19. private final ObjectMapper objectMapper;
  20. public KuaishouColdWorker(KuaishouHotStore hotStore, KuaishouColdStore coldStore,
  21. String group, String consumer, int batch,
  22. Duration pendingIdle, long streamMaxLen,
  23. ObjectMapper objectMapper) {
  24. this.hotStore = hotStore;
  25. this.coldStore = coldStore;
  26. this.group = group;
  27. this.consumer = consumer;
  28. this.batch = batch <= 0 ? 100 : batch;
  29. this.pendingIdle = pendingIdle == null || pendingIdle.isZero() ? Duration.ofMinutes(2) : pendingIdle;
  30. this.streamMaxLen = streamMaxLen;
  31. this.objectMapper = objectMapper;
  32. }
  33. public int runOnce() throws Exception {
  34. hotStore.ensureGroup(group);
  35. List<QueuedEvent> events = hotStore.claimStale(group, consumer, pendingIdle, batch);
  36. if (events.isEmpty()) events = hotStore.read(group, consumer, batch);
  37. if (events.isEmpty()) return 0;
  38. List<String> acked = new ArrayList<>();
  39. Exception failed = null;
  40. for (QueuedEvent event : events) {
  41. try {
  42. handleEvent(event);
  43. } catch (Exception e) {
  44. failed = e;
  45. break;
  46. }
  47. acked.add(event.getId());
  48. }
  49. if (failed != null) throw failed;
  50. hotStore.ack(group, acked);
  51. hotStore.delete(acked);
  52. if (streamMaxLen > 0) hotStore.trim(streamMaxLen);
  53. return events.size();
  54. }
  55. private void handleEvent(QueuedEvent event) throws Exception {
  56. switch (event.getType()) {
  57. case "bid" -> coldStore.saveBid(objectMapper.readValue(event.getPayload(), KuaishouBidRecord.class));
  58. case "tracking" -> coldStore.saveTracking(objectMapper.readValue(event.getPayload(), KuaishouTrackingRecord.class));
  59. default -> throw new IllegalArgumentException("unknown kuaishou event type: " + event.getType());
  60. }
  61. }
  62. }