PageRenderTime 65ms CodeModel.GetById 30ms RepoModel.GetById 0ms app.codeStats 0ms

/src/main/java/com/alibaba/rocketmq/storm/bolt/CRAggregationBolt.java

https://gitlab.com/penuel/rocketmq-storm
Java | 263 lines | 207 code | 43 blank | 13 comment | 32 complexity | b1d6f7968f2e658f7456e09b8bb621f2 MD5 | raw file
Possible License(s): Apache-2.0
  1. package com.alibaba.rocketmq.storm.bolt;
  2. import backtype.storm.task.OutputCollector;
  3. import backtype.storm.task.TopologyContext;
  4. import backtype.storm.topology.IRichBolt;
  5. import backtype.storm.topology.OutputFieldsDeclarer;
  6. import backtype.storm.tuple.Tuple;
  7. import com.alibaba.fastjson.JSON;
  8. import com.alibaba.rocketmq.common.message.MessageExt;
  9. import com.alibaba.rocketmq.storm.hbase.HBaseClient;
  10. import com.alibaba.rocketmq.storm.hbase.Helper;
  11. import com.alibaba.rocketmq.storm.hbase.exception.HBasePersistenceException;
  12. import com.alibaba.rocketmq.storm.model.CRLog;
  13. import com.alibaba.rocketmq.storm.model.HBaseData;
  14. import com.alibaba.rocketmq.storm.redis.CacheManager;
  15. import com.alibaba.rocketmq.storm.redis.Constant;
  16. import org.slf4j.Logger;
  17. import org.slf4j.LoggerFactory;
  18. import java.nio.charset.Charset;
  19. import java.util.*;
  20. import java.util.concurrent.atomic.AtomicLong;
  21. import java.util.concurrent.atomic.AtomicReference;
  22. /**
  23. * <p>
  24. * To aggregate CR grouping by offer ID, affiliate ID, event code.
  25. * </p>
  26. *
  27. * @version 1.0
  28. * @since 1.0
  29. * @author Li Zhanhui
  30. */
  31. public class CRAggregationBolt implements IRichBolt, Constant {
  32. private static final long serialVersionUID = 7591260982890048043L;
  33. private static final Logger LOG = LoggerFactory.getLogger(CRAggregationBolt.class);
  34. private OutputCollector collector;
  35. private static final String DATE_FORMAT = "yyyyMMddHHmmss";
  36. private static final String TABLE_NAME = "eagle_log";
  37. private static final String COLUMN_FAMILY = "t";
  38. private static final String COLUMN_CLICK = "click";
  39. private static final String COLUMN_CONVERSION = "conv";
  40. private static final Charset DEFAULT_CHARSET = Charset.forName("UTF-8");
  41. private static final int HBASE_MAX_RETRY_TIMES = 5;
  42. private static final int REDIS_MAX_RETRY_TIMES = 5;
  43. private AtomicReference<HashMap<String, HashMap<String, HashMap<String, Long>>>>
  44. atomicReference = new AtomicReference<>();
  45. private AtomicLong counter = new AtomicLong(1L);
  46. private volatile boolean stop = false;
  47. @Override
  48. public void prepare(@SuppressWarnings("rawtypes") Map stormConf, TopologyContext context,
  49. OutputCollector collector) {
  50. this.collector = collector;
  51. /**
  52. * We use one worker, one executor and one task strategy.
  53. */
  54. HashMap<String /* offer_id */, HashMap<String /* affiliate_id*/, HashMap<String /* event_code*/, Long>>>
  55. resultMap = new HashMap<>();
  56. while (!atomicReference.compareAndSet(null, resultMap)) {
  57. }
  58. Thread persistThread = new Thread(new PersistTask());
  59. persistThread.setName("PersistThread");
  60. persistThread.setDaemon(true);
  61. persistThread.start();
  62. }
  63. @Override
  64. public void execute(Tuple input) {
  65. if (counter.incrementAndGet() % 10000 == 0) {
  66. LOG.info("10000 tuples aggregated.");
  67. }
  68. Object msgObj = input.getValue(0);
  69. Object msgStat = input.getValue(1);
  70. try {
  71. if (msgObj instanceof MessageExt) {
  72. MessageExt msg = (MessageExt) msgObj;
  73. CRLog logEntry = JSON.parseObject(new String(msg.getBody(), Charset.forName("UTF-8")), CRLog.class);
  74. HashMap<String, HashMap<String, HashMap<String, Long>>> map = atomicReference.get();
  75. if (!map.containsKey(logEntry.getOffer_id())) {
  76. HashMap<String, HashMap<String, Long>> affMap = new HashMap<>();
  77. HashMap<String, Long> eventMap = new HashMap<>();
  78. eventMap.put(logEntry.getEvent_code(), BASE);
  79. affMap.put(logEntry.getAffiliate_id(), eventMap);
  80. map.put(logEntry.getOffer_id(), affMap);
  81. } else {
  82. HashMap<String, HashMap<String, Long>> affMap = map.get(logEntry.getOffer_id());
  83. if (affMap.containsKey(logEntry.getAffiliate_id())) {
  84. HashMap<String, Long> eventMap = affMap.get(logEntry.getAffiliate_id());
  85. String eventCode = logEntry.getEvent_code();
  86. if (eventMap.containsKey(eventCode)) {
  87. eventMap.put(eventCode, eventMap.get(eventCode) + INCREMENT);
  88. } else {
  89. eventMap.put(logEntry.getEvent_code(), BASE);
  90. }
  91. } else {
  92. HashMap<String, Long> eventMap = new HashMap<>();
  93. eventMap.put(logEntry.getEvent_code(), BASE);
  94. affMap.put(logEntry.getAffiliate_id(), eventMap);
  95. }
  96. }
  97. } else {
  98. LOG.error("The first value in tuple should be MessageExt object");
  99. }
  100. } catch (Exception e) {
  101. LOG.error("Failed to handle Message", e);
  102. collector.fail(input);
  103. return;
  104. }
  105. collector.ack(input);
  106. }
  107. @Override
  108. public void cleanup() {
  109. stop = true;
  110. }
  111. @Override
  112. public void declareOutputFields(OutputFieldsDeclarer declarer) {
  113. }
  114. @Override
  115. public Map<String, Object> getComponentConfiguration() {
  116. return null;
  117. }
  118. class PersistTask implements Runnable {
  119. private final CacheManager cacheManager = CacheManager.getInstance();
  120. private HBaseClient hBaseClient = new HBaseClient();
  121. public PersistTask() {
  122. hBaseClient.start();
  123. }
  124. @Override
  125. public void run() {
  126. while (!stop) {
  127. LOG.info("Start to persist aggregation result.");
  128. try {
  129. HashMap<String, HashMap<String, HashMap<String, Long>>> map =
  130. atomicReference.getAndSet(new HashMap<String, HashMap<String, HashMap<String, Long>>>());
  131. if (null == map || map.isEmpty()) {
  132. LOG.info("No data to persist. Sleep to wait for the next cycle.");
  133. Thread.sleep(PERIOD * 1000);
  134. continue;
  135. }
  136. Calendar calendar = Calendar.getInstance();
  137. //Use Beijing Time Zone: GMT+8
  138. calendar.setTimeZone(TimeZone.getTimeZone("GMT+8"));
  139. String timestamp = calendar.getTimeInMillis() + "";
  140. List<HBaseData> hBaseDataList = new ArrayList<>();
  141. Map<String, String> redisCacheMap = new HashMap<>();
  142. for (Map.Entry<String, HashMap<String, HashMap<String, Long>>> row : map.entrySet()) {
  143. String offerId = row.getKey();
  144. HashMap<String, HashMap<String, Long>> affMap = row.getValue();
  145. for (Map.Entry<String, HashMap<String, Long>> affRow : affMap.entrySet()) {
  146. String affId = affRow.getKey();
  147. HashMap<String, Long> eventMap = affRow.getValue();
  148. String rowKey = Helper.generateKey(offerId, affId, timestamp);
  149. StringBuilder click = new StringBuilder();
  150. click.append("{");
  151. StringBuilder conversion = new StringBuilder();
  152. conversion.append("{");
  153. Map<String, byte[]> data = new HashMap<String, byte[]>();
  154. for (Map.Entry<String, Long> eventRow : eventMap.entrySet()) {
  155. String event = eventRow.getKey();
  156. if (event.startsWith("C")) {
  157. click.append(event).append(": ").append(eventRow.getValue()).append(", ");
  158. } else {
  159. conversion.append(event).append(": ").append(eventRow.getValue()).append(", ");
  160. }
  161. }
  162. if (click.length() > 2) {
  163. click.replace(click.length() - 2, click.length(), "}");
  164. } else {
  165. click.append("}");
  166. }
  167. if (conversion.length() > 2) {
  168. conversion.replace(conversion.length() - 2, conversion.length(), "}");
  169. } else {
  170. conversion.append("}");
  171. }
  172. LOG.debug("[Click] Key = " + click.toString());
  173. LOG.debug("[Conversion] Key = " + conversion.toString());
  174. data.put(COLUMN_CLICK, click.toString().getBytes(DEFAULT_CHARSET));
  175. data.put(COLUMN_CONVERSION, conversion.toString().getBytes(DEFAULT_CHARSET));
  176. redisCacheMap.put(rowKey, "{click: " + click + ", conversion: " + conversion + "}");
  177. HBaseData hBaseData = new HBaseData(TABLE_NAME, rowKey, COLUMN_FAMILY, data);
  178. hBaseDataList.add(hBaseData);
  179. }
  180. }
  181. for (int i = 0; i < REDIS_MAX_RETRY_TIMES; i++) {
  182. if (!cacheManager.setKeyLive(redisCacheMap, PERIOD * NUMBERS)) {
  183. if (i < REDIS_MAX_RETRY_TIMES - 1) {
  184. LOG.error("Persisting to Redis failed, retry in " + (i + 1) + " seconds");
  185. } else {
  186. LOG.error("The following data are dropped due to failure to persist to Redis: %s", redisCacheMap);
  187. }
  188. Thread.sleep((i + 1) * 1000);
  189. } else {
  190. break;
  191. }
  192. }
  193. for (int i = 0; i < HBASE_MAX_RETRY_TIMES; i++) {
  194. try {
  195. hBaseClient.insertBatch(hBaseDataList);
  196. break;
  197. } catch (HBasePersistenceException e) {
  198. if (i < HBASE_MAX_RETRY_TIMES - 1) {
  199. LOG.error("Persisting aggregation data to HBase failed. Retry in " + (i + 1) + " second(s)");
  200. } else {
  201. LOG.error("The following aggregation data are dropped: %s", hBaseDataList);
  202. }
  203. }
  204. Thread.sleep((i + 1) * 1000);
  205. }
  206. cacheManager.publish(redisCacheMap, REDIS_CHANNEL);
  207. LOG.info("Persisting aggregation result done.");
  208. } catch (Exception e) {
  209. LOG.error("Persistence of aggregated result failed.", e);
  210. } finally {
  211. try {
  212. Thread.sleep(PERIOD * 1000);
  213. } catch (InterruptedException e) {
  214. LOG.error("PersistThread was interrupted.", e);
  215. }
  216. }
  217. }
  218. }
  219. }
  220. }