/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java

https://github.com/apache/rocketmq-externals · Java · 128 lines · 88 code · 23 blank · 17 comment · 11 complexity · e8ca427f28c32caddb3f1a8e0bf98e69 MD5 · raw file

  1. /*
  2. * Licensed to the Apache Software Foundation (ASF) under one or more
  3. * contributor license agreements. See the NOTICE file distributed with
  4. * this work for additional information regarding copyright ownership.
  5. * The ASF licenses this file to You under the Apache License, Version 2.0
  6. * (the "License"); you may not use this file except in compliance with
  7. * the License. You may obtain a copy of the License at
  8. *
  9. * http://www.apache.org/licenses/LICENSE-2.0
  10. *
  11. * Unless required by applicable law or agreed to in writing, software
  12. * distributed under the License is distributed on an "AS IS" BASIS,
  13. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. * See the License for the specific language governing permissions and
  15. * limitations under the License.
  16. */
  17. package org.apache.connect.mongo.replicator;
  18. import com.mongodb.CursorType;
  19. import com.mongodb.client.FindIterable;
  20. import com.mongodb.client.MongoClient;
  21. import com.mongodb.client.MongoCursor;
  22. import com.mongodb.client.MongoDatabase;
  23. import com.mongodb.client.model.Filters;
  24. import org.apache.connect.mongo.initsync.InitSync;
  25. import org.apache.connect.mongo.replicator.event.Document2EventConverter;
  26. import org.apache.connect.mongo.replicator.event.ReplicationEvent;
  27. import org.bson.BsonTimestamp;
  28. import org.bson.Document;
  29. import org.slf4j.Logger;
  30. import org.slf4j.LoggerFactory;
  31. public class ReplicatorTask implements Runnable {
  32. private Logger logger = LoggerFactory.getLogger(this.getClass());
  33. private ReplicaSet replicaSet;
  34. private MongoClient mongoClient;
  35. private ReplicaSetConfig replicaSetConfig;
  36. private ReplicaSetsContext replicaSetsContext;
  37. public ReplicatorTask(ReplicaSet replicaSet, MongoClient mongoClient, ReplicaSetConfig replicaSetConfig,
  38. ReplicaSetsContext replicaSetsContext) {
  39. this.replicaSet = replicaSet;
  40. this.replicaSetConfig = replicaSetConfig;
  41. this.mongoClient = mongoClient;
  42. this.replicaSetsContext = replicaSetsContext;
  43. }
  44. @Override
  45. public void run() {
  46. BsonTimestamp firstAvailablePosition = findFirstOplogPosition();
  47. Position userConfigOrRuntimePosition = replicaSetConfig.getPosition();
  48. boolean needDataSync = !userConfigOrRuntimePosition.isValid()
  49. || userConfigOrRuntimePosition.isInitSync()
  50. // userConfigOrRuntimePosition.position < firstAvailablePosition maybe lost some operations
  51. || userConfigOrRuntimePosition.converBsonTimeStamp().compareTo(firstAvailablePosition) < 0;
  52. if (needDataSync) {
  53. recordLastOplogPosition();
  54. InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
  55. initSync.start();
  56. }
  57. if (!replicaSet.isRuning() || replicaSetsContext.isInitSyncAbort()) {
  58. return;
  59. }
  60. MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
  61. FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(
  62. Filters.gt("ts", replicaSetConfig.getPosition().converBsonTimeStamp()));
  63. MongoCursor<Document> cursor = iterable.sort(new Document("$natural", 1))
  64. .noCursorTimeout(true)
  65. .cursorType(CursorType.TailableAwait)
  66. .batchSize(200)
  67. .iterator();
  68. while (replicaSet.isRuning()) {
  69. try {
  70. executorCursor(cursor);
  71. } catch (Exception e) {
  72. logger.error("replicaSet:{} shutdown.....", replicaSetConfig, e);
  73. } finally {
  74. if (cursor != null) {
  75. cursor.close();
  76. }
  77. replicaSet.shutdown();
  78. }
  79. }
  80. logger.info("replicaSet:{}, already shutdown, replicaTask end of life cycle", replicaSetConfig);
  81. }
  82. private BsonTimestamp findFirstOplogPosition() {
  83. MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
  84. FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
  85. Document lastOplog = iterable.sort(new Document("$natural", 1)).limit(1).first();
  86. BsonTimestamp timestamp = lastOplog.get(Constants.TIMESTAMP, BsonTimestamp.class);
  87. return timestamp;
  88. }
  89. private void recordLastOplogPosition() {
  90. MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
  91. FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
  92. Document lastOplog = iterable.sort(new Document("$natural", -1)).limit(1).first();
  93. BsonTimestamp timestamp = lastOplog.get(Constants.TIMESTAMP, BsonTimestamp.class);
  94. replicaSetConfig.setPosition(new Position(timestamp.getTime(), timestamp.getInc(), false));
  95. }
  96. private void executorCursor(MongoCursor<Document> cursor) {
  97. while (cursor.hasNext() && !replicaSet.isPause()) {
  98. Document document = cursor.next();
  99. ReplicationEvent event = Document2EventConverter.convert(document, replicaSetConfig.getReplicaSetName());
  100. if (replicaSetsContext.filterEvent(event)) {
  101. replicaSetsContext.publishEvent(event, replicaSetConfig);
  102. }
  103. }
  104. }
  105. }