/src/main/java/org/apache/kafka/connect/mongodb/MongodbSinkConnector.java

https://github.com/DataReply/kafka-connect-mongodb · Java · 143 lines · 94 code · 17 blank · 32 comment · 17 complexity · 616d5e903b6cb6d2abd2e99f0d7bf835 MD5 · raw file

  1. package org.apache.kafka.connect.mongodb;
  2. import org.apache.kafka.common.config.ConfigDef;
  3. import org.apache.kafka.common.utils.AppInfoParser;
  4. import org.apache.kafka.connect.connector.Task;
  5. import org.apache.kafka.connect.errors.ConnectException;
  6. import org.apache.kafka.connect.sink.SinkConnector;
  7. import org.apache.kafka.connect.util.ConnectorUtils;
  8. import org.apache.kafka.connect.utils.LogUtils;
  9. import org.apache.kafka.connect.utils.StringUtils;
  10. import org.slf4j.Logger;
  11. import org.slf4j.LoggerFactory;
  12. import java.util.*;
  13. import static org.apache.kafka.connect.mongodb.MongodbSinkConfig.HOST;
  14. import static org.apache.kafka.connect.mongodb.MongodbSinkConfig.PORT;
  15. import static org.apache.kafka.connect.mongodb.MongodbSinkConfig.BULK_SIZE;
  16. import static org.apache.kafka.connect.mongodb.MongodbSinkConfig.TOPICS;
  17. import static org.apache.kafka.connect.mongodb.MongodbSinkConfig.URI;
  18. import static org.apache.kafka.connect.mongodb.MongodbSinkConfig.DATABASE;
  19. import static org.apache.kafka.connect.mongodb.MongodbSinkConfig.COLLECTIONS;
  20. /**
  21. * MongodbSinkConnector implement the Connector interface to send Kafka
  22. * data to Mongodb
  23. *
  24. * @author Andrea Patelli
  25. */
  26. public class MongodbSinkConnector extends SinkConnector {
  27. private final static Logger log = LoggerFactory.getLogger(MongodbSinkConnector.class);
  28. private String uri;
  29. private String port;
  30. private String host;
  31. private String bulkSize;
  32. private String database;
  33. private String collections;
  34. private String topics;
  35. /**
  36. * Get the version of this connector.
  37. *
  38. * @return the version, formatted as a string
  39. */
  40. @Override
  41. public String version() {
  42. return AppInfoParser.getVersion();
  43. }
  44. /**
  45. * Start this Connector. This method will only be called on a clean Connector, i.e. it has
  46. * either just been instantiated and initialized or {@link #stop()} has been invoked.
  47. *
  48. * @param map configuration settings
  49. */
  50. @Override
  51. public void start(Map<String, String> map) {
  52. log.trace("Parsing configuration");
  53. uri = map.get(URI);
  54. if (uri == null || uri.isEmpty()){
  55. host = map.get(HOST);
  56. if (host == null || host.isEmpty()){
  57. throw new ConnectException("Missing " + HOST + " config");
  58. }
  59. port = map.get(PORT);
  60. if (port == null || port.isEmpty()){
  61. throw new ConnectException("Missing " + PORT + " config");
  62. }
  63. }
  64. bulkSize = map.get(BULK_SIZE);
  65. if (bulkSize == null || bulkSize.isEmpty())
  66. throw new ConnectException("Missing " + BULK_SIZE + " config");
  67. database = map.get(DATABASE);
  68. collections = map.get(COLLECTIONS);
  69. topics = map.get(TOPICS);
  70. if (collections.split(",").length != topics.split(",").length) {
  71. throw new ConnectException("The number of topics should be the same as the number of collections");
  72. }
  73. LogUtils.dumpConfiguration(map, log);
  74. }
  75. /**
  76. * Returns the task implementation for this Connector
  77. *
  78. * @return the task class
  79. */
  80. @Override
  81. public Class<? extends Task> taskClass() {
  82. return MongodbSinkTask.class;
  83. }
  84. /**
  85. * Returns a set of configurations for Tasks based on the current configuration,
  86. * producing at most maxTasks configurations.
  87. *
  88. * @param maxTasks maximum number of task to start
  89. * @return configurations for tasks
  90. */
  91. @Override
  92. public List<Map<String, String>> taskConfigs(int maxTasks) {
  93. List<Map<String, String>> configs = new ArrayList<>();
  94. List<String> coll = Arrays.asList(collections.split(","));
  95. int numGroups = Math.min(coll.size(), maxTasks);
  96. List<List<String>> dbsGrouped = ConnectorUtils.groupPartitions(coll, numGroups);
  97. List<String> topics = Arrays.asList(this.topics.split(","));
  98. List<List<String>> topicsGrouped = ConnectorUtils.groupPartitions(topics, numGroups);
  99. for (int i = 0; i < numGroups; i++) {
  100. Map<String, String> config = new HashMap<>();
  101. config.put(URI, uri);
  102. if(host!=null){
  103. config.put(HOST, host);
  104. }
  105. if(port!=null){
  106. config.put(PORT, port);
  107. }
  108. config.put(BULK_SIZE, bulkSize);
  109. config.put(DATABASE, database);
  110. config.put(COLLECTIONS, StringUtils.join(dbsGrouped.get(i), ","));
  111. config.put(TOPICS, StringUtils.join(topicsGrouped.get(i), ","));
  112. configs.add(config);
  113. }
  114. return configs;
  115. }
  116. /**
  117. * Stop this connector.
  118. */
  119. @Override
  120. public void stop() {
  121. }
  122. @Override
  123. public ConfigDef config () {
  124. return null;
  125. }
  126. }