/src/java/com/cloudera/sqoop/mapreduce/db/DBConfiguration.java

https://github.com/spinisetti/sqoop · Java · 312 lines · 173 code · 57 blank · 82 comment · 29 complexity · 1f7c566d26737d0bbd0e1bca33e45b19 MD5 · raw file

  1. /**
  2. * Copyright 2011 The Apache Software Foundation
  3. *
  4. * Licensed to the Apache Software Foundation (ASF) under one
  5. * or more contributor license agreements. See the NOTICE file
  6. * distributed with this work for additional information
  7. * regarding copyright ownership. The ASF licenses this file
  8. * to you under the Apache License, Version 2.0 (the
  9. * "License"); you may not use this file except in compliance
  10. * with the License. You may obtain a copy of the License at
  11. *
  12. * http://www.apache.org/licenses/LICENSE-2.0
  13. *
  14. * Unless required by applicable law or agreed to in writing, software
  15. * distributed under the License is distributed on an "AS IS" BASIS,
  16. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17. * See the License for the specific language governing permissions and
  18. * limitations under the License.
  19. */
  20. package com.cloudera.sqoop.mapreduce.db;
  21. import java.sql.Connection;
  22. import java.sql.DriverManager;
  23. import java.sql.SQLException;
  24. import org.apache.hadoop.conf.Configuration;
  25. import org.apache.hadoop.mapreduce.lib.db.DBWritable;
  26. import com.cloudera.sqoop.mapreduce.db.DBInputFormat.NullDBWritable;
  27. /**
  28. * A container for configuration property names for jobs with DB input/output.
  29. *
  30. * The job can be configured using the static methods in this class,
  31. * {@link DBInputFormat}, and {@link DBOutputFormat}.
  32. * Alternatively, the properties can be set in the configuration with proper
  33. * values.
  34. *
  35. * @see DBConfiguration#configureDB(Configuration, String, String, String,
  36. * String)
  37. * @see DBInputFormat#setInput(Job, Class, String, String)
  38. * @see DBInputFormat#setInput(Job, Class, String, String, String, String...)
  39. * @see DBOutputFormat#setOutput(Job, String, String...)
  40. */
  41. public class DBConfiguration {
  42. /** The JDBC Driver class name. */
  43. public static final String DRIVER_CLASS_PROPERTY =
  44. "mapreduce.jdbc.driver.class";
  45. /** JDBC Database access URL. */
  46. public static final String URL_PROPERTY = "mapreduce.jdbc.url";
  47. /** User name to access the database. */
  48. public static final String USERNAME_PROPERTY = "mapreduce.jdbc.username";
  49. /** Password to access the database. */
  50. public static final String PASSWORD_PROPERTY = "mapreduce.jdbc.password";
  51. /** Fetch size. */
  52. public static final String FETCH_SIZE = "mapreduce.jdbc.fetchsize";
  53. /** Input table name. */
  54. public static final String INPUT_TABLE_NAME_PROPERTY =
  55. "mapreduce.jdbc.input.table.name";
  56. /** Field names in the Input table. */
  57. public static final String INPUT_FIELD_NAMES_PROPERTY =
  58. "mapreduce.jdbc.input.field.names";
  59. /** WHERE clause in the input SELECT statement. */
  60. public static final String INPUT_CONDITIONS_PROPERTY =
  61. "mapreduce.jdbc.input.conditions";
  62. /** ORDER BY clause in the input SELECT statement. */
  63. public static final String INPUT_ORDER_BY_PROPERTY =
  64. "mapreduce.jdbc.input.orderby";
  65. /** Whole input query, exluding LIMIT...OFFSET. */
  66. public static final String INPUT_QUERY = "mapreduce.jdbc.input.query";
  67. /** Input query to get the count of records. */
  68. public static final String INPUT_COUNT_QUERY =
  69. "mapreduce.jdbc.input.count.query";
  70. /** Input query to get the max and min values of the jdbc.input.query. */
  71. public static final String INPUT_BOUNDING_QUERY =
  72. "mapred.jdbc.input.bounding.query";
  73. /** Class name implementing DBWritable which will hold input tuples. */
  74. public static final String INPUT_CLASS_PROPERTY =
  75. "mapreduce.jdbc.input.class";
  76. /** Output table name. */
  77. public static final String OUTPUT_TABLE_NAME_PROPERTY =
  78. "mapreduce.jdbc.output.table.name";
  79. /** Field names in the Output table. */
  80. public static final String OUTPUT_FIELD_NAMES_PROPERTY =
  81. "mapreduce.jdbc.output.field.names";
  82. /** Number of fields in the Output table. */
  83. public static final String OUTPUT_FIELD_COUNT_PROPERTY =
  84. "mapreduce.jdbc.output.field.count";
  85. /**
  86. * Sets the DB access related fields in the {@link Configuration}.
  87. * @param conf the configuration
  88. * @param driverClass JDBC Driver class name
  89. * @param dbUrl JDBC DB access URL
  90. * @param userName DB access username
  91. * @param passwd DB access passwd
  92. * @param fetchSize DB fetch size
  93. */
  94. public static void configureDB(Configuration conf, String driverClass,
  95. String dbUrl, String userName, String passwd, Integer fetchSize) {
  96. conf.set(DRIVER_CLASS_PROPERTY, driverClass);
  97. conf.set(URL_PROPERTY, dbUrl);
  98. if (userName != null) {
  99. conf.set(USERNAME_PROPERTY, userName);
  100. }
  101. if (passwd != null) {
  102. conf.set(PASSWORD_PROPERTY, passwd);
  103. }
  104. if (fetchSize != null) {
  105. conf.setInt(FETCH_SIZE, fetchSize);
  106. }
  107. }
  108. /**
  109. * Sets the DB access related fields in the JobConf.
  110. * @param job the job
  111. * @param driverClass JDBC Driver class name
  112. * @param dbUrl JDBC DB access URL
  113. * @param fetchSize DB fetch size
  114. */
  115. public static void configureDB(Configuration job, String driverClass,
  116. String dbUrl, Integer fetchSize) {
  117. configureDB(job, driverClass, dbUrl, null, null, fetchSize);
  118. }
  119. /**
  120. * Sets the DB access related fields in the {@link Configuration}.
  121. * @param conf the configuration
  122. * @param driverClass JDBC Driver class name
  123. * @param dbUrl JDBC DB access URL
  124. * @param userName DB access username
  125. * @param passwd DB access passwd
  126. */
  127. public static void configureDB(Configuration conf, String driverClass,
  128. String dbUrl, String userName, String passwd) {
  129. configureDB(conf, driverClass, dbUrl, userName, passwd, null);
  130. }
  131. /**
  132. * Sets the DB access related fields in the JobConf.
  133. * @param job the job
  134. * @param driverClass JDBC Driver class name
  135. * @param dbUrl JDBC DB access URL.
  136. */
  137. public static void configureDB(Configuration job, String driverClass,
  138. String dbUrl) {
  139. configureDB(job, driverClass, dbUrl, null);
  140. }
  141. private Configuration conf;
  142. public DBConfiguration(Configuration job) {
  143. this.conf = job;
  144. }
  145. /** Returns a connection object to the DB.
  146. * @throws ClassNotFoundException
  147. * @throws SQLException */
  148. public Connection getConnection()
  149. throws ClassNotFoundException, SQLException {
  150. Class.forName(conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY));
  151. if(conf.get(DBConfiguration.USERNAME_PROPERTY) == null) {
  152. return DriverManager.getConnection(
  153. conf.get(DBConfiguration.URL_PROPERTY));
  154. } else {
  155. return DriverManager.getConnection(
  156. conf.get(DBConfiguration.URL_PROPERTY),
  157. conf.get(DBConfiguration.USERNAME_PROPERTY),
  158. conf.get(DBConfiguration.PASSWORD_PROPERTY));
  159. }
  160. }
  161. public Configuration getConf() {
  162. return conf;
  163. }
  164. public Integer getFetchSize() {
  165. if (conf.get(DBConfiguration.FETCH_SIZE) == null) {
  166. return null;
  167. }
  168. return conf.getInt(DBConfiguration.FETCH_SIZE, 0);
  169. }
  170. public void setFetchSize(Integer fetchSize) {
  171. if (fetchSize != null) {
  172. conf.setInt(DBConfiguration.FETCH_SIZE, fetchSize);
  173. } else {
  174. conf.set(FETCH_SIZE, null);
  175. }
  176. }
  177. public String getInputTableName() {
  178. return conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY);
  179. }
  180. public void setInputTableName(String tableName) {
  181. conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, tableName);
  182. }
  183. public String[] getInputFieldNames() {
  184. return conf.getStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY);
  185. }
  186. public void setInputFieldNames(String... fieldNames) {
  187. conf.setStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, fieldNames);
  188. }
  189. public String getInputConditions() {
  190. return conf.get(DBConfiguration.INPUT_CONDITIONS_PROPERTY);
  191. }
  192. public void setInputConditions(String conditions) {
  193. if (conditions != null && conditions.length() > 0) {
  194. conf.set(DBConfiguration.INPUT_CONDITIONS_PROPERTY, conditions);
  195. }
  196. }
  197. public String getInputOrderBy() {
  198. return conf.get(DBConfiguration.INPUT_ORDER_BY_PROPERTY);
  199. }
  200. public void setInputOrderBy(String orderby) {
  201. if(orderby != null && orderby.length() >0) {
  202. conf.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, orderby);
  203. }
  204. }
  205. public String getInputQuery() {
  206. return conf.get(DBConfiguration.INPUT_QUERY);
  207. }
  208. public void setInputQuery(String query) {
  209. if(query != null && query.length() >0) {
  210. conf.set(DBConfiguration.INPUT_QUERY, query);
  211. }
  212. }
  213. public String getInputCountQuery() {
  214. return conf.get(DBConfiguration.INPUT_COUNT_QUERY);
  215. }
  216. public void setInputCountQuery(String query) {
  217. if(query != null && query.length() > 0) {
  218. conf.set(DBConfiguration.INPUT_COUNT_QUERY, query);
  219. }
  220. }
  221. public void setInputBoundingQuery(String query) {
  222. if (query != null && query.length() > 0) {
  223. conf.set(DBConfiguration.INPUT_BOUNDING_QUERY, query);
  224. }
  225. }
  226. public String getInputBoundingQuery() {
  227. return conf.get(DBConfiguration.INPUT_BOUNDING_QUERY);
  228. }
  229. public Class<?> getInputClass() {
  230. return conf.getClass(DBConfiguration.INPUT_CLASS_PROPERTY,
  231. NullDBWritable.class);
  232. }
  233. public void setInputClass(Class<? extends DBWritable> inputClass) {
  234. conf.setClass(DBConfiguration.INPUT_CLASS_PROPERTY, inputClass,
  235. DBWritable.class);
  236. }
  237. public String getOutputTableName() {
  238. return conf.get(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY);
  239. }
  240. public void setOutputTableName(String tableName) {
  241. conf.set(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, tableName);
  242. }
  243. public String[] getOutputFieldNames() {
  244. return conf.getStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY);
  245. }
  246. public void setOutputFieldNames(String... fieldNames) {
  247. conf.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, fieldNames);
  248. }
  249. public void setOutputFieldCount(int fieldCount) {
  250. conf.setInt(DBConfiguration.OUTPUT_FIELD_COUNT_PROPERTY, fieldCount);
  251. }
  252. public int getOutputFieldCount() {
  253. return conf.getInt(OUTPUT_FIELD_COUNT_PROPERTY, 0);
  254. }
  255. }