PageRenderTime 36ms CodeModel.GetById 38ms RepoModel.GetById 1ms app.codeStats 0ms

/src/main/java/com/cloudburo/test/db/RestAPIBasedDBStorageManager.java

https://bitbucket.org/talfco/clb-test
Java | 219 lines | 168 code | 20 blank | 31 comment | 32 complexity | 5142182a87b31feceb736a08248574f5 MD5 | raw file
  1. /**
  2. * The MIT License (MIT)
  3. *
  4. * Copyright (C) 2013 Felix Kuestahler <felix@cloudburo.com> http://cloudburo.com
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
  7. * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
  8. * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
  9. * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
  10. * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
  11. * the Software.
  12. *
  13. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
  14. * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  15. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  16. * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  17. */
  18. package com.cloudburo.test.db;
  19. import java.io.BufferedReader;
  20. import java.io.InputStreamReader;
  21. import java.util.HashMap;
  22. import java.util.Iterator;
  23. import java.util.Map.Entry;
  24. import java.util.Set;
  25. import java.util.logging.Level;
  26. import java.util.logging.Logger;
  27. import org.apache.http.client.methods.CloseableHttpResponse;
  28. import org.apache.http.client.methods.HttpDelete;
  29. import org.apache.http.client.methods.HttpGet;
  30. import org.apache.http.client.methods.HttpPost;
  31. import org.apache.http.entity.ContentType;
  32. import org.apache.http.entity.StringEntity;
  33. import org.apache.http.impl.client.CloseableHttpClient;
  34. import org.apache.http.impl.client.HttpClients;
  35. import org.bson.BSONObject;
  36. import com.mongodb.BasicDBObject;
  37. import com.cloudburo.test.base.BaseDBObject;
  38. import com.cloudburo.test.load.CSVTestDataLoader;
  39. import com.google.gson.JsonElement;
  40. import com.google.gson.JsonObject;
  41. import com.google.gson.JsonParser;
  42. /**
  43. * The RestAPIBasedDBStorageManager will store the test data set via a rest based API following the Cloudburo REST API specification
  44. */
  45. public class RestAPIBasedDBStorageManager implements PersistentStoreManager {
  46. private static String urlRequest = "";
  47. private static final Logger logger = Logger.getLogger(RestAPIBasedDBStorageManager.class.getCanonicalName());
  48. public void connectDB(HashMap<String, String> connectionProps)
  49. throws Exception {
  50. urlRequest = connectionProps.get(PersistentStoreManager.STORE_ENDPOINT)+":"+connectionProps.get(PersistentStoreManager.STORE_PORT)+"/"+
  51. connectionProps.get(PersistentStoreManager.STORE_INSTANCENAME)+"/";
  52. }
  53. public BSONObject getCollectionObject(String collection, String id)
  54. throws Exception {
  55. CloseableHttpClient httpclient = HttpClients.createDefault();
  56. HttpGet httpGet = new HttpGet(urlRequest+collection+"/"+id);
  57. String content = "";
  58. String line = "";
  59. CloseableHttpResponse resp;
  60. BufferedReader rd;
  61. JsonParser parser = new JsonParser();
  62. resp = httpclient.execute(httpGet);
  63. rd = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));
  64. content="";
  65. while ((line = rd.readLine()) != null) content+=line;
  66. logger.log(Level.INFO, "Got content {0}",content);
  67. JsonObject obj = parser.parse(content).getAsJsonObject();
  68. Set<Entry<String, JsonElement>> entrySet = obj.entrySet();
  69. Iterator<Entry<String, JsonElement>> it = entrySet.iterator();
  70. BasicDBObject boj = new BasicDBObject();
  71. while (it.hasNext()) {
  72. Entry<String,JsonElement> map = it.next();
  73. boj.append(map.getKey(), map.getValue());
  74. }
  75. return boj;
  76. }
  77. public long countCollectionObjects(String collection) throws Exception {
  78. CloseableHttpClient httpclient = HttpClients.createDefault();
  79. int count = 0;
  80. // Let's retrieve first all entries and then delete them
  81. HttpGet httpGet = new HttpGet(urlRequest+collection);
  82. boolean doFetch = true;
  83. String content = "";
  84. String line = "";
  85. CloseableHttpResponse resp;
  86. BufferedReader rd;
  87. JsonParser parser;
  88. while (doFetch) {
  89. doFetch = false;
  90. resp = httpclient.execute(httpGet);
  91. rd = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));
  92. content="";
  93. while ((line = rd.readLine()) != null) content+=line;
  94. logger.log(Level.INFO, "Got content {0}",content);
  95. parser = new JsonParser();
  96. Iterator<JsonElement> jselemIt = parser.parse(content).getAsJsonArray().iterator();
  97. while (jselemIt.hasNext()) {
  98. JsonObject obj = jselemIt.next().getAsJsonObject();
  99. if (obj.has("_id")) {
  100. count++;
  101. } else {
  102. // Ok this must be the meta record then
  103. // We differentiate two types of
  104. if (obj.has("_maxRec")) {
  105. // We can return the result immediately
  106. return obj.get("_maxRec").getAsInt();
  107. } else if (obj.has("_cursor")) {
  108. // We have to iterate with the cursor
  109. if (obj.get("_cursor").getAsString().equals("")) {
  110. doFetch = false;
  111. } else {
  112. doFetch = true;
  113. logger.log(Level.INFO, "Paging: Going to fetch records");
  114. }
  115. } else {
  116. doFetch = false;
  117. logger.log(Level.WARNING, "Unrecoginize JSON Element in return {0}",obj.toString());
  118. return -1;
  119. }
  120. }
  121. }
  122. }
  123. return count;
  124. }
  125. public void loadTestDataSet(String dataset, BaseDBObject dbo)
  126. throws Exception {
  127. CloseableHttpClient httpclient = HttpClients.createDefault();
  128. // Let's retrieve first all entries and then delete them
  129. HttpGet httpGet = new HttpGet(urlRequest+dbo.getDBCollectionName());
  130. boolean doFetch = true;
  131. String content = "";
  132. String line = "";
  133. CloseableHttpResponse resp;
  134. BufferedReader rd;
  135. JsonParser parser;
  136. while (doFetch) {
  137. doFetch = false;
  138. resp = httpclient.execute(httpGet);
  139. rd = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));
  140. content="";
  141. while ((line = rd.readLine()) != null) content+=line;
  142. logger.log(Level.INFO, "Got content {0}",content);
  143. parser = new JsonParser();
  144. Iterator<JsonElement> jselemIt = parser.parse(content).getAsJsonArray().iterator();
  145. while (jselemIt.hasNext()) {
  146. JsonObject obj = jselemIt.next().getAsJsonObject();
  147. if (obj.has("_id")) {
  148. String id = obj.get("_id").getAsString();
  149. logger.log(Level.INFO, "Delete JSON Object {0}",id);
  150. HttpDelete httpDelete = new HttpDelete(urlRequest+dbo.getDBCollectionName()+"/"+id);
  151. resp = httpclient.execute(httpDelete);
  152. resp.close();
  153. } else {
  154. // Ok this must be the meta record then
  155. // We differentiate two types of
  156. if (obj.has("_maxRec")) {
  157. int offset = obj.get("_offset").getAsInt()+obj.get("_limit").getAsInt();
  158. if (offset > obj.get("_maxRec").getAsInt()) {
  159. doFetch = false;
  160. } else {
  161. doFetch = true;
  162. logger.log(Level.INFO, "Paging: Going to fetch records with offset {0}",offset);
  163. //httpGet = new HttpGet(urlRequest+dbo.getDBCollectionName()+"?offset="+offset+"?maxRec="+obj.get("_maxRec").getAsString());
  164. }
  165. } else if (obj.has("_cursor")) {
  166. if (obj.get("_cursor").getAsString().equals("")) {
  167. doFetch = false;
  168. } else {
  169. doFetch = true;
  170. logger.log(Level.INFO, "Paging: Going to fetch records");
  171. }
  172. } else {
  173. doFetch = false;
  174. logger.log(Level.WARNING, "Unrecoginize JSON Element in return {0}",obj.toString());
  175. }
  176. }
  177. }
  178. }
  179. // Load the Test Data Set
  180. CSVTestDataLoader csvloader = new com.cloudburo.test.load.CSVTestDataLoader();
  181. Iterator<BSONObject> iter = csvloader.loadTestDataSet(dataset,dbo).iterator();
  182. // Now insert all elements
  183. while (iter.hasNext()) {
  184. String jsonStr = ((BaseDBObject)iter.next()).toString();
  185. HttpPost httpPost = new HttpPost(urlRequest+dbo.getDBCollectionName());
  186. httpPost.setEntity(new StringEntity(jsonStr,ContentType.create("application/json")));
  187. logger.log(Level.INFO, "Creating JSON Object {0}",jsonStr);
  188. resp = httpclient.execute(httpPost);
  189. rd = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));
  190. content = "";
  191. line = "";
  192. while ((line = rd.readLine()) != null) content+=line;
  193. parser = new JsonParser();
  194. JsonObject object = parser.parse(content).getAsJsonObject();
  195. logger.log(Level.INFO, "Created object {0}",object.get("_id"));
  196. resp.close();
  197. }
  198. }
  199. }