PageRenderTime 74ms CodeModel.GetById 27ms RepoModel.GetById 1ms app.codeStats 0ms

/gpoptimizer-server/src/ru/ifmo/gpo/server/storage/MongoDBStorageManager.java

https://bitbucket.org/e_smirnov/gp_optimizer
Java | 263 lines | 207 code | 35 blank | 21 comment | 30 complexity | d4d7fd1114a0bb3fe5b38c9ea9e4586f MD5 | raw file
Possible License(s): Apache-2.0, LGPL-2.1, MPL-2.0-no-copyleft-exception, LGPL-2.0
  1. package ru.ifmo.gpo.server.storage;
  2. import com.google.gson.Gson;
  3. import com.google.gson.GsonBuilder;
  4. import com.mongodb.*;
  5. import com.mongodb.util.JSON;
  6. import org.slf4j.Logger;
  7. import org.slf4j.LoggerFactory;
  8. import ru.ifmo.gpo.core.instructions.InstructionSequence;
  9. import ru.ifmo.gpo.core.instructions.generic.IGenericInstruction;
  10. import ru.ifmo.gpo.java.instructions.JVMInstruction;
  11. import ru.ifmo.gpo.server.IStorageManager;
  12. import ru.ifmo.gpo.util.CollectionUtils;
  13. import ru.ifmo.gpo.util.ProbabilitySet;
  14. import java.lang.reflect.Array;
  15. import java.util.HashMap;
  16. import java.util.Map;
  17. import java.util.Random;
  18. import java.util.Set;
  19. /**
  20. * User: e_smirnov
  21. * Date: 11.09.2010
  22. * Time: 17:08:44
  23. * <p/>
  24. * Saves optimization results into Mongo databse.
  25. * Results are saved with following fields:
  26. * size - size of source sequence
  27. * success - flag showing if optimization was successfull
  28. * hash - source hash
  29. * source - source instruction sequence, serialized into JSON by gson library
  30. * optimized - optimization result, may be null if optimization has failed
  31. * instructionClass - fully qualified name for IGenericInstruction subclass that is used in these instruction sequences
  32. */
  33. public class MongoDBStorageManager implements IStorageManager {
  34. private static final String collectionPrefix = "seq_";
  35. private DB mongoDb;
  36. private Logger logger = LoggerFactory.getLogger(getClass());
  37. private static final String sourceFieldName = "source";
  38. private static final String optimizedFieldName = "optimized";
  39. private static Gson jsonSerializer;
  40. @Override
  41. public void addOptimizationRecord(InstructionSequence source, InstructionSequence optimized) {
  42. if (source == null || optimized == null || source.isEmpty()) {
  43. logger.warn("Attempting to save null or empty sequences, ignoring");
  44. return;
  45. }
  46. logger.debug("Adding optimization record, source is {}, optimized is {}", source, optimized);
  47. DBCollection collection = mongoDb.getCollection(buildCollectionName(source));
  48. BasicDBObject newRecord = new BasicDBObject();
  49. newRecord.put("size", source.size());
  50. newRecord.put("success", true);
  51. newRecord.put("hash", source.hashCode());
  52. serializeInstructionSequence(newRecord, sourceFieldName, source);
  53. serializeInstructionSequence(newRecord, optimizedFieldName, optimized);
  54. newRecord.put("instructionClass", source.get(0).getClass().getCanonicalName());
  55. collection.insert(newRecord);
  56. }
  57. private DBObject findRecordForSource(InstructionSequence source) {
  58. DBCollection collection = mongoDb.getCollection(buildCollectionName(source));
  59. BasicDBObject query = new BasicDBObject();
  60. query.put("hash", source.hashCode());
  61. DBCursor cursor = collection.find(query);
  62. while (cursor.hasNext()) {
  63. DBObject obj = cursor.next();
  64. InstructionSequence sourceFromDb = deserialize(obj, sourceFieldName);
  65. if (sourceFromDb == null) {
  66. // record found but some error happened on deserialization
  67. return null;
  68. }
  69. if (sourceFromDb.equals(source)) {
  70. return obj;
  71. }
  72. }
  73. return null;
  74. }
  75. @Override
  76. public void addFailedOptimizationRecord(InstructionSequence source) {
  77. if (source == null || source.isEmpty()) {
  78. logger.warn("Attempting to save null or empty sequences, ignoring");
  79. return;
  80. }
  81. if (findRecordForSource(source) != null) {
  82. // database already contains record for this source
  83. return;
  84. }
  85. logger.debug("Adding failed optimization record, source is {}", source);
  86. DBCollection collection = mongoDb.getCollection(buildCollectionName(source));
  87. BasicDBObject newRecord = new BasicDBObject();
  88. newRecord.put("size", source.size());
  89. newRecord.put("success", false);
  90. newRecord.put("hash", source.hashCode());
  91. serializeInstructionSequence(newRecord, sourceFieldName, source);
  92. newRecord.put(optimizedFieldName, null);
  93. newRecord.put("instructionClass", source.get(0).getClass().getCanonicalName());
  94. collection.insert(newRecord);
  95. }
  96. @Override
  97. public InstructionSequence getOptimized(InstructionSequence source) {
  98. DBObject dbObj = findRecordForSource(source);
  99. if (dbObj == null) {
  100. return null;
  101. }
  102. return deserialize(dbObj, optimizedFieldName);
  103. }
  104. @Override
  105. public InstructionSequence getRandomRecordWithSolution() {
  106. return CollectionUtils.selectRandomElement(getAllResults().keySet()); // surely not very fast on big databases
  107. }
  108. @Override
  109. public InstructionSequence getRandomRecordWithoutSolution() {
  110. Random random = new Random(System.currentTimeMillis());
  111. BasicDBObject query = new BasicDBObject();
  112. query.put("success", false);
  113. ProbabilitySet<DBCollection> collectionSet = new ProbabilitySet<DBCollection>(random);
  114. for (String s : mongoDb.getCollectionNames()) {
  115. DBCollection coll = mongoDb.getCollection(s);
  116. collectionSet.put(coll, (double) coll.count(query));
  117. }
  118. if (collectionSet.isEmpty()) {
  119. return null;
  120. }
  121. DBCollection selectedCollection = collectionSet.getRandom();
  122. int recordIdx = random.nextInt(collectionSet.get(selectedCollection).intValue());
  123. DBCursor cur = selectedCollection.find(query);
  124. for (int idx = 0; cur.hasNext(); idx++) {
  125. DBObject obj = cur.next();
  126. if (idx == recordIdx) {
  127. return deserialize(obj, sourceFieldName);
  128. }
  129. }
  130. return null;
  131. }
  132. @Override
  133. public int getRecordsNum() {
  134. Set<String> collections = mongoDb.getCollectionNames();
  135. int rz = 0;
  136. for (String s : collections) {
  137. DBCollection coll = mongoDb.getCollection(s);
  138. rz += coll.count();
  139. }
  140. return rz;
  141. }
  142. @Override
  143. public boolean init(String dbName) {
  144. try {
  145. Mongo mongo = new Mongo("localhost");
  146. mongoDb = mongo.getDB(dbName);
  147. GsonBuilder gsonBuilder = new GsonBuilder();
  148. gsonBuilder.registerTypeAdapter(JVMInstruction.class, new JVMInstructionSerializer());
  149. jsonSerializer = gsonBuilder.create();
  150. return true;
  151. } catch (Exception ex) {
  152. logger.error("Mongo storage manager init failed", ex);
  153. return false;
  154. }
  155. }
  156. @Override
  157. public void term() {
  158. }
  159. @Override
  160. public void flush() {
  161. }
  162. @Override
  163. public void clear() {
  164. mongoDb.dropDatabase();
  165. }
  166. private String buildCollectionName(InstructionSequence sequence) {
  167. return collectionPrefix + sequence.size();
  168. }
  169. private void serializeInstructionSequence(DBObject parentObject, String fieldName, InstructionSequence instructionSequence) {
  170. if (instructionSequence == null) {
  171. parentObject.put(fieldName, null);
  172. return;
  173. }
  174. final String serializedJSON = jsonSerializer.toJson(instructionSequence);
  175. final Object objectToPut = JSON.parse(serializedJSON);
  176. parentObject.put(fieldName, objectToPut);
  177. }
  178. private InstructionSequence deserialize(DBObject parentObject, String fieldName) {
  179. String className = "";
  180. try {
  181. className = (String) parentObject.get("instructionClass");
  182. Class<? extends IGenericInstruction> instructionClass = (Class<? extends IGenericInstruction>) Class.forName(className);
  183. return deserialize0(parentObject, fieldName, instructionClass);
  184. } catch (ClassNotFoundException e) {
  185. logger.error("Can not deserialize InstructionSequence as its instruction class " + className + " can not be loaded:", e);
  186. return null;
  187. } catch (Exception e) {
  188. logger.error("Error while deserializing InstructionSequence", e);
  189. return null;
  190. }
  191. }
  192. private <T extends IGenericInstruction> InstructionSequence deserialize0(DBObject parentObject, String fieldName, Class<T> instructionClass) {
  193. Object obj = parentObject.get(fieldName);
  194. if (obj == null) {
  195. return null;
  196. }
  197. final String jsonString = JSON.serialize(obj);
  198. T[] coll = (T[]) jsonSerializer.fromJson(jsonString, Array.newInstance(instructionClass, 0).getClass());
  199. return new InstructionSequence(coll);
  200. }
  201. @Override
  202. public boolean containsFailedOptimizationRecord(InstructionSequence source) {
  203. DBObject obj = findRecordForSource(source);
  204. return obj != null && obj.get("success").equals(false);
  205. }
  206. /**
  207. * Returns pair of source-optimized for all successfull optimizations that exist in database
  208. *
  209. * @return All succesfull optimization results
  210. */
  211. public Map<InstructionSequence, InstructionSequence> getAllResults() {
  212. Map<InstructionSequence, InstructionSequence> result = new HashMap<InstructionSequence, InstructionSequence>();
  213. Set<String> collections = mongoDb.getCollectionNames();
  214. for (String s : collections) {
  215. DBCollection coll = mongoDb.getCollection(s);
  216. BasicDBObject query = new BasicDBObject();
  217. query.put("success", true);
  218. DBCursor cur = coll.find(query);
  219. for (; cur.hasNext();) {
  220. DBObject obj = cur.next();
  221. final InstructionSequence source = deserialize(obj, sourceFieldName);
  222. final InstructionSequence optimized = deserialize(obj, optimizedFieldName);
  223. result.put(source, optimized);
  224. }
  225. }
  226. return result;
  227. }
  228. }