PageRenderTime 95ms CodeModel.GetById 28ms RepoModel.GetById 1ms app.codeStats 0ms

/custom_processing/infinit.e.hadoop.examples/src/testing/InfiniteHadoopTestUtils.java

https://github.com/IKANOW/Infinit.e
Java | 124 lines | 97 code | 19 blank | 8 comment | 15 complexity | 49e835f05580bec95f459cfbbccb7466 MD5 | raw file
Possible License(s): BSD-3-Clause
  1. package testing;
  2. import java.io.BufferedReader;
  3. import java.io.File;
  4. import java.io.FileReader;
  5. import java.io.IOException;
  6. import java.util.List;
  7. import javax.xml.parsers.DocumentBuilder;
  8. import javax.xml.parsers.DocumentBuilderFactory;
  9. import org.apache.hadoop.conf.Configuration;
  10. import org.apache.hadoop.util.Tool;
  11. import org.apache.hadoop.util.ToolRunner;
  12. import org.w3c.dom.Document;
  13. import org.w3c.dom.Element;
  14. import org.w3c.dom.Node;
  15. import org.w3c.dom.NodeList;
  16. import com.google.gson.GsonBuilder;
  17. import com.ikanow.infinit.e.data_model.api.BaseApiPojo;
  18. import com.ikanow.infinit.e.data_model.api.knowledge.DocumentPojoApiMap;
  19. import com.ikanow.infinit.e.data_model.store.MongoDbConnection;
  20. import com.ikanow.infinit.e.data_model.store.document.DocumentPojo;
  21. import com.mongodb.MongoException;
  22. public class InfiniteHadoopTestUtils {
  23. //////////////////////////////////////////////////////////////////////////////////////////////////
  24. // Run Hadoop inside Eclipse
  25. public static int runStandalone(Tool runner, String configPath, String [] args) throws Exception
  26. {
  27. Configuration config = new Configuration();
  28. config.set("mapred.job.tracker", "local");
  29. config.set("fs.default.name", "local");
  30. // Now read in via XML
  31. File fXmlFile = new File(configPath);
  32. DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
  33. DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
  34. Document doc = dBuilder.parse(fXmlFile);
  35. NodeList nList = doc.getElementsByTagName("property");
  36. for (int temp = 0; temp < nList.getLength(); temp++) {
  37. Node nNode = nList.item(temp);
  38. if (nNode.getNodeType() == Node.ELEMENT_NODE) {
  39. Element eElement = (Element) nNode;
  40. String name = getTagValue("name", eElement);
  41. String value = getTagValue("value", eElement);
  42. if ((null != name) && (null != value)) {
  43. config.set(name, value);
  44. }
  45. }
  46. }
  47. return ToolRunner.run( config, runner, args );
  48. }
  49. //////////////////////////////////////////////////////////////////////////////////////////////////
  50. // Load document results from the GUI into a (local) mongoDB
  51. public static void loadSampleData(String dataPath, String mongoServer, int mongoPort, boolean resetBeforeLoading) throws MongoException, IOException
  52. {
  53. String json = readFile(dataPath);
  54. MongoDbConnection mongoConnection = new MongoDbConnection(mongoServer, mongoPort);
  55. if (resetBeforeLoading) {
  56. mongoConnection.getMongo().getDB("test").getCollection("docs").drop();
  57. }
  58. QueryResults res = QueryResults.fromApi(json, QueryResults.class);
  59. List<DocumentPojo> documents = res.documents;
  60. if (null == documents) {
  61. documents = res.data; // (maybe it was a direct JSON call not a GUI save file)
  62. }
  63. for (DocumentPojo doc: documents) {
  64. mongoConnection.getMongo().getDB("test").getCollection("docs").insert(doc.toDb());
  65. }
  66. }//TESTED
  67. public static class QueryResults extends BaseApiPojo
  68. {
  69. @Override
  70. public GsonBuilder extendBuilder(GsonBuilder gp) { // Extend the builder to apply standard document deserialization
  71. return new DocumentPojoApiMap().extendBuilder(gp);
  72. }
  73. public List<DocumentPojo> documents; // (from GUI save)
  74. public List<DocumentPojo> data; // (from direct JSON call to rest)
  75. //(and lots of other stuff that will get discarded for now)
  76. }
  77. //////////////////////////////////////////////////////////////////////////////////////////////////
  78. // Utilities
  79. private static String getTagValue(String sTag, Element eElement) {
  80. NodeList nlList = eElement.getElementsByTagName(sTag).item(0).getChildNodes();
  81. Node nValue = (Node) nlList.item(0);
  82. if (null != nValue) {
  83. return nValue.getNodeValue();
  84. }
  85. else {
  86. return null;
  87. }
  88. }
  89. private static String readFile( String file ) throws IOException {
  90. StringBuilder stringBuilder = new StringBuilder();
  91. BufferedReader reader = new BufferedReader( new FileReader (file));
  92. try {
  93. String line = null;
  94. String ls = System.getProperty("line.separator");
  95. while( ( line = reader.readLine() ) != null ) {
  96. stringBuilder.append( line );
  97. stringBuilder.append( ls );
  98. }
  99. }
  100. finally {
  101. reader.close();
  102. }
  103. return stringBuilder.toString();
  104. }
  105. }