/src/uk/ac/manchester/cs/patelt9/twitter/stream/StreamingApi.java

https://github.com/taxomania/TwitterMining · Java · 158 lines · 132 code · 24 blank · 2 comment · 15 complexity · ef32915352bd49ef691eeff95fa29d80 MD5 · raw file

  1. package uk.ac.manchester.cs.patelt9.twitter.stream;
  2. import java.io.BufferedReader;
  3. import java.io.IOException;
  4. import java.io.InputStreamReader;
  5. import java.net.MalformedURLException;
  6. import java.net.URL;
  7. import java.net.UnknownHostException;
  8. import java.sql.SQLException;
  9. import javax.net.ssl.HttpsURLConnection;
  10. import sun.misc.BASE64Encoder;
  11. import uk.ac.manchester.cs.patelt9.twitter.StaticFunctions;
  12. import uk.ac.manchester.cs.patelt9.twitter.data.DatabaseThread;
  13. import uk.ac.manchester.cs.patelt9.twitter.data.SQLThread;
  14. import uk.ac.manchester.cs.patelt9.twitter.data.Tweet;
  15. import uk.ac.manchester.cs.patelt9.twitter.data.db.task.DeleteTask;
  16. import uk.ac.manchester.cs.patelt9.twitter.data.db.task.InsertTask;
  17. import uk.ac.manchester.cs.patelt9.twitter.parse.ScannerThread;
  18. import uk.ac.manchester.cs.patelt9.twitter.parse.StreamParseThread;
  19. import uk.ac.manchester.cs.patelt9.twitter.parse.StreamParseThread.ParseListener;
  20. import com.google.gson.JsonParser;
  21. import com.google.gson.stream.JsonReader;
  22. import com.mongodb.MongoException;
  23. public abstract class StreamingApi implements ParseListener {
  24. private static String encoding = null;
  25. private final String urlString;
  26. private final int counterInterval;
  27. private final JsonParser jp;
  28. private HttpsURLConnection con = null;
  29. private JsonReader jsonReader = null;
  30. private/* volatile */boolean stillStream = true; // Only changes once so no need to waste time
  31. private StreamParseThread parseThread = null;
  32. private ScannerThread scanner = null;
  33. private DatabaseThread dbThread = null;
  34. protected InsertTask createInsertTask(final Tweet t) {
  35. return new InsertTask(t);
  36. } // createInsertTask(Tweet)
  37. @Override
  38. public void onParseComplete(final Tweet t) {
  39. dbThread.addTask(createInsertTask(t));
  40. } // onParseComplete(Tweet)
  41. @Override
  42. public void onParseComplete(final long id) {
  43. dbThread.addTask(new DeleteTask(id));
  44. } // onParseComplete(long)
  45. static {
  46. encoding = new BASE64Encoder()
  47. .encode(StaticFunctions.getDetails("userpass.txt").getBytes());
  48. } // static
  49. protected HttpsURLConnection getConnection() {
  50. return con;
  51. } // getConnection()
  52. protected StreamingApi(final String url, final int interval) throws MongoException,
  53. UnknownHostException, SQLException {
  54. counterInterval = interval;
  55. urlString = url;
  56. jp = new JsonParser();
  57. dbThread = new SQLThread();
  58. parseThread = new StreamParseThread();
  59. parseThread.addListener(this);
  60. scanner = new ScannerThread() {
  61. protected void performTask() {
  62. stillStream = false;
  63. } // performTask()
  64. };
  65. } // StreamingApi(String)
  66. // Public interface for setting up connection
  67. public void connect() throws IOException, MalformedURLException {
  68. connect(urlString);
  69. } // connect()
  70. private void connect(final String s) throws IOException, MalformedURLException {
  71. connect(new URL(s));
  72. } // connect(String)
  73. private void connect(final URL url) throws IOException {
  74. System.out.println("Connecting to " + url.toString());
  75. con = (HttpsURLConnection) url.openConnection();
  76. con.setRequestProperty("Authorization", "Basic " + encoding);
  77. connect(con);
  78. } // connect(URL)
  79. // This is to be overridden by subclasses implementing HTTPS POST
  80. protected void connect(final HttpsURLConnection con) throws IOException {
  81. con.connect();
  82. } // connect(HttpsURLConnection)
  83. private void close() {
  84. if (parseThread != null) {
  85. parseThread.interrupt();
  86. parseThread.removeListener(this);
  87. } // if
  88. if (scanner.isAlive()) {
  89. scanner.interrupt();
  90. scanner = null;
  91. } // if
  92. if (dbThread != null) {
  93. dbThread.interrupt();
  94. dbThread = null;
  95. } // if
  96. disconnect();
  97. } // close()
  98. private void disconnect() {
  99. if (jsonReader != null) {
  100. try {
  101. jsonReader.close();
  102. } catch (final IOException e) {
  103. e.printStackTrace();
  104. } // catch
  105. } // if
  106. if (con != null) {
  107. con.disconnect();
  108. } // if
  109. } // disconnect()
  110. public void initialiseReader() throws IOException {
  111. jsonReader = new JsonReader(new BufferedReader(new InputStreamReader(con.getInputStream())));
  112. } // initialiseReader()
  113. public void streamTweets() {
  114. try {
  115. initialiseReader();
  116. } catch (final IOException e) {
  117. e.printStackTrace();
  118. return;
  119. } // catch
  120. scanner.start();
  121. dbThread.start();
  122. parseThread.start();
  123. System.out.println("Started");
  124. for (int i = 1; stillStream; i++) {
  125. if (i % counterInterval == 0) {
  126. System.out.println(i);
  127. } else if (i == Integer.MAX_VALUE) {
  128. i = 1;
  129. } // else if
  130. parseThread.addTask(jp.parse(jsonReader).getAsJsonObject());
  131. } // for
  132. close();
  133. } // streamTweets()
  134. } // StreamingApi