/src/net/aminecraftdev/spigot/site/http/HTTPUnitRequest.java

https://bitbucket.org/AMinecraftDev/discordbot · Java · 182 lines · 158 code · 15 blank · 9 comment · 21 complexity · da8a88f9467a2adc11174c80f7a6d378 MD5 · raw file

  1. package net.aminecraftdev.spigot.site.http;
  2. import com.gargoylesoftware.htmlunit.*;
  3. import com.gargoylesoftware.htmlunit.html.HtmlPage;
  4. import com.gargoylesoftware.htmlunit.util.Cookie;
  5. import com.gargoylesoftware.htmlunit.util.NameValuePair;
  6. import net.aminecraftdev.spigot.site.SpigotSiteCore;
  7. import org.jsoup.Jsoup;
  8. import org.jsoup.nodes.Document;
  9. import java.io.InputStream;
  10. import java.net.URL;
  11. import java.util.ArrayList;
  12. import java.util.HashMap;
  13. import java.util.List;
  14. import java.util.Map;
  15. import java.util.logging.Level;
  16. public class HTTPUnitRequest {
  17. private static WebClient webClient = null;
  18. public static void initialize() {
  19. webClient = new WebClient(BrowserVersion.CHROME);
  20. webClient.getOptions().setJavaScriptEnabled(true);
  21. webClient.getOptions().setTimeout(15000);
  22. webClient.getOptions().setCssEnabled(false);
  23. webClient.getOptions().setRedirectEnabled(true);
  24. webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
  25. webClient.getOptions().setThrowExceptionOnScriptError(false);
  26. webClient.getOptions().setPrintContentOnFailingStatusCode(false);
  27. java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
  28. }
  29. public static HTTPDownloadResponse downloadFile(String url, Map<String, String> cookies) {
  30. try {
  31. WebRequest wr = new WebRequest(new URL(url), HttpMethod.GET);
  32. for (Map.Entry<String, String> entry : cookies.entrySet())
  33. webClient.getCookieManager().addCookie(new Cookie("spigotmc.org", entry.getKey(), entry.getValue()));
  34. InputStream stream = null;
  35. Page page = webClient.getPage(wr);
  36. if (page instanceof HtmlPage)
  37. if (((HtmlPage) page).asXml().contains("DDoS protection by Cloudflare")) {
  38. // DDOS protection
  39. try {
  40. Thread.sleep(9000);
  41. Request.setDdosBypass(false);
  42. } catch (InterruptedException e) {
  43. Thread.currentThread().interrupt(); // restore
  44. // interrupted
  45. // status
  46. }
  47. }
  48. URL outputURL = webClient.getCurrentWindow().getEnclosedPage().getUrl();
  49. stream = webClient.getCurrentWindow().getEnclosedPage().getWebResponse().getContentAsStream();
  50. return new HTTPDownloadResponse(stream, outputURL);
  51. } catch (Exception ex) {
  52. ex.printStackTrace();
  53. }
  54. return null;
  55. }
  56. public static HTTPResponse get(String url, Map<String, String> cookies, Map<String, String> params) {
  57. HTTPResponse response = new HTTPResponse();
  58. try {
  59. WebRequest wr = new WebRequest(new URL(url), HttpMethod.GET);
  60. for (Map.Entry<String, String> entry : cookies.entrySet())
  61. webClient.getCookieManager().addCookie(new Cookie("spigotmc.org", entry.getKey(), entry.getValue()));
  62. List<NameValuePair> paramsPair = new ArrayList<NameValuePair>();
  63. for (Map.Entry<String, String> entry : params.entrySet())
  64. paramsPair.add(new NameValuePair(entry.getKey(), entry.getValue()));
  65. wr.setRequestParameters(paramsPair);
  66. Page page = webClient.getPage(wr);
  67. if (page instanceof HtmlPage)
  68. if (((HtmlPage) page).asXml().contains("DDoS protection by Cloudflare")) {
  69. // DDOS protection
  70. try {
  71. Thread.sleep(9000);
  72. Request.setDdosBypass(false);
  73. } catch (InterruptedException e) {
  74. Thread.currentThread().interrupt(); // restore
  75. // interrupted
  76. // status
  77. }
  78. if (webClient.getPage(wr) instanceof UnexpectedPage) {
  79. UnexpectedPage unexpectedPage = webClient.getPage(wr);
  80. System.out.println("UNEXPECTED PAGE: " + unexpectedPage.getWebResponse().getStatusMessage());
  81. } else
  82. page = webClient.getPage(wr);
  83. }
  84. Map<String, String> cookiesMap = new HashMap<String, String>();
  85. for (Cookie cookie : webClient.getCookieManager().getCookies()) {
  86. cookiesMap.put(cookie.getName(), cookie.getValue());
  87. }
  88. Document doc = Jsoup.parse(((HtmlPage) page).asXml());
  89. response.setDocument(doc);
  90. response.setHtml(doc.html());
  91. Map<String, String> resultCookies = new HashMap<String, String>();
  92. for (Cookie cookie : webClient.getCookieManager().getCookies()) {
  93. resultCookies.put(cookie.getName(), cookie.getValue());
  94. }
  95. response.setCookies(resultCookies);
  96. webClient.close();
  97. } catch (Exception ex) {
  98. ex.printStackTrace();
  99. }
  100. return response;
  101. }
  102. public static HTTPResponse post(String url, Map<String, String> cookies, Map<String, String> params) {
  103. try {
  104. if (Request.isRateLimit()) {
  105. Thread.sleep(SpigotSiteCore.getRateLimitTimeout());
  106. Request.setRateLimit(false);
  107. }
  108. } catch (InterruptedException e1) {
  109. Request.setRateLimit(false);
  110. }
  111. HTTPResponse response = new HTTPResponse();
  112. try {
  113. WebRequest wr = new WebRequest(new URL(url), HttpMethod.POST);
  114. for (Map.Entry<String, String> entry : cookies.entrySet())
  115. webClient.getCookieManager().addCookie(new Cookie("spigotmc.org", entry.getKey(), entry.getValue()));
  116. List<NameValuePair> paramsPair = new ArrayList<NameValuePair>();
  117. for (Map.Entry<String, String> entry : params.entrySet())
  118. paramsPair.add(new NameValuePair(entry.getKey(), entry.getValue()));
  119. wr.setRequestParameters(paramsPair);
  120. Page page = webClient.getPage(wr);
  121. if (page instanceof HtmlPage)
  122. if (((HtmlPage) page).asXml().contains("DDoS protection by Cloudflare")) {
  123. // DDOS protection
  124. try {
  125. Thread.sleep(9000);
  126. Request.setDdosBypass(false);
  127. } catch (InterruptedException e) {
  128. Thread.currentThread().interrupt(); // restore
  129. // interrupted
  130. // status
  131. }
  132. wr = new WebRequest(new URL(url), HttpMethod.POST);
  133. paramsPair = new ArrayList<NameValuePair>();
  134. for (Map.Entry<String, String> entry : params.entrySet())
  135. paramsPair.add(new NameValuePair(entry.getKey(), entry.getValue()));
  136. wr.setRequestParameters(paramsPair);
  137. if (webClient.getPage(wr) instanceof UnexpectedPage) {
  138. UnexpectedPage unexpectedPage = webClient.getPage(wr);
  139. System.out.println("UNEXPECTED PAGE: " + unexpectedPage.getWebResponse().getStatusMessage());
  140. } else
  141. page = webClient.getPage(wr);
  142. }
  143. Map<String, String> cookiesMap = new HashMap<>();
  144. for (Cookie cookie : webClient.getCookieManager().getCookies()) {
  145. cookiesMap.put(cookie.getName(), cookie.getValue());
  146. }
  147. if (page instanceof HtmlPage) {
  148. Document doc = Jsoup.parse(((HtmlPage) page).asXml());
  149. response.setDocument(doc);
  150. response.setHtml(doc.html());
  151. } else {
  152. response.setHtml(page.getWebResponse().getContentAsString());
  153. Document doc = Jsoup.parse(response.getHtml());
  154. response.setDocument(doc);
  155. }
  156. Map<String, String> resultCookies = new HashMap<>();
  157. for (Cookie cookie : webClient.getCookieManager().getCookies()) {
  158. resultCookies.put(cookie.getName(), cookie.getValue());
  159. }
  160. response.setCookies(resultCookies);
  161. webClient.close();
  162. } catch (Exception ex) {
  163. ex.printStackTrace();
  164. }
  165. return response;
  166. }
  167. }