PageRenderTime 49ms CodeModel.GetById 13ms RepoModel.GetById 1ms app.codeStats 0ms

/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java

https://gitlab.com/xiaoliuliu2050/hadoop
Java | 459 lines | 384 code | 47 blank | 28 comment | 2 complexity | 1b0b541841843f8990b8f675af54f4c4 MD5 | raw file
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.yarn.server.nodemanager.webapp;
  19. import static org.junit.Assert.assertEquals;
  20. import static org.junit.Assert.assertTrue;
  21. import static org.junit.Assert.fail;
  22. import java.io.File;
  23. import java.io.IOException;
  24. import java.io.PrintWriter;
  25. import java.io.StringReader;
  26. import javax.ws.rs.core.MediaType;
  27. import javax.xml.parsers.DocumentBuilder;
  28. import javax.xml.parsers.DocumentBuilderFactory;
  29. import org.junit.Assert;
  30. import org.apache.hadoop.conf.Configuration;
  31. import org.apache.hadoop.fs.FileUtil;
  32. import org.apache.hadoop.fs.Path;
  33. import org.apache.hadoop.util.VersionInfo;
  34. import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
  35. import org.apache.hadoop.yarn.api.records.NodeId;
  36. import org.apache.hadoop.yarn.api.records.ApplicationId;
  37. import org.apache.hadoop.yarn.api.records.ContainerId;
  38. import org.apache.hadoop.yarn.conf.YarnConfiguration;
  39. import org.apache.hadoop.yarn.event.AsyncDispatcher;
  40. import org.apache.hadoop.yarn.server.nodemanager.Context;
  41. import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
  42. import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
  43. import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
  44. import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
  45. import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
  46. import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
  47. import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
  48. import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
  49. import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
  50. import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
  51. import org.apache.hadoop.yarn.server.utils.BuilderUtils;
  52. import org.apache.hadoop.yarn.util.YarnVersionInfo;
  53. import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
  54. import org.apache.hadoop.yarn.webapp.WebApp;
  55. import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
  56. import org.codehaus.jettison.json.JSONException;
  57. import org.codehaus.jettison.json.JSONObject;
  58. import org.junit.AfterClass;
  59. import org.junit.Before;
  60. import org.junit.Test;
  61. import org.w3c.dom.Document;
  62. import org.w3c.dom.Element;
  63. import org.w3c.dom.NodeList;
  64. import org.xml.sax.InputSource;
  65. import com.google.inject.Guice;
  66. import com.google.inject.Injector;
  67. import com.google.inject.servlet.GuiceServletContextListener;
  68. import com.google.inject.servlet.ServletModule;
  69. import com.sun.jersey.api.client.ClientResponse;
  70. import com.sun.jersey.api.client.ClientResponse.Status;
  71. import com.sun.jersey.api.client.UniformInterfaceException;
  72. import com.sun.jersey.api.client.WebResource;
  73. import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
  74. import com.sun.jersey.test.framework.JerseyTest;
  75. import com.sun.jersey.test.framework.WebAppDescriptor;
  76. /**
  77. * Test the nodemanager node info web services api's
  78. */
  79. public class TestNMWebServices extends JerseyTest {
  80. private static Context nmContext;
  81. private static ResourceView resourceView;
  82. private static ApplicationACLsManager aclsManager;
  83. private static LocalDirsHandlerService dirsHandler;
  84. private static WebApp nmWebApp;
  85. private static final File testRootDir = new File("target",
  86. TestNMWebServices.class.getSimpleName());
  87. private static File testLogDir = new File("target",
  88. TestNMWebServices.class.getSimpleName() + "LogDir");
  89. private Injector injector = Guice.createInjector(new ServletModule() {
  90. @Override
  91. protected void configureServlets() {
  92. Configuration conf = new Configuration();
  93. conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  94. conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  95. NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  96. healthChecker.init(conf);
  97. dirsHandler = healthChecker.getDiskHandler();
  98. aclsManager = new ApplicationACLsManager(conf);
  99. nmContext = new NodeManager.NMContext(null, null, dirsHandler,
  100. aclsManager, null);
  101. NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042);
  102. ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  103. resourceView = new ResourceView() {
  104. @Override
  105. public long getVmemAllocatedForContainers() {
  106. // 15.5G in bytes
  107. return new Long("16642998272");
  108. }
  109. @Override
  110. public long getPmemAllocatedForContainers() {
  111. // 16G in bytes
  112. return new Long("17179869184");
  113. }
  114. @Override
  115. public long getVCoresAllocatedForContainers() {
  116. return new Long("4000");
  117. }
  118. @Override
  119. public boolean isVmemCheckEnabled() {
  120. return true;
  121. }
  122. @Override
  123. public boolean isPmemCheckEnabled() {
  124. return true;
  125. }
  126. };
  127. nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  128. bind(JAXBContextResolver.class);
  129. bind(NMWebServices.class);
  130. bind(GenericExceptionHandler.class);
  131. bind(Context.class).toInstance(nmContext);
  132. bind(WebApp.class).toInstance(nmWebApp);
  133. bind(ResourceView.class).toInstance(resourceView);
  134. bind(ApplicationACLsManager.class).toInstance(aclsManager);
  135. bind(LocalDirsHandlerService.class).toInstance(dirsHandler);
  136. serve("/*").with(GuiceContainer.class);
  137. }
  138. });
  139. public class GuiceServletConfig extends GuiceServletContextListener {
  140. @Override
  141. protected Injector getInjector() {
  142. return injector;
  143. }
  144. }
  145. @Before
  146. @Override
  147. public void setUp() throws Exception {
  148. super.setUp();
  149. testRootDir.mkdirs();
  150. testLogDir.mkdir();
  151. }
  152. @AfterClass
  153. static public void stop() {
  154. FileUtil.fullyDelete(testRootDir);
  155. FileUtil.fullyDelete(testLogDir);
  156. }
  157. public TestNMWebServices() {
  158. super(new WebAppDescriptor.Builder(
  159. "org.apache.hadoop.yarn.server.nodemanager.webapp")
  160. .contextListenerClass(GuiceServletConfig.class)
  161. .filterClass(com.google.inject.servlet.GuiceFilter.class)
  162. .contextPath("jersey-guice-filter").servletPath("/").build());
  163. }
  164. @Test
  165. public void testInvalidUri() throws JSONException, Exception {
  166. WebResource r = resource();
  167. String responseStr = "";
  168. try {
  169. responseStr = r.path("ws").path("v1").path("node").path("bogus")
  170. .accept(MediaType.APPLICATION_JSON).get(String.class);
  171. fail("should have thrown exception on invalid uri");
  172. } catch (UniformInterfaceException ue) {
  173. ClientResponse response = ue.getResponse();
  174. assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
  175. WebServicesTestUtils.checkStringMatch(
  176. "error string exists and shouldn't", "", responseStr);
  177. }
  178. }
  179. @Test
  180. public void testInvalidAccept() throws JSONException, Exception {
  181. WebResource r = resource();
  182. String responseStr = "";
  183. try {
  184. responseStr = r.path("ws").path("v1").path("node")
  185. .accept(MediaType.TEXT_PLAIN).get(String.class);
  186. fail("should have thrown exception on invalid uri");
  187. } catch (UniformInterfaceException ue) {
  188. ClientResponse response = ue.getResponse();
  189. assertEquals(Status.INTERNAL_SERVER_ERROR,
  190. response.getClientResponseStatus());
  191. WebServicesTestUtils.checkStringMatch(
  192. "error string exists and shouldn't", "", responseStr);
  193. }
  194. }
  195. @Test
  196. public void testInvalidUri2() throws JSONException, Exception {
  197. WebResource r = resource();
  198. String responseStr = "";
  199. try {
  200. responseStr = r.accept(MediaType.APPLICATION_JSON).get(String.class);
  201. fail("should have thrown exception on invalid uri");
  202. } catch (UniformInterfaceException ue) {
  203. ClientResponse response = ue.getResponse();
  204. assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
  205. WebServicesTestUtils.checkStringMatch(
  206. "error string exists and shouldn't", "", responseStr);
  207. }
  208. }
  209. @Test
  210. public void testNode() throws JSONException, Exception {
  211. WebResource r = resource();
  212. ClientResponse response = r.path("ws").path("v1").path("node")
  213. .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  214. assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  215. JSONObject json = response.getEntity(JSONObject.class);
  216. verifyNodeInfo(json);
  217. }
  218. @Test
  219. public void testNodeSlash() throws JSONException, Exception {
  220. WebResource r = resource();
  221. ClientResponse response = r.path("ws").path("v1").path("node/")
  222. .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  223. assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  224. JSONObject json = response.getEntity(JSONObject.class);
  225. verifyNodeInfo(json);
  226. }
  227. // make sure default is json output
  228. @Test
  229. public void testNodeDefault() throws JSONException, Exception {
  230. WebResource r = resource();
  231. ClientResponse response = r.path("ws").path("v1").path("node")
  232. .get(ClientResponse.class);
  233. assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  234. JSONObject json = response.getEntity(JSONObject.class);
  235. verifyNodeInfo(json);
  236. }
  237. @Test
  238. public void testNodeInfo() throws JSONException, Exception {
  239. WebResource r = resource();
  240. ClientResponse response = r.path("ws").path("v1").path("node").path("info")
  241. .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  242. assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  243. JSONObject json = response.getEntity(JSONObject.class);
  244. verifyNodeInfo(json);
  245. }
  246. @Test
  247. public void testNodeInfoSlash() throws JSONException, Exception {
  248. WebResource r = resource();
  249. ClientResponse response = r.path("ws").path("v1").path("node")
  250. .path("info/").accept(MediaType.APPLICATION_JSON)
  251. .get(ClientResponse.class);
  252. assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  253. JSONObject json = response.getEntity(JSONObject.class);
  254. verifyNodeInfo(json);
  255. }
  256. // make sure default is json output
  257. @Test
  258. public void testNodeInfoDefault() throws JSONException, Exception {
  259. WebResource r = resource();
  260. ClientResponse response = r.path("ws").path("v1").path("node").path("info")
  261. .get(ClientResponse.class);
  262. assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  263. JSONObject json = response.getEntity(JSONObject.class);
  264. verifyNodeInfo(json);
  265. }
  266. @Test
  267. public void testSingleNodesXML() throws JSONException, Exception {
  268. WebResource r = resource();
  269. ClientResponse response = r.path("ws").path("v1").path("node")
  270. .path("info/").accept(MediaType.APPLICATION_XML)
  271. .get(ClientResponse.class);
  272. assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
  273. String xml = response.getEntity(String.class);
  274. DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
  275. DocumentBuilder db = dbf.newDocumentBuilder();
  276. InputSource is = new InputSource();
  277. is.setCharacterStream(new StringReader(xml));
  278. Document dom = db.parse(is);
  279. NodeList nodes = dom.getElementsByTagName("nodeInfo");
  280. assertEquals("incorrect number of elements", 1, nodes.getLength());
  281. verifyNodesXML(nodes);
  282. }
  283. @Test
  284. public void testContainerLogs() throws IOException {
  285. WebResource r = resource();
  286. final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0);
  287. final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0)
  288. .toString();
  289. final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
  290. final ApplicationId appId = appAttemptId.getApplicationId();
  291. final String appIdStr = appId.toString();
  292. final String filename = "logfile1";
  293. final String logMessage = "log message\n";
  294. nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
  295. appId, null, nmContext));
  296. MockContainer container = new MockContainer(appAttemptId,
  297. new AsyncDispatcher(), new Configuration(), "user", appId, 1);
  298. container.setState(ContainerState.RUNNING);
  299. nmContext.getContainers().put(containerId, container);
  300. // write out log file
  301. Path path = dirsHandler.getLogPathForWrite(
  302. ContainerLaunch.getRelativeContainerLogDir(
  303. appIdStr, containerIdStr) + "/" + filename, false);
  304. File logFile = new File(path.toUri().getPath());
  305. logFile.deleteOnExit();
  306. assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
  307. PrintWriter pw = new PrintWriter(logFile);
  308. pw.print(logMessage);
  309. pw.close();
  310. // ask for it
  311. ClientResponse response = r.path("ws").path("v1").path("node")
  312. .path("containerlogs").path(containerIdStr).path(filename)
  313. .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  314. String responseText = response.getEntity(String.class);
  315. assertEquals(logMessage, responseText);
  316. // ask for file that doesn't exist
  317. response = r.path("ws").path("v1").path("node")
  318. .path("containerlogs").path(containerIdStr).path("uhhh")
  319. .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
  320. Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus());
  321. responseText = response.getEntity(String.class);
  322. assertTrue(responseText.contains("Cannot find this log on the local disk."));
  323. // After container is completed, it is removed from nmContext
  324. nmContext.getContainers().remove(containerId);
  325. Assert.assertNull(nmContext.getContainers().get(containerId));
  326. response =
  327. r.path("ws").path("v1").path("node").path("containerlogs")
  328. .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN)
  329. .get(ClientResponse.class);
  330. responseText = response.getEntity(String.class);
  331. assertEquals(logMessage, responseText);
  332. }
  333. public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {
  334. for (int i = 0; i < nodes.getLength(); i++) {
  335. Element element = (Element) nodes.item(i);
  336. verifyNodeInfoGeneric(WebServicesTestUtils.getXmlString(element, "id"),
  337. WebServicesTestUtils.getXmlString(element, "healthReport"),
  338. WebServicesTestUtils.getXmlLong(element,
  339. "totalVmemAllocatedContainersMB"),
  340. WebServicesTestUtils.getXmlLong(element,
  341. "totalPmemAllocatedContainersMB"),
  342. WebServicesTestUtils.getXmlLong(element,
  343. "totalVCoresAllocatedContainers"),
  344. WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"),
  345. WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"),
  346. WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"),
  347. WebServicesTestUtils.getXmlBoolean(element, "nodeHealthy"),
  348. WebServicesTestUtils.getXmlString(element, "nodeHostName"),
  349. WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"),
  350. WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"),
  351. WebServicesTestUtils.getXmlString(element, "hadoopVersion"),
  352. WebServicesTestUtils.getXmlString(element,
  353. "nodeManagerVersionBuiltOn"), WebServicesTestUtils.getXmlString(
  354. element, "nodeManagerBuildVersion"),
  355. WebServicesTestUtils.getXmlString(element, "nodeManagerVersion"));
  356. }
  357. }
  358. public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
  359. assertEquals("incorrect number of elements", 1, json.length());
  360. JSONObject info = json.getJSONObject("nodeInfo");
  361. assertEquals("incorrect number of elements", 16, info.length());
  362. verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"),
  363. info.getLong("totalVmemAllocatedContainersMB"),
  364. info.getLong("totalPmemAllocatedContainersMB"),
  365. info.getLong("totalVCoresAllocatedContainers"),
  366. info.getBoolean("vmemCheckEnabled"),
  367. info.getBoolean("pmemCheckEnabled"),
  368. info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"),
  369. info.getString("nodeHostName"), info.getString("hadoopVersionBuiltOn"),
  370. info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"),
  371. info.getString("nodeManagerVersionBuiltOn"),
  372. info.getString("nodeManagerBuildVersion"),
  373. info.getString("nodeManagerVersion"));
  374. }
  375. public void verifyNodeInfoGeneric(String id, String healthReport,
  376. long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB,
  377. long totalVCoresAllocatedContainers,
  378. boolean vmemCheckEnabled, boolean pmemCheckEnabled,
  379. long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName,
  380. String hadoopVersionBuiltOn, String hadoopBuildVersion,
  381. String hadoopVersion, String resourceManagerVersionBuiltOn,
  382. String resourceManagerBuildVersion, String resourceManagerVersion) {
  383. WebServicesTestUtils.checkStringMatch("id", "testhost.foo.com:8042", id);
  384. WebServicesTestUtils.checkStringMatch("healthReport", "Healthy",
  385. healthReport);
  386. assertEquals("totalVmemAllocatedContainersMB incorrect", 15872,
  387. totalVmemAllocatedContainersMB);
  388. assertEquals("totalPmemAllocatedContainersMB incorrect", 16384,
  389. totalPmemAllocatedContainersMB);
  390. assertEquals("totalVCoresAllocatedContainers incorrect", 4000,
  391. totalVCoresAllocatedContainers);
  392. assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled);
  393. assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled);
  394. assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext
  395. .getNodeHealthStatus().getLastHealthReportTime());
  396. assertTrue("nodeHealthy isn't true", nodeHealthy);
  397. WebServicesTestUtils.checkStringMatch("nodeHostName", "testhost.foo.com",
  398. nodeHostName);
  399. WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn",
  400. VersionInfo.getDate(), hadoopVersionBuiltOn);
  401. WebServicesTestUtils.checkStringEqual("hadoopBuildVersion",
  402. VersionInfo.getBuildVersion(), hadoopBuildVersion);
  403. WebServicesTestUtils.checkStringMatch("hadoopVersion",
  404. VersionInfo.getVersion(), hadoopVersion);
  405. WebServicesTestUtils.checkStringMatch("resourceManagerVersionBuiltOn",
  406. YarnVersionInfo.getDate(), resourceManagerVersionBuiltOn);
  407. WebServicesTestUtils.checkStringEqual("resourceManagerBuildVersion",
  408. YarnVersionInfo.getBuildVersion(), resourceManagerBuildVersion);
  409. WebServicesTestUtils.checkStringMatch("resourceManagerVersion",
  410. YarnVersionInfo.getVersion(), resourceManagerVersion);
  411. }
  412. }