PageRenderTime 47ms CodeModel.GetById 24ms RepoModel.GetById 0ms app.codeStats 0ms

/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java

https://github.com/lalithsuresh/Scaling-HDFS-NameNode
Java | 294 lines | 239 code | 32 blank | 23 comment | 9 complexity | 917779e940690034a6fab8f6ccdd4697 MD5 | raw file
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.mapreduce.v2.hs;
  19. import java.io.IOException;
  20. import java.net.InetAddress;
  21. import java.net.InetSocketAddress;
  22. import java.net.UnknownHostException;
  23. import java.security.AccessControlException;
  24. import java.security.PrivilegedExceptionAction;
  25. import java.util.Arrays;
  26. import java.util.Collection;
  27. import org.apache.avro.ipc.Server;
  28. import org.apache.commons.logging.Log;
  29. import org.apache.commons.logging.LogFactory;
  30. import org.apache.hadoop.conf.Configuration;
  31. import org.apache.hadoop.fs.CommonConfigurationKeys;
  32. import org.apache.hadoop.mapreduce.JobACL;
  33. import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
  34. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
  35. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
  36. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
  37. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse;
  38. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest;
  39. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse;
  40. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
  41. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse;
  42. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest;
  43. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse;
  44. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest;
  45. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse;
  46. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest;
  47. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse;
  48. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest;
  49. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse;
  50. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest;
  51. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobResponse;
  52. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest;
  53. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
  54. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
  55. import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
  56. import org.apache.hadoop.mapreduce.v2.api.records.JobId;
  57. import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
  58. import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
  59. import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
  60. import org.apache.hadoop.mapreduce.v2.app.job.Job;
  61. import org.apache.hadoop.mapreduce.v2.app.job.Task;
  62. import org.apache.hadoop.mapreduce.v2.hs.webapp.HSWebApp;
  63. import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
  64. import org.apache.hadoop.mapreduce.v2.security.client.ClientHSSecurityInfo;
  65. import org.apache.hadoop.net.NetUtils;
  66. import org.apache.hadoop.security.SecurityInfo;
  67. import org.apache.hadoop.security.UserGroupInformation;
  68. import org.apache.hadoop.yarn.YarnException;
  69. import org.apache.hadoop.yarn.conf.YarnConfiguration;
  70. import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
  71. import org.apache.hadoop.yarn.factories.RecordFactory;
  72. import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
  73. import org.apache.hadoop.yarn.ipc.RPCUtil;
  74. import org.apache.hadoop.yarn.ipc.YarnRPC;
  75. import org.apache.hadoop.yarn.service.AbstractService;
  76. import org.apache.hadoop.yarn.webapp.WebApp;
  77. import org.apache.hadoop.yarn.webapp.WebApps;
  78. /**
  79. * This module is responsible for talking to the
  80. * JobClient (user facing).
  81. *
  82. */
  83. public class HistoryClientService extends AbstractService {
  84. private static final Log LOG = LogFactory.getLog(HistoryClientService.class);
  85. private MRClientProtocol protocolHandler;
  86. private Server server;
  87. private WebApp webApp;
  88. private InetSocketAddress bindAddress;
  89. private HistoryContext history;
  90. public HistoryClientService(HistoryContext history) {
  91. super("HistoryClientService");
  92. this.history = history;
  93. this.protocolHandler = new MRClientProtocolHandler();
  94. }
  95. public void start() {
  96. YarnRPC rpc = YarnRPC.create(getConfig());
  97. Configuration conf = new Configuration(getConfig());
  98. conf.setClass(
  99. YarnConfiguration.YARN_SECURITY_INFO,
  100. ClientHSSecurityInfo.class, SecurityInfo.class);
  101. initializeWebApp(getConfig());
  102. String serviceAddr = conf.get(JHConfig.HS_BIND_ADDRESS,
  103. JHConfig.DEFAULT_HS_BIND_ADDRESS);
  104. InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr);
  105. InetAddress hostNameResolved = null;
  106. try {
  107. hostNameResolved = InetAddress.getLocalHost(); //address.getAddress().getLocalHost();
  108. } catch (UnknownHostException e) {
  109. throw new YarnException(e);
  110. }
  111. server =
  112. rpc.getServer(MRClientProtocol.class, protocolHandler, address,
  113. conf, null,
  114. conf.getInt(JHConfig.HS_CLIENT_THREADS,
  115. JHConfig.DEFAULT_HS_CLIENT_THREADS));
  116. server.start();
  117. this.bindAddress =
  118. NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
  119. + ":" + server.getPort());
  120. LOG.info("Instantiated MRClientService at " + this.bindAddress);
  121. super.start();
  122. }
  123. private void initializeWebApp(Configuration conf) {
  124. webApp = new HSWebApp(history);
  125. String bindAddress = conf.get(JHConfig.HS_WEBAPP_BIND_ADDRESS,
  126. JHConfig.DEFAULT_HS_WEBAPP_BIND_ADDRESS);
  127. WebApps.$for("yarn", this).at(bindAddress).start(webApp);
  128. }
  129. @Override
  130. public void stop() {
  131. if (server != null) {
  132. server.close();
  133. }
  134. if (webApp != null) {
  135. webApp.stop();
  136. }
  137. super.stop();
  138. }
  139. private class MRClientProtocolHandler implements MRClientProtocol {
  140. private RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  141. private Job verifyAndGetJob(final JobId jobID) throws YarnRemoteException {
  142. UserGroupInformation loginUgi = null;
  143. Job job = null;
  144. try {
  145. loginUgi = UserGroupInformation.getLoginUser();
  146. job = loginUgi.doAs(new PrivilegedExceptionAction<Job>() {
  147. @Override
  148. public Job run() throws Exception {
  149. Job job = history.getJob(jobID);
  150. return job;
  151. }
  152. });
  153. } catch (IOException e) {
  154. throw RPCUtil.getRemoteException(e);
  155. } catch (InterruptedException e) {
  156. throw RPCUtil.getRemoteException(e);
  157. }
  158. if (job == null) {
  159. throw RPCUtil.getRemoteException("Unknown job " + jobID);
  160. }
  161. JobACL operation = JobACL.VIEW_JOB;
  162. //TODO disable check access for now.
  163. checkAccess(job, operation);
  164. return job;
  165. }
  166. @Override
  167. public GetCountersResponse getCounters(GetCountersRequest request) throws YarnRemoteException {
  168. JobId jobId = request.getJobId();
  169. Job job = verifyAndGetJob(jobId);
  170. GetCountersResponse response = recordFactory.newRecordInstance(GetCountersResponse.class);
  171. response.setCounters(job.getCounters());
  172. return response;
  173. }
  174. @Override
  175. public GetJobReportResponse getJobReport(GetJobReportRequest request) throws YarnRemoteException {
  176. JobId jobId = request.getJobId();
  177. Job job = verifyAndGetJob(jobId);
  178. GetJobReportResponse response = recordFactory.newRecordInstance(GetJobReportResponse.class);
  179. response.setJobReport(job.getReport());
  180. return response;
  181. }
  182. @Override
  183. public GetTaskAttemptReportResponse getTaskAttemptReport(GetTaskAttemptReportRequest request) throws YarnRemoteException {
  184. TaskAttemptId taskAttemptId = request.getTaskAttemptId();
  185. Job job = verifyAndGetJob(taskAttemptId.getTaskId().getJobId());
  186. GetTaskAttemptReportResponse response = recordFactory.newRecordInstance(GetTaskAttemptReportResponse.class);
  187. response.setTaskAttemptReport(job.getTask(taskAttemptId.getTaskId()).getAttempt(taskAttemptId).getReport());
  188. return response;
  189. }
  190. @Override
  191. public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) throws YarnRemoteException {
  192. TaskId taskId = request.getTaskId();
  193. Job job = verifyAndGetJob(taskId.getJobId());
  194. GetTaskReportResponse response = recordFactory.newRecordInstance(GetTaskReportResponse.class);
  195. response.setTaskReport(job.getTask(taskId).getReport());
  196. return response;
  197. }
  198. @Override
  199. public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(GetTaskAttemptCompletionEventsRequest request) throws YarnRemoteException {
  200. JobId jobId = request.getJobId();
  201. int fromEventId = request.getFromEventId();
  202. int maxEvents = request.getMaxEvents();
  203. Job job = verifyAndGetJob(jobId);
  204. GetTaskAttemptCompletionEventsResponse response = recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
  205. response.addAllCompletionEvents(Arrays.asList(job.getTaskAttemptCompletionEvents(fromEventId, maxEvents)));
  206. return response;
  207. }
  208. @Override
  209. public KillJobResponse killJob(KillJobRequest request) throws YarnRemoteException {
  210. throw RPCUtil.getRemoteException("Invalid operation on completed job");
  211. }
  212. @Override
  213. public KillTaskResponse killTask(KillTaskRequest request) throws YarnRemoteException {
  214. throw RPCUtil.getRemoteException("Invalid operation on completed job");
  215. }
  216. @Override
  217. public KillTaskAttemptResponse killTaskAttempt(KillTaskAttemptRequest request) throws YarnRemoteException {
  218. throw RPCUtil.getRemoteException("Invalid operation on completed job");
  219. }
  220. @Override
  221. public GetDiagnosticsResponse getDiagnostics(GetDiagnosticsRequest request) throws YarnRemoteException {
  222. TaskAttemptId taskAttemptId = request.getTaskAttemptId();
  223. Job job = verifyAndGetJob(taskAttemptId.getTaskId().getJobId());
  224. GetDiagnosticsResponse response = recordFactory.newRecordInstance(GetDiagnosticsResponse.class);
  225. response.addAllDiagnostics(job.getTask(taskAttemptId.getTaskId()).getAttempt(taskAttemptId).getDiagnostics());
  226. return response;
  227. }
  228. @Override
  229. public FailTaskAttemptResponse failTaskAttempt(FailTaskAttemptRequest request) throws YarnRemoteException {
  230. throw RPCUtil.getRemoteException("Invalid operation on completed job");
  231. }
  232. @Override
  233. public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws YarnRemoteException {
  234. JobId jobId = request.getJobId();
  235. TaskType taskType = request.getTaskType();
  236. GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class);
  237. Job job = verifyAndGetJob(jobId);
  238. Collection<Task> tasks = job.getTasks(taskType).values();
  239. for (Task task : tasks) {
  240. response.addTaskReport(task.getReport());
  241. }
  242. return response;
  243. }
  244. private void checkAccess(Job job, JobACL jobOperation)
  245. throws YarnRemoteException {
  246. if (!UserGroupInformation.isSecurityEnabled()) {
  247. return;
  248. }
  249. UserGroupInformation callerUGI;
  250. try {
  251. callerUGI = UserGroupInformation.getCurrentUser();
  252. } catch (IOException e) {
  253. throw RPCUtil.getRemoteException(e);
  254. }
  255. if (!job.checkAccess(callerUGI, jobOperation)) {
  256. throw RPCUtil.getRemoteException(new AccessControlException("User "
  257. + callerUGI.getShortUserName() + " cannot perform operation "
  258. + jobOperation.name() + " on " + job.getID()));
  259. }
  260. }
  261. }
  262. }