PageRenderTime 76ms CodeModel.GetById 33ms RepoModel.GetById 0ms app.codeStats 0ms

/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc

https://gitlab.com/xiaoliuliu2050/hadoop
C++ | 444 lines | 368 code | 49 blank | 27 comment | 117 complexity | 2b81793f8bd7f6403dea7e7a40c42124 MD5 | raw file
  1. /*
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. #include <signal.h>
  19. #ifndef __CYGWIN__
  20. #include <execinfo.h>
  21. #endif
  22. #include "lib/commons.h"
  23. #include "NativeTask.h"
  24. #include "lib/NativeObjectFactory.h"
  25. #include "lib/NativeLibrary.h"
  26. #include "lib/BufferStream.h"
  27. #include "util/StringUtil.h"
  28. #include "util/SyncUtils.h"
  29. #include "util/WritableUtils.h"
  30. #include "handler/BatchHandler.h"
  31. #include "handler/MCollectorOutputHandler.h"
  32. #include "handler/CombineHandler.h"
  33. using namespace NativeTask;
  34. // TODO: just for debug, should be removed
  35. extern "C" void handler(int sig) {
  36. void *array[10];
  37. size_t size;
  38. // print out all the frames to stderr
  39. fprintf(stderr, "Error: signal %d:\n", sig);
  40. #ifndef __CYGWIN__
  41. // get void*'s for all entries on the stack
  42. size = backtrace(array, 10);
  43. backtrace_symbols_fd(array, size, 2);
  44. #endif
  45. exit(1);
  46. }
  47. DEFINE_NATIVE_LIBRARY(NativeTask) {
  48. REGISTER_CLASS(BatchHandler, NativeTask);
  49. REGISTER_CLASS(CombineHandler, NativeTask);
  50. REGISTER_CLASS(MCollectorOutputHandler, NativeTask);
  51. NativeObjectFactory::SetDefaultClass(BatchHandlerType, "NativeTask.BatchHandler");
  52. }
  53. namespace NativeTask {
  54. static Config G_CONFIG;
  55. vector<NativeLibrary *> NativeObjectFactory::Libraries;
  56. map<NativeObjectType, string> NativeObjectFactory::DefaultClasses;
  57. Config * NativeObjectFactory::GlobalConfig = &G_CONFIG;
  58. float NativeObjectFactory::LastProgress = 0;
  59. Progress * NativeObjectFactory::TaskProgress = NULL;
  60. string NativeObjectFactory::LastStatus;
  61. set<Counter *, CounterPtrCompare> NativeObjectFactory::CounterSet;
  62. vector<Counter *> NativeObjectFactory::Counters;
  63. vector<uint64_t> NativeObjectFactory::CounterLastUpdateValues;
  64. bool NativeObjectFactory::Inited = false;
  65. static Lock FactoryLock;
  66. bool NativeObjectFactory::Init() {
  67. ScopeLock<Lock> autolocak(FactoryLock);
  68. if (Inited == false) {
  69. // setup log device
  70. string device = GetConfig().get(NATIVE_LOG_DEVICE, "stderr");
  71. if (device == "stdout") {
  72. LOG_DEVICE = stdout;
  73. } else if (device == "stderr") {
  74. LOG_DEVICE = stderr;
  75. } else {
  76. LOG_DEVICE = fopen(device.c_str(), "w");
  77. }
  78. NativeTaskInit();
  79. NativeLibrary * library = new NativeLibrary("libnativetask.so", "NativeTask");
  80. library->_getObjectCreatorFunc = NativeTaskGetObjectCreator;
  81. Libraries.push_back(library);
  82. Inited = true;
  83. // load extra user provided libraries
  84. string libraryConf = GetConfig().get(NATIVE_CLASS_LIBRARY_BUILDIN, "");
  85. if (libraryConf.length() > 0) {
  86. vector<string> libraries;
  87. vector<string> pair;
  88. StringUtil::Split(libraryConf, ",", libraries, true);
  89. for (size_t i = 0; i < libraries.size(); i++) {
  90. pair.clear();
  91. StringUtil::Split(libraries[i], "=", pair, true);
  92. if (pair.size() == 2) {
  93. string & name = pair[0];
  94. string & path = pair[1];
  95. LOG("[NativeObjectLibrary] Try to load library [%s] with file [%s]", name.c_str(),
  96. path.c_str());
  97. if (false == RegisterLibrary(path, name)) {
  98. LOG("[NativeObjectLibrary] RegisterLibrary failed: name=%s path=%s", name.c_str(),
  99. path.c_str());
  100. return false;
  101. } else {
  102. LOG("[NativeObjectLibrary] RegisterLibrary success: name=%s path=%s", name.c_str(),
  103. path.c_str());
  104. }
  105. } else {
  106. LOG("[NativeObjectLibrary] Illegal native.class.libray: [%s] in [%s]",
  107. libraries[i].c_str(), libraryConf.c_str());
  108. }
  109. }
  110. }
  111. const char * version = GetConfig().get(NATIVE_HADOOP_VERSION);
  112. LOG("[NativeObjectLibrary] NativeTask library initialized with hadoop %s",
  113. version == NULL ? "unkown" : version);
  114. }
  115. return true;
  116. }
  117. void NativeObjectFactory::Release() {
  118. ScopeLock<Lock> autolocak(FactoryLock);
  119. for (ssize_t i = Libraries.size() - 1; i >= 0; i--) {
  120. delete Libraries[i];
  121. Libraries[i] = NULL;
  122. }
  123. Libraries.clear();
  124. for (size_t i = 0; i < Counters.size(); i++) {
  125. delete Counters[i];
  126. }
  127. Counters.clear();
  128. if (LOG_DEVICE != stdout && LOG_DEVICE != stderr) {
  129. fclose(LOG_DEVICE);
  130. LOG_DEVICE = stderr;
  131. }
  132. Inited = false;
  133. }
  134. void NativeObjectFactory::CheckInit() {
  135. if (Inited == false) {
  136. if (!Init()) {
  137. throw new IOException("Init NativeTask library failed.");
  138. }
  139. }
  140. }
  141. Config & NativeObjectFactory::GetConfig() {
  142. return *GlobalConfig;
  143. }
  144. Config * NativeObjectFactory::GetConfigPtr() {
  145. return GlobalConfig;
  146. }
  147. void NativeObjectFactory::SetTaskProgressSource(Progress * progress) {
  148. TaskProgress = progress;
  149. }
  150. float NativeObjectFactory::GetTaskProgress() {
  151. if (TaskProgress != NULL) {
  152. LastProgress = TaskProgress->getProgress();
  153. }
  154. return LastProgress;
  155. }
  156. void NativeObjectFactory::SetTaskStatus(const string & status) {
  157. LastStatus = status;
  158. }
  159. static Lock CountersLock;
  160. void NativeObjectFactory::GetTaskStatusUpdate(string & statusData) {
  161. // Encoding:
  162. // progress:float
  163. // status:Text
  164. // Counter number
  165. // Counters[group:Text, name:Text, incrCount:Long]
  166. OutputStringStream os(statusData);
  167. float progress = GetTaskProgress();
  168. WritableUtils::WriteFloat(&os, progress);
  169. WritableUtils::WriteText(&os, LastStatus);
  170. LastStatus.clear();
  171. {
  172. ScopeLock<Lock> AutoLock(CountersLock);
  173. uint32_t numCounter = (uint32_t)Counters.size();
  174. WritableUtils::WriteInt(&os, numCounter);
  175. for (size_t i = 0; i < numCounter; i++) {
  176. Counter * counter = Counters[i];
  177. uint64_t newCount = counter->get();
  178. uint64_t incr = newCount - CounterLastUpdateValues[i];
  179. CounterLastUpdateValues[i] = newCount;
  180. WritableUtils::WriteText(&os, counter->group());
  181. WritableUtils::WriteText(&os, counter->name());
  182. WritableUtils::WriteLong(&os, incr);
  183. }
  184. }
  185. }
  186. Counter * NativeObjectFactory::GetCounter(const string & group, const string & name) {
  187. ScopeLock<Lock> AutoLock(CountersLock);
  188. Counter tmpCounter(group, name);
  189. set<Counter *>::iterator itr = CounterSet.find(&tmpCounter);
  190. if (itr != CounterSet.end()) {
  191. return *itr;
  192. }
  193. Counter * ret = new Counter(group, name);
  194. Counters.push_back(ret);
  195. CounterLastUpdateValues.push_back(0);
  196. CounterSet.insert(ret);
  197. return ret;
  198. }
  199. void NativeObjectFactory::RegisterClass(const string & clz, ObjectCreatorFunc func) {
  200. NativeTaskClassMap__[clz] = func;
  201. }
  202. NativeObject * NativeObjectFactory::CreateObject(const string & clz) {
  203. ObjectCreatorFunc creator = GetObjectCreator(clz);
  204. return creator ? creator() : NULL;
  205. }
  206. void * NativeObjectFactory::GetFunction(const string & funcName) {
  207. CheckInit();
  208. {
  209. for (vector<NativeLibrary*>::reverse_iterator ritr = Libraries.rbegin();
  210. ritr != Libraries.rend(); ritr++) {
  211. void * ret = (*ritr)->getFunction(funcName);
  212. if (NULL != ret) {
  213. return ret;
  214. }
  215. }
  216. return NULL;
  217. }
  218. }
  219. ObjectCreatorFunc NativeObjectFactory::GetObjectCreator(const string & clz) {
  220. CheckInit();
  221. {
  222. for (vector<NativeLibrary*>::reverse_iterator ritr = Libraries.rbegin();
  223. ritr != Libraries.rend(); ritr++) {
  224. ObjectCreatorFunc ret = (*ritr)->getObjectCreator(clz);
  225. if (NULL != ret) {
  226. return ret;
  227. }
  228. }
  229. return NULL;
  230. }
  231. }
  232. void NativeObjectFactory::ReleaseObject(NativeObject * obj) {
  233. delete obj;
  234. }
  235. bool NativeObjectFactory::RegisterLibrary(const string & path, const string & name) {
  236. CheckInit();
  237. {
  238. NativeLibrary * library = new NativeLibrary(path, name);
  239. bool ret = library->init();
  240. if (!ret) {
  241. delete library;
  242. return false;
  243. }
  244. Libraries.push_back(library);
  245. return true;
  246. }
  247. }
  248. static Lock DefaultClassesLock;
  249. void NativeObjectFactory::SetDefaultClass(NativeObjectType type, const string & clz) {
  250. ScopeLock<Lock> autolocak(DefaultClassesLock);
  251. DefaultClasses[type] = clz;
  252. }
  253. NativeObject * NativeObjectFactory::CreateDefaultObject(NativeObjectType type) {
  254. CheckInit();
  255. {
  256. if (DefaultClasses.find(type) != DefaultClasses.end()) {
  257. string clz = DefaultClasses[type];
  258. return CreateObject(clz);
  259. }
  260. LOG("[NativeObjectLibrary] Default class for NativeObjectType %s not found",
  261. NativeObjectTypeToString(type).c_str());
  262. return NULL;
  263. }
  264. }
  265. int NativeObjectFactory::BytesComparator(const char * src, uint32_t srcLength, const char * dest,
  266. uint32_t destLength) {
  267. uint32_t minlen = std::min(srcLength, destLength);
  268. int64_t ret = fmemcmp(src, dest, minlen);
  269. if (ret > 0) {
  270. return 1;
  271. } else if (ret < 0) {
  272. return -1;
  273. }
  274. return srcLength - destLength;
  275. }
  276. int NativeObjectFactory::ByteComparator(const char * src, uint32_t srcLength, const char * dest,
  277. uint32_t destLength) {
  278. return (*src) - (*dest);
  279. }
  280. int NativeObjectFactory::IntComparator(const char * src, uint32_t srcLength, const char * dest,
  281. uint32_t destLength) {
  282. int result = (*src) - (*dest);
  283. if (result == 0) {
  284. uint32_t from = bswap(*(uint32_t*)src);
  285. uint32_t to = bswap(*(uint32_t*)dest);
  286. if (from > to) {
  287. return 1;
  288. } else if (from == to) {
  289. return 0;
  290. } else {
  291. return -1;
  292. }
  293. }
  294. return result;
  295. }
  296. int NativeObjectFactory::LongComparator(const char * src, uint32_t srcLength, const char * dest,
  297. uint32_t destLength) {
  298. int result = (int)(*src) - (int)(*dest);
  299. if (result == 0) {
  300. uint64_t from = bswap64(*(uint64_t*)src);
  301. uint64_t to = bswap64(*(uint64_t*)dest);
  302. if (from > to) {
  303. return 1;
  304. } else if (from == to) {
  305. return 0;
  306. } else {
  307. return -1;
  308. }
  309. }
  310. return result;
  311. }
  312. int NativeObjectFactory::VIntComparator(const char * src, uint32_t srcLength, const char * dest,
  313. uint32_t destLength) {
  314. int32_t from = WritableUtils::ReadVInt(src, srcLength);
  315. int32_t to = WritableUtils::ReadVInt(dest, destLength);
  316. if (from > to) {
  317. return 1;
  318. } else if (from == to) {
  319. return 0;
  320. } else {
  321. return -1;
  322. }
  323. }
  324. int NativeObjectFactory::VLongComparator(const char * src, uint32_t srcLength, const char * dest,
  325. uint32_t destLength) {
  326. int64_t from = WritableUtils::ReadVLong(src, srcLength);
  327. int64_t to = WritableUtils::ReadVLong(dest, destLength);
  328. if (from > to) {
  329. return 1;
  330. } else if (from == to) {
  331. return 0;
  332. } else {
  333. return -1;
  334. }
  335. }
  336. int NativeObjectFactory::FloatComparator(const char * src, uint32_t srcLength, const char * dest,
  337. uint32_t destLength) {
  338. if (srcLength != 4 || destLength != 4) {
  339. THROW_EXCEPTION_EX(IOException, "float comparator, while src/dest lengt is not 4");
  340. }
  341. uint32_t from = bswap(*(uint32_t*)src);
  342. uint32_t to = bswap(*(uint32_t*)dest);
  343. float * srcValue = (float *)(&from);
  344. float * destValue = (float *)(&to);
  345. if ((*srcValue) < (*destValue)) {
  346. return -1;
  347. } else if ((*srcValue) == (*destValue)) {
  348. return 0;
  349. } else {
  350. return 1;
  351. }
  352. }
  353. int NativeObjectFactory::DoubleComparator(const char * src, uint32_t srcLength, const char * dest,
  354. uint32_t destLength) {
  355. if (srcLength != 8 || destLength != 8) {
  356. THROW_EXCEPTION_EX(IOException, "double comparator, while src/dest lengt is not 4");
  357. }
  358. uint64_t from = bswap64(*(uint64_t*)src);
  359. uint64_t to = bswap64(*(uint64_t*)dest);
  360. double * srcValue = (double *)(&from);
  361. double * destValue = (double *)(&to);
  362. if ((*srcValue) < (*destValue)) {
  363. return -1;
  364. } else if ((*srcValue) == (*destValue)) {
  365. return 0;
  366. } else {
  367. return 1;
  368. }
  369. }
  370. ComparatorPtr get_comparator(const KeyValueType keyType, const char * comparatorName) {
  371. if (NULL == comparatorName) {
  372. if (keyType == BytesType || keyType == TextType) {
  373. return &NativeObjectFactory::BytesComparator;
  374. } else if (keyType == ByteType || keyType == BoolType) {
  375. return &NativeObjectFactory::ByteComparator;
  376. } else if (keyType == IntType) {
  377. return &NativeObjectFactory::IntComparator;
  378. } else if (keyType == LongType) {
  379. return &NativeObjectFactory::LongComparator;
  380. } else if (keyType == FloatType) {
  381. return &NativeObjectFactory::FloatComparator;
  382. } else if (keyType == DoubleType) {
  383. return &NativeObjectFactory::DoubleComparator;
  384. } else if (keyType == VIntType) {
  385. return &NativeObjectFactory::VIntComparator;
  386. } else if (keyType == VLongType) {
  387. return &NativeObjectFactory::VLongComparator;
  388. }
  389. } else {
  390. void * func = NativeObjectFactory::GetFunction(string(comparatorName));
  391. return (ComparatorPtr)func;
  392. }
  393. return NULL;
  394. }
  395. } // namespace NativeTask