/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc
C++ | 444 lines | 368 code | 49 blank | 27 comment | 117 complexity | 2b81793f8bd7f6403dea7e7a40c42124 MD5 | raw file
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include <signal.h>
- #ifndef __CYGWIN__
- #include <execinfo.h>
- #endif
- #include "lib/commons.h"
- #include "NativeTask.h"
- #include "lib/NativeObjectFactory.h"
- #include "lib/NativeLibrary.h"
- #include "lib/BufferStream.h"
- #include "util/StringUtil.h"
- #include "util/SyncUtils.h"
- #include "util/WritableUtils.h"
- #include "handler/BatchHandler.h"
- #include "handler/MCollectorOutputHandler.h"
- #include "handler/CombineHandler.h"
- using namespace NativeTask;
- // TODO: just for debug, should be removed
- extern "C" void handler(int sig) {
- void *array[10];
- size_t size;
- // print out all the frames to stderr
- fprintf(stderr, "Error: signal %d:\n", sig);
- #ifndef __CYGWIN__
- // get void*'s for all entries on the stack
- size = backtrace(array, 10);
- backtrace_symbols_fd(array, size, 2);
- #endif
- exit(1);
- }
- DEFINE_NATIVE_LIBRARY(NativeTask) {
- REGISTER_CLASS(BatchHandler, NativeTask);
- REGISTER_CLASS(CombineHandler, NativeTask);
- REGISTER_CLASS(MCollectorOutputHandler, NativeTask);
- NativeObjectFactory::SetDefaultClass(BatchHandlerType, "NativeTask.BatchHandler");
- }
- namespace NativeTask {
- static Config G_CONFIG;
- vector<NativeLibrary *> NativeObjectFactory::Libraries;
- map<NativeObjectType, string> NativeObjectFactory::DefaultClasses;
- Config * NativeObjectFactory::GlobalConfig = &G_CONFIG;
- float NativeObjectFactory::LastProgress = 0;
- Progress * NativeObjectFactory::TaskProgress = NULL;
- string NativeObjectFactory::LastStatus;
- set<Counter *, CounterPtrCompare> NativeObjectFactory::CounterSet;
- vector<Counter *> NativeObjectFactory::Counters;
- vector<uint64_t> NativeObjectFactory::CounterLastUpdateValues;
- bool NativeObjectFactory::Inited = false;
- static Lock FactoryLock;
- bool NativeObjectFactory::Init() {
- ScopeLock<Lock> autolocak(FactoryLock);
- if (Inited == false) {
- // setup log device
- string device = GetConfig().get(NATIVE_LOG_DEVICE, "stderr");
- if (device == "stdout") {
- LOG_DEVICE = stdout;
- } else if (device == "stderr") {
- LOG_DEVICE = stderr;
- } else {
- LOG_DEVICE = fopen(device.c_str(), "w");
- }
- NativeTaskInit();
- NativeLibrary * library = new NativeLibrary("libnativetask.so", "NativeTask");
- library->_getObjectCreatorFunc = NativeTaskGetObjectCreator;
- Libraries.push_back(library);
- Inited = true;
- // load extra user provided libraries
- string libraryConf = GetConfig().get(NATIVE_CLASS_LIBRARY_BUILDIN, "");
- if (libraryConf.length() > 0) {
- vector<string> libraries;
- vector<string> pair;
- StringUtil::Split(libraryConf, ",", libraries, true);
- for (size_t i = 0; i < libraries.size(); i++) {
- pair.clear();
- StringUtil::Split(libraries[i], "=", pair, true);
- if (pair.size() == 2) {
- string & name = pair[0];
- string & path = pair[1];
- LOG("[NativeObjectLibrary] Try to load library [%s] with file [%s]", name.c_str(),
- path.c_str());
- if (false == RegisterLibrary(path, name)) {
- LOG("[NativeObjectLibrary] RegisterLibrary failed: name=%s path=%s", name.c_str(),
- path.c_str());
- return false;
- } else {
- LOG("[NativeObjectLibrary] RegisterLibrary success: name=%s path=%s", name.c_str(),
- path.c_str());
- }
- } else {
- LOG("[NativeObjectLibrary] Illegal native.class.libray: [%s] in [%s]",
- libraries[i].c_str(), libraryConf.c_str());
- }
- }
- }
- const char * version = GetConfig().get(NATIVE_HADOOP_VERSION);
- LOG("[NativeObjectLibrary] NativeTask library initialized with hadoop %s",
- version == NULL ? "unkown" : version);
- }
- return true;
- }
- void NativeObjectFactory::Release() {
- ScopeLock<Lock> autolocak(FactoryLock);
- for (ssize_t i = Libraries.size() - 1; i >= 0; i--) {
- delete Libraries[i];
- Libraries[i] = NULL;
- }
- Libraries.clear();
- for (size_t i = 0; i < Counters.size(); i++) {
- delete Counters[i];
- }
- Counters.clear();
- if (LOG_DEVICE != stdout && LOG_DEVICE != stderr) {
- fclose(LOG_DEVICE);
- LOG_DEVICE = stderr;
- }
- Inited = false;
- }
- void NativeObjectFactory::CheckInit() {
- if (Inited == false) {
- if (!Init()) {
- throw new IOException("Init NativeTask library failed.");
- }
- }
- }
- Config & NativeObjectFactory::GetConfig() {
- return *GlobalConfig;
- }
- Config * NativeObjectFactory::GetConfigPtr() {
- return GlobalConfig;
- }
- void NativeObjectFactory::SetTaskProgressSource(Progress * progress) {
- TaskProgress = progress;
- }
- float NativeObjectFactory::GetTaskProgress() {
- if (TaskProgress != NULL) {
- LastProgress = TaskProgress->getProgress();
- }
- return LastProgress;
- }
- void NativeObjectFactory::SetTaskStatus(const string & status) {
- LastStatus = status;
- }
- static Lock CountersLock;
- void NativeObjectFactory::GetTaskStatusUpdate(string & statusData) {
- // Encoding:
- // progress:float
- // status:Text
- // Counter number
- // Counters[group:Text, name:Text, incrCount:Long]
- OutputStringStream os(statusData);
- float progress = GetTaskProgress();
- WritableUtils::WriteFloat(&os, progress);
- WritableUtils::WriteText(&os, LastStatus);
- LastStatus.clear();
- {
- ScopeLock<Lock> AutoLock(CountersLock);
- uint32_t numCounter = (uint32_t)Counters.size();
- WritableUtils::WriteInt(&os, numCounter);
- for (size_t i = 0; i < numCounter; i++) {
- Counter * counter = Counters[i];
- uint64_t newCount = counter->get();
- uint64_t incr = newCount - CounterLastUpdateValues[i];
- CounterLastUpdateValues[i] = newCount;
- WritableUtils::WriteText(&os, counter->group());
- WritableUtils::WriteText(&os, counter->name());
- WritableUtils::WriteLong(&os, incr);
- }
- }
- }
- Counter * NativeObjectFactory::GetCounter(const string & group, const string & name) {
- ScopeLock<Lock> AutoLock(CountersLock);
- Counter tmpCounter(group, name);
- set<Counter *>::iterator itr = CounterSet.find(&tmpCounter);
- if (itr != CounterSet.end()) {
- return *itr;
- }
- Counter * ret = new Counter(group, name);
- Counters.push_back(ret);
- CounterLastUpdateValues.push_back(0);
- CounterSet.insert(ret);
- return ret;
- }
- void NativeObjectFactory::RegisterClass(const string & clz, ObjectCreatorFunc func) {
- NativeTaskClassMap__[clz] = func;
- }
- NativeObject * NativeObjectFactory::CreateObject(const string & clz) {
- ObjectCreatorFunc creator = GetObjectCreator(clz);
- return creator ? creator() : NULL;
- }
- void * NativeObjectFactory::GetFunction(const string & funcName) {
- CheckInit();
- {
- for (vector<NativeLibrary*>::reverse_iterator ritr = Libraries.rbegin();
- ritr != Libraries.rend(); ritr++) {
- void * ret = (*ritr)->getFunction(funcName);
- if (NULL != ret) {
- return ret;
- }
- }
- return NULL;
- }
- }
- ObjectCreatorFunc NativeObjectFactory::GetObjectCreator(const string & clz) {
- CheckInit();
- {
- for (vector<NativeLibrary*>::reverse_iterator ritr = Libraries.rbegin();
- ritr != Libraries.rend(); ritr++) {
- ObjectCreatorFunc ret = (*ritr)->getObjectCreator(clz);
- if (NULL != ret) {
- return ret;
- }
- }
- return NULL;
- }
- }
- void NativeObjectFactory::ReleaseObject(NativeObject * obj) {
- delete obj;
- }
- bool NativeObjectFactory::RegisterLibrary(const string & path, const string & name) {
- CheckInit();
- {
- NativeLibrary * library = new NativeLibrary(path, name);
- bool ret = library->init();
- if (!ret) {
- delete library;
- return false;
- }
- Libraries.push_back(library);
- return true;
- }
- }
- static Lock DefaultClassesLock;
- void NativeObjectFactory::SetDefaultClass(NativeObjectType type, const string & clz) {
- ScopeLock<Lock> autolocak(DefaultClassesLock);
- DefaultClasses[type] = clz;
- }
- NativeObject * NativeObjectFactory::CreateDefaultObject(NativeObjectType type) {
- CheckInit();
- {
- if (DefaultClasses.find(type) != DefaultClasses.end()) {
- string clz = DefaultClasses[type];
- return CreateObject(clz);
- }
- LOG("[NativeObjectLibrary] Default class for NativeObjectType %s not found",
- NativeObjectTypeToString(type).c_str());
- return NULL;
- }
- }
- int NativeObjectFactory::BytesComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- uint32_t minlen = std::min(srcLength, destLength);
- int64_t ret = fmemcmp(src, dest, minlen);
- if (ret > 0) {
- return 1;
- } else if (ret < 0) {
- return -1;
- }
- return srcLength - destLength;
- }
- int NativeObjectFactory::ByteComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- return (*src) - (*dest);
- }
- int NativeObjectFactory::IntComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- int result = (*src) - (*dest);
- if (result == 0) {
- uint32_t from = bswap(*(uint32_t*)src);
- uint32_t to = bswap(*(uint32_t*)dest);
- if (from > to) {
- return 1;
- } else if (from == to) {
- return 0;
- } else {
- return -1;
- }
- }
- return result;
- }
- int NativeObjectFactory::LongComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- int result = (int)(*src) - (int)(*dest);
- if (result == 0) {
- uint64_t from = bswap64(*(uint64_t*)src);
- uint64_t to = bswap64(*(uint64_t*)dest);
- if (from > to) {
- return 1;
- } else if (from == to) {
- return 0;
- } else {
- return -1;
- }
- }
- return result;
- }
- int NativeObjectFactory::VIntComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- int32_t from = WritableUtils::ReadVInt(src, srcLength);
- int32_t to = WritableUtils::ReadVInt(dest, destLength);
- if (from > to) {
- return 1;
- } else if (from == to) {
- return 0;
- } else {
- return -1;
- }
- }
- int NativeObjectFactory::VLongComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- int64_t from = WritableUtils::ReadVLong(src, srcLength);
- int64_t to = WritableUtils::ReadVLong(dest, destLength);
- if (from > to) {
- return 1;
- } else if (from == to) {
- return 0;
- } else {
- return -1;
- }
- }
- int NativeObjectFactory::FloatComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- if (srcLength != 4 || destLength != 4) {
- THROW_EXCEPTION_EX(IOException, "float comparator, while src/dest lengt is not 4");
- }
- uint32_t from = bswap(*(uint32_t*)src);
- uint32_t to = bswap(*(uint32_t*)dest);
- float * srcValue = (float *)(&from);
- float * destValue = (float *)(&to);
- if ((*srcValue) < (*destValue)) {
- return -1;
- } else if ((*srcValue) == (*destValue)) {
- return 0;
- } else {
- return 1;
- }
- }
- int NativeObjectFactory::DoubleComparator(const char * src, uint32_t srcLength, const char * dest,
- uint32_t destLength) {
- if (srcLength != 8 || destLength != 8) {
- THROW_EXCEPTION_EX(IOException, "double comparator, while src/dest lengt is not 4");
- }
- uint64_t from = bswap64(*(uint64_t*)src);
- uint64_t to = bswap64(*(uint64_t*)dest);
- double * srcValue = (double *)(&from);
- double * destValue = (double *)(&to);
- if ((*srcValue) < (*destValue)) {
- return -1;
- } else if ((*srcValue) == (*destValue)) {
- return 0;
- } else {
- return 1;
- }
- }
- ComparatorPtr get_comparator(const KeyValueType keyType, const char * comparatorName) {
- if (NULL == comparatorName) {
- if (keyType == BytesType || keyType == TextType) {
- return &NativeObjectFactory::BytesComparator;
- } else if (keyType == ByteType || keyType == BoolType) {
- return &NativeObjectFactory::ByteComparator;
- } else if (keyType == IntType) {
- return &NativeObjectFactory::IntComparator;
- } else if (keyType == LongType) {
- return &NativeObjectFactory::LongComparator;
- } else if (keyType == FloatType) {
- return &NativeObjectFactory::FloatComparator;
- } else if (keyType == DoubleType) {
- return &NativeObjectFactory::DoubleComparator;
- } else if (keyType == VIntType) {
- return &NativeObjectFactory::VIntComparator;
- } else if (keyType == VLongType) {
- return &NativeObjectFactory::VLongComparator;
- }
- } else {
- void * func = NativeObjectFactory::GetFunction(string(comparatorName));
- return (ComparatorPtr)func;
- }
- return NULL;
- }
- } // namespace NativeTask