/be/src/udf/udf-test-harness.h

https://gitlab.com/s9perween/Impala · C Header · 310 lines · 244 code · 28 blank · 38 comment · 23 complexity · 04561492fcf2d971016b9e6824afa3c8 MD5 · raw file

  1. // Copyright 2012 Cloudera Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. #ifndef IMPALA_UDF_TEST_HARNESS_H
  15. #define IMPALA_UDF_TEST_HARNESS_H
  16. #include <iostream>
  17. #include <vector>
  18. #include <boost/function.hpp>
  19. #include <boost/scoped_ptr.hpp>
  20. #include "udf/udf.h"
  21. #include "udf/udf-debug.h"
  22. namespace impala_udf {
  23. // Utility class to help test UDFs.
  24. class UdfTestHarness {
  25. public:
  26. // Create a test FunctionContext object. 'arg_types' should contain a TypeDesc for each
  27. // argument of the UDF not including the FunctionContext*. The caller is responsible
  28. // for calling delete on it. This context has additional debugging validation enabled.
  29. static FunctionContext* CreateTestContext(
  30. const std::vector<FunctionContext::TypeDesc>& arg_types);
  31. // Use with test contexts to test use of IsArgConstant() and GetConstantArg().
  32. // constant_args should contain an AnyVal* for each argument of the UDF not including
  33. // the FunctionContext*; constant_args[i] corresponds to the i-th argument.
  34. // Non-constant arguments should be set to NULL, and constant arguments should be set
  35. // to the constant value.
  36. //
  37. // The AnyVal* values are owned by the caller.
  38. //
  39. // Can only be called on contexts created by CreateTestContext().
  40. static void SetConstantArgs(
  41. FunctionContext* context, const std::vector<AnyVal*>& constant_args);
  42. // Test contexts should be closed in order to check for UDF memory leaks. Leaks cause
  43. // the error to be set on context.
  44. static void CloseContext(FunctionContext* context);
  45. // Template function to execute a UDF and validate the result. They should be
  46. // used like:
  47. // ValidateUdf(udf_fn, arg1, arg2, ..., expected_result);
  48. // Only functions with up to 8 arguments are supported
  49. //
  50. // For variable argument udfs, the variable arguments should be passed as
  51. // a std::vector:
  52. // ValidateUdf(udf_fn, arg1, arg2, const vector<arg3>& args, expected_result);
  53. template<typename RET>
  54. static bool ValidateUdf(boost::function<RET(FunctionContext*)> fn,
  55. const RET& expected, UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  56. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  57. std::vector<FunctionContext::TypeDesc> types;
  58. boost::scoped_ptr<FunctionContext> context(CreateTestContext(types));
  59. SetConstantArgs(context.get(), constant_args);
  60. if (!RunPrepareFn(init_fn, context.get())) return false;
  61. RET ret = fn(context.get());
  62. RunCloseFn(close_fn, context.get());
  63. CloseContext(context.get());
  64. return Validate(context.get(), expected, ret);
  65. }
  66. template<typename RET, typename A1>
  67. static bool ValidateUdf(boost::function<RET(FunctionContext*, const A1&)> fn,
  68. const A1& a1, const RET& expected, UdfPrepare init_fn = NULL,
  69. UdfClose close_fn = NULL,
  70. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  71. std::vector<FunctionContext::TypeDesc> types; // TODO
  72. boost::scoped_ptr<FunctionContext> context(CreateTestContext(types));
  73. SetConstantArgs(context.get(), constant_args);
  74. if (!RunPrepareFn(init_fn, context.get())) return false;
  75. RET ret = fn(context.get(), a1);
  76. RunCloseFn(close_fn, context.get());
  77. return Validate(context.get(), expected, ret);
  78. }
  79. template<typename RET, typename A1>
  80. static bool ValidateUdf(boost::function<RET(FunctionContext*, int, const A1*)> fn,
  81. const std::vector<A1>& a1, const RET& expected, UdfPrepare init_fn = NULL,
  82. UdfClose close_fn = NULL,
  83. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  84. std::vector<FunctionContext::TypeDesc> types; // TODO
  85. boost::scoped_ptr<FunctionContext> context(CreateTestContext(types));
  86. SetConstantArgs(context.get(), constant_args);
  87. if (!RunPrepareFn(init_fn, context.get())) return false;
  88. RET ret = fn(context.get(), a1.size(), &a1[0]);
  89. RunCloseFn(close_fn, context.get());
  90. return Validate(context.get(), expected, ret);
  91. }
  92. template<typename RET, typename A1, typename A2>
  93. static bool ValidateUdf(
  94. boost::function<RET(FunctionContext*, const A1&, const A2&)> fn,
  95. const A1& a1, const A2& a2, const RET& expected, UdfPrepare init_fn = NULL,
  96. UdfClose close_fn = NULL,
  97. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  98. std::vector<FunctionContext::TypeDesc> types; // TODO
  99. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  100. SetConstantArgs(context.get(), constant_args);
  101. if (!RunPrepareFn(init_fn, context.get())) return false;
  102. RET ret = fn(context.get(), a1, a2);
  103. RunCloseFn(close_fn, context.get());
  104. return Validate(context.get(), expected, ret);
  105. }
  106. template<typename RET, typename A1, typename A2>
  107. static bool ValidateUdf(
  108. boost::function<RET(FunctionContext*, const A1&, int, const A2*)> fn,
  109. const A1& a1, const std::vector<A2>& a2, const RET& expected,
  110. UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  111. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  112. std::vector<FunctionContext::TypeDesc> types; // TODO
  113. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  114. SetConstantArgs(context.get(), constant_args);
  115. if (!RunPrepareFn(init_fn, context.get())) return false;
  116. RET ret = fn(context.get(), a1, a2.size(), &a2[0]);
  117. RunCloseFn(close_fn, context.get());
  118. return Validate(context.get(), expected, ret);
  119. }
  120. template<typename RET, typename A1, typename A2, typename A3>
  121. static bool ValidateUdf(
  122. boost::function<RET(FunctionContext*, const A1&, const A2&, const A3&)> fn,
  123. const A1& a1, const A2& a2, const A3& a3, const RET& expected,
  124. UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  125. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  126. std::vector<FunctionContext::TypeDesc> types; // TODO
  127. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  128. SetConstantArgs(context.get(), constant_args);
  129. if (!RunPrepareFn(init_fn, context.get())) return false;
  130. RET ret = fn(context.get(), a1, a2, a3);
  131. RunCloseFn(close_fn, context.get());
  132. return Validate(context.get(), expected, ret);
  133. }
  134. template<typename RET, typename A1, typename A2, typename A3>
  135. static bool ValidateUdf(
  136. boost::function<RET(FunctionContext*, const A1&, const A2&, int, const A3*)> fn,
  137. const A1& a1, const A2& a2, const std::vector<A3>& a3, const RET& expected,
  138. UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  139. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  140. std::vector<FunctionContext::TypeDesc> types; // TODO
  141. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  142. SetConstantArgs(context.get(), constant_args);
  143. if (!RunPrepareFn(init_fn, context.get())) return false;
  144. RET ret = fn(context.get(), a1, a2, a3.size(), &a3[0]);
  145. RunCloseFn(close_fn, context.get());
  146. return Validate(context.get(), expected, ret);
  147. }
  148. template<typename RET, typename A1, typename A2, typename A3, typename A4>
  149. static bool ValidateUdf(
  150. boost::function<RET(FunctionContext*, const A1&, const A2&, const A3&,
  151. const A4&)> fn,
  152. const A1& a1, const A2& a2, const A3& a3, const A4& a4, const RET& expected,
  153. UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  154. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  155. std::vector<FunctionContext::TypeDesc> types; // TODO
  156. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  157. SetConstantArgs(context.get(), constant_args);
  158. if (!RunPrepareFn(init_fn, context.get())) return false;
  159. RET ret = fn(context.get(), a1, a2, a3, a4);
  160. RunCloseFn(close_fn, context.get());
  161. return Validate(context.get(), expected, ret);
  162. }
  163. template<typename RET, typename A1, typename A2, typename A3, typename A4>
  164. static bool ValidateUdf(
  165. boost::function<RET(FunctionContext*, const A1&, const A2&, const A3&,
  166. int, const A4*)> fn,
  167. const A1& a1, const A2& a2, const A3& a3, const std::vector<A4>& a4,
  168. const RET& expected, UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  169. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  170. std::vector<FunctionContext::TypeDesc> types; // TODO
  171. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  172. SetConstantArgs(context.get(), constant_args);
  173. if (!RunPrepareFn(init_fn, context.get())) return false;
  174. RET ret = fn(context.get(), a1, a2, a3, a4.size(), &a4[0]);
  175. RunCloseFn(close_fn, context.get());
  176. return Validate(context.get(), expected, ret);
  177. }
  178. template<typename RET, typename A1, typename A2, typename A3, typename A4,
  179. typename A5>
  180. static bool ValidateUdf(
  181. boost::function<RET(FunctionContext*, const A1&, const A2&, const A3&,
  182. const A4&, const A5&)> fn,
  183. const A1& a1, const A2& a2, const A3& a3, const A4& a4,const A5& a5,
  184. const RET& expected, UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  185. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  186. std::vector<FunctionContext::TypeDesc> types; // TODO
  187. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  188. SetConstantArgs(context.get(), constant_args);
  189. if (!RunPrepareFn(init_fn, context.get())) return false;
  190. RET ret = fn(context.get(), a1, a2, a3, a4, a5);
  191. RunCloseFn(close_fn, context.get());
  192. return Validate(context.get(), expected, ret);
  193. }
  194. template<typename RET, typename A1, typename A2, typename A3, typename A4,
  195. typename A5, typename A6>
  196. static bool ValidateUdf(
  197. boost::function<RET(FunctionContext*, const A1&, const A2&, const A3&,
  198. const A4&, const A5&, const A6&)> fn,
  199. const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& a5,
  200. const A6& a6, const RET& expected, UdfPrepare init_fn = NULL,
  201. UdfClose close_fn = NULL,
  202. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  203. std::vector<FunctionContext::TypeDesc> types; // TODO
  204. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  205. SetConstantArgs(context.get(), constant_args);
  206. if (!RunPrepareFn(init_fn, context.get())) return false;
  207. RET ret = fn(context.get(), a1, a2, a3, a4, a5, a6);
  208. RunCloseFn(close_fn, context.get());
  209. return Validate(context.get(), expected, ret);
  210. }
  211. template<typename RET, typename A1, typename A2, typename A3, typename A4,
  212. typename A5, typename A6, typename A7>
  213. static bool ValidateUdf(
  214. boost::function<RET(FunctionContext*, const A1&, const A2&, const A3&,
  215. const A4&, const A5&, const A6&, const A7&)> fn,
  216. const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& a5,
  217. const A6& a6, const A7& a7, const RET& expected, UdfPrepare init_fn = NULL,
  218. UdfClose close_fn = NULL,
  219. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  220. std::vector<FunctionContext::TypeDesc> types; // TODO
  221. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  222. SetConstantArgs(context.get(), constant_args);
  223. if (!RunPrepareFn(init_fn, context.get())) return false;
  224. RET ret = fn(context.get(), a1, a2, a3, a4, a5, a6, a7);
  225. RunCloseFn(close_fn, context.get());
  226. return Validate(context.get(), expected, ret);
  227. }
  228. template<typename RET, typename A1, typename A2, typename A3, typename A4,
  229. typename A5, typename A6, typename A7, typename A8>
  230. static bool ValidateUdf(
  231. boost::function<RET(FunctionContext*, const A1&, const A2&, const A3&,
  232. const A4&, const A5&, const A6&, const A7&)> fn,
  233. const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& a5,
  234. const A6& a6, const A7& a7, const A8& a8, const RET& expected,
  235. UdfPrepare init_fn = NULL, UdfClose close_fn = NULL,
  236. const std::vector<AnyVal*>& constant_args = std::vector<AnyVal*>()) {
  237. std::vector<FunctionContext::TypeDesc> types; // TODO
  238. boost::scoped_ptr<FunctionContext> context(UdfTestHarness::CreateTestContext(types));
  239. SetConstantArgs(context.get(), constant_args);
  240. if (!RunPrepareFn(init_fn, context.get())) return false;
  241. RET ret = fn(context.get(), a1, a2, a3, a4, a5, a6, a7, a8);
  242. RunCloseFn(close_fn, context.get());
  243. return Validate(context.get(), expected, ret);
  244. }
  245. private:
  246. static bool ValidateError(FunctionContext* context) {
  247. if (context->has_error()) {
  248. std::cerr << "Udf Failed: " << context->error_msg() << std::endl;
  249. return false;
  250. }
  251. return true;
  252. }
  253. template<typename RET>
  254. static bool Validate(FunctionContext* context, const RET& expected, const RET& actual) {
  255. bool valid = true;
  256. if (!context->has_error() && actual != expected) {
  257. std::cerr << "UDF did not return the correct result:" << std::endl
  258. << " Expected: " << DebugString(expected) << std::endl
  259. << " Actual: " << DebugString(actual) << std::endl;
  260. valid = false;
  261. }
  262. CloseContext(context);
  263. if (!ValidateError(context)) valid = false;
  264. return valid;
  265. }
  266. static bool RunPrepareFn(UdfPrepare prepare_fn, FunctionContext* context) {
  267. if (prepare_fn != NULL) {
  268. // TODO: FRAGMENT_LOCAL
  269. prepare_fn(context, FunctionContext::THREAD_LOCAL);
  270. if (!ValidateError(context)) return false;
  271. }
  272. return true;
  273. }
  274. static void RunCloseFn(UdfClose close_fn, FunctionContext* context) {
  275. if (close_fn != NULL) {
  276. // TODO: FRAGMENT_LOCAL
  277. close_fn(context, FunctionContext::THREAD_LOCAL);
  278. }
  279. }
  280. };
  281. }
  282. #endif