PageRenderTime 80ms CodeModel.GetById 15ms RepoModel.GetById 0ms app.codeStats 1ms

/test/e2e/pig/tests/nightly.conf

https://github.com/zjffdu/pig
Perl | 4623 lines | 4186 code | 112 blank | 325 comment | 102 complexity | 735ef9c41158ef2319bd7bea7c6a5f6f MD5 | raw file
Possible License(s): Apache-2.0, CPL-1.0 
    

  1. #!/usr/bin/env perl
    2. 

  2. 

  3. ############################################################################           
    4. 

  4. #  Licensed to the Apache Software Foundation (ASF) under one or more                  
    5. 

  5. #  contributor license agreements.  See the NOTICE file distributed with               
    6. 

  6. #  this work for additional information regarding copyright ownership.                 
    7. 

  7. #  The ASF licenses this file to You under the Apache License, Version 2.0             
    8. 

  8. #  (the "License"); you may not use this file except in compliance with                
    9. 

  9. #  the License.  You may obtain a copy of the License at                               
    10. 

  10. #                                                                                      
    11. 

  11. #      http://www.apache.org/licenses/LICENSE-2.0                                      
    12. 

  12. #                                                                                      
    13. 

  13. #  Unless required by applicable law or agreed to in writing, software                 
    14. 

  14. #  distributed under the License is distributed on an "AS IS" BASIS,                   
    15. 

  15. #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.            
    16. 

  16. #  See the License for the specific language governing permissions and                 
    17. 

  17. #  limitations under the License.                                                      
    18. 

  18.                                                                                        
    19. 

  19. ###############################################################################
    20. 

  20. # Nightly tests for pig.
    21. 

  21. #
    22. 

  22. #
    23. 

  23. 

  24. #PigSetup::setup();
    25. 

  25. 

  26. #my $me = `whoami`;
    27. 

  27. #chomp $me;
    28. 

  28. 

  29. $cfg = {
    30. 

  30. 	'driver' => 'Pig',
    31. 

  31. 	'nummachines' => 5,
    32. 

  32. 	'verify_with_pig' => 1,
    33. 

  33. 	'verify_pig_version' => 'old',
    34. 

  34. 

  35. 	'groups' => [
    36. 

  36. 		{
    37. 

  37. 		'name' => 'Checkin',
    38. 

  38. 		'tests' => [
    39. 

  39. 			{
    40. 

  40. 			'num' => 1,
    41. 

  41. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    42. 

  42. store a into ':OUTPATH:';\,
    43. 

  43. 			},
    44. 

  44. 			{
    45. 

  45. 			'num' => 2,
    46. 

  46. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    47. 

  47. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    48. 

  48. c = filter a by age < 50;
    49. 

  49. d = filter b by age < 50;
    50. 

  50. e = cogroup c by (name, age), d by (name, age) ;
    51. 

  51. f = foreach e generate flatten(c), flatten(d);
    52. 

  52. g = group f by registration;
    53. 

  53. h = foreach g generate group, SUM(f.d::contributions);
    54. 

  54. i = order h by $1;
    55. 

  55. store i into ':OUTPATH:';\,
    56. 

  56.                         'floatpostprocess' => 1,
    57. 

  57.                         'delimiter' => '	',
    58. 

  58. 			'sortArgs' => ['-t', '	', '+1', '-2'],
    59. 

  59. 			}
    60. 

  60. 		]
    61. 

  61. 		},
    62. 

  62. 		{
    63. 

  63. 		'name' => 'LoaderDefaultDir',
    64. 

  64. 		'tests' => [
    65. 

  65. 			{
    66. 

  66. 			'num' => 1,
    67. 

  67. 			'pig' => q\a = load ':INPATH:/dir/studenttab10k' as (name, age, gpa);
    68. 

  68. store a into ':OUTPATH:';\,
    69. 

  69. 			},
    70. 

  70. 		]
    71. 

  71. 		},
    72. 

  72. 		{
    73. 

  73. 		'name' => 'LoaderPigStorageArg',
    74. 

  74. 		'tests' => [
    75. 

  75. 			{
    76. 

  76. 			'num' => 1,
    77. 

  77. 			'pig' => q\a = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa);
    78. 

  78. store a into ':OUTPATH:';\,
    79. 

  79. 			},
    80. 

  80. 			{
    81. 

  81. 			# load with control character
    82. 

  82. 			'num' => 2,
    83. 

  83. 			'pig' => q#a = load ':INPATH:/singlefile/studentctrla10k' using PigStorage('\\u0001') as (name, age, gpa);
    84. 

  84. store a into ':OUTPATH:';#,
    85. 

  85. 			},
    86. 

  86. 			{
    87. 

  87. 	        	# load and store with control character
    88. 

  88. 			'num' => 3,
    89. 

  89. 			'pig' => q#a = load ':INPATH:/singlefile/studentctrla10k' using PigStorage('\\u0001') as (name, age, gpa);
    90. 

  90. store a into ':OUTPATH:.intermediate' using PigStorage('\\u0001');
    91. 

  91. b = load ':OUTPATH:.intermediate' using PigStorage('\\u0001') as (name, age, gpa); 
    92. 

  92. store b into ':OUTPATH:'; #,
    93. 

  93.             'notmq' => 1,
    94. 

  94. 			},
    95. 

  95. 		]
    96. 

  96. 		},
    97. 

  97. 		{
    98. 

  98. 		# Results doctored, if you change this query you need to copy the
    99. 

  99. 		# expected results into test/nightly/benchmarks
    100. 

  100. 		'name' => 'LoaderBinStorage',
    101. 

  101. 		'tests' => [
    102. 

  102. 			{
    103. 

  103. 			'num' => 1,
    104. 

  104. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    105. 

  105. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    106. 

  106. b = foreach a generate name, org.apache.pig.test.udf.evalfunc.Swap(name, age), TOKENIZE((chararray)name), org.apache.pig.test.udf.evalfunc.CreateMap((chararray)name, age); 
    107. 

  107. store b into ':OUTPATH:.intermediate' using BinStorage();
    108. 

  108. c = load ':OUTPATH:.intermediate' using BinStorage();
    109. 

  109. store c into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
    110. 

  110.             'notmq' => 1,
    111. 

  111. 			},
    112. 

  112. 		]
    113. 

  113. 		},
    114. 

  114. 		{
    115. 

  115. 		# Results doctored, if you change this query you need to copy the
    116. 

  116. 		# expected results into test/nightly/benchmarks
    117. 

  117. 		'name' => 'LoaderTextLoader',
    118. 

  118. 		'tests' => [
    119. 

  119. 			{
    120. 

  120. 			'num' => 1,
    121. 

  121. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    122. 

  122. a = load ':INPATH:/singlefile/textdoc' using TextLoader();
    123. 

  123. b = foreach a generate TOKENIZE((chararray)$0);
    124. 

  124. store b into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
    125. 

  125. 			},
    126. 

  126. 		]
    127. 

  127. 		},
    128. 

  128. 		{
    129. 

  129. 		'name' => 'FilterBoolean',
    130. 

  130. 		'tests' => [
    131. 

  131. 			{
    132. 

  132. 			'num' => 1,
    133. 

  133. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    134. 

  134. b = filter a by name == 'fred allen' and age > 50;
    135. 

  135. store b into ':OUTPATH:' using PigStorage;\,
    136. 

  136. 			},
    137. 

  137. 			{
    138. 

  138. 			'num' => 2,
    139. 

  139. 			'pig' => q\a = load ':INPATH:/dir/studenttab10k' using PigStorage() as (name, age, gpa);
    140. 

  140. b = filter a by name != 'fred allen' or age < 10;
    141. 

  141. store b into ':OUTPATH:' using PigStorage;\,
    142. 

  142. 			},
    143. 

  143. 			{
    144. 

  144. 			'num' => 3,
    145. 

  145. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    146. 

  146. b = filter a by not (age == 50);
    147. 

  147. store b into ':OUTPATH:' using PigStorage;\,
    148. 

  148. 			},
    149. 

  149. 			{
    150. 

  150. 			'num' => 4,
    151. 

  151. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    152. 

  152. b = filter a by (age >= 50 or name > 'fred') and (gpa <= 3.0 or name >= 'bob');
    153. 

  153. store b into ':OUTPATH:' using PigStorage;\,
    154. 

  154. 			},
    155. 

  155. 			{
    156. 

  156. 			'num' => 5,
    157. 

  157. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    158. 

  158. b = filter a by age >= 50 or name > 'fred' and gpa <= 3.0 or name >= 'bob';
    159. 

  159. store b into ':OUTPATH:' using PigStorage;\,
    160. 

  160. 			},
    161. 

  161.             # test filter <= and >= for chararray, int and double
    162. 

  162. 			{
    163. 

  163. 			'num' => 6,
    164. 

  164. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    165. 

  165. b = filter a by age >= 40 and age <=50 and gpa >= 2.0 and gpa <= 3.0 and  name >= 'bob' and name <= 'fred';
    166. 

  166. store b into ':OUTPATH:' using PigStorage;\,
    167. 

  167. 			},
    168. 

  168.             # test filter <= and >= for bytearray, long and float
    169. 

  169. 			{
    170. 

  170. 			'num' => 7,
    171. 

  171. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:long, gpa:float);
    172. 

  172. b = filter a by age >= 40 and age <=50 and gpa >= 2.0f and gpa <= 3.0f and  name >= 'bob' and name <= 'fred';
    173. 

  173. store b into ':OUTPATH:' using PigStorage;\,
    174. 

  174. 			},
    175. 

  175.             # test filter < and > for chararray, int and double
    176. 

  176. 			{
    177. 

  177. 			'num' => 8,
    178. 

  178. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    179. 

  179. b = filter a by age > 40 and age <50 and gpa > 2.0 and gpa < 3.0 and  name > 'bob' and name < 'fred';
    180. 

  180. store b into ':OUTPATH:' using PigStorage;\,
    181. 

  181. 			},
    182. 

  182.             # test filter < and > for bytearray, long and float
    183. 

  183. 			{
    184. 

  184. 			'num' => 9,
    185. 

  185. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:long, gpa:float);
    186. 

  186. b = filter a by age > 40 and age <50 and gpa > 2.0f and gpa < 3.0f and  name > 'bob' and name < 'fred';
    187. 

  187. store b into ':OUTPATH:' using PigStorage;\,
    188. 

  188. 			},
    189. 

  189.             # test filter <= and >= for explicit cast for chararray, int and double
    190. 

  190. 			{
    191. 

  191. 			'num' => 10,
    192. 

  192. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    193. 

  193. b = filter a by (int)age >= 40 and (int)age <=50 and (double)gpa >= 2.0 and (double)gpa <= 3.0 and  (chararray)name >= 'bob' and (chararray)name <= 'fred';
    194. 

  194. store b into ':OUTPATH:' using PigStorage;\,
    195. 

  195. 			},
    196. 

  196.             # test filter <= and >= for explicit cast for bytearray, long and float
    197. 

  197. 			{
    198. 

  198. 			'num' => 11,
    199. 

  199. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    200. 

  200. b = filter a by (long)age >= 40 and (long)age <=50 and (float)gpa >= 2.0f and (float)gpa <= 3.0f and  name >= 'bob' and name <= 'fred';
    201. 

  201. store b into ':OUTPATH:' using PigStorage;\,
    202. 

  202. 			},
    203. 

  203.             # test filter < and > for explicit cast for chararray, int and double
    204. 

  204. 			{
    205. 

  205. 			'num' => 12,
    206. 

  206. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    207. 

  207. b = filter a by (int)age > 40 and (int)age <50 and (double)gpa > 2.0 and (double)gpa < 3.0 and  (chararray)name > 'bob' and (chararray)name < 'fred';
    208. 

  208. store b into ':OUTPATH:' using PigStorage;\,
    209. 

  209. 			},
    210. 

  210.             # test filter < and > for explicit cast for bytearray, long and float
    211. 

  211. 			{
    212. 

  212. 			'num' => 13,
    213. 

  213. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    214. 

  214. b = filter a by (long)age > 40 and (long)age <50 and (float)gpa > 2.0f and (float)gpa < 3.0f and  name > 'bob' and name < 'fred';
    215. 

  215. store b into ':OUTPATH:' using PigStorage;\,
    216. 

  216. 			},
    217. 

  217.             # test AND with nulls
    218. 

  218. 			{
    219. 

  219. 			'num' => 14,
    220. 

  220. 			'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' using PigStorage() as (name, age, gpa);
    221. 

  221. b = filter a by name == 'fred allen' and age > 50;
    222. 

  222. store b into ':OUTPATH:' using PigStorage;\,
    223. 

  223. 			},
    224. 

  224.             # test OR with nulls
    225. 

  225. 			{
    226. 

  226. 			'num' => 15,
    227. 

  227. 			'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' using PigStorage() as (name, age, gpa);
    228. 

  228. b = filter a by name != 'fred allen' or age < 10;
    229. 

  229. store b into ':OUTPATH:' using PigStorage;\,
    230. 

  230. 			},
    231. 

  231.             # test with nulls filter <= and >= for chararray, int and double
    232. 

  232. 			{
    233. 

  233. 			'num' => 16,
    234. 

  234. 			'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    235. 

  235. b = filter a by age >= 40 and age <=50 and gpa >= 2.0 and gpa <= 3.0 and  name >= 'bob' and name <= 'fred';
    236. 

  236. store b into ':OUTPATH:' using PigStorage;\,
    237. 

  237. 			},
    238. 

  238.             # test with nulls filter < and > for explicit cast for chararray, int and double
    239. 

  239. 			{
    240. 

  240. 			'num' => 17,
    241. 

  241. 			'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' using PigStorage() as (name, age, gpa);
    242. 

  242. b = filter a by (int)age > 40 and (int)age <50 and (double)gpa > 2.0 and (double)gpa < 3.0 and  (chararray)name > 'bob' and (chararray)name < 'fred';
    243. 

  243. store b into ':OUTPATH:' using PigStorage;\,
    244. 

  244. 			},
    245. 

  245.                         {
    246. 

  246.                         'num' => 18,
    247. 

  247.                         'ignore' => 1, # PIG-2593
    248. 

  248.                         'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    249. 

  249. b = filter a by instate;
    250. 

  250. store b into ':OUTPATH:' using PigStorage;\,
    251. 

  251.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    252. 

  252. b = filter a by instate == 'true';
    253. 

  253. store b into ':OUTPATH:' using PigStorage;\,
    254. 

  254.                         },
    255. 

  255.                         {
    256. 

  256.                         'num' => 19,
    257. 

  257.                         'ignore' => 1, # PIG-2593
    258. 

  258. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    259. 

  259. b = filter a by not instate;
    260. 

  260. store b into ':OUTPATH:' using PigStorage;\,
    261. 

  261.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    262. 

  262. b = filter a by instate == 'false';
    263. 

  263. store b into ':OUTPATH:' using PigStorage;\,
    264. 

  264.                         },
    265. 

  265.                         {
    266. 

  266.                         'num' => 20,
    267. 

  267. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    268. 

  268. b = filter a by instate is null;
    269. 

  269. store b into ':OUTPATH:' using PigStorage;\,
    270. 

  270.                         },
    271. 

  271.                         {
    272. 

  272.                         'num' => 21,
    273. 

  273.                         'ignore' => 1, # TODO Need to file a JIRA-2
    274. 

  274. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    275. 

  275. b = filter a by instate == true;
    276. 

  276. store b into ':OUTPATH:' using PigStorage;\,
    277. 

  277.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    278. 

  278. b = filter a by instate == 'true';
    279. 

  279. store b into ':OUTPATH:' using PigStorage;\,
    280. 

  280.                         },
    281. 

  281.                         {
    282. 

  282.                         'num' => 22,
    283. 

  283.                         'ignore' => 1, # TODO Need to file a JIRA-2
    284. 

  284. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    285. 

  285. b = filter a by instate == false;
    286. 

  286. store b into ':OUTPATH:' using PigStorage;\,
    287. 

  287.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name, age, gpa, instate);
    288. 

  288. b = filter a by instate == 'false';
    289. 

  289. store b into ':OUTPATH:' using PigStorage;\,
    290. 

  290.                         },
    291. 

  291.                         {
    292. 

  292.                         'num' => 23,
    293. 

  293.                         'ignore' => 1, # TODO Need to file a JIRA-1
    294. 

  294. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    295. 

  295. b = filter a by instate;
    296. 

  296. store b into ':OUTPATH:' using PigStorage;\,
    297. 

  297.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    298. 

  298. b = filter a by instate == 'true';
    299. 

  299. store b into ':OUTPATH:' using PigStorage;\,
    300. 

  300.                         },
    301. 

  301.                         {
    302. 

  302.                         'num' => 24,
    303. 

  303.                         'ignore' => 1, # TODO Need to file a JIRA-1
    304. 

  304. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    305. 

  305. b = filter a by not instate;
    306. 

  306. store b into ':OUTPATH:' using PigStorage;\,
    307. 

  307.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    308. 

  308. b = filter a by instate == 'false';
    309. 

  309. store b into ':OUTPATH:' using PigStorage;\,
    310. 

  310.                         },
    311. 

  311.                         {
    312. 

  312.                         'num' => 25,
    313. 

  313. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    314. 

  314. b = filter a by instate is null;
    315. 

  315. store b into ':OUTPATH:' using PigStorage;\,
    316. 

  316.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    317. 

  317. b = filter a by instate is null;
    318. 

  318. store b into ':OUTPATH:' using PigStorage;\,
    319. 

  319.                         },
    320. 

  320.                         {
    321. 

  321.                         'num' => 26,
    322. 

  322. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    323. 

  323. b = filter a by instate == true;
    324. 

  324. store b into ':OUTPATH:' using PigStorage;\,
    325. 

  325.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    326. 

  326. b = filter a by instate == 'true';
    327. 

  327. store b into ':OUTPATH:' using PigStorage;\,
    328. 

  328.                         },
    329. 

  329.                         {
    330. 

  330.                         'num' => 27,
    331. 

  331. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    332. 

  332. b = filter a by instate == false;
    333. 

  333. store b into ':OUTPATH:' using PigStorage;\,
    334. 

  334.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    335. 

  335. b = filter a by instate == 'false';
    336. 

  336. store b into ':OUTPATH:' using PigStorage;\,
    337. 

  337.                         },
    338. 

  338. 		    ],
    339. 

  339. 		},
    340. 

  340. 		{
    341. 

  341. 		'name' => 'FilterEq',
    342. 

  342. 		'tests' => [
    343. 

  343. 			{
    344. 

  344. 			'num' => 1,
    345. 

  345. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    346. 

  346. b = filter a by name == 'alice johnson' and age == 64 and gpa == 3.99;
    347. 

  347. store b into ':OUTPATH:' using PigStorage;\,
    348. 

  348. 			},
    349. 

  349. 			{
    350. 

  350. 			'num' => 2,
    351. 

  351. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    352. 

  352. b = filter a by name > 'fred allen' and age > 40 and gpa > 2.50;
    353. 

  353. store b into ':OUTPATH:' using PigStorage;\,
    354. 

  354. 			},
    355. 

  355. 			{
    356. 

  356. 			'num' => 3,
    357. 

  357. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    358. 

  358. b = filter a by name >= 'fred allen' and age >= 40 and gpa >= 2.50;
    359. 

  359. store b into ':OUTPATH:' using PigStorage;\,
    360. 

  360. 			},
    361. 

  361. 			{
    362. 

  362. 			'num' => 4,
    363. 

  363. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    364. 

  364. b = filter a by name lt 'fred allen' and age < 40 and gpa < 2.50;
    365. 

  365. store b into ':OUTPATH:' using PigStorage;\,
    366. 

  366. 			},
    367. 

  367. 			{
    368. 

  368. 			'num' => 5,
    369. 

  369. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    370. 

  370. b = filter a by name lte 'fred allen' and age <= 40 and gpa <= 2.50;
    371. 

  371. store b into ':OUTPATH:' using PigStorage;\,
    372. 

  372. 			},
    373. 

  373. 			{
    374. 

  374. 			'num' => 6,
    375. 

  375. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage();
    376. 

  376. b = filter a by $0 neq 'fred allen' and $1 != '40' and $2 != '2.50';
    377. 

  377. store b into ':OUTPATH:' using PigStorage;\,
    378. 

  378. 			},
    379. 

  379.             # test for filter == for chararray, int and double
    380. 

  380. 			{
    381. 

  381. 			'num' => 7,
    382. 

  382. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    383. 

  383. b = filter a by  name == 'fred allen' and age == 61 and gpa == 1.42;
    384. 

  384. store b into ':OUTPATH:' using PigStorage;\,
    385. 

  385. 			},
    386. 

  386.             # test for filter == for bytearray, long and float
    387. 

  387. 			{
    388. 

  388. 			'num' => 8,
    389. 

  389. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:long, gpa:float);
    390. 

  390. b = filter a by  name == 'fred allen' and age == 61 and gpa == 1.42f;
    391. 

  391. store b into ':OUTPATH:' using PigStorage;\,
    392. 

  392. 			},
    393. 

  393.             # test for filter != for chararray, int and double
    394. 

  394. 			{
    395. 

  395. 			'num' => 9,
    396. 

  396. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    397. 

  397. b = filter a by $0 != 'fred allen' and $1 != 40 and $2 != 2.50;
    398. 

  398. store b into ':OUTPATH:' using PigStorage;\,
    399. 

  399. 			},
    400. 

  400.             # test for filter != for bytearray, long and float
    401. 

  401. 			{
    402. 

  402. 			'num' => 10,
    403. 

  403. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:long, gpa:float);
    404. 

  404. b = filter a by $0 != 'fred allen' and $1 != 40 and $2 != 2.50f;
    405. 

  405. store b into ':OUTPATH:' using PigStorage;\,
    406. 

  406. 			},
    407. 

  407.             # test for filter == for explicit casts to chararray, int and double
    408. 

  408. 			{
    409. 

  409. 			'num' => 11,
    410. 

  410. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    411. 

  411. b = filter a by  (chararray)name == 'fred allen' and (int)age == 61 and (double)gpa == 1.42;
    412. 

  412. store b into ':OUTPATH:' using PigStorage;\,
    413. 

  413. 			},
    414. 

  414.             # test for filter == for explicit casts to bytearray, long and float
    415. 

  415. 			{
    416. 

  416. 			'num' => 12,
    417. 

  417. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    418. 

  418. b = filter a by  name == 'fred allen' and (long)age == 61 and (float)gpa == 1.42f;
    419. 

  419. store b into ':OUTPATH:' using PigStorage;\,
    420. 

  420. 			},
    421. 

  421.             # test for filter != for explicit casts to chararray, int and double
    422. 

  422. 			{
    423. 

  423. 			'num' => 13,
    424. 

  424. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() ;
    425. 

  425. b = filter a by (chararray)$0 != 'fred allen' and (int)$1 != 40 and (double)$2 != 2.50;
    426. 

  426. store b into ':OUTPATH:' using PigStorage;\,
    427. 

  427. 			},
    428. 

  428.             # test for filter != for explicit casts to bytearray, long and float
    429. 

  429. 			{
    430. 

  430. 			'num' => 14,
    431. 

  431. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() ;
    432. 

  432. b = filter a by $0 != 'fred allen' and (long)$1 != 40 and (float)$2 != 2.50f;
    433. 

  433. store b into ':OUTPATH:' using PigStorage;\,
    434. 

  434. 			},
    435. 

  435. 		]
    436. 

  436. 		},
    437. 

  437. 		{
    438. 

  438. 		'name' => 'FilterMatches',
    439. 

  439. 		'tests' => [
    440. 

  440. 			{
    441. 

  441. 			'num' => 1,
    442. 

  442. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    443. 

  443. b = filter a by name matches '^fred.*';
    444. 

  444. store b into ':OUTPATH:' using PigStorage;\,
    445. 

  445. 			},
    446. 

  446. 			{
    447. 

  447. 			'num' => 2,
    448. 

  448. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage();
    449. 

  449. b = filter a by not $0 matches '^fred.*';
    450. 

  450. store b into ':OUTPATH:' using PigStorage;\,
    451. 

  451. 			},
    452. 

  452. 			{
    453. 

  453.             # test for filter on matches for chararray (declared and explicit cast)
    454. 

  454. 			'num' => 3,
    455. 

  455. 			'pig' => q\a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name:chararray, age:int, registration, contributions:double);
    456. 

  456. b = filter a by name matches '^fred.*' and (chararray)registration matches '^dem.*';
    457. 

  457. store b into ':OUTPATH:' using PigStorage;\,
    458. 

  458. 			},
    459. 

  459. 			{
    460. 

  460. 			'num' => 4,
    461. 

  461. 			'pig' => q\a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name:chararray, age:int, registration, contributions:double);
    462. 

  462. b = filter a by name matches 'f.ed' and (chararray)registration matches 'd.m';
    463. 

  463. store b into ':OUTPATH:' using PigStorage;\,
    464. 

  464. 			},
    465. 

  465. 			{
    466. 

  466. 			'num' => 5,
    467. 

  467. 			'pig' => q\a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name:chararray, age:int, registration, contributions:double);
    468. 

  468. b = filter a by name matches 'f[^f]ed.*';
    469. 

  469. store b into ':OUTPATH:' using PigStorage;\,
    470. 

  470. 			},
    471. 

  471. 			{
    472. 

  472. 			'num' => 6,
    473. 

  473. 			'pig' => "a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name:chararray, age:int, registration, contributions:double);\nb = filter a by name matches '.*\\\\wan.*';\nstore b into ':OUTPATH:' using PigStorage;",
    474. 

  474. 			},
    475. 

  475. 			{
    476. 

  476. 			'num' => 7,
    477. 

  477. 			'pig' => "a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name:chararray, age:int, registration, contributions:double);\nb = filter a by name matches '^e.*\\\\sc.*';\nstore b into ':OUTPATH:' using PigStorage;",
    478. 

  478. 			},
    479. 

  479. 			{
    480. 

  480. 			'num' => 8,
    481. 

  481. 			'pig' => "a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name:chararray, age:int, registration, contributions:double);\nb = filter a by name matches 'ethan white';\nstore b into ':OUTPATH:' using PigStorage;",
    482. 

  482. 			},
    483. 

  483. 			{
    484. 

  484. 			'num' => 9,
    485. 

  485. 			'pig' => "a = load ':INPATH:/singlefile/studentnulltab10k' using PigStorage() as (name, age, gpa);\nb = filter a by gpa matches '\\\\d\\\\.45';\nstore b into ':OUTPATH:' using PigStorage;",
    486. 

  486. 			},
    487. 

  487. 		]
    488. 

  488. 		},
    489. 

  489. 		{
    490. 

  490. 		'name' => 'FilterUdf',
    491. 

  491. 		'tests' => [
    492. 

  492. 			{
    493. 

  493. 			'num' => 1,
    494. 

  494. 			'pig' => q\
    495. 

  495. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    496. 

  496. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    497. 

  497. c = cogroup a by (name, age), b by (name, age);
    498. 

  498. d = filter c by not IsEmpty(a);
    499. 

  499. e = filter d by not IsEmpty(b);
    500. 

  500. f = foreach e generate flatten(a), flatten(b);
    501. 

  501. store f into ':OUTPATH:';\,
    502. 

  502. 			},
    503. 

  503. 			{
    504. 

  504. 			    'num' => 2,
    505. 

  505. 			    'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    506. 

  506. 				           b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    507. 

  507. 				           c = filter a by age < 50;
    508. 

  508. 				           d = filter b by age < 50;
    509. 

  509. 				           e = cogroup c by (name, age), d by (name, age);
    510. 

  510. 				           f = filter e by COUNT(c)> 0 AND COUNT(d)>0;
    511. 

  511. 				           store f into ':OUTPATH:';\,
    512. 

  512. 			    'rc' => 0
    513. 

  513. 			},
    514. 

  514. 		]
    515. 

  515. 		},
    516. 

  516. 		# TODO Group that don't flatten via Agg functions
    517. 

  517. 		{
    518. 

  518. 		'name' => 'GroupAggFunc',
    519. 

  519. 		'tests' => [
    520. 

  520. 			{
    521. 

  521. 			'num' => 1,
    522. 

  522. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    523. 

  523. b = group a by name;
    524. 

  524. c = foreach b generate group, COUNT(a.age);
    525. 

  525. store c into ':OUTPATH:';\,
    526. 

  526. 			},
    527. 

  527. 			{
    528. 

  528. 			'num' => 2,
    529. 

  529. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    530. 

  530. b = group a by $0;
    531. 

  531. c = foreach b generate group, COUNT(a.$1);
    532. 

  532. store c into ':OUTPATH:';\,
    533. 

  533. 			},
    534. 

  534. 			{
    535. 

  535. 			'num' => 3,
    536. 

  536. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    537. 

  537. b = group a by (name, age);
    538. 

  538. c = foreach b generate group.name, group.age, COUNT(a.gpa);
    539. 

  539. store c into ':OUTPATH:';\,
    540. 

  540. 			},
    541. 

  541. 			{
    542. 

  542. 			'num' => 5,
    543. 

  543. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    544. 

  544. b = group a all;
    545. 

  545. c = foreach b generate COUNT(a.$0);
    546. 

  546. store c into ':OUTPATH:';\,
    547. 

  547. 			},
    548. 

  548. 			{
    549. 

  549. 			'num' => 6,
    550. 

  550. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    551. 

  551. b = group a by name;
    552. 

  552. c = foreach b generate group, SUM(a.age);
    553. 

  553. store c into ':OUTPATH:';\,
    554. 

  554. 			},
    555. 

  555. 			{
    556. 

  556. 			'num' => 7,
    557. 

  557. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    558. 

  558. b = group a by name;
    559. 

  559. c = foreach b generate group, SUM(a.gpa);
    560. 

  560. store c into ':OUTPATH:';\,
    561. 

  561.                         'floatpostprocess' => 1,
    562. 

  562.                         'delimiter' => '	',
    563. 

  563. 			},
    564. 

  564. 			{
    565. 

  565. 			'num' => 8,
    566. 

  566. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    567. 

  567. b = group a by name;
    568. 

  568. c = foreach b generate group, AVG(a.age);
    569. 

  569. store c into ':OUTPATH:';\,
    570. 

  570. 			},
    571. 

  571. 			{
    572. 

  572. 			'num' => 9,
    573. 

  573.                         'ignore23' => 'I cannot get it right due to float precision, temporarily disable',
    574. 

  574. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    575. 

  575. b = group a by name;
    576. 

  576. c = foreach b generate group, AVG(a.gpa);
    577. 

  577. store c into ':OUTPATH:';\,
    578. 

  578.                         'floatpostprocess' => 1,
    579. 

  579.                         'delimiter' => '	',
    580. 

  580. 			},
    581. 

  581. 			{
    582. 

  582. 			'num' => 10,
    583. 

  583. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    584. 

  584. b = group a by name;
    585. 

  585. c = foreach b generate group, MIN(a.gpa);
    586. 

  586. store c into ':OUTPATH:';\,
    587. 

  587. 			},
    588. 

  588. 			{
    589. 

  589. 			'num' => 11,
    590. 

  590. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    591. 

  591. b = group a by name;
    592. 

  592. c = foreach b generate group, MAX(a.gpa);
    593. 

  593. store c into ':OUTPATH:';\,
    594. 

  594. 			},
    595. 

  595.             {
    596. 

  596. 			'num' => 12,
    597. 

  597. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    598. 

  598. b = group a by (name, age);
    599. 

  599. c = foreach b generate flatten(group), SUM(a.gpa);
    600. 

  600. store c into ':OUTPATH:';\,
    601. 

  601.                         'floatpostprocess' => 1,
    602. 

  602.                         'delimiter' => '	',
    603. 

  603. 			},
    604. 

  604.             {
    605. 

  605. 			'num' => 13,
    606. 

  606. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    607. 

  607. b = group a by (name);
    608. 

  608. c = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    609. 

  609. d = cogroup b by group, c by name;
    610. 

  610. e = foreach d generate flatten(group), SUM(c.gpa), COUNT(c.name);
    611. 

  611. store e into ':OUTPATH:';\,
    612. 

  612.                         'floatpostprocess' => 1,
    613. 

  613.                         'delimiter' => '	',
    614. 

  614. 			},
    615. 

  615.             {
    616. 

  616. 			'num' => 14,
    617. 

  617. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    618. 

  618. b = group a by (name);
    619. 

  619. e = foreach b generate COUNT(a.name);
    620. 

  620. store e into ':OUTPATH:';\,
    621. 

  621. 			'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    622. 

  622. b = group a by (name);
    623. 

  623. e = foreach b generate COUNT(a.name);
    624. 

  624. store e into ':OUTPATH:';\,
    625. 

  625. 			}
    626. 

  626. 			],
    627. 

  627. 		},
    628. 

  628. 		{
    629. 

  629.         'name' => 'MapPartialAgg',
    630. 

  630.         'tests' => [
    631. 

  631.                     {
    632. 

  632.             'num' => 1,
    633. 

  633.             'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    634. 

  634. b = group a by name;
    635. 

  635. c = foreach b generate group, COUNT(a.age);
    636. 

  636. store c into ':OUTPATH:';\,
    637. 

  637.              'java_params' => ['-Dpig.exec.mapPartAgg=true']
    638. 

  638.             },
    639. 

  639.             {
    640. 

  640.             #multiquery with group in one sub query
    641. 

  641.             'num' => 2,
    642. 

  642.             'pig' => q# a = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float); 
    643. 

  643.                         b = filter a by age < 22; store b into ':OUTPATH:.1'; 
    644. 

  644.                         c = group b by age; 
    645. 

  645.                         d = foreach c generate group, SUM(b.gpa);
    646. 

  646.                         store d into ':OUTPATH:.2'; #,
    647. 

  647.             'java_params' => ['-Dpig.exec.mapPartAgg=true']
    648. 

  648.             
    649. 

  649.             },
    650. 

  650.             {
    651. 

  651.              #multi query with two group on diff columns
    652. 

  652.             'num' => 3,
    653. 

  653.             'pig' => q# a = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float); 
    654. 

  654.                         g1 = group a by name;
    655. 

  655.                         f1 = foreach g1 generate group as name, MAX(a.gpa);
    656. 

  656.                         store f1 into ':OUTPATH:.1'; 
    657. 

  657.                         g2 = group a by age;
    658. 

  658.                         f2 = foreach g2 generate group as age, AVG(a.gpa);
    659. 

  659.                         store f2 into ':OUTPATH:.2'; #,
    660. 

  660.             'java_params' => ['-Dpig.exec.mapPartAgg=true']
    661. 

  661.             
    662. 

  662.             },
    663. 

  663.             {
    664. 

  664.              #multi query with three groups on diff columns, group key being an expression
    665. 

  665.             'num' => 4,
    666. 

  666.             'pig' => q# a = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float); 
    667. 

  667.                         g1 = group a by name;
    668. 

  668.                         f1 = foreach g1 generate group as name, MAX(a.gpa);
    669. 

  669.                         store f1 into ':OUTPATH:.1'; 
    670. 

  670.                         g2 = group a by age%10;
    671. 

  671.                         f2 = foreach g2 generate group as age_mod10, AVG(a.gpa);
    672. 

  672.                         store f2 into ':OUTPATH:.2'; 
    673. 

  673.                         g3 = group a by age;
    674. 

  674.                         f3 = foreach g3 generate group%10, AVG(a.gpa);
    675. 

  675.                         store f3 into ':OUTPATH:.3';                         
    676. 

  676.                         g4 = group a by gpa;
    677. 

  677.                         f4 = foreach g4 generate group as gpa, COUNT(a);
    678. 

  678.                         store f4 into ':OUTPATH:.4';                                                 
    679. 

  679.                         
    680. 

  680.                         #,
    681. 

  681.             'java_params' => ['-Dpig.exec.mapPartAgg=true']
    682. 

  682.             
    683. 

  683.             },
    684. 

  684.             {
    685. 

  685.             #aggregation gets more than one tuple for every tuple from load func 
    686. 

  686.             
    687. 

  687.             'num' => 5,
    688. 

  688.             'pig' => q# a = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float); 
    689. 

  689.                         b = foreach a generate name, age, gpa, flatten(TOBAG(age,age)) as x;
    690. 

  690.                         c = group b by age; 
    691. 

  691.                         d = foreach c generate group, AVG(b.gpa);
    692. 

  692.                         store d into ':OUTPATH:'; #,
    693. 

  693.             'java_params' => ['-Dpig.exec.mapPartAgg=true']
    694. 

  694.             
    695. 

  695.             },            
    696. 

  696.         
    697. 

  697.             ],
    698. 

  698.         },
    699. 

  699. 		{
    700. 

  700. 		'name' => 'EvalFunc',
    701. 

  701. 		'tests' => [
    702. 

  702. 			{
    703. 

  703. 			'num' => 1,
    704. 

  704. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    705. 

  705. b = filter a by name lt 'b';
    706. 

  706. c = foreach b generate ARITY(name, age, gpa);
    707. 

  707. store c into ':OUTPATH:';\,
    708. 

  708. 			},
    709. 

  709. 			{
    710. 

  710. 			'num' => 2,
    711. 

  711. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age, gpa);
    712. 

  712. b = filter a by name lt 'b';
    713. 

  713. c = foreach b generate TOKENIZE(name);
    714. 

  714. d = foreach c generate flatten($0);
    715. 

  715. store d into ':OUTPATH:';\,
    716. 

  716. 			},
    717. 

  717. 			{
    718. 

  718. 			'num' => 3,
    719. 

  719. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    720. 

  720. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    721. 

  721. b = filter a by name lt 'b';
    722. 

  722. c = foreach b generate org.apache.pig.test.udf.evalfunc.Swap(name, age);
    723. 

  723. store c into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
    724. 

  724. 			},
    725. 

  725. 			{
    726. 

  726. 			'num' => 4,
    727. 

  727. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    728. 

  728. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    729. 

  729. b = filter a by name lt 'b';
    730. 

  730. c = foreach b generate org.apache.pig.test.udf.evalfunc.CreateMap((chararray)name, age);
    731. 

  731. store c into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
    732. 

  732. 			},
    733. 

  733.                         {
    734. 

  734.                         'num' => 5,
    735. 

  735. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    736. 

  736. a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    737. 

  737. b = foreach a generate org.apache.pig.test.udf.evalfunc.TestBoolean(instate);
    738. 

  738. store b into ':OUTPATH:';\,
    739. 

  739.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    740. 

  740. b = foreach a generate (instate is null ? '' : (instate == 'true' ? 'false' : 'true'));
    741. 

  741. store b into ':OUTPATH:';\,
    742. 

  742.                         }
    743. 

  743. 

  744. 		]
    745. 

  745. 		},
    746. 

  746. 		# TODO DIFF
    747. 

  747. 		# TODO User defined grouping function
    748. 

  748. 		{
    749. 

  749. 		'name' => 'CoGroupFlatten',
    750. 

  750. 		'tests' => [
    751. 

  751. 			{
    752. 

  752. 			'num' => 1,
    753. 

  753. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    754. 

  754. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    755. 

  755. c = filter a by age < 20;
    756. 

  756. d = filter b by age < 20;
    757. 

  757. e = cogroup c by name, d by name;
    758. 

  758. f = foreach e generate flatten (c), flatten(d);
    759. 

  759. store f into ':OUTPATH:';\,
    760. 

  760. 			},
    761. 

  761. 			{
    762. 

  762. 			'num' => 2,
    763. 

  763. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    764. 

  764. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    765. 

  765. c = filter a by $1 < 20;
    766. 

  766. d = filter b by $1 < 20;
    767. 

  767. e = cogroup c by $0, d by $0;
    768. 

  768. f = foreach e generate flatten (c), flatten(d);
    769. 

  769. store f into ':OUTPATH:';\,
    770. 

  770. 			},
    771. 

  771. 			{
    772. 

  772. 			'num' => 3,
    773. 

  773. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    774. 

  774. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    775. 

  775. c = filter a by age < 20;
    776. 

  776. d = filter b by age < 20;
    777. 

  777. e = cogroup c by (name, age), d by (name, age);
    778. 

  778. f = foreach e generate flatten (c), flatten(d);
    779. 

  779. store f into ':OUTPATH:';\,
    780. 

  780. 			},
    781. 

  781. 			{
    782. 

  782. 			'num' => 4,
    783. 

  783. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    784. 

  784. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    785. 

  785. d = filter b by age < 20;
    786. 

  786. e = cogroup a by (name, age) inner, d by (name, age);
    787. 

  787. f = foreach e generate flatten (a), flatten(d);
    788. 

  788. store f into ':OUTPATH:';\,
    789. 

  789. 			},
    790. 

  790. 			{
    791. 

  791. 			'num' => 5,
    792. 

  792. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    793. 

  793. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    794. 

  794. c = filter a by age < 20;
    795. 

  795. e = cogroup c by (name, age), b by (name, age) inner;
    796. 

  796. f = foreach e generate flatten (c), flatten(b);
    797. 

  797. store f into ':OUTPATH:';\,
    798. 

  798. 			},
    799. 

  799. 			{
    800. 

  800. 			'num' => 6,
    801. 

  801. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    802. 

  802. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    803. 

  803. e = cogroup a by (name, age) inner, b by (name, age) inner;
    804. 

  804. f = foreach e generate flatten (a), flatten(b);
    805. 

  805. store f into ':OUTPATH:';\,
    806. 

  806. 			},
    807. 

  807.             {
    808. 

  808.             # Test cogrouping data loaded from two separate loaders.  We don't have any data that can join with studenttab that isn't also loaded with PigStorage, so the
    809. 

  809.             # first step is an intermediate load and store using BinStorage.
    810. 

  810.     		'num' => 7,
    811. 

  811. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    812. 

  812. store a into ':OUTPATH:.intermediate' using BinStorage();
    813. 

  813. b = load ':OUTPATH:.intermediate' using BinStorage() as (name, age, gpa);
    814. 

  814. c = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    815. 

  815. e = cogroup b by (name, age) inner, c by (name, age) inner;
    816. 

  816. f = foreach e generate flatten (b), flatten(c);
    817. 

  817. store f into ':OUTPATH:';\,
    818. 

  818.             'notmq' => 1,
    819. 

  819.             },
    820. 

  820.             
    821. 

  821. 		]
    822. 

  822. 		},
    823. 

  823. 		{
    824. 

  824. 		'name' => 'CoGroup',
    825. 

  825. 		'tests' => [
    826. 

  826. 			{
    827. 

  827. 			'num' => 1,
    828. 

  828. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    829. 

  829. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    830. 

  830. c = cogroup a by name, b by name;
    831. 

  831. d = foreach c generate flatten(group), COUNT(a) + COUNT(b);
    832. 

  832. store d into ':OUTPATH:';\,
    833. 

  833. 			},
    834. 

  834. 		]
    835. 

  835. 		},
    836. 

  836. 		{
    837. 

  837. 		'name' => 'Join',
    838. 

  838. 		'tests' => [
    839. 

  839. 			{
    840. 

  840. 			'num' => 1,
    841. 

  841. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    842. 

  842. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    843. 

  843. c = filter a by age < 20;
    844. 

  844. d = filter b by age < 20;
    845. 

  845. e = join c by name, d by name;
    846. 

  846. store e into ':OUTPATH:';\,
    847. 

  847. 			},
    848. 

  848. 			{
    849. 

  849. 			'num' => 2,
    850. 

  850. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    851. 

  851. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    852. 

  852. c = filter a by age < 20;
    853. 

  853. d = filter b by age < 20;
    854. 

  854. e = join c by $0, d by $0;
    855. 

  855. store e into ':OUTPATH:';\,
    856. 

  856. 			},
    857. 

  857. 			{
    858. 

  858. 			'num' => 3,
    859. 

  859. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    860. 

  860. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    861. 

  861. c = filter a by age < 20;
    862. 

  862. d = filter b by age < 20;
    863. 

  863. e = join c by (name, age), d by (name, age);
    864. 

  864. store e into ':OUTPATH:';\,
    865. 

  865. 			},
    866. 

  866.             # self join with implict split
    867. 

  867.             # JIRA PIG-429
    868. 

  868. 			{
    869. 

  869. 			'num' => 4,
    870. 

  870. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    871. 

  871. b = filter a by $1 > 25;
    872. 

  872. c = join a by $0, b by $0;
    873. 

  873. store c into ':OUTPATH:';\,
    874. 

  874. 			},
    875. 

  875.             # join with one input having schema and another without
    876. 

  876.             # JIRA PIG-428
    877. 

  877. 			{
    878. 

  878. 			'num' => 5,
    879. 

  879. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray,age:int, gpa:double);
    880. 

  880. another = load ':INPATH:/singlefile/studenttab10k';
    881. 

  881. c = foreach another generate $0, $1+ 10, $2 + 10.0;
    882. 

  882. d = join a by $0, c by $0;
    883. 

  883. store d into ':OUTPATH:';\,
    884. 

  884. 			},
    885. 

  885.             # self join using fragment replicate join
    886. 

  886.             # no types
    887. 

  887. 			{
    888. 

  888. 			'num' => 6,
    889. 

  889. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    890. 

  890. b = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    891. 

  891. c = join a by name, b by name using 'repl';
    892. 

  892. store c into ':OUTPATH:';\,
    893. 

  893.             'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    894. 

  894. b = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    895. 

  895. c = join a by name, b by name ;
    896. 

  896. store c into ':OUTPATH:';\,
    897. 

  897. 			},
    898. 

  898.             # self join using fragment replicate join
    899. 

  899.             # with types and no cast for join key
    900. 

  900. 			{
    901. 

  901. 			'num' => 7,
    902. 

  902. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    903. 

  903. b = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    904. 

  904. c = join a by name, b by name using 'repl';
    905. 

  905. store c into ':OUTPATH:';\,
    906. 

  906.             'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    907. 

  907. b = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    908. 

  908. c = join a by name, b by name ;
    909. 

  909. store c into ':OUTPATH:';\,
    910. 

  910. 

  911. 			},
    912. 

  912.             # self join using fragment replicate join
    913. 

  913.             # with types and cast for join key
    914. 

  914. 			{
    915. 

  915. 			'num' => 8,
    916. 

  916. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    917. 

  917. b = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa);
    918. 

  918. c = join a by gpa, b by gpa using 'repl';
    919. 

  919. store c into ':OUTPATH:';\,
    920. 

  920.             'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    921. 

  921. b = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa);
    922. 

  922. c = join a by gpa, b by gpa ;
    923. 

  923. store c into ':OUTPATH:';\,
    924. 

  924. 

  925. 			},
    926. 

  926.             # left outer join
    927. 

  927. 			{
    928. 

  928. 			'num' => 9,
    929. 

  929. 			'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    930. 

  930. b = load ':INPATH:/singlefile/voternulltab10k' as  (name:chararray, age:long, registration:chararray, contributions:double); 
    931. 

  931. c = join a by name left outer, b by name;
    932. 

  932. store c into ':OUTPATH:';\,
    933. 

  933. 			},
    934. 

  934.             # right outer join
    935. 

  935. 			{
    936. 

  936. 			'num' => 10,
    937. 

  937. 			'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    938. 

  938. b = load ':INPATH:/singlefile/voternulltab10k' as  (name:chararray, age:long, registration:chararray, contributions:double); 
    939. 

  939. c = join a by name right outer, b by name;
    940. 

  940. store c into ':OUTPATH:';\,
    941. 

  941. 			},
    942. 

  942.             # full outer join
    943. 

  943. 			{
    944. 

  944. 			'num' => 11,
    945. 

  945. 			'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    946. 

  946. b = load ':INPATH:/singlefile/voternulltab10k' as  (name:chararray, age:long, registration:chararray, contributions:double); 
    947. 

  947. c = join a by name full outer, b by name;
    948. 

  948. store c into ':OUTPATH:';\,
    949. 

  949. 			},
    950. 

  950.             # see PIG-1209 join package now uses internalcachedBag, so every tuple on reduce side in this test will spilled to disk.
    951. 

  951. 			{
    952. 

  952. 			'num' => 12,
    953. 

  953. 	 		'java_params' => ['-Dpig.cachedbag.memusage=0'],
    954. 

  954. 			 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    955. 

  955. 			 b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    956. 

  956. 			 c = filter a by age < 20;
    957. 

  957. 			 d = filter b by age < 20;
    958. 

  958. 			 e = join c by name, d by name;
    959. 

  959. 			 store e into ':OUTPATH:';\,
    960. 

  960. 			},
    961. 

  961.                         {
    962. 

  962. 			'num' => 13,
    963. 

  963. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    964. 

  964. b = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    965. 

  965. c = filter a by age < 20;
    966. 

  966. d = filter b by age < 20;
    967. 

  967. e = join c by instate, d by instate parallel 5;
    968. 

  968. store e into ':OUTPATH:';\,
    969. 

  969. 			'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    970. 

  970. b = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    971. 

  971. c = filter a by age < 20;
    972. 

  972. d = filter b by age < 20;
    973. 

  973. e = join c by instate, d by instate parallel 5;
    974. 

  974. store e into ':OUTPATH:';\,
    975. 

  975. 			}
    976. 

  976. 		]
    977. 

  977. 		},
    978. 

  978. 		{
    979. 

  979. 		'name' => 'Foreach',
    980. 

  980. 		'tests' => [
    981. 

  981. 			{
    982. 

  982. 			'num' => 1,
    983. 

  983. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    984. 

  984. b = foreach a generate *;
    985. 

  985. store b into ':OUTPATH:';\,
    986. 

  986. 			},
    987. 

  987. 			{
    988. 

  988. 			'num' => 2,
    989. 

  989. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    990. 

  990. b = foreach a generate *;
    991. 

  991. store b into ':OUTPATH:';\,
    992. 

  992. 			},
    993. 

  993. 			{
    994. 

  994. 			'num' => 3,
    995. 

  995. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    996. 

  996. b = foreach a generate name, age;
    997. 

  997. store b into ':OUTPATH:';\,
    998. 

  998. 			},
    999. 

  999. 			{
    1000. 

  1000. 			'num' => 4,
    1001. 

  1001. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1002. 

  1002. b = foreach a generate $0, $2;
    1003. 

  1003. store b into ':OUTPATH:';\,
    1004. 

  1004. 			},
    1005. 

  1005. 			{
    1006. 

  1006.                 # test filter, projection, sort , duplicate elimination
    1007. 

  1007.                 'num' => 5,
    1008. 

  1008.                 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1009. 

  1009. b = filter a by age < 20;
    1010. 

  1010. c = group b by age;
    1011. 

  1011. d = foreach c { 
    1012. 

  1012.     cf = filter b by gpa < 3.0; 
    1013. 

  1013.     cp = cf.gpa; 
    1014. 

  1014. 	cd = distinct cp;
    1015. 

  1015.     co = order cd by $0; 
    1016. 

  1016.     generate group, flatten(co);
    1017. 

  1017.     }
    1018. 

  1018. store d into ':OUTPATH:';\,
    1019. 

  1019.             },
    1020. 

  1020. 			{
    1021. 

  1021. 			# test flatten for map and scalar
    1022. 

  1022. 			'num' => 6,
    1023. 

  1023. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    1024. 

  1024. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1025. 

  1025. b = foreach a generate flatten(name) as n, flatten(org.apache.pig.test.udf.evalfunc.CreateMap((chararray)name, gpa)) as m;
    1026. 

  1026. store b into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
    1027. 

  1027. 			},
    1028. 

  1028. 			{
    1029. 

  1029. 			# test flatten for UDF that returns bag with multiple tuples with multiple columns
    1030. 

  1030. 			'num' => 7,
    1031. 

  1031. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    1032. 

  1032. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1033. 

  1033. b = foreach a generate name, flatten(org.apache.pig.test.udf.evalfunc.CreateTupleBag(age, gpa)) as foo;
    1034. 

  1034. store b into ':OUTPATH:';\,
    1035. 

  1035. 			},
    1036. 

  1036. 			{
    1037. 

  1037. 			'num' => 8,
    1038. 

  1038. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age: int, gpa);
    1039. 

  1039. c = group a by name;
    1040. 

  1040. d = foreach c generate flatten(group), MAX(a.age) + MIN(a.age);
    1041. 

  1041. store d into ':OUTPATH:';\,
    1042. 

  1042. 			},
    1043. 

  1043. 			{
    1044. 

  1044.                 # test filter, projection, sort , duplicate elimination
    1045. 

  1045.                 'num' => 9,
    1046. 

  1046.                 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1047. 

  1047. b = filter a by age < 20;
    1048. 

  1048. c = group b by age;
    1049. 

  1049. d = foreach c { 
    1050. 

  1050.     cf = filter b by gpa >= 3.0 and gpa <= 3.5; 
    1051. 

  1051.     cp = cf.gpa; 
    1052. 

  1052. 	cd = distinct cp;
    1053. 

  1053.     co = order cd by $0; 
    1054. 

  1054.     generate group, flatten(co);
    1055. 

  1055.     }
    1056. 

  1056. store d into ':OUTPATH:';\,
    1057. 

  1057.             },
    1058. 

  1058. 			{
    1059. 

  1059.                 # test filter, projection, sort , duplicate elimination
    1060. 

  1060.                 'num' => 10,
    1061. 

  1061.                 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1062. 

  1062. b = filter a by age < 20;
    1063. 

  1063. c = group b by age;
    1064. 

  1064. d = foreach c { 
    1065. 

  1065.     cf = filter b by (gpa == 4.0 or gpa != 2.0) and name > 'a'; 
    1066. 

  1066.     cp = cf.gpa; 
    1067. 

  1067. 	cd = distinct cp;
    1068. 

  1068.     co = order cd by $0; 
    1069. 

  1069.     generate group, flatten(co);
    1070. 

  1070.     }
    1071. 

  1071. store d into ':OUTPATH:';\,
    1072. 

  1072.             },
    1073. 

  1073. 			{
    1074. 

  1074.                 # test filter, projection, sort , duplicate elimination
    1075. 

  1075.                 'num' => 11,
    1076. 

  1076.                 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1077. 

  1077. b = filter a by age < 20;
    1078. 

  1078. c = foreach b { 
    1079. 

  1079.     exp1 = age + gpa;
    1080. 

  1080.     exp2 = exp1 + age;
    1081. 

  1081.     generate exp1, exp2;
    1082. 

  1082.     }
    1083. 

  1083. store c into ':OUTPATH:';\,
    1084. 

  1084.             },
    1085. 

  1085. 			{
    1086. 

  1086.                 # test a udf with no args
    1087. 

  1087.                 'num' => 12,
    1088. 

  1088.                 'pig' => q\register :FUNCPATH:/testudf.jar;
    1089. 

  1089. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1090. 

  1090. b = foreach a generate name, org.apache.pig.test.udf.evalfunc.Fred() as fred;
    1091. 

  1091. store b into ':OUTPATH:';\,
    1092. 

  1092.             },
    1093. 

  1093.             		{
    1094. 

  1094. 			'num' => 13,
    1095. 

  1095. 		'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    1096. 

  1096. b = foreach a generate *;
    1097. 

  1097. store b into ':OUTPATH:';\,
    1098. 

  1098. 		'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:chararray);
    1099. 

  1099. b = foreach a generate *;
    1100. 

  1100. store b into ':OUTPATH:';\,
    1101. 

  1101. 			}
    1102. 

  1102. 		]
    1103. 

  1103. 		},
    1104. 

  1104. 		{
    1105. 

  1105. 		'name' => 'Order',
    1106. 

  1106. 		'tests' => [
    1107. 

  1107. 			{
    1108. 

  1108. 			'num' => 1,
    1109. 

  1109. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1110. 

  1110. b = foreach a generate name;
    1111. 

  1111. c = order b by name;
    1112. 

  1112. store c into ':OUTPATH:';\,
    1113. 

  1113. 			'sortArgs' => ['-t', '	', '+0', '-1'],
    1114. 

  1114. 			},
    1115. 

  1115. 			{
    1116. 

  1116. 			'num' => 2,
    1117. 

  1117. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1118. 

  1118. b = foreach a generate $1;
    1119. 

  1119. c = order b by $0;
    1120. 

  1120. store c into ':OUTPATH:';\,
    1121. 

  1121. 			'sortArgs' => ['-t', '	', '+0', '-1'],
    1122. 

  1122. 			},
    1123. 

  1123. 			{
    1124. 

  1124. 			'num' => 3,
    1125. 

  1125. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1126. 

  1126. b = foreach a generate gpa;
    1127. 

  1127. c = order b by gpa;
    1128. 

  1128. store c into ':OUTPATH:';\,
    1129. 

  1129. 			'sortArgs' => ['-t', '	', '+0', '-1'],
    1130. 

  1130. 			},
    1131. 

  1131. 			{
    1132. 

  1132. 			'num' => 4,
    1133. 

  1133. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1134. 

  1134. b = order a by *;
    1135. 

  1135. store b into ':OUTPATH:';\,
    1136. 

  1136. 			'sortArgs' => ['-t', '	'],
    1137. 

  1137. 			},
    1138. 

  1138. 			{
    1139. 

  1139. 			'num' => 5,
    1140. 

  1140. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1141. 

  1141. b = foreach a generate name, age;
    1142. 

  1142. c = order b by name, age;
    1143. 

  1143. store c into ':OUTPATH:';\,
    1144. 

  1144. 			'sortArgs' => ['-t', '	', '+0', '-2'],
    1145. 

  1145. 			},
    1146. 

  1146.             {
    1147. 

  1147. 			'num' => 6,
    1148. 

  1148. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1149. 

  1149. c = order a by $0;
    1150. 

  1150. store c into ':OUTPATH:';\,
    1151. 

  1151. 			'sortArgs' => ['-t', '	', '+0', '-1'],
    1152. 

  1152. 			},
    1153. 

  1153.             {
    1154. 

  1154. 			'num' => 7,
    1155. 

  1155. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1156. 

  1156. c = order a by $1;
    1157. 

  1157. store c into ':OUTPATH:';\,
    1158. 

  1158. 			'sortArgs' => ['-t', '	', '+1', '-2'],
    1159. 

  1159. 			},
    1160. 

  1160.             {
    1161. 

  1161. 			'num' => 8,
    1162. 

  1162. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1163. 

  1163. c = order a by $0, $1;
    1164. 

  1164. store c into ':OUTPATH:';\,
    1165. 

  1165. 			'sortArgs' => ['-t', '	', '+0', '-2'],
    1166. 

  1166. 			},
    1167. 

  1167.             {
    1168. 

  1168. 			'num' => 9,
    1169. 

  1169. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1170. 

  1170. c = order a by $1, $0;
    1171. 

  1171. store c into ':OUTPATH:';\,
    1172. 

  1172. 			'sortArgs' => ['-t', '	', '+1', '-2', '+0', '-1'],
    1173. 

  1173. 			},
    1174. 

  1174.             {
    1175. 

  1175. 			'num' => 10,
    1176. 

  1176. 			'ignore' => 'order by UDF is not supported',
    1177. 

  1177. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    1178. 

  1178. a = load ':INPATH:/singlefile/studenttab10k';
    1179. 

  1179. c = order a by * using org.apache.pig.test.udf.orderby.OrdDesc;
    1180. 

  1180. store c into ':OUTPATH:';\,
    1181. 

  1181. 			'sortArgs' => ['-t', '	', '-r'],
    1182. 

  1182. 			},
    1183. 

  1183.             {
    1184. 

  1184. 			'num' => 11,
    1185. 

  1185. 			'ignore' => 'order by UDF is not supported',
    1186. 

  1186. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    1187. 

  1187. a = load ':INPATH:/singlefile/studenttab10k';
    1188. 

  1188. c = order a by $0 using org.apache.pig.test.udf.orderby.OrdDesc;
    1189. 

  1189. store c into ':OUTPATH:';\,
    1190. 

  1190. 			'sortArgs' => ['-t', '	', '-r', '+0', '-1'],
    1191. 

  1191. 			},
    1192. 

  1192.             {
    1193. 

  1193. 			'num' => 12,
    1194. 

  1194. 			'ignore' => 'order by UDF is not supported',
    1195. 

  1195. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    1196. 

  1196. a = load ':INPATH:/singlefile/studenttab10k';
    1197. 

  1197. c = order a by $0, $1 using org.apache.pig.test.udf.orderby.OrdDesc;
    1198. 

  1198. store c into ':OUTPATH:';\,
    1199. 

  1199. 			'sortArgs' => ['-t', '	', '-r', '+0', '-2'],
    1200. 

  1200. 			},
    1201. 

  1201. # ALERT All these tests with inner order bys aren't testing the inner
    1202. 

  1202. # ordering.  We need to develop a sorting tool to do that.
    1203. 

  1203.             {
    1204. 

  1204. 			'num' => 13,
    1205. 

  1205. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1206. 

  1206. b = group a by $0;
    1207. 

  1207. c = foreach b {c1 = order $1 by $1; generate flatten(c1); };
    1208. 

  1208. store c into ':OUTPATH:';\,
    1209. 

  1209. 			},
    1210. 

  1210.             {
    1211. 

  1211. 			'num' => 14,
    1212. 

  1212. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1213. 

  1213. b = group a by $0;
    1214. 

  1214. c = foreach b {c1 = order $1 by *; generate flatten(c1); };
    1215. 

  1215. store c into ':OUTPATH:';\,
    1216. 

  1216. 			},
    1217. 

  1217.             {
    1218. 

  1218. 			'num' => 15,
    1219. 

  1219. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    1220. 

  1220. a = load ':INPATH:/singlefile/studenttab10k';
    1221. 

  1221. b = group a by $0;
    1222. 

  1222. c = foreach b {c1 = order $1 by * using org.apache.pig.test.udf.orderby.OrdDesc; generate flatten(c1); };
    1223. 

  1223. store c into ':OUTPATH:';\,
    1224. 

  1224. 			},
    1225. 

  1225.             {
    1226. 

  1226. 			'num' => 16,
    1227. 

  1227. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    1228. 

  1228. a = load ':INPATH:/singlefile/studenttab10k';
    1229. 

  1229. b = group a by $0;
    1230. 

  1230. c = foreach b {c1 = order $1 by $1 using org.apache.pig.test.udf.orderby.OrdDesc; generate flatten(c1);};
    1231. 

  1231. store c into ':OUTPATH:';\,
    1232. 

  1232. 			},
    1233. 

  1233.             {
    1234. 

  1234. 			'num' => 17,
    1235. 

  1235. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1236. 

  1236. b = group a by $0;
    1237. 

  1237. c = foreach b {c1 = order $1 by $1; generate flatten(c1), MAX($1.$1); };
    1238. 

  1238. store c into ':OUTPATH:';\,
    1239. 

  1239. 			},
    1240. 

  1240.             {
    1241. 

  1241.             # test to make sure the weighted range patitioning
    1242. 

  1242.             # works correctly when a sort key value repeats across
    1243. 

  1243.             # reduce partitions
    1244. 

  1244. 			'num' => 18,
    1245. 

  1245. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1246. 

  1246. b = order a by $1 parallel 100;
    1247. 

  1247. store b into ':OUTPATH:';\,
    1248. 

  1248. 			'sortArgs' => ['-t', '	', '+1', '-2'],
    1249. 

  1249. 			},
    1250. 

  1250. 			{
    1251. 

  1251. 			'num' => 19,
    1252. 

  1252. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    1253. 

  1253. b = foreach a generate instate;
    1254. 

  1254. c = order b by instate;
    1255. 

  1255. store c into ':OUTPATH:';\,
    1256. 

  1256. 			'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    1257. 

  1257. b = foreach a generate instate;
    1258. 

  1258. c = order b by instate;
    1259. 

  1259. store c into ':OUTPATH:';\,
    1260. 

  1260. 			'sortArgs' => ['-t', '	', '+0', '-1'],
    1261. 

  1261. 			},
    1262. 

  1262. 		]
    1263. 

  1263. 		},
    1264. 

  1264. 		{
    1265. 

  1265. 		'name' => 'Distinct',
    1266. 

  1266. 		'tests' => [
    1267. 

  1267. 			{
    1268. 

  1268. 			'num' => 1,
    1269. 

  1269. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1270. 

  1270. b = foreach a generate name;
    1271. 

  1271. c = distinct b;
    1272. 

  1272. store c into ':OUTPATH:';\,
    1273. 

  1273. 			},
    1274. 

  1274. 			{
    1275. 

  1275. 			'num' => 2,
    1276. 

  1276. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1277. 

  1277. b = foreach a generate $1;
    1278. 

  1278. c = distinct b;
    1279. 

  1279. store c into ':OUTPATH:';\,
    1280. 

  1280. 			},
    1281. 

  1281. 			{
    1282. 

  1282. 			'num' => 3,
    1283. 

  1283. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1284. 

  1284. b = foreach a generate gpa;
    1285. 

  1285. c = distinct b;
    1286. 

  1286. store c into ':OUTPATH:';\,
    1287. 

  1287. 			},
    1288. 

  1288. 			{
    1289. 

  1289. 			'num' => 4,
    1290. 

  1290. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    1291. 

  1291. b = distinct a;
    1292. 

  1292. store b into ':OUTPATH:';\,
    1293. 

  1293. 			},
    1294. 

  1294. 			{
    1295. 

  1295. 			'num' => 5,
    1296. 

  1296. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1297. 

  1297. b = foreach a generate name, age;
    1298. 

  1298. c = distinct b;
    1299. 

  1299. store c into ':OUTPATH:';\,
    1300. 

  1300. 			},
    1301. 

  1301. 			{
    1302. 

  1302. 			'num' => 6,
    1303. 

  1303. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1304. 

  1304. b = group a by name;
    1305. 

  1305. c = foreach b { aa = distinct a.age; generate group, COUNT(aa); }
    1306. 

  1306. store c into ':OUTPATH:';\,
    1307. 

  1307. 			}
    1308. 

  1308. 		]
    1309. 

  1309. 		},
    1310. 

  1310. 		{
    1311. 

  1311. 		'name' => 'Cross',
    1312. 

  1312. 		'tests' => [
    1313. 

  1313. 			{
    1314. 

  1314. 			'num' => 1,
    1315. 

  1315. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1316. 

  1316. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    1317. 

  1317. c = filter a by age < 19 and gpa < 1.0;
    1318. 

  1318. d = filter b by age < 19;
    1319. 

  1319. e = cross c, d;
    1320. 

  1320. store e into ':OUTPATH:';\,
    1321. 

  1321. 			},
    1322. 

  1322. 			{
    1323. 

  1323. 			'num' => 2,
    1324. 

  1324. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1325. 

  1325. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    1326. 

  1326. c = filter a by age < 19 and gpa < 1.0;
    1327. 

  1327. d = filter b by age < 19;
    1328. 

  1328. e = cross c, d parallel 10;
    1329. 

  1329. store e into ':OUTPATH:';\,
    1330. 

  1330. 			},
    1331. 

  1331. 			{
    1332. 

  1332. 			'num' => 3,
    1333. 

  1333. 			'pig' => q\set default_parallel 10;
    1334. 

  1334. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1335. 

  1335. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    1336. 

  1336. c = filter a by age < 19 and gpa < 1.0;
    1337. 

  1337. d = filter b by age < 19;
    1338. 

  1338. e = cross c, d;
    1339. 

  1339. store e into ':OUTPATH:';\,
    1340. 

  1340. 			},
    1341. 

  1341. 			{
    1342. 

  1342. 			'num' => 4,
    1343. 

  1343. 			'pig' => q\
    1344. 

  1344. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1345. 

  1345. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    1346. 

  1346. c = filter a by age < 25;
    1347. 

  1347. d = filter b by age < 25;
    1348. 

  1348. e = cross c, d;
    1349. 

  1349. f = filter e by c::age < d::age;
    1350. 

  1350. store f into ':OUTPATH:';\,
    1351. 

  1351. 			}
    1352. 

  1352. 		]
    1353. 

  1353. 		},
    1354. 

  1354. 		{
    1355. 

  1355. 		'name' => 'Union',
    1356. 

  1356. 		'tests' => [
    1357. 

  1357. 			{
    1358. 

  1358. 			'num' => 1,
    1359. 

  1359. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1360. 

  1360. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    1361. 

  1361. c = foreach a generate name, age;
    1362. 

  1362. d = foreach b generate name, age;
    1363. 

  1363. e = union c, d;
    1364. 

  1364. store e into ':OUTPATH:';\,
    1365. 

  1365. 			},
    1366. 

  1366. 		]
    1367. 

  1367. 		},
    1368. 

  1368. 		{
    1369. 

  1369. 		'name' => 'Bincond',
    1370. 

  1370. 		'tests' => [
    1371. 

  1371. 			{
    1372. 

  1372. 			'num' => 1,
    1373. 

  1373. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1374. 

  1374. b = foreach a generate name, (name matches 'yuri.*' ? age - 10 : (int)age);
    1375. 

  1375. store b into ':OUTPATH:';\,
    1376. 

  1376. 			},
    1377. 

  1377. 		]
    1378. 

  1378. 		},
    1379. 

  1379. 		{
    1380. 

  1380. 		'name' => 'Glob',
    1381. 

  1381. 		'tests' => [
    1382. 

  1382. 			{
    1383. 

  1383. 			'num' => 1,
    1384. 

  1384. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10?' as (name, age, gpa);
    1385. 

  1385. b = filter a by name == 'nick miller';
    1386. 

  1386. store b into ':OUTPATH:';\,
    1387. 

  1387. 			},
    1388. 

  1388. 			{
    1389. 

  1389. 			'num' => 2,
    1390. 

  1390. 			'pig' => q\a = load ':INPATH:/singlefile/st*ttab10k' as (name, age, gpa);
    1391. 

  1391. b = filter a by name == 'nick miller';
    1392. 

  1392. store b into ':OUTPATH:';\,
    1393. 

  1393. 			},
    1394. 

  1394. 			{
    1395. 

  1395. 			'num' => 3,
    1396. 

  1396. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab*' as (name, age, gpa);
    1397. 

  1397. b = filter a by name == 'nick miller';
    1398. 

  1398. store b into ':OUTPATH:';\,
    1399. 

  1399. 			},
    1400. 

  1400. 			{
    1401. 

  1401. 			'num' => 4,
    1402. 

  1402. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab???' as (name, age, gpa);
    1403. 

  1403. b = filter a by name == 'nick miller';
    1404. 

  1404. store b into ':OUTPATH:';\,
    1405. 

  1405. 			},
    1406. 

  1406. 			{
    1407. 

  1407. 			'num' => 5,
    1408. 

  1408. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab[1-9]0[km]' as (name, age, gpa);
    1409. 

  1409. b = filter a by name == 'nick miller';
    1410. 

  1410. store b into ':OUTPATH:';\,
    1411. 

  1411. 			},
    1412. 

  1412. 			{
    1413. 

  1413. 			'num' => 6,
    1414. 

  1414. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab[13]0[km]' as (name, age, gpa);
    1415. 

  1415. b = filter a by name == 'nick miller';
    1416. 

  1416. store b into ':OUTPATH:';\,
    1417. 

  1417. 			},
    1418. 

  1418. 			{
    1419. 

  1419. 			'num' => 7,
    1420. 

  1420. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab[12]0[a-l]' as (name, age, gpa);
    1421. 

  1421. b = filter a by name == 'nick miller';
    1422. 

  1422. store b into ':OUTPATH:';\,
    1423. 

  1423. 			},
    1424. 

  1424. 			{
    1425. 

  1425. 			'num' => 8,
    1426. 

  1426. 			'pig' => q\a = load ':INPATH:/glob/star/*good' as (name, age, gpa);
    1427. 

  1427. b = filter a by name == 'nick miller';
    1428. 

  1428. store b into ':OUTPATH:';\,
    1429. 

  1429. 			},
    1430. 

  1430. 			{
    1431. 

  1431. 			'num' => 9,
    1432. 

  1432. 			'pig' => q\a = load ':INPATH:/glob/star/*' as (name, age, gpa);
    1433. 

  1433. b = filter a by name == 'nick miller';
    1434. 

  1434. store b into ':OUTPATH:';\,
    1435. 

  1435. 			}
    1436. 

  1436. 		]
    1437. 

  1437. 		},
    1438. 

  1438. 		{
    1439. 

  1439. 		'name' => 'Arithmetic',
    1440. 

  1440. 		'tests' => [
    1441. 

  1441. 			{
    1442. 

  1442. 			'num' => 1,
    1443. 

  1443. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1444. 

  1444. c = foreach a generate age + 1, (int)gpa + 1;
    1445. 

  1445. store c into ':OUTPATH:';\,
    1446. 

  1446. 			},
    1447. 

  1447. 			{
    1448. 

  1448. 			'num' => 2,
    1449. 

  1449. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1450. 

  1450. c = foreach a generate (double)age + 1.5, gpa + 1.5;
    1451. 

  1451. store c into ':OUTPATH:';\,
    1452. 

  1452. 			},
    1453. 

  1453. 			{
    1454. 

  1454. 			'num' => 3,
    1455. 

  1455. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1456. 

  1456. c = foreach a generate age - 30, (int)gpa - 3;
    1457. 

  1457. store c into ':OUTPATH:';\,
    1458. 

  1458. 			},
    1459. 

  1459. 			{
    1460. 

  1460. 			'num' => 4,
    1461. 

  1461. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1462. 

  1462. c = foreach a generate (double)age - 30.1, gpa - 3.199;
    1463. 

  1463. store c into ':OUTPATH:';\,
    1464. 

  1464. 			},
    1465. 

  1465. 			{
    1466. 

  1466. 			'num' => 5,
    1467. 

  1467. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1468. 

  1468. c = foreach a generate age * 10, (int)gpa * 2;
    1469. 

  1469. store c into ':OUTPATH:';\,
    1470. 

  1470. 			},
    1471. 

  1471. 			{
    1472. 

  1472. 			'num' => 6,
    1473. 

  1473. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1474. 

  1474. c = foreach a generate (double)age * 10.1, gpa * 2.752342;
    1475. 

  1475. store c into ':OUTPATH:';\,
    1476. 

  1476. 			},
    1477. 

  1477. 			{
    1478. 

  1478. 			'num' => 7,
    1479. 

  1479. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1480. 

  1480. c = foreach a generate age  / 30, (int)gpa / 3;
    1481. 

  1481. store c into ':OUTPATH:';\,
    1482. 

  1482. 			},
    1483. 

  1483. 			{
    1484. 

  1484. 			'num' => 8,
    1485. 

  1485. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1486. 

  1486. c = foreach a generate (double)age / 30.323, gpa / 3.22;
    1487. 

  1487. store c into ':OUTPATH:';\,
    1488. 

  1488. 			},
    1489. 

  1489. 			{
    1490. 

  1490. 			'num' => 9,
    1491. 

  1491. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1492. 

  1492. c = foreach a generate 3 * age + gpa / 9.1 - 2;
    1493. 

  1493. store c into ':OUTPATH:';\,
    1494. 

  1494. 			},
    1495. 

  1495. 			{
    1496. 

  1496. 			'num' => 10,
    1497. 

  1497. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    1498. 

  1498. c = foreach a generate 3 * (age + gpa) / (9.1 - 2);
    1499. 

  1499. store c into ':OUTPATH:';\,
    1500. 

  1500. 			}
    1501. 

  1501. 		]
    1502. 

  1502. 		},
    1503. 

  1503. 		{
    1504. 

  1504. 		'name' => 'Regression',
    1505. 

  1505. 		'tests' => [
    1506. 

  1506. 			{
    1507. 

  1507. 			'num' => 1459894,
    1508. 

  1508. 			'pig' => q\a = load ':INPATH:/singlefile/reg1459894';
    1509. 

  1509. b = group a by $0;
    1510. 

  1510. c = foreach b generate group, COUNT(a.$1);
    1511. 

  1511. store c into ':OUTPATH:';\,
    1512. 

  1512. 			},
    1513. 

  1513.             {
    1514. 

  1514. 			'num' => 97,
    1515. 

  1515. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    1516. 

  1516. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    1517. 

  1517. e = cogroup a by name, b by name;
    1518. 

  1518. f = foreach e generate group, COUNT(a), COUNT(b);
    1519. 

  1519. store f into ':OUTPATH:';\,
    1520. 

  1520. 			},
    1521. 

  1521.             {
    1522. 

  1522. 			'num' => 203,
    1523. 

  1523. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    1524. 

  1524. b = group a by name;
    1525. 

  1525. c = foreach b generate group, COUNT($1);
    1526. 

  1526. store c into ':OUTPATH:';
    1527. 

  1527. --This is a really long script to test that when script size exceeds 1k we can still parse it.
    1528. 

  1528. --The quick sly fox jumped over the lazy brown dog.
    1529. 

  1529. --he quick sly fox jumped over the lazy brown dog.T
    1530. 

  1530. --e quick sly fox jumped over the lazy brown dog.Th
    1531. 

  1531. -- quick sly fox jumped over the lazy brown dog.The
    1532. 

  1532. --quick sly fox jumped over the lazy brown dog.The 
    1533. 

  1533. --uick sly fox jumped over the lazy brown dog.The q
    1534. 

  1534. --ick sly fox jumped over the lazy brown dog.The qu
    1535. 

  1535. --ck sly fox jumped over the lazy brown dog.The qui
    1536. 

  1536. --k sly fox jumped over the lazy brown dog.The quic
    1537. 

  1537. -- sly fox jumped over the lazy brown dog.The quick
    1538. 

  1538. --sly fox jumped over the lazy brown dog.The quick 
    1539. 

  1539. --ly fox jumped over the lazy brown dog.The quick s
    1540. 

  1540. --y fox jumped over the lazy brown dog.The quick sl
    1541. 

  1541. -- fox jumped over the lazy brown dog.The quick sly
    1542. 

  1542. --fox jumped over the lazy brown dog.The quick sly 
    1543. 

  1543. --ox jumped over the lazy brown dog.The quick sly f
    1544. 

  1544. --x jumped over the lazy brown dog.The quick sly fo
    1545. 

  1545. -- jumped over the lazy brown dog.The quick sly fox
    1546. 

  1546. --jumped over the lazy brown dog.The quick sly fox 
    1547. 

  1547. --umped over the lazy brown dog.The quick sly fox j
    1548. 

  1548. --mped over the lazy brown dog.The quick sly fox ju
    1549. 

  1549. --ped over the lazy brown dog.The quick sly fox jum\,
    1550. 

  1550. 			}
    1551. 

  1551. 		]
    1552. 

  1552. 		},
    1553. 

  1553.         {
    1554. 

  1554. 		'name' => 'Unicode',
    1555. 

  1555. 		'tests' => [
    1556. 

  1556. 			{
    1557. 

  1557. 			'num' => 1,
    1558. 

  1558. 			'pig' => q\a = load ':INPATH:/singlefile/unicode100';
    1559. 

  1559. store a into ':OUTPATH:';\,
    1560. 

  1560. 			},
    1561. 

  1561. 		]
    1562. 

  1562. 		},
    1563. 

  1563. 		{
    1564. 

  1564. 		'name' => 'Parameters',
    1565. 

  1565. 		'tests' => [
    1566. 

  1566. 			{
    1567. 

  1567. 				# test default
    1568. 

  1568. 				'num' => 1,
    1569. 

  1569. 				'pig' => q\%default fname 'studenttab10k'
    1570. 

  1570. a = load ':INPATH:/singlefile/$fname' using PigStorage() as (name, age, gpa);
    1571. 

  1571. b = foreach a generate name;
    1572. 

  1572. store b into ':OUTPATH:';\,
    1573. 

  1573. 			},
    1574. 

  1574. 			{
    1575. 

  1575. 				# test paramter from command line
    1576. 

  1576. 				'num' => 2,
    1577. 

  1577. 				'pig_params' => ['-p', qq(fname='studenttab10k')],
    1578. 

  1578. 				'pig' => q\a = load ':INPATH:/singlefile/$fname' using PigStorage() as (name, age, gpa);
    1579. 

  1579. b = foreach a generate name;
    1580. 

  1580. store b into ':OUTPATH:';\,
    1581. 

  1581. 			},
    1582. 

  1582. 			{
    1583. 

  1583. 				# test paramter from param file
    1584. 

  1584. 				'num' => 3,
    1585. 

  1585. 				'pig_params' => ['-m', ":PARAMPATH:/params_3"],
    1586. 

  1586. 				'pig' => q\a = load ':INPATH:/singlefile/$fname' using PigStorage() as (name, age, gpa);
    1587. 

  1587. b = foreach a generate name;
    1588. 

  1588. store b into ':OUTPATH:';\,
    1589. 

  1589. 			},
    1590. 

  1590. 			{
    1591. 

  1591. 				# test command
    1592. 

  1592. 				'num' => 4,
    1593. 

  1593. 				'pig' => q\%declare cmd `/usr/local/bin/perl -e 'print "studenttab10k"'`
    1594. 

  1594. a = load ':INPATH:/singlefile/$cmd' using PigStorage() as (name, age, gpa);
    1595. 

  1595. b = foreach a generate name;
    1596. 

  1596. store b into ':OUTPATH:';\,
    1597. 

  1597. 			},
    1598. 

  1598.                         {
    1599. 

  1599. 				# test parameter with a space
    1600. 

  1600. 				'num' => 5,
    1601. 

  1601. 				'pig_params' => ['-p', qq(setting='set default_parallel 100;'),'-p',qq(fname='studenttab10k')],
    1602. 

  1602. 				'pig' => q\a = load ':INPATH:/singlefile/$fname' using PigStorage() as (name, age, gpa);
    1603. 

  1603. $setting
    1604. 

  1604. b = foreach a generate name;
    1605. 

  1605. store b into ':OUTPATH:';\,
    1606. 

  1606.                                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/$fname' using PigStorage() as (name, age, gpa);
    1607. 

  1607. b = foreach a generate name;
    1608. 

  1608. store b into ':OUTPATH:';\,
    1609. 

  1609. 			},
    1610. 

  1610. 		]
    1611. 

  1611. 		},
    1612. 

  1612. 

  1613. 		{
    1614. 

  1614. 		'name' => 'Types',
    1615. 

  1615. 		'tests' => [
    1616. 

  1616. 			{
    1617. 

  1617. 				#  constants
    1618. 

  1618. 				'num' => 1,
    1619. 

  1619. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1620. 

  1620. b = foreach a generate age + 1 + 0.2f + 253645L, gpa+1;
    1621. 

  1621. store b into ':OUTPATH:';\,
    1622. 

  1622. 			},
    1623. 

  1623.  			{
    1624. 

  1624. 				#  NULL and cast
    1625. 

  1625. 				'num' => 2,
    1626. 

  1626. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1627. 

  1627. b = foreach a generate (int)((int)gpa/((int)gpa - 1)) as norm_gpa:int;
    1628. 

  1628. c = foreach b generate (norm_gpa is null? 0 :norm_gpa);
    1629. 

  1629. store c into ':OUTPATH:';\,
    1630. 

  1630. 			     #  'expected_err_regex' => "Encountered Warning DIVIDE_BY_ZERO 2387 time.*",
    1631. 

  1631. 			     #  Driver does currently not support both 'sql' and 'expected_...' verification directives.
    1632. 

  1632. 			},
    1633. 

  1633. 			{
    1634. 

  1634. 				#  arithmetic operators and SIZE for int, double and size and concat operators for chararrays
    1635. 

  1635. 				'num' => 3,
    1636. 

  1636. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1637. 

  1637. b = foreach a generate age, gpa, age % 25, age + 25, age - 25, age/2, age * 2, SIZE(age), gpa + 10.1, gpa - 1.1 , gpa / 1.2, gpa * 2.5, SIZE(gpa), SIZE(name), CONCAT(name, 'test');
    1638. 

  1638. store b into ':OUTPATH:';\,
    1639. 

  1639. 			},
    1640. 

  1640. 			{
    1641. 

  1641. 				#  arithmetic operators and SIZE for long, float and size and concat operators for bytearrays
    1642. 

  1642. 				'num' => 4,
    1643. 

  1643. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:long, gpa:float);
    1644. 

  1644. b = foreach a generate age, gpa, age % 2L, age + 2500000000L, age - 2500000000L, age/2L, age * 250000000L, SIZE(age), gpa + 10.1f, gpa - 1.1f , gpa / 1.2f, gpa * 2.6f, SIZE(gpa), SIZE(name), CONCAT(name, name);
    1645. 

  1645. store b into ':OUTPATH:';\,
    1646. 

  1646. 			},
    1647. 

  1647. 			{
    1648. 

  1648. 				#  equlity and implicit cast
    1649. 

  1649. 				'num' => 5,
    1650. 

  1650. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age, gpa);
    1651. 

  1651. b = filter a by age == '25' and gpa < 3; 
    1652. 

  1652. store b into ':OUTPATH:';\,
    1653. 

  1653. 

  1654. 			},
    1655. 

  1655. 			{
    1656. 

  1656.                 # will need to test against previous version of pig 
    1657. 

  1657.                 # because in pig currently count includes nulls - this affects
    1658. 

  1658.                 # avg
    1659. 

  1659. 				'num' => 6,
    1660. 

  1660. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1661. 

  1661. b = group a ALL;
    1662. 

  1662. c = foreach b generate SUM(a.age), MIN(a.age), MAX(a.age), AVG(a.age), MIN(a.name), MAX(a.name), SUM(a.gpa), MIN(a.gpa), MAX(a.gpa), AVG(a.gpa);
    1663. 

  1663. store c into ':OUTPATH:';\,
    1664. 

  1664.                                 'floatpostprocess' => 1,
    1665. 

  1665.                                 'delimiter' => '	',
    1666. 

  1666. 			},
    1667. 

  1667. 			{
    1668. 

  1668. 				#  sum, min, max, avg for long and float (declared)
    1669. 

  1669. 				'num' => 7,
    1670. 

  1670. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:long, gpa:float);
    1671. 

  1671. b = group a ALL;
    1672. 

  1672. c = foreach b generate SUM(a.age), MIN(a.age), MAX(a.age), AVG(a.age), SUM(a.gpa), MIN(a.gpa), MAX(a.gpa), AVG(a.gpa);
    1673. 

  1673. store c into ':OUTPATH:';\,
    1674. 

  1674. 			},
    1675. 

  1675. 			{
    1676. 

  1676. 				#  Explicit casts - arithmetic operators and SIZE for int, double and size and concat operators for chararrays
    1677. 

  1677. 				'num' => 8,
    1678. 

  1678. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age, gpa);
    1679. 

  1679. b = foreach a generate (int)age % 25, (int)age + 25, (int)age - 25, (int)age/2, (int)age * 2, SIZE((int)age), (double)gpa + 10.1, (double)gpa - 1.1 , (double)gpa / 1.2, (double)gpa * 2.5, SIZE((double)gpa), SIZE((chararray)name), CONCAT((chararray)name, 'test');
    1680. 

  1680. store b into ':OUTPATH:';\,
    1681. 

  1681. 			},
    1682. 

  1682. 			{
    1683. 

  1683. 				#  Explicit casts - arithmetic operators and SIZE for long, float
    1684. 

  1684. 				'num' => 9,
    1685. 

  1685. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age, gpa);
    1686. 

  1686. b = foreach a generate (long)age, (long)age % 2L, (long)age + 2500000000L, (long)age - 2500000000L, (long)age/2L, (long)age * 250000000L, SIZE((long)age), (float)gpa + 10.1f, (float)gpa - 1.1f , (float)gpa / 1.2f, (float)gpa * 2.6f, SIZE((float)gpa);
    1687. 

  1687. store b into ':OUTPATH:';\,
    1688. 

  1688. 			},
    1689. 

  1689.             {
    1690. 

  1690.                 # Filter is null for chararray and double and is not null for int
    1691. 

  1691.                 'num' => 10,
    1692. 

  1692. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1693. 

  1693. b = filter a by name is null and age is not null and gpa is null;
    1694. 

  1694. c = group b ALL;
    1695. 

  1695. d = foreach c generate COUNT(b);
    1696. 

  1696. store d into ':OUTPATH:';\,
    1697. 

  1697.             },
    1698. 

  1698.             {
    1699. 

  1699.                 # Filter is not null for chararray and double and is null for int
    1700. 

  1700.                 'num' => 11,
    1701. 

  1701. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1702. 

  1702. b = filter a by name is not null and age is null and gpa is not null;
    1703. 

  1703. c = group b ALL;
    1704. 

  1704. d = foreach c generate COUNT(b);
    1705. 

  1705. store d into ':OUTPATH:';\,
    1706. 

  1706.             },
    1707. 

  1707.             {
    1708. 

  1708.                 # Filter is null for bytearray and float and is not null for long
    1709. 

  1709.                 'num' => 12,
    1710. 

  1710. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:long, gpa:float);
    1711. 

  1711. b = filter a by name is null and age is not null and gpa is null;
    1712. 

  1712. c = group b ALL;
    1713. 

  1713. d = foreach c generate COUNT(b);
    1714. 

  1714. store d into ':OUTPATH:';\,
    1715. 

  1715.             },
    1716. 

  1716.             {
    1717. 

  1717.                 # Filter is not null for bytearray and float and is null for long
    1718. 

  1718.                 'num' => 13,
    1719. 

  1719. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:long, gpa:float);
    1720. 

  1720. b = filter a by name is not null and age is null and gpa is not null;
    1721. 

  1721. c = group b ALL;
    1722. 

  1722. d = foreach c generate COUNT(b);
    1723. 

  1723. store d into ':OUTPATH:';\,
    1724. 

  1724.             },
    1725. 

  1725.             {
    1726. 

  1726. 				# test that sorting is based on the type for chararray, int and double
    1727. 

  1727. 				'num' => 14,
    1728. 

  1728. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1729. 

  1729. b = order a by name, age, gpa;
    1730. 

  1730. store b into ':OUTPATH:';\,
    1731. 

  1731.                 'sortArgs' => ['-t', '	', '+0', '-1', '+1n', '-2'],
    1732. 

  1732. 			},
    1733. 

  1733. 			{
    1734. 

  1734. 				# test that sorting descending is based on the type for chararray, int and double
    1735. 

  1735. 				'num' => 15,
    1736. 

  1736. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1737. 

  1737. b = order a by name desc, age desc, gpa desc;
    1738. 

  1738. store b into ':OUTPATH:';\,
    1739. 

  1739.                 'sortArgs' => ['-t', '	', '+0r', '-1', '+1nr', '-2'],
    1740. 

  1740. 			},
    1741. 

  1741. 			{
    1742. 

  1742. 				# test that sorting is based on the type for bytearray, long and float
    1743. 

  1743. 				'num' => 16,
    1744. 

  1744. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:long, gpa:float);
    1745. 

  1745. b = order a by name, age, gpa;
    1746. 

  1746. store b into ':OUTPATH:';\,
    1747. 

  1747.                 'sortArgs' => ['-t', '	', '+0', '-1', '+1n', '-2'],
    1748. 

  1748. 			},
    1749. 

  1749. 			{
    1750. 

  1750. 				# test that sorting descending is based on the type for chararray, age and float
    1751. 

  1751. 				'num' => 17,
    1752. 

  1752. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:long, gpa:float);
    1753. 

  1753. b = order a by name desc, age desc, gpa desc;
    1754. 

  1754. store b into ':OUTPATH:';\,
    1755. 

  1755.                 'sortArgs' => ['-t', '	', '+0r', '-1', '+1nr', '-2'],
    1756. 

  1756. 			},
    1757. 

  1757. 			{
    1758. 

  1758. 				# test precision for doubles is atleast 15 digits
    1759. 

  1759. 				'num' => 18,
    1760. 

  1760. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1761. 

  1761. b = foreach a generate 0.123456789123456+0.123456789123456;
    1762. 

  1762. store b into ':OUTPATH:';\,
    1763. 

  1763. 			},
    1764. 

  1764. 			{
    1765. 

  1765. 				#  order by string
    1766. 

  1766. 				'num' => 20,
    1767. 

  1767. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1768. 

  1768. b = order a by name;
    1769. 

  1769. store b into ':OUTPATH:';\,
    1770. 

  1770.                 'sortArgs' => ['-t', '	', '+0', '-1'],
    1771. 

  1771. 			},
    1772. 

  1772. 			{
    1773. 

  1773. 				#  order by string desc
    1774. 

  1774. 				'num' => 21,
    1775. 

  1775. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1776. 

  1776. b = order a by name desc;
    1777. 

  1777. store b into ':OUTPATH:';\,
    1778. 

  1778.                 'sortArgs' => ['-t', '	', '+0r', '-1'],
    1779. 

  1779. 			},
    1780. 

  1780. 			{
    1781. 

  1781. 				#  order by int
    1782. 

  1782. 				'num' => 22,
    1783. 

  1783. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1784. 

  1784. b = order a by age;
    1785. 

  1785. store b into ':OUTPATH:';\,
    1786. 

  1786.                 'sortArgs' => ['-t', '	', '+1n', '-2'],
    1787. 

  1787. 			},
    1788. 

  1788. 			{
    1789. 

  1789. 				#  order by int desc
    1790. 

  1790. 				'num' => 23,
    1791. 

  1791. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1792. 

  1792. b = order a by age desc;
    1793. 

  1793. store b into ':OUTPATH:';\,
    1794. 

  1794.                 'sortArgs' => ['-t', '	', '+1nr', '-2'],
    1795. 

  1795. 			},
    1796. 

  1796. 			{
    1797. 

  1797. 				#  order by long
    1798. 

  1798. 				'num' => 24,
    1799. 

  1799. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:long, gpa:double);
    1800. 

  1800. b = order a by age;
    1801. 

  1801. store b into ':OUTPATH:';\,
    1802. 

  1802.                 'sortArgs' => ['-t', '	', '+1n', '-2'],
    1803. 

  1803. 			},
    1804. 

  1804. 			{
    1805. 

  1805. 				#  order by long desc
    1806. 

  1806. 				'num' => 25,
    1807. 

  1807. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:long, gpa:double);
    1808. 

  1808. b = order a by age desc;
    1809. 

  1809. store b into ':OUTPATH:';\,
    1810. 

  1810.                 'sortArgs' => ['-t', '	', '+1nr', '-2'],
    1811. 

  1811. 			},
    1812. 

  1812. 			{
    1813. 

  1813. 				#  order by float
    1814. 

  1814. 				'num' => 26,
    1815. 

  1815. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:float);
    1816. 

  1816. b = order a by gpa;
    1817. 

  1817. store b into ':OUTPATH:';\,
    1818. 

  1818.                 'sortArgs' => ['-t', '	', '-k 3n'],
    1819. 

  1819. 			},
    1820. 

  1820. 			{
    1821. 

  1821. 				#  order by float desc
    1822. 

  1822. 				'num' => 27,
    1823. 

  1823. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:float);
    1824. 

  1824. b = order a by gpa desc;
    1825. 

  1825. store b into ':OUTPATH:';\,
    1826. 

  1826.                 'sortArgs' => ['-t', '	', '-k 3nr'],
    1827. 

  1827. 			},
    1828. 

  1828. 			{
    1829. 

  1829. 				#  order by double
    1830. 

  1830. 				'num' => 28,
    1831. 

  1831. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1832. 

  1832. b = order a by gpa;
    1833. 

  1833. store b into ':OUTPATH:';\,
    1834. 

  1834.                 'sortArgs' => ['-t', '	', '-k 3n'],
    1835. 

  1835. 			},
    1836. 

  1836. 			{
    1837. 

  1837. 				#  order by double desc
    1838. 

  1838. 				'num' => 29,
    1839. 

  1839. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1840. 

  1840. b = order a by gpa desc;
    1841. 

  1841. store b into ':OUTPATH:';\,
    1842. 

  1842.                 'sortArgs' => ['-t', '	', '-k 3nr'],
    1843. 

  1843. 			},
    1844. 

  1844. 			{
    1845. 

  1845. 				#  order by *
    1846. 

  1846. 				'num' => 30,
    1847. 

  1847. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1848. 

  1848. b = order a by *;
    1849. 

  1849. store b into ':OUTPATH:';\,
    1850. 

  1850.                 'sortArgs' => ['-t', '	', '+0', '-1', '+1n', '-2'],
    1851. 

  1851. 			},
    1852. 

  1852. 			{
    1853. 

  1853. 				#  order by * desc
    1854. 

  1854. 				'num' => 31,
    1855. 

  1855. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1856. 

  1856. b = order a by * desc;
    1857. 

  1857. store b into ':OUTPATH:';\,
    1858. 

  1858.                 'sortArgs' => ['-t', '	', '+0r', '-1', '+1nr', '-2'],
    1859. 

  1859. 			},
    1860. 

  1860. 			{
    1861. 

  1861. 			'num' => 32,
    1862. 

  1862. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    1863. 

  1863. b = load ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration:chararray, contributions:double);
    1864. 

  1864. c = filter a by age < 20;
    1865. 

  1865. d = filter b by age < 20;
    1866. 

  1866. e = cogroup c by name, d by name;
    1867. 

  1867. f = foreach e generate flatten (c), flatten(d);
    1868. 

  1868. store f into ':OUTPATH:';\,
    1869. 

  1869. 			},
    1870. 

  1870. 			{
    1871. 

  1871. 			'num' => 33,
    1872. 

  1872. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    1873. 

  1873. b = load ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration:chararray, contributions:double);
    1874. 

  1874. c = filter a by age < 20;
    1875. 

  1875. d = filter b by age < 20;
    1876. 

  1876. e = cogroup c by age, d by age;
    1877. 

  1877. f = foreach e generate flatten (c), flatten(d);
    1878. 

  1878. store f into ':OUTPATH:';\,
    1879. 

  1879. 			},
    1880. 

  1880. 			{
    1881. 

  1881. 			'num' => 34,
    1882. 

  1882. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:long, gpa:double);
    1883. 

  1883. b = load ':INPATH:/singlefile/votertab10k' as (name:chararray, age:long, registration:chararray, contributions:double);
    1884. 

  1884. c = filter a by age < 20;
    1885. 

  1885. d = filter b by age < 20;
    1886. 

  1886. e = cogroup c by age, d by age;
    1887. 

  1887. f = foreach e generate flatten (c), flatten(d);
    1888. 

  1888. store f into ':OUTPATH:';\,
    1889. 

  1889. 			},
    1890. 

  1890. 			{
    1891. 

  1891. 			'num' => 35,
    1892. 

  1892. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:float, gpa:double);
    1893. 

  1893. b = load ':INPATH:/singlefile/votertab10k' as (name:chararray, age:float, registration:chararray, contributions:double);
    1894. 

  1894. c = filter a by age < 20;
    1895. 

  1895. d = filter b by age < 20;
    1896. 

  1896. e = cogroup c by age, d by age;
    1897. 

  1897. f = foreach e generate flatten (c), flatten(d);
    1898. 

  1898. store f into ':OUTPATH:';\,
    1899. 

  1899. 			},
    1900. 

  1900. 			{
    1901. 

  1901. 			'num' => 36,
    1902. 

  1902. 			'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:double, gpa:double);
    1903. 

  1903. b = load ':INPATH:/singlefile/votertab10k' as (name:chararray, age:double, registration:chararray, contributions:double);
    1904. 

  1904. c = filter a by age < 20;
    1905. 

  1905. d = filter b by age < 20;
    1906. 

  1906. e = cogroup c by age, d by age;
    1907. 

  1907. f = foreach e generate flatten (c), flatten(d);
    1908. 

  1908. store f into ':OUTPATH:';\,
    1909. 

  1909. 			},
    1910. 

  1910.  			{
    1911. 

  1911. 				#  NULL and cast
    1912. 

  1912. 				'num' => 37,
    1913. 

  1913. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1914. 

  1914. b = foreach a generate (int)((int)gpa/((int)gpa - 1)) as norm_gpa:int;
    1915. 

  1915. c = foreach b generate (norm_gpa is not null? norm_gpa: 0);
    1916. 

  1916. store c into ':OUTPATH:';\,
    1917. 

  1917. 			},
    1918. 

  1918. 			{
    1919. 

  1919. 				#  constants
    1920. 

  1920. 				'num' => 38,
    1921. 

  1921. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    1922. 

  1922. b = foreach a generate -(age + 1 + 0.2f + 253645L), -(gpa+1);
    1923. 

  1923. store b into ':OUTPATH:';\,
    1924. 

  1924. 			},
    1925. 

  1925. 			{
    1926. 

  1926. 				'num' => 39,
    1927. 

  1927. 				'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:boolean);
    1928. 

  1928. b = foreach a generate instate, true, false;
    1929. 

  1929. store b into ':OUTPATH:';\,
    1930. 

  1930. 				'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    1931. 

  1931. b = foreach a generate instate, 'true', 'false';
    1932. 

  1932. store b into ':OUTPATH:';\,
    1933. 

  1933. 			},
    1934. 

  1934. 		]
    1935. 

  1935. 		},
    1936. 

  1936. 		{
    1937. 

  1937. 		'name' => 'Limit',
    1938. 

  1938. 		'tests' => [
    1939. 

  1939. 			{
    1940. 

  1940. 				'num' => 1,
    1941. 

  1941. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k';
    1942. 

  1942. b = order a by $0, $1;
    1943. 

  1943. c = filter b by $0 > 'a'; -- break the sort/limit optimization
    1944. 

  1944. d = limit c 100;
    1945. 

  1945. store d into ':OUTPATH:';\,
    1946. 

  1946.             	'sortArgs' => ['-t', '	', '+0', '-1'],
    1947. 

  1947. 			},
    1948. 

  1948. 			{
    1949. 

  1949. 				'num' => 2,
    1950. 

  1950.                                 'ignore23' => 'The record limit pick is different in 23',
    1951. 

  1951. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k';
    1952. 

  1952. b = order a by $0, $1;
    1953. 

  1953. c = limit b 100;
    1954. 

  1954. store c into ':OUTPATH:';\,
    1955. 

  1955.             	'sortArgs' => ['-t', '	', '+0', '-1'],
    1956. 

  1956. 			},
    1957. 

  1957. 			{
    1958. 

  1958. 				# Make sure that limit higher than number of rows doesn't mess stuff up
    1959. 

  1959. 				'num' => 3,
    1960. 

  1960. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k';
    1961. 

  1961. b = order a by $0, $1;
    1962. 

  1962. c = filter b by $1 < 1000;
    1963. 

  1963. d = limit c 100000;
    1964. 

  1964. store d into ':OUTPATH:';\,
    1965. 

  1965. 			},
    1966. 

  1966. 			{
    1967. 

  1967. 				'num' => 4,
    1968. 

  1968. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k';
    1969. 

  1969. b = distinct a;
    1970. 

  1970. c = limit b 100;
    1971. 

  1971. store c into ':OUTPATH:';\,
    1972. 

  1972. 			},
    1973. 

  1973. 			{
    1974. 

  1974. 				'num' => 5,
    1975. 

  1975. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k';
    1976. 

  1976. b = load ':INPATH:/singlefile/votertab10k';
    1977. 

  1977. a1 = foreach a generate $0, $1;
    1978. 

  1978. b1 = foreach b generate $0, $1;
    1979. 

  1979. c = union a1, b1;
    1980. 

  1980. d = limit c 100;
    1981. 

  1981. store d into ':OUTPATH:';\,
    1982. 

  1982. 			},
    1983. 

  1983. 			{
    1984. 

  1984. 				'num' => 6,
    1985. 

  1985. 				'pig' =>q\A = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float);
    1986. 

  1986. B = limit A 40;
    1987. 

  1987. C = filter B by age == 40;
    1988. 

  1988. D = group C by name;
    1989. 

  1989. E = foreach D generate group, COUNT(C);
    1990. 

  1990. store E into ':OUTPATH:';\,
    1991. 

  1991. 			},
    1992. 

  1992. 			{
    1993. 

  1993. 				'num' => 7,
    1994. 

  1994. 				'pig' =>q\A = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float);
    1995. 

  1995. B = group A by name;
    1996. 

  1996. C = foreach B {
    1997. 

  1997. 	C1 = limit A 10;
    1998. 

  1998. 	generate group, COUNT(C1);
    1999. 

  1999. }
    2000. 

  2000. store C into ':OUTPATH:';\,
    2001. 

  2001. 			},
    2002. 

  2002. 			{
    2003. 

  2003. 				'num' => 8,
    2004. 

  2004. 				'pig' =>q\A = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float);
    2005. 

  2005. B = group A by name;
    2006. 

  2006. C = foreach B {
    2007. 

  2007. 	C1 = filter A by age < 40;
    2008. 

  2008. 	C2 = limit C1 10;
    2009. 

  2009. 	generate group, COUNT(C2);
    2010. 

  2010. }
    2011. 

  2011. D = filter C by $1 > 0;
    2012. 

  2012. store D into ':OUTPATH:';\,
    2013. 

  2013. 			},
    2014. 

  2014.  			{
    2015. 

  2015. 				'num' => 9,
    2016. 

  2016. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k';
    2017. 

  2017. b = order a by $0, $1;
    2018. 

  2018. c = limit b 1000/10;
    2019. 

  2019. store c into ':OUTPATH:';\,
    2020. 

  2020. 

  2021. 				'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentnulltab10k';
    2022. 

  2022. b = order a by $0, $1;
    2023. 

  2023. c = limit b 100;
    2024. 

  2024. store c into ':OUTPATH:';\,
    2025. 

  2025. 

  2026.             	'sortArgs' => ['-t', '	', '-k1,2'],
    2027. 

  2027. 			},
    2028. 

  2028.  			{
    2029. 

  2029. 				'num' => 10,
    2030. 

  2030. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2031. 

  2031. b = group a all;
    2032. 

  2032. c = foreach b generate COUNT(a) as count;
    2033. 

  2033. d = limit a c.count/10;
    2034. 

  2034. store d into ':OUTPATH:';\,
    2035. 

  2035. 

  2036. 				'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2037. 

  2037. b = limit a 1000;
    2038. 

  2038. store b into ':OUTPATH:';\,
    2039. 

  2039.                         },
    2040. 

  2040.  			{
    2041. 

  2041. 				'num' => 11,
    2042. 

  2042. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2043. 

  2043. b = group a all;
    2044. 

  2044. c = foreach b generate COUNT(a) as count;
    2045. 

  2045. d = load ':INPATH:/singlefile/votertab10k';
    2046. 

  2046. e = group d all;
    2047. 

  2047. f = foreach e generate COUNT(d) as count;
    2048. 

  2048. d = limit a c.count/10+f.count/10;
    2049. 

  2049. store d into ':OUTPATH:';\,
    2050. 

  2050. 

  2051. 				'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2052. 

  2052. b = limit a 2000;
    2053. 

  2053. store b into ':OUTPATH:';\,
    2054. 

  2054.                         }
    2055. 

  2055. 		]
    2056. 

  2056. 		},
    2057. 

  2057. 		{
    2058. 

  2058. 		'name' => 'Split',
    2059. 

  2059. 		'tests' => [
    2060. 

  2060. 			{
    2061. 

  2061. 				'num' => 1,
    2062. 

  2062. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k';
    2063. 

  2063. split a into a1 if $0 > 'm', a2 if $0 <= 'm'; 
    2064. 

  2064. store a1 into ':OUTPATH:';\,
    2065. 

  2065. 			},
    2066. 

  2066. 			{
    2067. 

  2067. 				'num' => 2,
    2068. 

  2068. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k';
    2069. 

  2069. split a into a1 if $0 > 'm', a2 if $0 <= 'm'; 
    2070. 

  2070. store a2 into ':OUTPATH:';\,
    2071. 

  2071. 			},
    2072. 

  2072. 			{
    2073. 

  2073. 				'num' => 3,
    2074. 

  2074. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k';
    2075. 

  2075. split a into a1 if $0 > 'm', a2 if $0 <= 'm'; 
    2076. 

  2076. b = cogroup a1 by $1, a2 by $1;
    2077. 

  2077. c = foreach b generate flatten(a1), flatten(a2);
    2078. 

  2078. store c into ':OUTPATH:';\,
    2079. 

  2079. 			},
    2080. 

  2080. 			{
    2081. 

  2081. 				'num' => 4,
    2082. 

  2082. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k';
    2083. 

  2083. split a into a1 if $0 > 'm', a2 if $0 <= 'm'; 
    2084. 

  2084. b = cogroup a1 by $1, a2 by $1;
    2085. 

  2085. c = foreach b generate flatten($1), flatten($2);
    2086. 

  2086. store c into ':OUTPATH:';\,
    2087. 

  2087. 			},
    2088. 

  2088. 			{
    2089. 

  2089. 				'num' => 5,
    2090. 

  2090. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age, gpa);
    2091. 

  2091. split a into a1 if name > 'm', a2 if name <= 'm'; 
    2092. 

  2092. b = distinct a1;
    2093. 

  2093. store b into ':OUTPATH:';\,
    2094. 

  2094. 			},
    2095. 

  2095. 			{
    2096. 

  2096. 				'num' => 6,
    2097. 

  2097. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age, gpa);
    2098. 

  2098. split a into a1 if age > 50, a2 if age <= 25; 
    2099. 

  2099. b = order a2 by name;
    2100. 

  2100. store b into ':OUTPATH:';\,
    2101. 

  2101. 				'sortArgs' => ['-t', '	', '+0', '-1'],
    2102. 

  2102. 			},
    2103. 

  2103. 			{
    2104. 

  2104. 				'num' => 7,
    2105. 

  2105. 				'pig' =>q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
    2106. 

  2106. split a into a1 if name > 'm', a2 if age < 50; 
    2107. 

  2107. b = distinct a1;
    2108. 

  2108. store b into ':OUTPATH:';\,
    2109. 

  2109. 			},
    2110. 

  2110. 			{
    2111. 

  2111. 				'num' => 8,
    2112. 

  2112. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2113. 

  2113. split a into a1 if age > 50, a2 if name < 'm'; 
    2114. 

  2114. b2 = foreach a2 generate name, 1;
    2115. 

  2115. b1 = foreach a1 generate name, 2;
    2116. 

  2116. c = cogroup b2 by name, b1 by name;
    2117. 

  2117. d = foreach c generate flatten(group), COUNT($1), COUNT($2);
    2118. 

  2118. store d into ':OUTPATH:';\,
    2119. 

  2119. 			},
    2120. 

  2120. 			{
    2121. 

  2121. 				'num' => 9,
    2122. 

  2122. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2123. 

  2123. split a into a1 if age > 50, a2 if name < 'm'; 
    2124. 

  2124. b2 = distinct a2;
    2125. 

  2125. b1 = order a1 by name;
    2126. 

  2126. c = cogroup b2 by name, b1 by name;
    2127. 

  2127. d = foreach c generate flatten(group), COUNT($1), COUNT($2);
    2128. 

  2128. store d into ':OUTPATH:';\,
    2129. 

  2129. 			},
    2130. 

  2130. 			{
    2131. 

  2131. 				'num' => 10,
    2132. 

  2132. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2133. 

  2133. split a into a1 if age > 50, a2 otherwise;
    2134. 

  2134. store a1 into ':OUTPATH:.1';
    2135. 

  2135. store a2 into ':OUTPATH:.2';\,
    2136. 

  2136. 				'verify_pig_script' =>q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2137. 

  2137. split a into a1 if age > 50, a2 if age<=50;
    2138. 

  2138. store a1 into ':OUTPATH:.1';
    2139. 

  2139. store a2 into ':OUTPATH:.2';\,
    2140. 

  2140. 			}
    2141. 

  2141. 		]
    2142. 

  2142. 		},
    2143. 

  2143. 		{
    2144. 

  2144. 		'name' => 'ImplicitSplit',
    2145. 

  2145. 		'tests' => [
    2146. 

  2146. 			{
    2147. 

  2147. 				'num' => 1,
    2148. 

  2148. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k';
    2149. 

  2149. b = filter a by $1 > 50;
    2150. 

  2150. c = filter a by $2 > 3.0;
    2151. 

  2151. d = cogroup b by $0, c by $0;
    2152. 

  2152. e = foreach d generate flatten(b), flatten(c);
    2153. 

  2153. store e into ':OUTPATH:';\,
    2154. 

  2154. 			},
    2155. 

  2155. 			{
    2156. 

  2156. 				'num' => 2,
    2157. 

  2157. 				'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    2158. 

  2158. b = filter a by age > 50;
    2159. 

  2159. c = filter a by gpa > 3.0;
    2160. 

  2160. d = cogroup b by name, c by name;
    2161. 

  2161. e = foreach d generate flatten(b), flatten(c);
    2162. 

  2162. f = filter e by b::age < 75;
    2163. 

  2163. store f into ':OUTPATH:';\,
    2164. 

  2164. 			}
    2165. 

  2165. 		]
    2166. 

  2166. 		},
    2167. 

  2167. 		{
    2168. 

  2168. 		'name' => 'describe',
    2169. 

  2169. 		'tests' => [
    2170. 

  2170.                         #JIRA[PIG-373]
    2171. 

  2171. 			{
    2172. 

  2172. 			'num' => 1,
    2173. 

  2173. 			'pig' => q\ 
    2174. 

  2174. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2175. 

  2175. describe A;
    2176. 

  2176. store A into ':OUTPATH:';\,
    2177. 

  2177. 

  2178. 			},
    2179. 

  2179. 		],
    2180. 

  2180. 		},
    2181. 

  2181. 		{
    2182. 

  2182. 		'name' => 'Sample',
    2183. 

  2183. 		'tests' => [
    2184. 

  2184. 			{
    2185. 

  2185. 			'num' => 1,
    2186. 

  2186. 			'pig' => q\ 
    2187. 

  2187. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2188. 

  2188. S = sample A 2-1-1;
    2189. 

  2189. store S into ':OUTPATH:';\,
    2190. 

  2190. 'verify_pig_script' => q\ 
    2191. 

  2191. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2192. 

  2192. S = sample A 0;
    2193. 

  2193. store S into ':OUTPATH:';\,
    2194. 

  2194. 

  2195. 

  2196. 			},
    2197. 

  2197. 			{
    2198. 

  2198. 			'num' => 2,
    2199. 

  2199. 			'pig' => q\ 
    2200. 

  2200. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2201. 

  2201. B = group A all;
    2202. 

  2202. C = foreach B generate COUNT(A) as count;
    2203. 

  2203. D = group A all;
    2204. 

  2204. E = foreach D generate (double)COUNT(A) as count;
    2205. 

  2205. S = sample A E.count/C.count;
    2206. 

  2206. store S into ':OUTPATH:';\,
    2207. 

  2207. 'verify_pig_script' => q\ 
    2208. 

  2208. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2209. 

  2209. S = sample A 1;
    2210. 

  2210. store S into ':OUTPATH:';\,
    2211. 

  2211. 			},
    2212. 

  2212. 		],
    2213. 

  2213. 		},
    2214. 

  2214. 		{
    2215. 

  2215. 		'name' => 'MissingColumns',
    2216. 

  2216. 		'tests' => [
    2217. 

  2217. 			{
    2218. 

  2218. 			'num' => 1,
    2219. 

  2219. 			'pig' => q\ 
    2220. 

  2220. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age: int, gpa: double, extra: chararray);
    2221. 

  2221. B = filter A by age > 50 or extra > 'm';
    2222. 

  2222. D = order B by age, extra;
    2223. 

  2223. store D into ':OUTPATH:';\,
    2224. 

  2224. 

  2225. 			'sortArgs' => ['-t', '	', '+1n', '-2'],
    2226. 

  2226. 			},
    2227. 

  2227. 			{
    2228. 

  2228. 			'num' => 2,
    2229. 

  2229. 			'pig' => q\ 
    2230. 

  2230. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage();
    2231. 

  2231. B = foreach A generate $0, $1 + 1, $3 + 1;
    2232. 

  2232. C = group B by ($0, $2);
    2233. 

  2233. D = foreach C generate flatten(group), COUNT($1);
    2234. 

  2234. store D into ':OUTPATH:';\,
    2235. 

  2235. 

  2236. 			},
    2237. 

  2237. 			{
    2238. 

  2238. 			'num' => 3,
    2239. 

  2239. 			'pig' => q\ 
    2240. 

  2240. A = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: double);
    2241. 

  2241. B = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa, extra1, extra2);
    2242. 

  2242. C = join A by (name, age), B by (name, extra1);
    2243. 

  2243. store C into ':OUTPATH:';\,
    2244. 

  2244. 

  2245. 			# The following SQL should produce empty results, which will match what our pig query should produce.
    2246. 

  2246. 			}
    2247. 

  2247. 		],
    2248. 

  2248. 		},
    2249. 

  2249. 		{
    2250. 

  2250. 		'name' => 'Aliases',
    2251. 

  2251.         # check access of a field using multiple valid aliases
    2252. 

  2252. 		'tests' => [
    2253. 

  2253. 			{
    2254. 

  2254.             # check that a free standing alias reference works
    2255. 

  2255.             # when it is unambiguous
    2256. 

  2256.             # check that a fully qualified alias reference works
    2257. 

  2257.             # check that a partially qualified unambiguous alias reference works
    2258. 

  2258. 			'num' => 1,
    2259. 

  2259. 			'pig' => q\ 
    2260. 

  2260. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2261. 

  2261. b = group a by name;
    2262. 

  2262. c = foreach b generate flatten(a);
    2263. 

  2263. d = filter c by name != 'fred';
    2264. 

  2264. e = group d by name;
    2265. 

  2265. f = foreach e generate flatten(d);
    2266. 

  2266. g = foreach f generate name, d::a::name as dname, a::name as aname;
    2267. 

  2267. store g into ':OUTPATH:';\,
    2268. 

  2268. 

  2269. 			},
    2270. 

  2270. 			{
    2271. 

  2271.             # check that the "group" alias is available
    2272. 

  2272.             # after a flatten(group)
    2273. 

  2273. 			'num' => 2,
    2274. 

  2274. 			'pig' => q\ 
    2275. 

  2275. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2276. 

  2276. b = group a by name;
    2277. 

  2277. c = foreach b generate flatten(group), COUNT(a) as cnt;
    2278. 

  2278. d = foreach c generate group;
    2279. 

  2279. store d into ':OUTPATH:';\,
    2280. 

  2280. 

  2281. 			},
    2282. 

  2282. 		],
    2283. 

  2283. 		},
    2284. 

  2284. 

  2285. 		{
    2286. 

  2286. 		'name' => 'Lineage',
    2287. 

  2287.         #test if the right cast function is picked
    2288. 

  2288. 		'tests' => [
    2289. 

  2289. 			{
    2290. 

  2290. 			'num' => 1,
    2291. 

  2291. 			'pig' => q\ 
    2292. 

  2292. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2293. 

  2293. b = load ':INPATH:/singlefile/textdoc' using TextLoader() as (sentence);
    2294. 

  2294. c = cogroup a ALL, b ALL;
    2295. 

  2295. d = foreach c generate flatten(a), flatten(b);
    2296. 

  2296. e = foreach d generate name, flatten(TOKENIZE((chararray)sentence)) as sentence;
    2297. 

  2297. f = foreach e generate CONCAT((chararray)name, sentence);
    2298. 

  2298. store f into ':OUTPATH:';\,
    2299. 

  2299. 

  2300. 			},
    2301. 

  2301. 			{
    2302. 

  2302. 			'num' => 2,
    2303. 

  2303. 			'pig' => q\ 
    2304. 

  2304. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa: double);
    2305. 

  2305. b = load ':INPATH:/singlefile/textdoc' using TextLoader() as (sentence);
    2306. 

  2306. c = cross a, b;
    2307. 

  2307. d = foreach c generate name, flatten(TOKENIZE((chararray)sentence)) as sentence;
    2308. 

  2308. e = foreach d generate CONCAT((chararray)name, sentence);
    2309. 

  2309. store e into ':OUTPATH:';\,
    2310. 

  2310. 

  2311. 			},
    2312. 

  2312.                         {
    2313. 

  2313.                         'num' => 3,
    2314. 

  2314.                         'pig' => q\
    2315. 

  2315. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa: double);
    2316. 

  2316. b = foreach a generate age as student_age;
    2317. 

  2317. c = filter b by student_age > 50;
    2318. 

  2318. d = foreach c generate student_age + 10;
    2319. 

  2319. store d into ':OUTPATH:';\,
    2320. 

  2320. 

  2321.                         },
    2322. 

  2322. 			{
    2323. 

  2323. 			'num' => 4,
    2324. 

  2324. 			'pig' => q\register :FUNCPATH:/testudf.jar;
    2325. 

  2325. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    2326. 

  2326. b = filter a by name lt 'b';
    2327. 

  2327. c = foreach b generate org.apache.pig.test.udf.evalfunc.CreateMap((chararray)name, (int)age);
    2328. 

  2328. d = foreach c generate $0#'alice young';
    2329. 

  2329. split d into e if $0 is not null, f if $0 is null;
    2330. 

  2330. store e into ':OUTPATH:';\,
    2331. 

  2331. 			}
    2332. 

  2332. 		],
    2333. 

  2333. 		},
    2334. 

  2334.         {
    2335. 

  2335. 		'name' => 'Casts',
    2336. 

  2336. 		'tests' => [
    2337. 

  2337. 			{
    2338. 

  2338.             # check that a cast of a value of type
    2339. 

  2339.             # same as the result type of the cast works
    2340. 

  2340.             # when the value is treated as a bytearray
    2341. 

  2341. 			'num' => 1,
    2342. 

  2342. 			'floatpostprocess' => 1,
    2343. 

  2343. 			'delimiter' => '	',
    2344. 

  2344. 			'pig' => q\ 
    2345. 

  2345. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2346. 

  2346. b = foreach a generate name, age, gpa;
    2347. 

  2347. store b into ':OUTPATH:.intermediate' using BinStorage();
    2348. 

  2348. c = load ':OUTPATH:.intermediate' using BinStorage();
    2349. 

  2349. -- after this load, the fields are treated as bytearrays though
    2350. 

  2350. -- they are actually "typed", test that the implicit casts 
    2351. 

  2351. -- introduced by the operations in the foreach below will work fine
    2352. 

  2352. d = foreach c generate CONCAT((chararray)$0, 'test'), $1 + 1, $2 + 0.2;
    2353. 

  2353. store d into ':OUTPATH:';\,
    2354. 

  2354. 

  2355.             'notmq' => 1,
    2356. 

  2356. 			},
    2357. 

  2357. 			{
    2358. 

  2358.             # check that a cast of a value of type
    2359. 

  2359.             # same as the result type of the cast works
    2360. 

  2360.             # when the value is treated as a bytearray
    2361. 

  2361. 			'num' => 2,
    2362. 

  2362. 			'floatpostprocess' => 1,
    2363. 

  2363. 			'delimiter' => '	',
    2364. 

  2364. 			'pig' => q\ 
    2365. 

  2365. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:long, gpa: float);
    2366. 

  2366. b = foreach a generate name, age, gpa;
    2367. 

  2367. store b into ':OUTPATH:.intermediate' using BinStorage();
    2368. 

  2368. c = load ':OUTPATH:.intermediate' using BinStorage();
    2369. 

  2369. -- after this load, the fields are treated as bytearrays though
    2370. 

  2370. -- they are actually "typed", test that the implicit casts 
    2371. 

  2371. -- introduced by the operations in the foreach below will work fine
    2372. 

  2372. d = foreach c generate CONCAT((chararray)$0, 'test'), $1 + 1L, $2 + 0.2f;
    2373. 

  2373. store d into ':OUTPATH:';\,
    2374. 

  2374. 

  2375.             'notmq' => 1,
    2376. 

  2376. 			},
    2377. 

  2377. 			{
    2378. 

  2378.              #check that a cast of a value of type
    2379. 

  2379.              #same as the result type of the cast works
    2380. 

  2380.              #when the value is treated as a bytearray
    2381. 

  2381. 			'num' => 3,
    2382. 

  2382. 			'floatpostprocess' => 1,
    2383. 

  2383. 			'delimiter' => '	',
    2384. 

  2384. 			'pig' => q\ 
    2385. 

  2385. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:long, gpa: float);
    2386. 

  2386. b = group a by  name;
    2387. 

  2387. c = foreach b generate a, (1,2,3), ['key1'#'value1','key2'#'value2'];
    2388. 

  2388. -- store the bag, tuple and map
    2389. 

  2389. store c into ':OUTPATH:.intermediate' using BinStorage();
    2390. 

  2390. d = load ':OUTPATH:.intermediate' using BinStorage() as (b:bag{t:tuple(x,y,z)}, t2:tuple(a,b,c), m:map[]);
    2391. 

  2391. -- after this load, the fields are treated as bytearrays though
    2392. 

  2392. -- they are actually "typed", test that the implicit casts 
    2393. 

  2393. -- introduced by the operations in the foreach below will work fine
    2394. 

  2394. e = foreach d generate COUNT(b), t2.a, t2.b, t2.c, m#'key1', m#'key2';
    2395. 

  2395. store e into ':OUTPATH:';\,
    2396. 

  2396. 

  2397.             'notmq' => 1,
    2398. 

  2398. 			},
    2399. 

  2399. 			{
    2400. 

  2400.             # check that a cast of a value of type
    2401. 

  2401.             # same as the result type of the cast works
    2402. 

  2402.             # when the value is treated as a bytearray
    2403. 

  2403. 			'num' => 4,
    2404. 

  2404. 			'floatpostprocess' => 1,
    2405. 

  2405. 			'delimiter' => '	',
    2406. 

  2406. 			'pig' => q\ 
    2407. 

  2407. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2408. 

  2408. b = foreach a generate name, age, gpa;
    2409. 

  2409. store b into ':OUTPATH:.intermediate' using PigStorage();
    2410. 

  2410. c = load ':OUTPATH:.intermediate' using PigStorage();
    2411. 

  2411. -- after this load, the fields are treated as bytearrays though
    2412. 

  2412. -- they are actually "typed", test that the implicit casts 
    2413. 

  2413. -- introduced by the operations in the foreach below will work fine
    2414. 

  2414. d = foreach c generate CONCAT((chararray)$0, 'test'), $1 + 1, $2 + 0.2;
    2415. 

  2415. store d into ':OUTPATH:';\,
    2416. 

  2416. 

  2417.             'notmq' => 1,
    2418. 

  2418. 			},
    2419. 

  2419. 			{
    2420. 

  2420.             # check that a cast of a value of type
    2421. 

  2421.             # same as the result type of the cast works
    2422. 

  2422.             # when the value is treated as a bytearray
    2423. 

  2423. 			'num' => 5,
    2424. 

  2424. 			'floatpostprocess' => 1,
    2425. 

  2425. 			'delimiter' => '	',
    2426. 

  2426. 			'pig' => q\ 
    2427. 

  2427. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:long, gpa: float);
    2428. 

  2428. b = foreach a generate name, age, gpa;
    2429. 

  2429. store b into ':OUTPATH:.intermediate' using PigStorage();
    2430. 

  2430. c = load ':OUTPATH:.intermediate' using PigStorage();
    2431. 

  2431. -- after this load, the fields are treated as bytearrays though
    2432. 

  2432. -- they are actually "typed", test that the implicit casts 
    2433. 

  2433. -- introduced by the operations in the foreach below will work fine
    2434. 

  2434. d = foreach c generate CONCAT((chararray)$0, 'test'), $1 + 1L, $2 + 0.2f;
    2435. 

  2435. store d into ':OUTPATH:';\,
    2436. 

  2436. 

  2437.             'notmq' => 1,
    2438. 

  2438. 			},
    2439. 

  2439. 			{
    2440. 

  2440.              #check that a cast of a value of type
    2441. 

  2441.              #same as the result type of the cast works
    2442. 

  2442.              #when the value is treated as a bytearray
    2443. 

  2443. 			'num' => 6,
    2444. 

  2444. 			'floatpostprocess' => 1,
    2445. 

  2445. 			'delimiter' => '	',
    2446. 

  2446. 			'pig' => q\ 
    2447. 

  2447. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:long, gpa: float);
    2448. 

  2448. b = group a by  name;
    2449. 

  2449. c = foreach b generate a, (1,2,3), ['key1'#'value1','key2'#'value2'];
    2450. 

  2450. -- store the bag, tuple and map
    2451. 

  2451. store c into ':OUTPATH:.intermediate' using PigStorage();
    2452. 

  2452. d = load ':OUTPATH:.intermediate' using PigStorage() as (b:bag{t:tuple(x,y,z)}, t2:tuple(a,b,c), m:map[]);
    2453. 

  2453. -- after this load, the fields are treated as bytearrays though
    2454. 

  2454. -- they are actually "typed", test that the implicit casts 
    2455. 

  2455. -- introduced by the operations in the foreach below will work fine
    2456. 

  2456. e = foreach d generate COUNT(b), t2.a, t2.b, t2.c, m#'key1', m#'key2';
    2457. 

  2457. store e into ':OUTPATH:';\,
    2458. 

  2458. 

  2459.             'notmq' => 1,
    2460. 

  2460. 			},
    2461. 

  2461. 			{
    2462. 

  2462. 			'num' => 7,
    2463. 

  2463. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name, age, gpa, instate);
    2464. 

  2464. b = foreach a generate (boolean)instate;
    2465. 

  2465. c = filter b by instate == true;
    2466. 

  2466. store c into ':OUTPATH:';\,
    2467. 

  2467. 			'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    2468. 

  2468. b = foreach a generate instate;
    2469. 

  2469. c = filter b by instate == 'true';
    2470. 

  2470. store c into ':OUTPATH:';\,
    2471. 

  2471. 			}
    2472. 

  2472. 		],
    2473. 

  2473.         },
    2474. 

  2474.         {
    2475. 

  2475. 		'name' => 'ClassResolution',
    2476. 

  2476. 		'tests' => [
    2477. 

  2477. 			{
    2478. 

  2478.             # check that Loader specified without a package
    2479. 

  2479.             # name works if that package name is specified
    2480. 

  2480.             # in udf.import.list
    2481. 

  2481. 			'num' => 1,
    2482. 

  2482. 			'floatpostprocess' => 1,
    2483. 

  2483. 			'delimiter' => '	',
    2484. 

  2484. 	'java_params' => ['-Dudf.import.list=org.apache.pig.test.udf.storefunc'],
    2485. 

  2485. 			'pig' => q\ 
    2486. 

  2486. register :FUNCPATH:/testudf.jar;
    2487. 

  2487. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa: double);
    2488. 

  2488. b = foreach a generate CONCAT('(', name), CONCAT((chararray)age, ' )');
    2489. 

  2489. store b into ':OUTPATH:.intermediate' using PigStorage(',');
    2490. 

  2490. c = load ':OUTPATH:.intermediate' using DumpLoader();
    2491. 

  2491. store c into ':OUTPATH:';\,
    2492. 

  2492. 

  2493.             'notmq' => 1,
    2494. 

  2494. 			},
    2495. 

  2495. 		],
    2496. 

  2496.         },
    2497. 

  2497. 

  2498. 	{
    2499. 

  2499. 		'name' => 'MergeJoin',
    2500. 

  2500. 		'tests' => [
    2501. 

  2501. 				# Simplest merge-join.
    2502. 

  2502. 				{
    2503. 

  2503. 				'num' => 1,
    2504. 

  2504. 				'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2505. 

  2505. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2506. 

  2506. 					c = order a by $0;
    2507. 

  2507. 					d = order b by $0;
    2508. 

  2508. 					store c into ':OUTPATH:.intermediate1';
    2509. 

  2509. 					store d into ':OUTPATH:.intermediate2';
    2510. 

  2510. 					exec;
    2511. 

  2511. 					e = load ':OUTPATH:.intermediate1';
    2512. 

  2512. 					f = load ':OUTPATH:.intermediate2';
    2513. 

  2513. 					g = join e by $0, f by $0 using 'merge';
    2514. 

  2514. 					store g into ':OUTPATH:';\,
    2515. 

  2515.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2516. 

  2516. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2517. 

  2517. 					g = join a by $0, b by $0;
    2518. 

  2518. 					store g into ':OUTPATH:';\,
    2519. 

  2519.                 'notmq' => 1,
    2520. 

  2520. 				},
    2521. 

  2521. 				# Merge-join with left-side filter
    2522. 

  2522. 				{
    2523. 

  2523. 				'num' => 2,
    2524. 

  2524. 				'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2525. 

  2525. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2526. 

  2526. 					c = order a by $0;
    2527. 

  2527. 					d = order b by $0;
    2528. 

  2528. 					store c into ':OUTPATH:.intermediate1';
    2529. 

  2529. 					store d into ':OUTPATH:.intermediate2';
    2530. 

  2530. 					exec;					
    2531. 

  2531. 					e = load ':OUTPATH:.intermediate1';
    2532. 

  2532. 					h = filter e by $1 > 30;					
    2533. 

  2533. 					f = load ':OUTPATH:.intermediate2';
    2534. 

  2534. 					g = join h by $0, f by $0 using 'merge';
    2535. 

  2535. 					store g into ':OUTPATH:';\,
    2536. 

  2536.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2537. 

  2537. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2538. 

  2538. 					h = filter a by $1 > 30;					
    2539. 

  2539. 					g = join h by $0, b by $0;
    2540. 

  2540. 					store g into ':OUTPATH:';\,
    2541. 

  2541.                 'notmq' => 1,
    2542. 

  2542. 

  2543. 				},
    2544. 

  2544. 				# Merge-join with right-side filter
    2545. 

  2545. 				{
    2546. 

  2546. 				'num' => 3,
    2547. 

  2547. 				'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2548. 

  2548. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2549. 

  2549. 					c = order a by $0;
    2550. 

  2550. 					d = order b by $0;
    2551. 

  2551. 					store c into ':OUTPATH:.intermediate1';
    2552. 

  2552. 					store d into ':OUTPATH:.intermediate2';
    2553. 

  2553. 					exec;					
    2554. 

  2554. 					e = load ':OUTPATH:.intermediate1';
    2555. 

  2555. 					f = load ':OUTPATH:.intermediate2';
    2556. 

  2556. 					i = filter f by $2 != 'democrat';
    2557. 

  2557. 					g = join e by $0, i by $0 using 'merge';
    2558. 

  2558. 					store g into ':OUTPATH:';\,
    2559. 

  2559.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2560. 

  2560. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2561. 

  2561. 					i = filter b by $2 != 'democrat';
    2562. 

  2562. 					g = join a by $0, i by $0;
    2563. 

  2563. 					store g into ':OUTPATH:';\,
    2564. 

  2564. 

  2565.                 'notmq' => 1,
    2566. 

  2566. 				},
    2567. 

  2567. 				# Merge-join with schemas
    2568. 

  2568. 				{
    2569. 

  2569. 				'num' => 4,
    2570. 

  2570. 		        'floatpostprocess' => 1,
    2571. 

  2571.                 'delimiter' => '	',
    2572. 

  2572. 				'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2573. 

  2573. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2574. 

  2574. 					c = order a by $0;
    2575. 

  2575. 					d = order b by $0;
    2576. 

  2576. 					store c into ':OUTPATH:.intermediate1';
    2577. 

  2577. 					store d into ':OUTPATH:.intermediate2';
    2578. 

  2578. 					exec;					
    2579. 

  2579. 					e = load ':OUTPATH:.intermediate1'  as (name:chararray, age:int, gpa:float);
    2580. 

  2580. 					f = load ':OUTPATH:.intermediate2'  as (name:chararray, age:int, reg:chararray, contrib:float);
    2581. 

  2581. 					g = join e by $0, f by $0 using 'merge';
    2582. 

  2582. 					store g into ':OUTPATH:';\,
    2583. 

  2583.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2584. 

  2584. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2585. 

  2585. 					g = join a by $0, b by $0;
    2586. 

  2586. 					store g into ':OUTPATH:';\,
    2587. 

  2587. 

  2588.                 'notmq' => 1,
    2589. 

  2589. 				},
    2590. 

  2590. 				# Merge-join with key as expression
    2591. 

  2591. 				{
    2592. 

  2592. 				'num' => 5,
    2593. 

  2593. 				'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2594. 

  2594. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2595. 

  2595. 					c = order a by $0,$1;
    2596. 

  2596. 					d = order b by $0,$1;
    2597. 

  2597. 					store c into ':OUTPATH:.intermediate1';
    2598. 

  2598. 					store d into ':OUTPATH:.intermediate2';
    2599. 

  2599. 					exec;
    2600. 

  2600. 					e = load ':OUTPATH:.intermediate1';
    2601. 

  2601. 					f = load ':OUTPATH:.intermediate2';
    2602. 

  2602. 					g = join e by ($0,$1), f by ($0,$1) using 'merge';
    2603. 

  2603. 					store g into ':OUTPATH:';\,
    2604. 

  2604.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2605. 

  2605. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2606. 

  2606. 					g = join a by ($0,$1), b by ($0,$1);
    2607. 

  2607. 					store g into ':OUTPATH:';\,
    2608. 

  2608. 

  2609.                 'notmq' => 1,
    2610. 

  2610. 				},
    2611. 

  2611. 				# Merge-join with key as expression This expression guarantees ordering 
    2612. 

  2612. 				{
    2613. 

  2613. 				'num' => 6,
    2614. 

  2614. 				'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2615. 

  2615. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2616. 

  2616. 					c = order a by $1;
    2617. 

  2617. 					d = order b by $1;
    2618. 

  2618. 					store c into ':OUTPATH:.intermediate1';
    2619. 

  2619. 					store d into ':OUTPATH:.intermediate2';
    2620. 

  2620. 					exec;
    2621. 

  2621. 					e = load ':OUTPATH:.intermediate1';
    2622. 

  2622. 					f = load ':OUTPATH:.intermediate2';
    2623. 

  2623. 					g = join e by ($1+10), f by ($1+10) using 'merge';
    2624. 

  2624. 					store g into ':OUTPATH:';\,
    2625. 

  2625.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2626. 

  2626. 					b = load ':INPATH:/singlefile/votertab10k'; 
    2627. 

  2627. 					g = join a by ($1+10), b by ($1+10) ;
    2628. 

  2628. 					store g into ':OUTPATH:';\,
    2629. 

  2629. 

  2630.                 'notmq' => 1,
    2631. 

  2631. 				},
    2632. 

  2632. 				# Merge-join with nulls in keys and data.
    2633. 

  2633. 				{
    2634. 

  2634. 				'num' => 7,
    2635. 

  2635. 				'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k';
    2636. 

  2636. 					b = load ':INPATH:/singlefile/voternulltab10k'; 
    2637. 

  2637. 					c = order a by $0;
    2638. 

  2638. 					d = order b by $0;
    2639. 

  2639. 					store c into ':OUTPATH:.intermediate1';
    2640. 

  2640. 					store d into ':OUTPATH:.intermediate2';
    2641. 

  2641. 					exec;
    2642. 

  2642. 					e = load ':OUTPATH:.intermediate1';
    2643. 

  2643. 					f = load ':OUTPATH:.intermediate2';
    2644. 

  2644. 					g = join e by $0, f by $0 using 'merge';
    2645. 

  2645. 					store g into ':OUTPATH:';\,
    2646. 

  2646.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentnulltab10k';
    2647. 

  2647. 					b = load ':INPATH:/singlefile/voternulltab10k'; 
    2648. 

  2648. 					g = join a by $0, b by $0;
    2649. 

  2649. 					store g into ':OUTPATH:';\,
    2650. 

  2650. 

  2651.                 'notmq' => 1,
    2652. 

  2652. 				},
    2653. 

  2653. 				# Merge-join with one file across multiple blocks
    2654. 

  2654.         	    {
    2655. 

  2655.                 'num' => 8,
    2656. 

  2656. 			    'execonly' => 'mapred', # since this join will run out of memory in local mode
    2657. 

  2657. 		        'floatpostprocess' => 1,
    2658. 

  2658. 		        'delimiter' => '	',
    2659. 

  2659.                 'pig' => q\a = load ':INPATH:/singlefile/votertab10k';
    2660. 

  2660.                     b = load ':INPATH:/singlefile/studenttab20m';
    2661. 

  2661.                     h = filter b by $2 < 1.5;
    2662. 

  2662.                     c = order a by $0;
    2663. 

  2663.                     d = order h by $0 parallel 1;
    2664. 

  2664.                     store c into ':OUTPATH:.intermediate1';
    2665. 

  2665.                     store d into ':OUTPATH:.intermediate2';
    2666. 

  2666.                     exec;
    2667. 

  2667.                     e = load ':OUTPATH:.intermediate1' as (name:chararray, age:int, reg:chararray, contrib:float);
    2668. 

  2668.                     f = load ':OUTPATH:.intermediate2'as (name:chararray, age:int, gpa:float);
    2669. 

  2669.                     g = join e by $0, f by $0 using 'merge';
    2670. 

  2670.                     i = filter g by $2 == 'democrat' and $1 > 76;
    2671. 

  2671.                     store i into ':OUTPATH:';\,
    2672. 

  2672.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/votertab10k';
    2673. 

  2673.                     b = load ':INPATH:/singlefile/studenttab20m';
    2674. 

  2674.                     h = filter b by $2 < 1.5;
    2675. 

  2675.                     g = join a by $0, h by $0;
    2676. 

  2676.                     i = filter g by $2 == 'democrat' and $1 > 76;
    2677. 

  2677.                     store i into ':OUTPATH:';\,
    2678. 

  2678. 

  2679.                 'notmq' => 1,
    2680. 

  2680.                 }, 
    2681. 

  2681. 				# Merge-join with join on numeric key
    2682. 

  2682. 				{
    2683. 

  2683. 				'num' => 9,
    2684. 

  2684. 				'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    2685. 

  2685. 					b = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    2686. 

  2686. 					c = order a by age;
    2687. 

  2687. 					d = order b by age;
    2688. 

  2688. 					store c into ':OUTPATH:.intermediate1';
    2689. 

  2689. 					store d into ':OUTPATH:.intermediate2';
    2690. 

  2690. 					exec;					
    2691. 

  2691. 					e = load ':OUTPATH:.intermediate1'  as (name:chararray, age:int, gpa:float);
    2692. 

  2692. 					f = load ':OUTPATH:.intermediate2'  as (name:chararray, age:int, reg:chararray, contrib:float);
    2693. 

  2693. 					g = join e by age, f by age using 'merge';
    2694. 

  2694. 					store g into ':OUTPATH:';\,
    2695. 

  2695.                 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    2696. 

  2696. 					b = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    2697. 

  2697. 					g = join a by age, b by age;
    2698. 

  2698. 					store g into ':OUTPATH:';\,
    2699. 

  2699. 

  2700.                 'notmq' => 1,
    2701. 

  2701. 				},
    2702. 

  2702. 

  2703. 			]
    2704. 

  2704. 		},
    2705. 

  2705.                 {
    2706. 

  2706.                 'name' => 'SkewedJoin',
    2707. 

  2707. 		'floatpostprocess' => 1,
    2708. 

  2708. 		'delimiter' => '	',
    2709. 

  2709.                 'tests' => [
    2710. 

  2710.                         {
    2711. 

  2711.                         'num' => 1,
    2712. 

  2712.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=100'],
    2713. 

  2713.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2714. 

  2714. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2715. 

  2715. e = join a by name, b by name using 'skewed' parallel 8;
    2716. 

  2716. store e into ':OUTPATH:';\,
    2717. 

  2717.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2718. 

  2718. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2719. 

  2719. e = join a by name, b by name;
    2720. 

  2720. store e into ':OUTPATH:';\,
    2721. 

  2721. 

  2722.                         },
    2723. 

  2723. 

  2724.                 # basic join with no skewed keys
    2725. 

  2725.                        {
    2726. 

  2726.                         'num' => 2,
    2727. 

  2727.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=10000'],
    2728. 

  2728.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age,
    2729. 

  2729. gpa);
    2730. 

  2730. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2731. 

  2731. e = join a by name, b by name using 'skewed';
    2732. 

  2732. store e into ':OUTPATH:';\,
    2733. 

  2733.                         'verify_pig_script' =>q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age,
    2734. 

  2734. gpa);
    2735. 

  2735. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2736. 

  2736. e = join a by name, b by name ;
    2737. 

  2737. store e into ':OUTPATH:';\,
    2738. 

  2738. 

  2739.                         },
    2740. 

  2740. 

  2741.                 # join after filtering
    2742. 

  2742.                         {
    2743. 

  2743.                         'num' => 3,
    2744. 

  2744.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=3'],
    2745. 

  2745.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age,
    2746. 

  2746. gpa);
    2747. 

  2747. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2748. 

  2748. c = filter a by age < 20;
    2749. 

  2749. d = filter b by age < 20;
    2750. 

  2750. e = join c by $0, d by $0 using 'skewed' parallel 8;
    2751. 

  2751. store e into ':OUTPATH:';\,
    2752. 

  2752.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age,
    2753. 

  2753. gpa);
    2754. 

  2754. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2755. 

  2755. c = filter a by age < 20;
    2756. 

  2756. d = filter b by age < 20;
    2757. 

  2757. e = join c by $0, d by $0 ;
    2758. 

  2758. store e into ':OUTPATH:';\,
    2759. 

  2759.                         },
    2760. 

  2760. 

  2761.                 # join by two columns
    2762. 

  2762.                         {
    2763. 

  2763.                         'num' => 4,
    2764. 

  2764.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=3'],
    2765. 

  2765.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2766. 

  2766. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2767. 

  2767. c = filter a by age < 20;
    2768. 

  2768. d = filter b by age < 20;
    2769. 

  2769. e = join c by (name, age), d by (name, age) using 'skewed' parallel 8;
    2770. 

  2770. store e into ':OUTPATH:';\,
    2771. 

  2771.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2772. 

  2772. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2773. 

  2773. c = filter a by age < 20;
    2774. 

  2774. d = filter b by age < 20;
    2775. 

  2775. e = join c by (name, age), d by (name, age) ;
    2776. 

  2776. store e into ':OUTPATH:';\,
    2777. 

  2777. 

  2778.                         },
    2779. 

  2779. 

  2780.                 # join with add
    2781. 

  2781.                        {
    2782. 

  2782.                         'num' => 5,
    2783. 

  2783.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=50'],
    2784. 

  2784.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray,age:int, gpa:double);
    2785. 

  2785. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2786. 

  2786. c = filter a by age < 20;
    2787. 

  2787. d = filter b by age < 20;
    2788. 

  2788. e = join c by age+10, d by age + 20 using 'skewed' parallel 10;
    2789. 

  2789. store e into ':OUTPATH:';\,
    2790. 

  2790.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray,age:int, gpa:double);
    2791. 

  2791. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2792. 

  2792. c = filter a by age < 20;
    2793. 

  2793. d = filter b by age < 20;
    2794. 

  2794. e = join c by age+10, d by age + 20 ;
    2795. 

  2795. store e into ':OUTPATH:';\,
    2796. 

  2796. 

  2797.                         },
    2798. 

  2798. 

  2799.                 # join with split
    2800. 

  2800.                        {
    2801. 

  2801.                         'num' => 6,
    2802. 

  2802.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=100'],
    2803. 

  2803.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2804. 

  2804. b = filter a by $1 > 25;
    2805. 

  2805. c = join a by $0, b by $0 using 'skewed' parallel 7;
    2806. 

  2806. store c into ':OUTPATH:';\,
    2807. 

  2807.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    2808. 

  2808. b = filter a by $1 > 25;
    2809. 

  2809. c = join a by $0, b by $0 ;
    2810. 

  2810. store c into ':OUTPATH:';\,
    2811. 

  2811. 

  2812.                         },
    2813. 

  2813. 

  2814. 		# join with UDF
    2815. 

  2815.                        {
    2816. 

  2816.                         'num' => 7,
    2817. 

  2817.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=20'],
    2818. 

  2818.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    2819. 

  2819. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2820. 

  2820. c = filter a by age < 20;
    2821. 

  2821. d = filter b by age < 20;
    2822. 

  2822. e = join c by SIZE(name), d by SIZE(name) using 'skewed' parallel 7;
    2823. 

  2823. store e into ':OUTPATH:';\,
    2824. 

  2824.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    2825. 

  2825. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2826. 

  2826. c = filter a by age < 20;
    2827. 

  2827. d = filter b by age < 20;
    2828. 

  2828. e = join c by SIZE(name), d by SIZE(name) ;
    2829. 

  2829. store e into ':OUTPATH:';\,
    2830. 

  2830. 

  2831.                         },
    2832. 

  2832. 		# left outer join
    2833. 

  2833.                         {
    2834. 

  2834.                         'num' => 8,
    2835. 

  2835.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=100'],
    2836. 

  2836.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2837. 

  2837. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2838. 

  2838. e = join a by name left outer, b by name using 'skewed' parallel 8;
    2839. 

  2839. store e into ':OUTPATH:';\,
    2840. 

  2840.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2841. 

  2841. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2842. 

  2842. e = join a by name left outer, b by name ;
    2843. 

  2843. store e into ':OUTPATH:';\,
    2844. 

  2844. 

  2845.                         },
    2846. 

  2846. 		# right outer join
    2847. 

  2847.                         {
    2848. 

  2848.                         'num' => 9,
    2849. 

  2849.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=100'],
    2850. 

  2850.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2851. 

  2851. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2852. 

  2852. e = join a by name right outer, b by name using 'skewed' parallel 8;
    2853. 

  2853. store e into ':OUTPATH:';\,
    2854. 

  2854.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2855. 

  2855. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2856. 

  2856. e = join a by name right outer, b by name ;
    2857. 

  2857. store e into ':OUTPATH:';\,
    2858. 

  2858. 

  2859.                         },
    2860. 

  2860. 		# full outer join
    2861. 

  2861.                         {
    2862. 

  2862.                         'num' => 10,
    2863. 

  2863.                         'java_params' => ['-Dpig.skewedjoin.reduce.maxtuple=100'],
    2864. 

  2864.                         'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2865. 

  2865. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2866. 

  2866. e = join a by name full outer, b by name using 'skewed' parallel 8;
    2867. 

  2867. store e into ':OUTPATH:';\,
    2868. 

  2868.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    2869. 

  2869. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    2870. 

  2870. e = join a by name full outer, b by name ;
    2871. 

  2871. store e into ':OUTPATH:';\,
    2872. 

  2872. 

  2873.                         },
    2874. 

  2874.                 ]
    2875. 

  2875. 

  2876.             },
    2877. 

  2877. 

  2878.             {
    2879. 

  2879.             'name' => 'CollectedGroup',
    2880. 

  2880.             'tests' => [
    2881. 

  2881.                     # Simplest collected group.
    2882. 

  2882.                     {
    2883. 

  2883.                     'num' => 1,
    2884. 

  2884.                     'pig' => q\a = load ':INPATH:/singlefile/studenttab10k'; 
    2885. 

  2885.                             b = order a by $0;
    2886. 

  2886.                             store b into ':OUTPATH:.intermediate';
    2887. 

  2887.                             exec;
    2888. 

  2888.                        	    register :FUNCPATH:/testudf.jar;	     
    2889. 

  2889. 			    c = load ':OUTPATH:.intermediate' using org.apache.pig.test.udf.storefunc.SimpleCollectableLoader();
    2890. 

  2890.                             d = group c by $0 using 'collected';
    2891. 

  2891.                             e = foreach d generate group, COUNT(c);
    2892. 

  2892.                             store e into ':OUTPATH:';\,
    2893. 

  2893.                     'notmq' => 1,
    2894. 

  2894.                     'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k'; 
    2895. 

  2895.                             d = group a by $0 ;
    2896. 

  2896.                             e = foreach d generate group, COUNT(a);
    2897. 

  2897.                             store e into ':OUTPATH:';\,
    2898. 

  2898. 

  2899.                     },
    2900. 

  2900.                     # Collected group with filter
    2901. 

  2901.                     {
    2902. 

  2902.                     'num' => 2,
    2903. 

  2903.                     'pig' => q\a = load ':INPATH:/singlefile/studenttab10k'; 
    2904. 

  2904.                             b = order a by $0;
    2905. 

  2905.                             store b into ':OUTPATH:.intermediate';
    2906. 

  2906.                             exec;
    2907. 

  2907.                        	    register :FUNCPATH:/testudf.jar;	     
    2908. 

  2908. 			    c = load ':OUTPATH:.intermediate' using org.apache.pig.test.udf.storefunc.SimpleCollectableLoader();
    2909. 

  2909.                             d = filter c by $1 > 30;
    2910. 

  2910.                             e = group d by $0 using 'collected';
    2911. 

  2911.                             f = foreach e generate group, COUNT(d);
    2912. 

  2912.                             store f into ':OUTPATH:';\,
    2913. 

  2913.                     'notmq' => 1,
    2914. 

  2914.                     'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k'; 
    2915. 

  2915.                             d = filter a by $1 > 30;
    2916. 

  2916.                             e = group d by $0 ;
    2917. 

  2917.                             f = foreach e generate group, COUNT(d);
    2918. 

  2918.                             store f into ':OUTPATH:';\,
    2919. 

  2919. 

  2920.                     },
    2921. 

  2921.                     # Collected group with schemas
    2922. 

  2922.                     {
    2923. 

  2923.                     'num' => 3,
    2924. 

  2924.                     'pig' => q\a = load ':INPATH:/singlefile/studenttab10k'; 
    2925. 

  2925.                             b = order a by $0;
    2926. 

  2926.                             store b into ':OUTPATH:.intermediate';
    2927. 

  2927.                             exec;
    2928. 

  2928.                        	    register :FUNCPATH:/testudf.jar;	     
    2929. 

  2929. 			    c = load ':OUTPATH:.intermediate' using org.apache.pig.test.udf.storefunc.SimpleCollectableLoader() as (name:chararray, age:int, gpa:float);
    2930. 

  2930.                             d = group c by $0 using 'collected';
    2931. 

  2931.                             e = foreach d generate group, MAX(c.age);
    2932. 

  2932.                             store e into ':OUTPATH:';\,
    2933. 

  2933.                     'notmq' => 1,
    2934. 

  2934.                     'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float); 
    2935. 

  2935.                             d = group a by $0 ;
    2936. 

  2936.                             e = foreach d generate group, MAX(a.$1);
    2937. 

  2937.                             store e into ':OUTPATH:';\,
    2938. 

  2938. 

  2939.                     },
    2940. 

  2940.                     # Collected group with multiple columns
    2941. 

  2941.                     {
    2942. 

  2942.                     'num' => 4,
    2943. 

  2943.                     'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    2944. 

  2944.                             b = order a by name, age;
    2945. 

  2945.                             store b into ':OUTPATH:.intermediate';
    2946. 

  2946.                             exec;
    2947. 

  2947.                        	    register :FUNCPATH:/testudf.jar;	     
    2948. 

  2948. 			    c = load ':OUTPATH:.intermediate' using org.apache.pig.test.udf.storefunc.SimpleCollectableLoader() as (name:chararray, age:int, gpa:float);
    2949. 

  2949.                             d = group c by (name, age) using 'collected';
    2950. 

  2950.                             e = foreach d generate group.name, group.age, MIN(c.gpa);
    2951. 

  2951.                             store e into ':OUTPATH:';\,
    2952. 

  2952.                     'notmq' => 1,
    2953. 

  2953.                     'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    2954. 

  2954.                             d = group a by (name, age) ;
    2955. 

  2955.                             e = foreach d generate group.name, group.age, MIN(a.gpa);
    2956. 

  2956.                             store e into ':OUTPATH:';\,
    2957. 

  2957. 

  2958.                     },
    2959. 

  2959.                     # Collected group with nulls in keys and data.
    2960. 

  2960.                     {
    2961. 

  2961.                     'num' => 5,
    2962. 

  2962.                     'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k'; 
    2963. 

  2963.                             b = order a by $0;
    2964. 

  2964.                             store b into ':OUTPATH:.intermediate';
    2965. 

  2965.                             exec;
    2966. 

  2966.                        	    register :FUNCPATH:/testudf.jar;	     
    2967. 

  2967. 			    c = load ':OUTPATH:.intermediate' using org.apache.pig.test.udf.storefunc.SimpleCollectableLoader() as (name:chararray, age:int, gpa:float);
    2968. 

  2968.                             d = group c by $0 using 'collected';
    2969. 

  2969.                             e = foreach d generate group, SUM(c.$1);
    2970. 

  2970.                             store e into ':OUTPATH:';\,
    2971. 

  2971.                     'notmq' => 1,
    2972. 

  2972.                     'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:float); 
    2973. 

  2973.                             d = group a by $0 ;
    2974. 

  2974.                             e = foreach d generate group, SUM(a.$1);
    2975. 

  2975.                             store e into ':OUTPATH:';\,
    2976. 

  2976. 

  2977.                     },
    2978. 

  2978.                     # Collected group with numeric key
    2979. 

  2979.                     {
    2980. 

  2980.                     'num' => 6,
    2981. 

  2981.                     'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    2982. 

  2982.                             b = order a by age;
    2983. 

  2983.                             store b into ':OUTPATH:.intermediate';
    2984. 

  2984.                             exec;
    2985. 

  2985.                        	    register :FUNCPATH:/testudf.jar;	     
    2986. 

  2986. 			    c = load ':OUTPATH:.intermediate' using org.apache.pig.test.udf.storefunc.SimpleCollectableLoader() as (name:chararray, age:int, gpa:float);
    2987. 

  2987.                             d = group c by age using 'collected';
    2988. 

  2988.                             e = foreach d generate group, AVG(c.gpa), COUNT(c.name);
    2989. 

  2989.                             store e into ':OUTPATH:';\,
    2990. 

  2990.                     'notmq' => 1,
    2991. 

  2991.                     'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    2992. 

  2992.                             d = group a by age ;
    2993. 

  2993.                             e = foreach d generate group, AVG(a.gpa), COUNT(a.name);
    2994. 

  2994.                             store e into ':OUTPATH:';\,
    2995. 

  2995. 

  2996.                     },
    2997. 

  2997.                 ]
    2998. 

  2998.             },
    2999. 

  2999.             {
    3000. 

  3000.             'name' => 'SecondarySort',
    3001. 

  3001.             'tests' => [
    3002. 

  3002.                     {
    3003. 

  3003.                     # simple order by
    3004. 

  3004.                     'num' => 1,
    3005. 

  3005.                     'java_params' => ['-Dpig.accumulative.batchsize=5'],
    3006. 

  3006.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3007. 

  3007.                             a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    3008. 

  3008.                             b = group a by age parallel 10;
    3009. 

  3009.                             c = foreach b {
    3010. 

  3010.                                 d = order a by name;
    3011. 

  3011.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(d);
    3012. 

  3012.                             };
    3013. 

  3013.                             store c into ':OUTPATH:';\,
    3014. 

  3014.                     },
    3015. 

  3015.                     {
    3016. 

  3016.                     # order by desc
    3017. 

  3017.                     'num' => 2,
    3018. 

  3018.                     'java_params' => ['-Dpig.accumulative.batchsize=5'],
    3019. 

  3019.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3020. 

  3020.                             a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    3021. 

  3021.                             b = group a by age parallel 10;
    3022. 

  3022.                             c = foreach b {
    3023. 

  3023.                                 d = order a by name desc;
    3024. 

  3024.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(d);
    3025. 

  3025.                             };
    3026. 

  3026.                             store c into ':OUTPATH:';\,
    3027. 

  3027.                     },
    3028. 

  3028.                     {
    3029. 

  3029.                     # order by float type
    3030. 

  3030.                     'num' => 3,
    3031. 

  3031.                     'java_params' => ['-Dpig.accumulative.batchsize=5'],
    3032. 

  3032.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3033. 

  3033.                             a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    3034. 

  3034.                             b = group a by age parallel 10;
    3035. 

  3035.                             c = foreach b {
    3036. 

  3036.                                 d = order a by gpa;
    3037. 

  3037.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(d.gpa);
    3038. 

  3038.                             };
    3039. 

  3039.                             store c into ':OUTPATH:';\,
    3040. 

  3040.                     },
    3041. 

  3041.                     # order by string type
    3042. 

  3042.                     {
    3043. 

  3043.                     'num' => 4,
    3044. 

  3044.                     'java_params' => ['-Dpig.accumulative.batchsize=5'],
    3045. 

  3045.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3046. 

  3046.                             a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    3047. 

  3047.                             b = group a by age parallel 10;
    3048. 

  3048.                             c = foreach b {
    3049. 

  3049.                                 d = order a by name;
    3050. 

  3050.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(d.name);
    3051. 

  3051.                             };
    3052. 

  3052.                             store c into ':OUTPATH:';\,
    3053. 

  3053.                     },
    3054. 

  3054.                     # simple distinct
    3055. 

  3055.                     {
    3056. 

  3056.                     'num' => 5,
    3057. 

  3057.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3058. 

  3058.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3059. 

  3059.                             a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    3060. 

  3060.                             b = group a by age parallel 10;
    3061. 

  3061.                             c = foreach b {
    3062. 

  3062.                                 d = a.name;
    3063. 

  3063.                                 e = distinct d;
    3064. 

  3064.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(e);
    3065. 

  3065.                             };
    3066. 

  3066.                             store c into ':OUTPATH:';\,
    3067. 

  3067.                     },
    3068. 

  3068.                     # distinct on tuple
    3069. 

  3069.                     {
    3070. 

  3070.                     'num' => 6,
    3071. 

  3071.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3072. 

  3072.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3073. 

  3073.                             a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    3074. 

  3074.                             b = group a by age parallel 10;
    3075. 

  3075.                             c = foreach b {
    3076. 

  3076.                                 d = distinct a;
    3077. 

  3077.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(d);
    3078. 

  3078.                             };
    3079. 

  3079.                             store c into ':OUTPATH:';\,
    3080. 

  3080.                     },
    3081. 

  3081.                     # sort by two columns
    3082. 

  3082.                     {
    3083. 

  3083.                     'num' => 7,
    3084. 

  3084.                     'java_params' => ['-Dpig.accumulative.batchsize=5'],
    3085. 

  3085.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3086. 

  3086.                             a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    3087. 

  3087.                             b = group a by age parallel 10;
    3088. 

  3088.                             c = foreach b {
    3089. 

  3089.                                 d = order a by gpa, name desc;
    3090. 

  3090.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(d.gpa), org.apache.pig.test.udf.evalfunc.AllFirstLetter(d.name);
    3091. 

  3091.                             };
    3092. 

  3092.                             store c into ':OUTPATH:';\,
    3093. 

  3093.                     },
    3094. 

  3094.                     # sort, distinct mix
    3095. 

  3095.                     {
    3096. 

  3096.                     'num' => 8,
    3097. 

  3097.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3098. 

  3098.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3099. 

  3099.                             a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    3100. 

  3100.                             b = group a by age parallel 10;
    3101. 

  3101.                             c = foreach b {
    3102. 

  3102.                                 d = order a by name;
    3103. 

  3103.                                 e = d.gpa;
    3104. 

  3104.                                 f = distinct e;
    3105. 

  3105.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(f); 
    3106. 

  3106.                             };
    3107. 

  3107.                             store c into ':OUTPATH:';\,
    3108. 

  3108.                     },
    3109. 

  3109.                     # sort, distinct mix
    3110. 

  3110.                     {
    3111. 

  3111.                     'num' => 9,
    3112. 

  3112.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3113. 

  3113.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3114. 

  3114.                             a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    3115. 

  3115.                             b = group a by age parallel 10;
    3116. 

  3116.                             c = foreach b {
    3117. 

  3117.                                 d = order a by gpa;
    3118. 

  3118.                                 e = d.gpa;
    3119. 

  3119.                                 f = distinct e;
    3120. 

  3120.                                 generate group, org.apache.pig.test.udf.evalfunc.AllFirstLetter(f); 
    3121. 

  3121.                             };
    3122. 

  3122.                             store c into ':OUTPATH:';\,
    3123. 

  3123.                     },
    3124. 

  3124.                     {
    3125. 

  3125.                     # secondary sort boolean
    3126. 

  3126.                     'num' => 10,
    3127. 

  3127.                     'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:boolean);
    3128. 

  3128.                             b = group a by age;
    3129. 

  3129.                             c = foreach b {
    3130. 

  3130.                                 d = order a by instate;
    3131. 

  3131.                                 generate group, flatten(d);
    3132. 

  3132.                             };
    3133. 

  3133.                             store c into ':OUTPATH:';\,
    3134. 

  3134.                     'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    3135. 

  3135.                             b = group a by age;
    3136. 

  3136.                             c = foreach b {
    3137. 

  3137.                                 d = order a by instate;
    3138. 

  3138.                                 generate group, flatten(d);
    3139. 

  3139.                             };
    3140. 

  3140.                             store c into ':OUTPATH:';\,
    3141. 

  3141.                     }
    3142. 

  3142.                 ]
    3143. 

  3143.             },
    3144. 

  3144.             {
    3145. 

  3145.             'name' => 'Accumulator',
    3146. 

  3146.             'tests' => [
    3147. 

  3147.                     {
    3148. 

  3148.                     'num' => 1,
    3149. 

  3149.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3150. 

  3150.                     'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa);
    3151. 

  3151. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    3152. 

  3152. e = cogroup a by name, b by name parallel 8;
    3153. 

  3153. f = foreach e generate group,  SUM(a.age) as s;
    3154. 

  3154. g = filter f by s>0;
    3155. 

  3155. store g into ':OUTPATH:';\,
    3156. 

  3156.                     },
    3157. 

  3157.                     {
    3158. 

  3158.                     'num' => 2,
    3159. 

  3159.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3160. 

  3160.                     'pig' => q\a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name, age:int, registration, contributions);
    3161. 

  3161. e = group a by name  parallel 8;
    3162. 

  3162. f = foreach e generate group,  COUNT(a), MAX(a.contributions), MIN(a.contributions) ;
    3163. 

  3163. store f into ':OUTPATH:';\,
    3164. 

  3164.                     },
    3165. 

  3165.                     {
    3166. 

  3166.                     'num' => 3,
    3167. 

  3167.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3168. 

  3168.                     'pig' => q\a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name, age:int, registration, contributions);
    3169. 

  3169. e = group a by name  parallel 8;
    3170. 

  3170. f = foreach e generate group,  (MAX(a.contributions)-MIN(a.contributions))*COUNT(a) ;
    3171. 

  3171. store f into ':OUTPATH:';\,
    3172. 

  3172.                     },
    3173. 

  3173.                     {
    3174. 

  3174.                     'num' => 4,
    3175. 

  3175.                     'java_params' => ['-Dpig.exec.nocombiner=true', '-Dpig.accumulative.batchsize=5'],
    3176. 

  3176.                     'pig' => q\a = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name, age:int, registration, contributions);
    3177. 

  3177. e = group a by name  parallel 8;
    3178. 

  3178. f = foreach e { g = distinct a.age; generate group,  COUNT(g);} 
    3179. 

  3179. store f into ':OUTPATH:';\,
    3180. 

  3180.                     }
    3181. 

  3181.                 ]
    3182. 

  3182.             },
    3183. 

  3183.             {
    3184. 

  3184.             'name' => 'PruneColumns',
    3185. 

  3185.             'tests' => [
    3186. 

  3186.                     {
    3187. 

  3187.                     'num' => 1,
    3188. 

  3188.                     'execonly' => 'mapred', # studenttab20m not available in local mode
    3189. 

  3189.                     'pig' => q\
    3190. 

  3190. a = load ':INPATH:/singlefile/studenttab20m' using PigStorage() as (name, age, gpa);
    3191. 

  3191. b = foreach a generate age;
    3192. 

  3192. store b into ':OUTPATH:';\,
    3193. 

  3193.                     }
    3194. 

  3194.                 ]
    3195. 

  3195.             },
    3196. 

  3196.             {
    3197. 

  3197.             'name' => 'Bzip',
    3198. 

  3198.             'tests' => [
    3199. 

  3199.                     {
    3200. 

  3200.                     # test reading and writing out files with .bz2 extension
    3201. 

  3201.                     'num' => 1,
    3202. 

  3202.                     'pig' => q\
    3203. 

  3203. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3204. 

  3204. store a into ':OUTPATH:.intermediate.bz2';
    3205. 

  3205. b = load ':OUTPATH:.intermediate.bz2';
    3206. 

  3206. store b into ':OUTPATH:';\,
    3207. 

  3207.                     'notmq' => 1,
    3208. 

  3208.                     },
    3209. 

  3209.                     {
    3210. 

  3210.                     # test reading and writing with .bz extension
    3211. 

  3211.                     'num' => 2,
    3212. 

  3212.                     'pig' => q\
    3213. 

  3213. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3214. 

  3214. store a into ':OUTPATH:.intermediate.bz';
    3215. 

  3215. b = load ':OUTPATH:.intermediate.bz';
    3216. 

  3216. store b into ':OUTPATH:';\,
    3217. 

  3217.                     'notmq' => 1,
    3218. 

  3218.                     },
    3219. 

  3219.                 ]
    3220. 

  3220.             },
    3221. 

  3221.             {
    3222. 

  3222.             'name' => 'Scalar',
    3223. 

  3223.             'tests' => [
    3224. 

  3224.                     {
    3225. 

  3225.                     # test scalar in foreach (most common)
    3226. 

  3226.                     'num' => 1,
    3227. 

  3227.                     'pig' => q\
    3228. 

  3228. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3229. 

  3229. b = group a all;
    3230. 

  3230. c = foreach b generate AVG(a.gpa) as avg, MAX(a.gpa) as max;
    3231. 

  3231. y = foreach a generate name, (gpa - c.avg) / c.max;
    3232. 

  3232. store y into ':OUTPATH:';\,
    3233. 

  3233.                     'floatpostprocess' => 1,
    3234. 

  3234.                     'delimiter' => '	',
    3235. 

  3235.                     },
    3236. 

  3236.                     {
    3237. 

  3237.                     # test scalar in filter
    3238. 

  3238.                     'num' => 2,
    3239. 

  3239.                     'pig' => q\
    3240. 

  3240. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3241. 

  3241. b = group a all;
    3242. 

  3242. c = foreach b generate AVG(a.gpa) as avg;
    3243. 

  3243. y = filter a by gpa > c.avg;
    3244. 

  3244. store y into ':OUTPATH:';\,
    3245. 

  3245.                     },
    3246. 

  3246.                     {
    3247. 

  3247.                     # test scalar with two branch
    3248. 

  3248.                     'num' => 3,
    3249. 

  3249.                     'pig' => q\
    3250. 

  3250. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3251. 

  3251. b = group a all;
    3252. 

  3252. c = foreach b generate AVG(a.age) as avg;
    3253. 

  3253. x = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name, age, registration, contributions);
    3254. 

  3254. y = filter x by age > c.avg;
    3255. 

  3255. store y into ':OUTPATH:';\,
    3256. 

  3256.                     },
    3257. 

  3257.                     {
    3258. 

  3258.                     # test with scalar from two inputs
    3259. 

  3259.                     'num' => 4,
    3260. 

  3260.                     'pig' => q\
    3261. 

  3261. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3262. 

  3262. b = group a all;
    3263. 

  3263. c = foreach b generate AVG(a.age) as avg;
    3264. 

  3264. d = load ':INPATH:/singlefile/votertab10k' using PigStorage() as (name, age, registration, contributions);
    3265. 

  3265. e = group d all;
    3266. 

  3266. f = foreach e generate AVG(d.age) as avg;
    3267. 

  3267. y = foreach a generate age/c.avg, age/f.avg;
    3268. 

  3268. store y into ':OUTPATH:';\,
    3269. 

  3269.                     },
    3270. 

  3270.                 ]
    3271. 

  3271.             },
    3272. 

  3272.             {
    3273. 

  3273.             'name' => 'Scripting',
    3274. 

  3274.             'tests' => [
    3275. 

  3275.                     {
    3276. 

  3276.                     # test integer square
    3277. 

  3277.                     'num' => 1,
    3278. 

  3278.                     'ignore23' => 'MAPREDUCE-3700',
    3279. 

  3279.                     'pig' => q\
    3280. 

  3280. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3281. 

  3281. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3282. 

  3282. b = foreach a generate myfuncs.square(age);
    3283. 

  3283. store b into ':OUTPATH:';\,
    3284. 

  3284.                     'verify_pig_script' => q\
    3285. 

  3285. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3286. 

  3286. b = foreach a generate age * age;
    3287. 

  3287. store b into ':OUTPATH:';\,
    3288. 

  3288.                     },
    3289. 

  3289.                     {
    3290. 

  3290.                     # test string concat and referencing function without a namespace
    3291. 

  3291.                     'num' => 2,
    3292. 

  3292.                     'ignore23' => 'MAPREDUCE-3700',
    3293. 

  3293.                     'pig' => q\
    3294. 

  3294. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython;
    3295. 

  3295. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age, gpa);
    3296. 

  3296. b = foreach a generate concat(name) as name;
    3297. 

  3297. store b into ':OUTPATH:';\,
    3298. 

  3298. 		    'verify_pig_script' => q\
    3299. 

  3299. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    3300. 

  3300. b = foreach a generate CONCAT(name, name);
    3301. 

  3301. store b into ':OUTPATH:';\,
    3302. 

  3302.                     },
    3303. 

  3303.                     {
    3304. 

  3304.                     # test long and double square, plus two references to the same UDF with different schemas
    3305. 

  3305.                     'num' => 3,
    3306. 

  3306.                     'ignore23' => 'MAPREDUCE-3700',
    3307. 

  3307.                     'pig' => q\
    3308. 

  3308. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3309. 

  3309. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:long, gpa:double);
    3310. 

  3310. b = foreach a generate myfuncs.square(age), myfuncs.square(gpa);
    3311. 

  3311. store b into ':OUTPATH:';\,
    3312. 

  3312. 		    'verify_pig_script' => q\
    3313. 

  3313. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3314. 

  3314. b = foreach a generate age * age, gpa * gpa;
    3315. 

  3315. store b into ':OUTPATH:';\,
    3316. 

  3316. 	        'floatpostprocess' => 1,
    3317. 

  3317. 	        'delimiter' => '	',
    3318. 

  3318.                     },
    3319. 

  3319.                     {
    3320. 

  3320.                     # test method with no schema decorator (ie, returns bytearray)
    3321. 

  3321.                     'num' => 4,
    3322. 

  3322.                     'ignore23' => 'MAPREDUCE-3700',
    3323. 

  3323.                     'pig' => q\
    3324. 

  3324. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3325. 

  3325. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age, gpa);
    3326. 

  3326. b = foreach a generate myfuncs.byteconcat(name);
    3327. 

  3327. store b into ':OUTPATH:';\,
    3328. 

  3328. 		    'verify_pig_script' => q\
    3329. 

  3329. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3330. 

  3330. b = foreach a generate CONCAT(name, name);
    3331. 

  3331. store b into ':OUTPATH:';\,
    3332. 

  3332.                     },
    3333. 

  3333.                     {
    3334. 

  3334.                     # test method with no schema decorator (ie, returns bytearray)
    3335. 

  3335.                     'num' => 5,
    3336. 

  3336.                     'ignore23' => 'MAPREDUCE-3700',
    3337. 

  3337.                     'pig' => q\
    3338. 

  3338. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3339. 

  3339. a = load ':INPATH:/singlefile/studentcomplextab10k' using PigStorage() as (m:[], t:(name:chararray, age:int, gpa:double), b:{t:(name:chararray, age:int, gpa:double)});
    3340. 

  3340. b = foreach a generate flatten(myfuncs.complexTypes(m, t, b)) as (mm, mt, mb);
    3341. 

  3341. c = foreach b generate mm#'name', mt.$0, mb.$0;
    3342. 

  3342. store c into ':OUTPATH:';\,
    3343. 

  3343. 		    'verify_pig_script' => q\
    3344. 

  3344. a = load ':INPATH:/singlefile/studentcomplextab10k' using PigStorage() as (m:[], t:(name:chararray, age:int, gpa:double), b:{t:(name:chararray, age:int, gpa:double)});
    3345. 

  3345. b = foreach a generate SIZE(m#'name'), t.$2, b.$2;
    3346. 

  3346. store b into ':OUTPATH:';\,
    3347. 

  3347.                     },
    3348. 

  3348.                     {
    3349. 

  3349.                     # test null input and output
    3350. 

  3350.                     'num' => 6,
    3351. 

  3351.                     'ignore23' => 'MAPREDUCE-3700',
    3352. 

  3352.                     'pig' => q\
    3353. 

  3353. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3354. 

  3354. a = load ':INPATH:/singlefile/studentnulltab10k' using PigStorage() as (name, age:int, gpa:double);
    3355. 

  3355. b = foreach a generate myfuncs.square(age);
    3356. 

  3356. store b into ':OUTPATH:';\,
    3357. 

  3357. 		    'verify_pig_script' => q\
    3358. 

  3358. a = load ':INPATH:/singlefile/studentnulltab10k' using PigStorage() as (name, age:int, gpa:double);
    3359. 

  3359. b = foreach a generate age * age;
    3360. 

  3360. store b into ':OUTPATH:';\,
    3361. 

  3361.                     },
    3362. 

  3362.                     {
    3363. 

  3363.                     # test functions that call other functions and include other files
    3364. 

  3364.                     'num' => 7,
    3365. 

  3365.                     'ignore23' => 'MAPREDUCE-3700',
    3366. 

  3366.                     'pig' => q\
    3367. 

  3367. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3368. 

  3368. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3369. 

  3369. b = foreach a generate myfuncs.redirect(age);
    3370. 

  3370. store b into ':OUTPATH:';\,
    3371. 

  3371. 		    'verify_pig_script' => q\
    3372. 

  3372. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3373. 

  3373. b = foreach a generate age * age;
    3374. 

  3374. store b into ':OUTPATH:';\,
    3375. 

  3375.                     },
    3376. 

  3376.                     {
    3377. 

  3377.                     # test that functions with same names resolve correctly across name spaces
    3378. 

  3378.                     'num' => 8,
    3379. 

  3379.                     'ignore23' => 'MAPREDUCE-3700',
    3380. 

  3380.                     'pig' => q\
    3381. 

  3381. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3382. 

  3382. register ':SCRIPTHOMEPATH:/python/morepythonudfs.py' using jython as morefuncs;
    3383. 

  3383. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3384. 

  3384. b = foreach a generate myfuncs.square(age), morefuncs.square(age);
    3385. 

  3385. store b into ':OUTPATH:';\,
    3386. 

  3386. 		    'verify_pig_script' => q\
    3387. 

  3387. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3388. 

  3388. b = foreach a generate age * age, age * age * age;
    3389. 

  3389. store b into ':OUTPATH:';\,
    3390. 

  3390.                     },
    3391. 

  3391.                     {
    3392. 

  3392.                     # test that functions with same names resolve correctly across name spaces
    3393. 

  3393.                     'num' => 9,
    3394. 

  3394.                     'ignore23' => 'MAPREDUCE-3700',
    3395. 

  3395.                     'pig' => q\
    3396. 

  3396. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3397. 

  3397. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3398. 

  3398. b = group a by name;
    3399. 

  3399. c = foreach b generate group, myfuncs.count(a);
    3400. 

  3400. store c into ':OUTPATH:';\,
    3401. 

  3401. 		    'verify_pig_script' => q\
    3402. 

  3402. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa:double);
    3403. 

  3403. b = group a by name;
    3404. 

  3404. c = foreach b generate group, COUNT(a);
    3405. 

  3405. store c into ':OUTPATH:';\,
    3406. 

  3406.                     },
    3407. 

  3407.                     {
    3408. 

  3408.                     # test that functions with same names resolve correctly across name spaces
    3409. 

  3409.                     'num' => 10,
    3410. 

  3410.                     'ignore23' => 'MAPREDUCE-3700',
    3411. 

  3411.                     'pig' => q\
    3412. 

  3412. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3413. 

  3413. a = load ':INPATH:/singlefile/allscalar10k' as (name:chararray, age:int, gpa:double, instate:boolean);
    3414. 

  3414. b = foreach a generate name, myfuncs.adjustgpa(gpa, instate);
    3415. 

  3415. store b into ':OUTPATH:';\,
    3416. 

  3416. 		    'verify_pig_script' => q\
    3417. 

  3417. a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    3418. 

  3418. b = foreach a generate name, (instate=='true'?gpa:gpa+1);
    3419. 

  3419. store b into ':OUTPATH:';\,
    3420. 

  3420.                     },
    3421. 

  3421.                     {
    3422. 

  3422.                     # test that functions with same names resolve correctly across name spaces
    3423. 

  3423.                     'num' => 11,
    3424. 

  3424.                     'ignore' => 1, # PIG-2596
    3425. 

  3425.                     'pig' => q\
    3426. 

  3426. register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
    3427. 

  3427. a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    3428. 

  3428. b = foreach a generate name, myfuncs.isretired(age);
    3429. 

  3429. store b into ':OUTPATH:';\,
    3430. 

  3430.                     'verify_pig_script' => q\
    3431. 

  3431. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
    3432. 

  3432. b = foreach a generate name, (age>=60?1:0);
    3433. 

  3433. store b into ':OUTPATH:';\,
    3434. 

  3434.                     }
    3435. 

  3435.                 ]
    3436. 

  3436.             },
    3437. 

  3437.             {
    3438. 

  3438.             'name' => 'Native',
    3439. 

  3439.             'tests' => [
    3440. 

  3440.                     {
    3441. 

  3441.                     # test common
    3442. 

  3442.                     'num' => 1,
    3443. 

  3443.                     'pig' => q\
    3444. 

  3444. rmf table_testNativeMRJobSimple_input
    3445. 

  3445. rmf table_testNativeMRJobSimple_output
    3446. 

  3446. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3447. 

  3447. b = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
    3448. 

  3448. store b into ':OUTPATH:';\,
    3449. 

  3449.                     'notmq' => 1,
    3450. 

  3450.                     },
    3451. 

  3451.                     {
    3452. 

  3452.                     # test complex
    3453. 

  3453.                     'num' => 2,
    3454. 

  3454.                     'pig' => q\
    3455. 

  3455. rmf table_testNativeMRJobSimple_input
    3456. 

  3456. rmf table_testNativeMRJobSimple_output
    3457. 

  3457. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3458. 

  3458. b = foreach a generate name;
    3459. 

  3459. c = distinct b;
    3460. 

  3460. d = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store c into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
    3461. 

  3461. e = order d by name;
    3462. 

  3462. store e into ':OUTPATH:';\,
    3463. 

  3463.                     'sortArgs' => ['-t', '	'],
    3464. 

  3464.                     'notmq' => 1,
    3465. 

  3465.                     },
    3466. 

  3466.                     {
    3467. 

  3467.                     # test streaming
    3468. 

  3468.                     'num' => 3,
    3469. 

  3469.                     'pig' => q\
    3470. 

  3470. rmf table_testNativeMRJobSimple_input
    3471. 

  3471. rmf table_testNativeMRJobSimple_output
    3472. 

  3472. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3473. 

  3473. b = mapreduce ':MAPREDJARS:/hadoop-streaming.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `-input table_testNativeMRJobSimple_input -output table_testNativeMRJobSimple_output -mapper /bin/cat -reducer /usr/bin/wc`;
    3474. 

  3474. store b into ':OUTPATH:';\,
    3475. 

  3475.                     'pig23' => q\
    3476. 

  3476. rmf table_testNativeMRJobSimple_input
    3477. 

  3477. rmf table_testNativeMRJobSimple_output
    3478. 

  3478. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3479. 

  3479. b = mapreduce ':MAPREDJARS:/hadoop-0.23.0-streaming.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `-input table_testNativeMRJobSimple_input -output table_testNativeMRJobSimple_output -mapper /bin/cat -reducer /usr/bin/wc`;
    3480. 

  3480. store b into ':OUTPATH:';\,
    3481. 

  3481.                     'notmq' => 1,
    3482. 

  3482.                     },
    3483. 

  3483.                 ]
    3484. 

  3484.             },
    3485. 

  3485.             {
    3486. 

  3486.             'name' => 'Partitioner',
    3487. 

  3487.             'tests' => [
    3488. 

  3488.                     {
    3489. 

  3489.                     # test group
    3490. 

  3490.                     'num' => 1,
    3491. 

  3491.                     'execonly' => 'mapred', # since this join will run out of memory in local mode
    3492. 

  3492.                     'pig' => q\register :FUNCPATH:/testudf.jar;
    3493. 

  3493. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa);
    3494. 

  3494. b = group a by age PARTITION BY org.apache.pig.test.utils.SimpleCustomPartitioner2 parallel 2;
    3495. 

  3495. c = foreach b generate group, COUNT(a);
    3496. 

  3496. store c into ':OUTPATH:';\,
    3497. 

  3497.                     },
    3498. 

  3498.                 ]
    3499. 

  3499.             },
    3500. 

  3500.             { 
    3501. 

  3501. ####################################################################
    3502. 

  3502. # SUB    : CastScalar
    3503. 

  3503. # FEATURE: adds functionality that allows to cast elements of a single-tuple relation into a scalar value.
    3504. 

  3504. # JIRA: Pig-1434 
    3505. 

  3505. #
    3506. 

  3506. # TEST ITEMS:
    3507. 

  3507. # 1 Test syntax
    3508. 

  3508. # 2 Test scalar for simple data type
    3509. 

  3509. # 3 Test scalar for complex data type: tuple, bag, map
    3510. 

  3510. # 4 Test implicit cast
    3511. 

  3511. # 5 Test explicit cast
    3512. 

  3512. # 6 Positional parameter
    3513. 

  3513. # 7 Cast within an aggregate function
    3514. 

  3514. # 8 Cast within an UDF function
    3515. 

  3515. # 9 Cast with a FOREACH
    3516. 

  3516. # 10 Cast with a FILTER
    3517. 

  3517. # 11 Cast with a SPLIT
    3518. 

  3518. # 12 Cast in a JOIN
    3519. 

  3519. # 13 Multiquery
    3520. 

  3520. # 14 Cast on a schema that cannot be inferred should result in bytearray
    3521. 

  3521. # 15 Replicated Join
    3522. 

  3522. # 16 Test operations such as R1 * (int)R1
    3523. 

  3523. # 17 CheckSingular(*)
    3524. 

  3524. # 18 missing field in scalar file
    3525. 

  3525. # 19 scalar referenced from an empty file
    3526. 

  3526. # 20 empty input directory
    3527. 

  3527. # 21 Single row vs Multiple Row
    3528. 

  3528. # 22 Cast on a multi-field tuple
    3529. 

  3529. # 23 Reference a non-scalar as a scalar
    3530. 

  3530. # 24 Test multiple loaders
    3531. 

  3531. 

  3532.         'name' => 'CastScalar',
    3533. 

  3533.         'tests' => [
    3534. 

  3534.             {
    3535. 

  3535. # 2 Test scalar for simple data type
    3536. 

  3536. # 3 Test scalar for complex data type: tuple, bag, map
    3537. 

  3537. # 9 Cast with a FOREACH
    3538. 

  3538. #INPATH = /user/hadoopqa/pig/tests/data
    3539. 

  3539. 

  3540.             'num' => 1,
    3541. 

  3541.             'pig' => q# 
    3542. 

  3542. a = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float); 
    3543. 

  3543. b = group a all;
    3544. 

  3544. c = foreach b generate SUM(a.age) as total;
    3545. 

  3545. d = foreach a generate name, age+(double)c.total as d_sum;
    3546. 

  3546. e = order d  by name, d_sum;
    3547. 

  3547. store d into ':OUTPATH:';
    3548. 

  3548. #,
    3549. 

  3549. # 6 Positional parameter
    3550. 

  3550. 

  3551.             }, {
    3552. 

  3552. 

  3553.             'num' => 2,
    3554. 

  3554.             'pig' => q# 
    3555. 

  3555. a = load ':INPATH:/singlefile/studenttab10k' as (name: chararray, age: int, gpa: float); 
    3556. 

  3556. b = group a all;
    3557. 

  3557. c = foreach b generate SUM(a.age) as total;
    3558. 

  3558. d = foreach a generate name, age+(double)c.$0 as d_sum;
    3559. 

  3559. e = order d  by name, d_sum;
    3560. 

  3560. store d into ':OUTPATH:';
    3561. 

  3561. #,
    3562. 

  3562. # 2 Test scalar for simple data type
    3563. 

  3563. # 3 Test scalar for complex data type:map
    3564. 

  3564. # 9 Cast with a FOREACH
    3565. 

  3565. # 13 Multiquery
    3566. 

  3566. # 24 Test multiple loaders
    3567. 

  3567. #INPATH = /user/hadoopqa/pig/tests/data
    3568. 

  3568. 

  3569.             }, {
    3570. 

  3570. 

  3571. # 4 Test implicit cast
    3572. 

  3572. # 10 Cast with a FILTER
    3573. 

  3573. #
    3574. 

  3574. # I set the benchmark to use "19" because pig trunkates during cast and sql rounds up.
    3575. 

  3575.             'num' => 7,
    3576. 

  3576.             'pig' => q\
    3577. 

  3577. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    3578. 

  3578. b = group a by name;
    3579. 

  3579. c = foreach b generate group, AVG(a.gpa)+20 as avg_gpa;
    3580. 

  3580. d = order c by avg_gpa;
    3581. 

  3581. simple_scalar = limit d 1;
    3582. 

  3582. f = filter a by age < (int) simple_scalar.avg_gpa;
    3583. 

  3583. g = order f by name, age, gpa;
    3584. 

  3584. store g into ':OUTPATH:';\,
    3585. 

  3585. 

  3586.             }, {
    3587. 

  3587. # 5 Test explicit cast
    3588. 

  3588. # 10 Cast with a FILTER
    3589. 

  3589.             'num' => 8,
    3590. 

  3590. 

  3591.                         'pig' => q\
    3592. 

  3592. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    3593. 

  3593. b = group a by name;
    3594. 

  3594. c = foreach b generate group, AVG(a.age) AS average;
    3595. 

  3595. d = order c by average;
    3596. 

  3596. simple_scalar = limit d 1;
    3597. 

  3597. d = filter a by age > (int) simple_scalar.average;
    3598. 

  3598. e = foreach d generate name, age;
    3599. 

  3599. store e into ':OUTPATH:';
    3600. 

  3600. \,
    3601. 

  3601.             }, {
    3602. 

  3602. # 5 Test explicit cast
    3603. 

  3603. # 6 Positional parameter
    3604. 

  3604.                         'num' => 9,
    3605. 

  3605.                         'pig' => q\
    3606. 

  3606. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    3607. 

  3607. b = group a by name;
    3608. 

  3608. c = foreach b generate group, AVG(a.age) AS average;
    3609. 

  3609. d = order c by average;
    3610. 

  3610. simple_scalar = limit d 1;
    3611. 

  3611. d = filter a by age > (int) simple_scalar.$1;
    3612. 

  3612. e = foreach d generate name, age;
    3613. 

  3613. store e into ':OUTPATH:';
    3614. 

  3614. \,
    3615. 

  3615.             }, {
    3616. 

  3616. # 4 Test implicit cast
    3617. 

  3617. # 6 Positional parameter
    3618. 

  3618. # 10 Cast with a FILTER
    3619. 

  3619.                         'num' => 10,
    3620. 

  3620.                         'pig' => q\
    3621. 

  3621. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    3622. 

  3622. b = group a by name;
    3623. 

  3623. c = foreach b generate group, AVG(a.age) AS average;
    3624. 

  3624. d = order c by average;
    3625. 

  3625. simple_scalar = limit d 1;
    3626. 

  3626. d = filter a by age >  simple_scalar.$1;
    3627. 

  3627. e = foreach d generate name, age;
    3628. 

  3628. store e into ':OUTPATH:';
    3629. 

  3629. \,
    3630. 

  3630.             }, {
    3631. 

  3631. # 4 Test implicit cast
    3632. 

  3632. # 6 Positional parameter
    3633. 

  3633. # 11 Cast with a SPLIT
    3634. 

  3634.             'num' => 11,
    3635. 

  3635.             'pig' => q\
    3636. 

  3636. a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    3637. 

  3637. b = group a by name;
    3638. 

  3638. c = foreach b generate group, AVG(a.age) AS average;
    3639. 

  3639. d = order c by average;
    3640. 

  3640. simple_scalar = limit d 1;
    3641. 

  3641. split a into X1 if age > (int) simple_scalar.$1, X2 if age < 20;
    3642. 

  3642. split a into X3 if age > (int) simple_scalar.$1, X4 if age > 70;
    3643. 

  3643. 

  3644. store X1 into ':OUTPATH:.1';
    3645. 

  3645. store X2 into ':OUTPATH:.2';
    3646. 

  3646. store X3 into ':OUTPATH:.3';
    3647. 

  3647. store X4 into ':OUTPATH:.4';
    3648. 

  3648. \,
    3649. 

  3649.                  }, {
    3650. 

  3650. # 4 Test implicit cast
    3651. 

  3651. # 6 Positional parameter
    3652. 

  3652. # 12 Cast with a JOIN
    3653. 

  3653.             'num' => 12,
    3654. 

  3654.             'pig' => q\
    3655. 

  3655. a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
    3656. 

  3656. b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
    3657. 

  3657. c = filter a by age < 20;
    3658. 

  3658. d = filter b by age < 20;
    3659. 

  3659. simple_scalar = limit d 1;
    3660. 

  3660. e = join c by name, d by name;
    3661. 

  3661. f= filter e by c::age <(int)simple_scalar.age;
    3662. 

  3662. store f into ':OUTPATH:';\,
    3663. 

  3663.                    },
    3664. 

  3664.                 ]
    3665. 

  3665. 

  3666.                 },{
    3667. 

  3667.                  
    3668. 

  3668.                 'name' => 'udf_TOBAGandTOTUPLE',
    3669. 

  3669.                
    3670. 

  3670.                 'sortResults' => 1,
    3671. 

  3671.                 'floatpostprocess' => 1,
    3672. 

  3672.                 'delimiter' => '	',
    3673. 

  3673.                 'tests' => [
    3674. 

  3674.                         {
    3675. 

  3675. # TEST : resulting schema for TOBAG/TOTUPLE with simple types
    3676. 

  3676. # TEST : resulting schema for TOBAG/TOTUPLE with positional parameters
    3677. 

  3677. # TEST : resulting schema for various projects using a combination of TOBAG/TOTUPLE and standard projections
    3678. 

  3678. # TEST : resulting schema for various projects using a combination of TOBAG/TOTUPLE using AS clause
    3679. 

  3679. 

  3680. 						'num' => 1
    3681. 

  3681.                         ,'pig' => q?
    3682. 

  3682. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3683. 

  3683. B = limit A 10;
    3684. 

  3684. Gen1 = FOREACH B GENERATE $0, $1, $2 ;
    3685. 

  3685. GroupById = GROUP B BY id;
    3686. 

  3686. 

  3687. B1  = foreach B generate TOBAG( intnum1000, id, intnum5); 
    3688. 

  3688. B3 = foreach B generate TOBAG( $0, $1, $2); 
    3689. 

  3689. T1= foreach B generate TOTUPLE( intnum1000, id, intnum5); 
    3690. 

  3690. T2= foreach B generate TOTUPLE( $0, $1, $2); 
    3691. 

  3691. T3 = foreach B generate TOTUPLE( $0, $0, $0); 
    3692. 

  3692. T4= foreach B generate TOBAG($0, $1, $2), TOTUPLE($3, $4, $5), $6, $7;
    3693. 

  3693. T5= foreach B generate $0, $1, TOTUPLE($2, $3, $4), TOBAG($5, $6), $7;
    3694. 

  3694. T6= foreach B generate $0,  TOTUPLE($0, $0, $0), TOBAG($0, $0), $0 AS duplicate;
    3695. 

  3695. describe Gen1;
    3696. 

  3696. describe GroupById;
    3697. 

  3697. describe B1;
    3698. 

  3698. describe B3;
    3699. 

  3699. describe T1;
    3700. 

  3700. describe T2;
    3701. 

  3701. describe T3;
    3702. 

  3702. describe T4;
    3703. 

  3703. describe T5;
    3704. 

  3704. describe T6;
    3705. 

  3705. ?
    3706. 

  3706.                          ,'expected_out_regex'  => 'B1: {{int}}'
    3707. 

  3707.                          ,'expected_out_regex'  => 'B3: {{int}}'
    3708. 

  3708.                          ,'expected_out_regex'  => 'T1: {org.apache.pig.builtin.totuple_id_.*: (intnum1000: int,id: int,intnum5: int)}'
    3709. 

  3709.                          ,'expected_out_regex'  => 'T2: {org.apache.pig.builtin.totuple_id_.*: (intnum1000: int,id: int,intnum5: int)}'
    3710. 

  3710.                          ,'expected_out_regex'  => 'T3: {org.apache.pig.builtin.totuple_intnum1000.*: (intnum1000: int,intnum1000: int,intnum1000: int)}'
    3711. 

  3711.                          ,'expected_out_regex'  => 'T4: {{int},org.apache.pig.builtin.totuple_intnum100.*: (intnum100: int,intnum: int,longnum: long),floatnum: float,doublenum: double}'
    3712. 

  3712.                          ,'expected_out_regex'  => 'T5: {intnum1000: int,id: int,org.apache.pig.builtin.totuple_intnum100.*: (intnum5: int,intnum100: int,intnum: int).*{NULL}.*doublenum: double}'
    3713. 

  3713.                          ,'expected_out_regex'  => "T6: {intnum1000: int,org.apache.pig.builtin.totuple_intnum1000.*: \\(intnum1000: int,intnum1000: int,intnum1000: int\\),{\\(int\\)},duplicate: int}"
    3714. 

  3714.                         }, {
    3715. 

  3715. # TEST : bag of mixed data types
    3716. 

  3716. # TEST : Order 
    3717. 

  3717. # TEST : positional parameters
    3718. 

  3718. 			'num' => 2
    3719. 

  3719.                         ,'pig' => q?
    3720. 

  3720. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3721. 

  3721. C   = foreach A generate TOBAG( id, floatnum, doublenum ); 
    3722. 

  3722. D   = foreach A generate TOBAG( id, intnum);
    3723. 

  3723. E   = foreach A generate TOBAG( (float) id,floatnum );
    3724. 

  3724. F   = foreach A generate TOBAG( (long) id,longnum );
    3725. 

  3725. G   = foreach A generate TOBAG( (double) id,doublenum );
    3726. 

  3726. describe C;
    3727. 

  3727. describe D;
    3728. 

  3728. describe E;
    3729. 

  3729. describe F;
    3730. 

  3730. describe G;
    3731. 

  3731. ?
    3732. 

  3732.                          ,'expected_out_regex'  => 'C: {{NULL}}'
    3733. 

  3733.                          ,'expected_out_regex'  => 'D: {{int}}'
    3734. 

  3734.                          ,'expected_out_regex'  => 'E: {{float}}'
    3735. 

  3735.                          ,'expected_out_regex'  => 'F: {{long}}'
    3736. 

  3736.                          ,'expected_out_regex'  => 'G: {{double}}'
    3737. 

  3737. 

  3738. 			}, {
    3739. 

  3739. # TEST : TOBAG/TOTUPLE with simple types
    3740. 

  3740. # TEST : TOBAG/TOTUPLE with positional parameters
    3741. 

  3741. # TEST : various projects using a combination of TOBAG/TOTUPLE and standard projections
    3742. 

  3742. # TEST : various projects using a combination of TOBAG/TOTUPLE using AS clause
    3743. 

  3743. 			 			'num' => 3
    3744. 

  3744.                         ,'pig' => q?
    3745. 

  3745. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3746. 

  3746. B = limit A 10;
    3747. 

  3747. B1  = foreach B generate TOBAG( intnum1000, id, intnum5); 
    3748. 

  3748. B2 = foreach B generate TOBAG( $0, $1, $2); 
    3749. 

  3749. T1= foreach B generate TOTUPLE( intnum1000, id, intnum5); 
    3750. 

  3750. T2= foreach B generate TOTUPLE( $0, $1, $2); 
    3751. 

  3751. T3 = foreach B generate TOTUPLE( $0, $0, $0); 
    3752. 

  3752. T4= foreach B generate TOBAG($0, $1, $2), TOTUPLE($3, $4, $5), $6, $7;
    3753. 

  3753. T5= foreach B generate $0, $1, TOTUPLE($2, $3, $4), TOBAG($5, $6), $7;
    3754. 

  3754. T6= foreach B generate $0,  TOTUPLE($0, $0, $0), TOBAG($0, $0), $0 AS duplicate;
    3755. 

  3755. Gen1 = FOREACH B GENERATE $0, $1, $2 ;
    3756. 

  3756. GroupById = GROUP B BY id;
    3757. 

  3757. store Gen1 into ':OUTPATH:.1';
    3758. 

  3758. store GroupById into ':OUTPATH:.2';
    3759. 

  3759. store B1 into  ':OUTPATH:.3';
    3760. 

  3760. store B2 into  ':OUTPATH:.4';
    3761. 

  3761. store T1 into ':OUTPATH:.5';
    3762. 

  3762. store T2 into ':OUTPATH:.6';
    3763. 

  3763. store T3 into ':OUTPATH:.7';
    3764. 

  3764. store T4 into ':OUTPATH:.8';
    3765. 

  3765. ?
    3766. 

  3766.                         }, {
    3767. 

  3767. # TEST : cast for TOTUPLE/TOBAG
    3768. 

  3768. 						 'num' => 4
    3769. 

  3769.                         ,'pig' => q?
    3770. 

  3770. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3771. 

  3771. B= limit A 10;
    3772. 

  3772. C = foreach B generate $0,  TOTUPLE((int) $0, (long) $0, (double) $0), TOBAG( (float) $0, (chararray) $0), $0;
    3773. 

  3773. store C into  ':OUTPATH:';
    3774. 

  3774. ?
    3775. 

  3775.                          ,'expected_err_regex'  => 'ERROR 1108: Duplicate schema alias'
    3776. 

  3776.                          ,'rc' => 6
    3777. 

  3777. 

  3778.                         }, {
    3779. 

  3779. # TEST : cast for TOTUPLE/TOBAG
    3780. 

  3780. 						'num' => 5
    3781. 

  3781.                         ,'pig' => q?
    3782. 

  3782. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3783. 

  3783. B= limit A 1;
    3784. 

  3784. C = foreach B generate $0,  TOTUPLE((int) $0);
    3785. 

  3785. D = foreach B generate $0,  TOTUPLE((long) $0);
    3786. 

  3786. E = foreach B generate $0,  TOTUPLE((double) $0);
    3787. 

  3787. F = foreach B generate $0,  TOTUPLE((float) $0);
    3788. 

  3788. G = foreach B generate $0,  TOTUPLE((chararray) $0);
    3789. 

  3789. store B into  ':OUTPATH:.1';
    3790. 

  3790. store C into  ':OUTPATH:.2';
    3791. 

  3791. store D into  ':OUTPATH:.3';
    3792. 

  3792. store E into  ':OUTPATH:.4';
    3793. 

  3793. store F into  ':OUTPATH:.5';
    3794. 

  3794. store G into  ':OUTPATH:.6';
    3795. 

  3795. ?
    3796. 

  3796.                         }, {
    3797. 

  3797. 

  3798. #TEST more complicated nested functions such as TOTUPLE(TOBAG())
    3799. 

  3799. #TEST more complicated nested functions such as TOBAG(TOTUPLE())
    3800. 

  3800. #TEST more complicated nested functions such as TOTUPLE(TOTUPLE())
    3801. 

  3801. #TEST more complicated nested functions such as TOBAG(TOBAG())
    3802. 

  3802. 			'num' => 6
    3803. 

  3803.                         ,'pig' => q?
    3804. 

  3804. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3805. 

  3805. B = limit A 10;
    3806. 

  3806. tint = foreach B generate  TOTUPLE( TOBAG( $1, $2, $3),TOTUPLE($3, $4, $5) );
    3807. 

  3807. bint = foreach B generate  TOTUPLE( TOBAG( $1, $2, $3),TOBAG($3, $4, $5) );
    3808. 

  3808. binb = foreach B generate  TOBAG( TOBAG( $1, $2, $3),TOBAG($3, $4, $5) );
    3809. 

  3809. tinb = foreach B generate  TOTUPLE( TOBAG( $1, $2, $3),TOBAG($3, $4, $5) );
    3810. 

  3810. store B into  ':OUTPATH:.1';
    3811. 

  3811. store tint into  ':OUTPATH:.2';
    3812. 

  3812. store bint into  ':OUTPATH:.3';
    3813. 

  3813. store binb into  ':OUTPATH:.4';
    3814. 

  3814. store tinb into  ':OUTPATH:.5';
    3815. 

  3815. ?
    3816. 

  3816. 

  3817.                         }, {
    3818. 

  3818. #TEST arithmetic operation in TOTUPLE and TOBAG
    3819. 

  3819. #TEST aggregate funcion - NOT IMPLEMENTED
    3820. 

  3820. #TEST tuple with 50+ items
    3821. 

  3821. #TEST with null
    3822. 

  3822. 						'num' => 7
    3823. 

  3823.                         ,'pig' => q?
    3824. 

  3824. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3825. 

  3825. B = limit A 10;
    3826. 

  3826. B1= foreach B generate  TOTUPLE( $1, $2, $3);
    3827. 

  3827. T1= foreach B generate  TOTUPLE( $1, $2, $3);
    3828. 

  3828. R1= foreach B generate  TOTUPLE( $1, $0+1, $0+2, $0+3),TOBAG($0+4, $0+1 );
    3829. 

  3829. R2= foreach B generate  TOTUPLE( $0, $1, $2, $3, $4, $5, $6, $7, (int) 8, (int) 9 , $1, $2, $3, $4, $5, $6, $7, (int) 19, (int) 20, $0, $1, $2, $3, $4, $5, $6, $7 , (int) 29, (int) 30, $0, $1, $2, $3, $4, $5, $6, $7, (int) 39, (int) 40 , $1, $2, $3, $4, $5, $6, $7, (int) 19, (int) 20, $0, $1, $2, $3, $4, $5, $5, $7 );
    3830. 

  3830. 

  3831. R3= foreach B generate $0, TOTUPLE(0,0,0),  TOBAG( 0, 0 );
    3832. 

  3832. R4= foreach B generate $0, TOTUPLE(null, id, null),  TOBAG( id, null, id,null );
    3833. 

  3833. 

  3834. describe R1;
    3835. 

  3835. describe R2;
    3836. 

  3836. describe R3;
    3837. 

  3837. describe R4;
    3838. 

  3838. 

  3839. store B into  ':OUTPATH:.1';
    3840. 

  3840. store B1 into ':OUTPATH:.2';
    3841. 

  3841. store R1 into ':OUTPATH:.3';
    3842. 

  3842. store R2 into ':OUTPATH:.4';
    3843. 

  3843. store R3 into ':OUTPATH:.5';
    3844. 

  3844. store R4 into ':OUTPATH:.6';
    3845. 

  3845. ?
    3846. 

  3846. 

  3847.                         }, {
    3848. 

  3848. # TEST more TOTUPLE and TOBAG nested combinations
    3849. 

  3849. 						'num' => 8
    3850. 

  3850.                         ,'pig' => q?
    3851. 

  3851. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3852. 

  3852. B = limit A 10;
    3853. 

  3853. C = foreach B generate TOBAG( $0, $1, $2); 
    3854. 

  3854. T1= foreach B generate  TOTUPLE( TOBAG( $1, $2, $3),TOTUPLE($3, $4, $5) );
    3855. 

  3855. T2= foreach B generate  TOTUPLE( TOBAG( $1, $2, $3),TOBAG($3, $4, $5) );
    3856. 

  3856. T3= foreach B generate  TOBAG( TOTUPLE( $1, $2, $3), TOTUPLE($4,$5), TOTUPLE($6,$7));
    3857. 

  3857. store B  into ':OUTPATH:.1';
    3858. 

  3858. store C  into ':OUTPATH:.2';
    3859. 

  3859. store T1 into ':OUTPATH:.3';
    3860. 

  3860. store T2 into ':OUTPATH:.4';
    3861. 

  3861. store T3 into ':OUTPATH:.5';
    3862. 

  3862. ?
    3863. 

  3863. 

  3864.                         }, {
    3865. 

  3865. #TEST negative test case: out of bounds positional parameter
    3866. 

  3866. # EVERYTHING IS CORRECT
    3867. 

  3867. 			'num' => 9
    3868. 

  3868.                         ,'pig' => q?
    3869. 

  3869. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3870. 

  3870. B = limit A 10;
    3871. 

  3871. C = foreach B generate $0, $1, TOTUPLE($2, $998, $4), TOBAG($5, $6), $7;
    3872. 

  3872. ?
    3873. 

  3873.                          ,'expected_err_regex'  => 'Out of bound access.*non-existent column: 998'
    3874. 

  3874.                         }, {
    3875. 

  3875. #TEST negative test case: out of bounds positional parameter
    3876. 

  3876. # EVERYTHING IS CORRECT
    3877. 

  3877. 			'num' => 10
    3878. 

  3878.                         ,'pig' => q?
    3879. 

  3879. A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
    3880. 

  3880. B = limit A 10;
    3881. 

  3881. C = foreach B generate $0, $1, TOBAG($5, $999), $7;
    3882. 

  3882. ?
    3883. 

  3883.                          ,'expected_err_regex'  => 'Out of bound access.*non-existent column: 999'
    3884. 

  3884. 

  3885.                         }, 
    3886. 

  3886. 

  3887.                 ] # end of tests
    3888. 

  3888. 

  3889.             },{
    3890. 

  3890. 

  3891. 		'name' => 'ToStuffSyntaxSugar',
    3892. 

  3892. 		'tests' => [
    3893. 

  3893. 				{
    3894. 

  3894. 				#TEST TOTUPLE syntax sugar
    3895. 

  3895. 				'num' => 1,
    3896. 

  3896. 				'pig' => q\
    3897. 

  3897. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3898. 

  3898. 				B = foreach A generate (name, age);
    3899. 

  3899. 				store B into ':OUTPATH:';\, 
    3900. 

  3900. 				                    'verify_pig_script' => q\
    3901. 

  3901. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3902. 

  3902. 				B = foreach A generate TOTUPLE(name, age);
    3903. 

  3903. 				store B into ':OUTPATH:';\,
    3904. 

  3904. 				}, {
    3905. 

  3905. 				#TEST TOBAG syntax sugar
    3906. 

  3906. 				'num' => 2,
    3907. 

  3907. 				'pig' => q\
    3908. 

  3908. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3909. 

  3909. 				B = foreach A generate {name, age};
    3910. 

  3910. 				store B into ':OUTPATH:';\, 
    3911. 

  3911. 				                    'verify_pig_script' => q\
    3912. 

  3912. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3913. 

  3913. 				B = foreach A generate TOBAG(name, age);
    3914. 

  3914. 				store B into ':OUTPATH:';\,			
    3915. 

  3915. 				}, {
    3916. 

  3916. 				#TEST TOMAP syntax sugar
    3917. 

  3917. 				'num' => 3,
    3918. 

  3918. 				'pig' => q\
    3919. 

  3919. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3920. 

  3920. 				B = foreach A generate [name, age];
    3921. 

  3921. 				store B into ':OUTPATH:';\, 
    3922. 

  3922. 				                    'verify_pig_script' => q\
    3923. 

  3923. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3924. 

  3924. 				B = foreach A generate TOMAP(name, age);
    3925. 

  3925. 				store B into ':OUTPATH:';\,			
    3926. 

  3926. 				}, {
    3927. 

  3927. 				#TEST verify single element inside parenthesis does NOT call TOTUPLE
    3928. 

  3928. 				'num' => 4,
    3929. 

  3929. 				'pig' => q\
    3930. 

  3930. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3931. 

  3931. 				B = foreach A generate (age) + 1;
    3932. 

  3932. 				store B into ':OUTPATH:';\, 
    3933. 

  3933. 				                    'verify_pig_script' => q\
    3934. 

  3934. 				A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3935. 

  3935. 				B = foreach A generate (age + 1);
    3936. 

  3936. 				store B into ':OUTPATH:';\,
    3937. 

  3937. 				}
    3938. 

  3938. 			] # end of tests
    3939. 

  3939. 		},{
    3940. 

  3940. 	
    3941. 

  3941. 	        'name' => 'MergeOperator',
    3942. 

  3942. 

  3943. 	        'tests' => [
    3944. 

  3944. 			    {
    3945. 

  3945.                     # Test Union using merge where schema is identical  | A&B have identical schema
    3946. 

  3946. 			        'num' => 1,
    3947. 

  3947. 			        'pig' => q\
    3948. 

  3948. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3949. 

  3949. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3950. 

  3950. C = union onschema A, B;
    3951. 

  3951. store C into ':OUTPATH:';\, 
    3952. 

  3952.                     'verify_pig_script' => q\
    3953. 

  3953. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3954. 

  3954. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3955. 

  3955. C = union A, B;
    3956. 

  3956. store C into ':OUTPATH:';\, 
    3957. 

  3957. 				},{
    3958. 

  3958.                     # Test Union using merge with type promotions, int->long and float->double
    3959. 

  3959. 			        'num' => 2,
    3960. 

  3960. 			        'floatpostprocess' => 1,
    3961. 

  3961. 			        'delimiter' => '	',
    3962. 

  3962. 			        'pig' => q\
    3963. 

  3963. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3964. 

  3964. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:long, gpa:double);
    3965. 

  3965. C = union onschema A, B;
    3966. 

  3966. store C into ':OUTPATH:';\, 
    3967. 

  3967.                     'verify_pig_script' => q\
    3968. 

  3968. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3969. 

  3969. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3970. 

  3970. C = union A, B;
    3971. 

  3971. D = foreach C generate name, (long)age, (double)gpa;
    3972. 

  3972. store C into ':OUTPATH:';\, 
    3973. 

  3973. 				},{
    3974. 

  3974.                     # Test Union using merge with type promotions, int->float
    3975. 

  3975. 			        'num' => 3,
    3976. 

  3976. 			        'floatpostprocess' => 1,
    3977. 

  3977. 			        'delimiter' => '	',
    3978. 

  3978. 			        'pig' => q\
    3979. 

  3979. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3980. 

  3980. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:float, gpa:float);
    3981. 

  3981. C = union onschema A, B;
    3982. 

  3982. store C into ':OUTPATH:';\, 
    3983. 

  3983.                     'verify_pig_script' => q\
    3984. 

  3984. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:float, gpa:float);
    3985. 

  3985. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:float, gpa:float);
    3986. 

  3986. C = union A, B;
    3987. 

  3987. D = foreach C generate name, (float)age, gpa;
    3988. 

  3988. store C into ':OUTPATH:';\, 
    3989. 

  3989. 				},{
    3990. 

  3990.                     # Test Union using merge with type promotions, int->double
    3991. 

  3991. 			        'num' => 4,
    3992. 

  3992. 			        'floatpostprocess' => 1,
    3993. 

  3993. 			        'delimiter' => '	',
    3994. 

  3994. 			        'pig' => q\
    3995. 

  3995. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    3996. 

  3996. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:double, gpa:float);
    3997. 

  3997. C = union onschema A, B;
    3998. 

  3998. store C into ':OUTPATH:';\, 
    3999. 

  3999.                     'verify_pig_script' => q\
    4000. 

  4000. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:double, gpa:float);
    4001. 

  4001. B = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:double, gpa:float);
    4002. 

  4002. C = union A, B;
    4003. 

  4003. D = foreach C generate name, (double)age, gpa;
    4004. 

  4004. store C into ':OUTPATH:';\, 
    4005. 

  4005. 				},{
    4006. 

  4006.                     # Test Union of an intersection
    4007. 

  4007. 			        'num' => 5,
    4008. 

  4008. 			        'pig' => q\
    4009. 

  4009. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    4010. 

  4010. B = load ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration:chararray, contributions:float);
    4011. 

  4011. C = union onschema A, B;
    4012. 

  4012. store C into ':OUTPATH:';\, 
    4013. 

  4013.                     'verify_pig_script' => q\
    4014. 

  4014. register :FUNCPATH:/testudf.jar;
    4015. 

  4015. define Nil org.apache.pig.test.udf.evalfunc.Nil();
    4016. 

  4016. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    4017. 

  4017. B = load ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration:chararray, contributions:float);
    4018. 

  4018. C = foreach A generate name, age, (chararray)gpa, Nil(), Nil();
    4019. 

  4019. D = foreach B generate name, age, Nil(), registration, (chararray)contributions;
    4020. 

  4020. E = union C, D;
    4021. 

  4021. store E into ':OUTPATH:';\, 
    4022. 

  4022. 				},
    4023. 

  4023.                 {
    4024. 

  4024.                     # Test Union where the intersection is null
    4025. 

  4025. 			        'num' => 6,
    4026. 

  4026. 			        'pig' => q\
    4027. 

  4027. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    4028. 

  4028. B = load ':INPATH:/singlefile/textdoc' as (line:chararray);
    4029. 

  4029. C = union onschema A, B;
    4030. 

  4030. store C into ':OUTPATH:';\, 
    4031. 

  4031.                     'verify_pig_script' => q\
    4032. 

  4032. register :FUNCPATH:/testudf.jar;
    4033. 

  4033. define Nil org.apache.pig.test.udf.evalfunc.Nil();
    4034. 

  4034. A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
    4035. 

  4035. B = load ':INPATH:/singlefile/textdoc' as (line:chararray);
    4036. 

  4036. C = foreach A generate name, (chararray)age, (chararray)gpa, Nil(name);
    4037. 

  4037. D = foreach B generate Nil(line), Nil(line), Nil(line), line;
    4038. 

  4038. E = union C, D;
    4039. 

  4039. store E into ':OUTPATH:';\, 
    4040. 

  4040. 				},
    4041. 

  4041.                 {
    4042. 

  4042.                     # Test Union using merge where schema is identical  | A&B have identical schema
    4043. 

  4043. 			        'num' => 7,
    4044. 

  4044. 			        'pig' => q\
    4045. 

  4045. a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:boolean);
    4046. 

  4046. b = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:boolean);
    4047. 

  4047. C = union onschema a, b;
    4048. 

  4048. store C into ':OUTPATH:';\, 
    4049. 

  4049.                     'verify_pig_script' => q\
    4050. 

  4050. a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    4051. 

  4051. b = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    4052. 

  4052. C = union a, b;
    4053. 

  4053. store C into ':OUTPATH:';\, 
    4054. 

  4054.  				}
    4055. 

  4055. 		]
    4056. 

  4056. 

  4057.             },
    4058. 

  4058. 			{
    4059. 

  4059. 

  4060. 			# Test Union using merge with Simple data types
    4061. 

  4061. 	        'name' => 'UdfDistributedCache',
    4062. 

  4062.             'tests' => [
    4063. 

  4063.             	{
    4064. 

  4064.             		'num' => 1,
    4065. 

  4065.                     'execonly' => 'mapred', # since distributed cache is not supported in local mode
    4066. 

  4066.                     'pig' => q?
    4067. 

  4067.                         register :FUNCPATH:/testudf.jar;
    4068. 

  4068.                         define udfdc org.apache.pig.test.udf.evalfunc.Udfcachetest(':INPATH:/singlefile/votertab10k#foodle');
    4069. 

  4069.                         a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4070. 

  4070.                         b = limit a 1;
    4071. 

  4071.                         c = foreach b generate udfdc(age);
    4072. 

  4072.                         STORE c into ':OUTPATH:';?,
    4073. 

  4073.                     'verify_pig_script' => q?
    4074. 

  4074.                         a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4075. 

  4075.                         b = limit a 1;
    4076. 

  4076.                         c = foreach b generate 'tom van buren', 68, 'socialist', 390.19;
    4077. 

  4077.                         STORE c into ':OUTPATH:';?,
    4078. 

  4078.             	},
    4079. 

  4079. 	        ]
    4080. 

  4080.             }, {
    4081. 

  4081.                 'name' => 'MonitoredUDF',
    4082. 

  4082.                 'tests' => [
    4083. 

  4083.                     {
    4084. 

  4084.                         'num' => 1,
    4085. 

  4085.                         'ignore23' => 'guava version of Pig is higher than hadoop 23',
    4086. 

  4086.                         'pig' => q?register :FUNCPATH:/testudf.jar;
    4087. 

  4087.                                 define gm org.apache.pig.test.udf.evalfunc.GoodMonitored();
    4088. 

  4088.                                 a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4089. 

  4089.                                 b = foreach a generate gm(name);
    4090. 

  4090.                                 store b into ':OUTPATH:';?,
    4091. 

  4091.                         'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4092. 

  4092.                                                 b = foreach a generate 'fred';
    4093. 

  4093.                                                 store b into ':OUTPATH:';?,
    4094. 

  4094.                     },{
    4095. 

  4095.                         'num' => 2,
    4096. 

  4096.                         'pig' => q?register :FUNCPATH:/testudf.jar;
    4097. 

  4097.                                 define bad org.apache.pig.test.udf.evalfunc.BadMonitored();
    4098. 

  4098.                                 a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4099. 

  4099.                                 b = limit a 1;
    4100. 

  4100.                                 c = foreach b generate bad(name);
    4101. 

  4101.                                 store b into ':OUTPATH:';?,
    4102. 

  4102.                         'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4103. 

  4103.                                                 b = limit a 1;
    4104. 

  4104.                                                 c = foreach b generate '';
    4105. 

  4105.                                                 store b into ':OUTPATH:';?,
    4106. 

  4106.                     },{
    4107. 

  4107.                         'num' => 3,
    4108. 

  4108.                         'pig' => q?register :FUNCPATH:/testudf.jar;
    4109. 

  4109.                                 define bad org.apache.pig.test.udf.evalfunc.BadMonitored();
    4110. 

  4110.                                 a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4111. 

  4111.                                 b = limit a 1;
    4112. 

  4112.                                 c = foreach b generate bad(name);
    4113. 

  4113.                                 store b into ':OUTPATH:';?,
    4114. 

  4114.                         'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4115. 

  4115.                                                 b = limit a 1;
    4116. 

  4116.                                                 c = foreach b generate 'barney';
    4117. 

  4117.                                                 store b into ':OUTPATH:';?,
    4118. 

  4118.                     }
    4119. 

  4119.                 ],
    4120. 

  4120.             },{
    4121. 

  4121.                 'name' => 'MergeSparseJoin',
    4122. 

  4122.                 'tests' => [
    4123. 

  4123.                     # Simplest merge-sparse-join.
    4124. 

  4124.                     {
    4125. 

  4125.                         'num' => 1,
    4126. 

  4126.                         'pig' => q\register :PIGPATH:/contrib/piggybank/java/piggybank.jar
    4127. 

  4127.                                 a = load ':INPATH:/singlefile/studenttab10k';
    4128. 

  4128.                                 b = load ':INPATH:/singlefile/votertab10k';
    4129. 

  4129.                                 c = order a by $0;
    4130. 

  4130.                                 d = order b by $0;
    4131. 

  4131.                                 store c into ':OUTPATH:.intermediate1';
    4132. 

  4132.                                 store d into ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4133. 

  4133.                                 exec;
    4134. 

  4134.                                 e = load ':OUTPATH:.intermediate1';
    4135. 

  4135.                                 f = load ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4136. 

  4136.                                 g = join e by $0, f by $0 using 'merge-sparse';
    4137. 

  4137.                                 store g into ':OUTPATH:';\,
    4138. 

  4138.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    4139. 

  4139.                                 b = load ':INPATH:/singlefile/votertab10k';
    4140. 

  4140.                                 g = join a by $0, b by $0;
    4141. 

  4141.                                 store g into ':OUTPATH:';\,
    4142. 

  4142.                         'notmq' => 1,
    4143. 

  4143.                     },
    4144. 

  4144.                     # Merge-sparse-join with left-side filter
    4145. 

  4145.                     {
    4146. 

  4146.                         'num' => 2,
    4147. 

  4147.                         'pig' => q\register :PIGPATH:/contrib/piggybank/java/piggybank.jar
    4148. 

  4148.                                 a = load ':INPATH:/singlefile/studenttab10k';
    4149. 

  4149.                                 b = load ':INPATH:/singlefile/votertab10k';
    4150. 

  4150.                                 c = order a by $0;
    4151. 

  4151.                                 d = order b by $0;
    4152. 

  4152.                                 store c into ':OUTPATH:.intermediate1';
    4153. 

  4153.                                 store d into ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4154. 

  4154.                                 exec;
    4155. 

  4155.                                 e = load ':OUTPATH:.intermediate1';
    4156. 

  4156.                                 h = filter e by $1 > 30;
    4157. 

  4157.                                 f = load ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4158. 

  4158.                                 g = join h by $0, f by $0 using 'merge-sparse';
    4159. 

  4159.                                 store g into ':OUTPATH:';\,
    4160. 

  4160.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    4161. 

  4161.                                 b = load ':INPATH:/singlefile/votertab10k';
    4162. 

  4162.                                 h = filter a by $1 > 30;
    4163. 

  4163.                                 g = join h by $0, b by $0;
    4164. 

  4164.                                 store g into ':OUTPATH:';\,
    4165. 

  4165.                         'notmq' => 1,
    4166. 

  4166.                     },
    4167. 

  4167.                     # Merge-sparse-join with right-side filter
    4168. 

  4168.                     {
    4169. 

  4169.                         'num' => 3,
    4170. 

  4170.                         'pig' => q\register :PIGPATH:/contrib/piggybank/java/piggybank.jar
    4171. 

  4171.                                 a = load ':INPATH:/singlefile/studenttab10k';
    4172. 

  4172.                                 b = load ':INPATH:/singlefile/votertab10k';
    4173. 

  4173.                                 c = order a by $0;
    4174. 

  4174.                                 d = order b by $0;
    4175. 

  4175.                                 store c into ':OUTPATH:.intermediate1';
    4176. 

  4176.                                 store d into ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4177. 

  4177.                                 exec;
    4178. 

  4178.                                 e = load ':OUTPATH:.intermediate1';
    4179. 

  4179.                                 f = load ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4180. 

  4180.                                 i = filter f by $2 != 'democrat';
    4181. 

  4181.                                 g = join e by $0, i by $0 using 'merge-sparse';
    4182. 

  4182.                                 store g into ':OUTPATH:';\,
    4183. 

  4183.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    4184. 

  4184.                                 b = load ':INPATH:/singlefile/votertab10k';
    4185. 

  4185.                                 i = filter b by $2 != 'democrat';
    4186. 

  4186.                                 g = join a by $0, i by $0;
    4187. 

  4187.                                 store g into ':OUTPATH:';\,
    4188. 

  4188. 

  4189.                         'notmq' => 1,
    4190. 

  4190.                     },
    4191. 

  4191.                     # Merge-sparse-join with key as expression
    4192. 

  4192.                     {
    4193. 

  4193.                         'num' => 4,
    4194. 

  4194.                         'pig' => q\register :PIGPATH:/contrib/piggybank/java/piggybank.jar
    4195. 

  4195.                                 a = load ':INPATH:/singlefile/studenttab10k';
    4196. 

  4196.                                 b = load ':INPATH:/singlefile/votertab10k';
    4197. 

  4197.                                 c = order a by $0,$1;
    4198. 

  4198.                                 d = order b by $0,$1;
    4199. 

  4199.                                 store c into ':OUTPATH:.intermediate1';
    4200. 

  4200.                                 store d into ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0,1');
    4201. 

  4201.                                 exec;
    4202. 

  4202.                                 e = load ':OUTPATH:.intermediate1';
    4203. 

  4203.                                 f = load ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0,1');
    4204. 

  4204.                                 g = join e by ($0,$1), f by ($0,$1) using 'merge-sparse';
    4205. 

  4205.                                 store g into ':OUTPATH:';\,
    4206. 

  4206.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k';
    4207. 

  4207.                                 b = load ':INPATH:/singlefile/votertab10k';
    4208. 

  4208.                                 g = join a by ($0,$1), b by ($0,$1);
    4209. 

  4209.                                 store g into ':OUTPATH:';\,
    4210. 

  4210.                         'notmq' => 1,
    4211. 

  4211.                     },
    4212. 

  4212.                     # Merge-sparse-join with nulls in keys and data.
    4213. 

  4213.                     {
    4214. 

  4214.                         'num' => 5,
    4215. 

  4215.                         'pig' => q\register :PIGPATH:/contrib/piggybank/java/piggybank.jar
    4216. 

  4216.                                 a = load ':INPATH:/singlefile/studentnulltab10k';
    4217. 

  4217.                                 b = load ':INPATH:/singlefile/voternulltab10k';
    4218. 

  4218.                                 c = order a by $0;
    4219. 

  4219.                                 d = order b by $0;
    4220. 

  4220.                                 store c into ':OUTPATH:.intermediate1';
    4221. 

  4221.                                 store d into ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4222. 

  4222.                                 exec;
    4223. 

  4223.                                 e = load ':OUTPATH:.intermediate1';
    4224. 

  4224.                                 f = load ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4225. 

  4225.                                 g = join e by $0, f by $0 using 'merge-sparse';
    4226. 

  4226.                                 store g into ':OUTPATH:';\,
    4227. 

  4227.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentnulltab10k';
    4228. 

  4228.                                 b = load ':INPATH:/singlefile/voternulltab10k';
    4229. 

  4229.                                 g = join a by $0, b by $0;
    4230. 

  4230.                                 store g into ':OUTPATH:';\,
    4231. 

  4231.                         'notmq' => 1,
    4232. 

  4232.                     },
    4233. 

  4233.                     # Merge-sparse-join with join on numeric key
    4234. 

  4234.                     {
    4235. 

  4235.                         'num' => 6,
    4236. 

  4236.                         'pig' => q\register :PIGPATH:/contrib/piggybank/java/piggybank.jar
    4237. 

  4237.                                 a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    4238. 

  4238.                                 b = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    4239. 

  4239.                                 c = order a by age;
    4240. 

  4240.                                 d = order b by age;
    4241. 

  4241.                                 store c into ':OUTPATH:.intermediate1';
    4242. 

  4242.                                 store d into ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0');
    4243. 

  4243.                                 exec;
    4244. 

  4244.                                 e = load ':OUTPATH:.intermediate1'  as (name:chararray, age:int, gpa:float);
    4245. 

  4245.                                 f = load ':OUTPATH:.intermediate2' using org.apache.pig.piggybank.storage.IndexedStorage(',', '0') as (name:chararray, age:int, reg:chararray, contrib:float);
    4246. 

  4246.                                 g = join e by age, f by age using 'merge-sparse';
    4247. 

  4247.                                 store g into ':OUTPATH:';\,
    4248. 

  4248.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
    4249. 

  4249.                                 b = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    4250. 

  4250.                                 g = join a by age, b by age;
    4251. 

  4251.                                 store g into ':OUTPATH:';\,
    4252. 

  4252.                         'notmq' => 1,
    4253. 

  4253.                     }
    4254. 

  4254.                 ],
    4255. 

  4255.             },{
    4256. 

  4256.                 'name' => 'BugFix',
    4257. 

  4257.                 'tests' => [
    4258. 

  4258.                     {
    4259. 

  4259.                         # PIG-2286
    4260. 

  4260.                         'num' => 1,
    4261. 

  4261.                         'pig' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name, age:double, gpa:double);
    4262. 

  4262.                                 B = group A all;
    4263. 

  4263.                                 C = foreach B generate group, COR(A.age, A.gpa);
    4264. 

  4264.                                 store C into ':OUTPATH:';?,
    4265. 

  4265.                         'verify_pig_script' => q?set pig.exec.nocombiner true
    4266. 

  4266.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name, age:double ,gpa:double);
    4267. 

  4267.                                 B = group A all;
    4268. 

  4268.                                 C = foreach B generate group, COR(A.age, A.gpa);
    4269. 

  4269.                                 store C into ':OUTPATH:';?,
    4270. 

  4270.                     }, {
    4271. 

  4271.                         # PIG-2286, with 3 inputs to COR
    4272. 

  4272.                         'num' => 2,
    4273. 

  4273.                         'pig' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name, age:double ,gpa:double);
    4274. 

  4274.                                 B = foreach A generate age, gpa, gpa*gpa as gpa2;
    4275. 

  4275.                                 C = group B all;
    4276. 

  4276.                                 D = foreach C generate group, COR(B.age, B.gpa, B.gpa2);
    4277. 

  4277.                                 store D into ':OUTPATH:';?,
    4278. 

  4278.                         'verify_pig_script' => q?set pig.exec.nocombiner true
    4279. 

  4279.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name, age:double ,gpa:double);
    4280. 

  4280.                                 B = foreach A generate age, gpa, gpa*gpa as gpa2;
    4281. 

  4281.                                 C = group B all;
    4282. 

  4282.                                 D = foreach C generate group, COR(B.age, B.gpa, B.gpa2);
    4283. 

  4283.                                 store D into ':OUTPATH:';?,
    4284. 

  4284.                     }, {
    4285. 

  4285.                         # PIG-2385
    4286. 

  4286.                         'num' => 3,
    4287. 

  4287.                         'pig_params' => ['-M'],
    4288. 

  4288.                         'pig' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int,gpa:double);
    4289. 

  4289.                                 Z = group A all;
    4290. 

  4290.                                 Z1 = foreach Z generate AVG(A.gpa) as avg;
    4291. 

  4291.                                 B = foreach A generate name, age, gpa-Z1.avg as diff;
    4292. 

  4292.                                 STORE B INTO ':OUTPATH:.1';
    4293. 

  4293.                                 C = DISTINCT B ;
    4294. 

  4294.                                 store C into ':OUTPATH:.2';?,
    4295. 

  4295.                         'verify_pig_script' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int,gpa:double);
    4296. 

  4296.                                 Z = group A all;
    4297. 

  4297.                                 Z1 = foreach Z generate AVG(A.gpa) as avg;
    4298. 

  4298.                                 B = cross A, Z1;
    4299. 

  4299.                                 B1 = foreach B generate name, age, gpa-Z1.avg as diff;
    4300. 

  4300.                                 STORE B1 INTO ':OUTPATH:.1';
    4301. 

  4301.                                 C = DISTINCT B1 ;
    4302. 

  4302.                                 store C into ':OUTPATH:.2';?,
    4303. 

  4303.                     }, {
    4304. 

  4304.                         # PIG-2576
    4305. 

  4305.                         'num' => 4,
    4306. 

  4306.                         'execonly' => 'mapred',
    4307. 

  4307.                         'pig' => q?register :FUNCPATH:/testudf.jar;
    4308. 

  4308.                                 define printconf org.apache.pig.test.udf.evalfunc.UdfContextFrontend('dummy');
    4309. 

  4309.                                 a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
    4310. 

  4310.                                 b = limit a 1;
    4311. 

  4311.                                 c = foreach b generate printconf(name);
    4312. 

  4312. 	                        store c into ':OUTPATH:';
    4313. 

  4313.                                 fs -ls;
    4314. 

  4314.                                 ?,
    4315. 

  4315.                         'rc' => 0,
    4316. 

  4316.                         'not_expected_out_regex' => "checkJobConf: conf is null: false",
    4317. 

  4317.                         'expected_out_regex' => "checkJobConf: conf is null: true",
    4318. 

  4318.                     }
    4319. 

  4319.                 ],
    4320. 

  4320.             },{
    4321. 

  4321.                 'name' => 'Bloom',
    4322. 

  4322. 			    'execonly' => 'mapred', # distributed cache does not work in local mode
    4323. 

  4323.                 'tests' => [
    4324. 

  4324.                     {
    4325. 

  4325.                         'num' => 1,
    4326. 

  4326.                         'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', 'fixed', '128', '3');
    4327. 

  4327.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4328. 

  4328.                                 B = filter A by name == 'alice allen';
    4329. 

  4329.                                 C = group B all;
    4330. 

  4330.                                 D = foreach C generate bb(B.name);
    4331. 

  4331.                                 store D into ':HDFSTMP:/mybloom_1';
    4332. 

  4332.                                 exec;
    4333. 

  4333.                                 define bloom Bloom(':HDFSTMP:/mybloom_1');
    4334. 

  4334.                                 E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4335. 

  4335.                                 F = filter E by bloom(name);
    4336. 

  4336.                                 store F into ':OUTPATH:';",
    4337. 

  4337.                         'notmq' => 1,
    4338. 

  4338.                         'verify_pig_script' => "
    4339. 

  4339.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name, age:int ,gpa:double);
    4340. 

  4340.                                 B = filter A by name == 'alice allen';
    4341. 

  4341.                                 store B into ':OUTPATH:';",
    4342. 

  4342.                     }, {
    4343. 

  4343.                         'num' => 2,
    4344. 

  4344.                         'pig' => "define bb BuildBloom('Hash.MURMUR_HASH', 'fixed', '128', '3');
    4345. 

  4345.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4346. 

  4346.                                 B = filter A by name == 'alice allen';
    4347. 

  4347.                                 C = group B all;
    4348. 

  4348.                                 D = foreach C generate bb(B.name);
    4349. 

  4349.                                 store D into ':HDFSTMP:/mybloom_2';
    4350. 

  4350.                                 exec;
    4351. 

  4351.                                 define bloom Bloom(':HDFSTMP:/mybloom_2');
    4352. 

  4352.                                 E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4353. 

  4353.                                 F = filter E by bloom(name);
    4354. 

  4354.                                 G = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    4355. 

  4355.                                 H = join F by name, G by name;
    4356. 

  4356.                                 store H into ':OUTPATH:';",
    4357. 

  4357.                         'notmq' => 1,
    4358. 

  4358.                         'verify_pig_script' => "
    4359. 

  4359.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name, age:int ,gpa:double);
    4360. 

  4360.                                 B = filter A by name == 'alice allen';
    4361. 

  4361.                                 C = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    4362. 

  4362.                                 D = join B by name, C by name;
    4363. 

  4363.                                 store D into ':OUTPATH:';",
    4364. 

  4364.                     },{
    4365. 

  4365.                         'num' => 3,
    4366. 

  4366.                         'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', '1', '0.0001');
    4367. 

  4367.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4368. 

  4368.                                 B = filter A by name == 'alice allen';
    4369. 

  4369.                                 C = group B all;
    4370. 

  4370.                                 D = foreach C generate bb(B.name);
    4371. 

  4371.                                 store D into ':HDFSTMP:/mybloom_3';
    4372. 

  4372.                                 exec;
    4373. 

  4373.                                 define bloom Bloom(':HDFSTMP:/mybloom_3');
    4374. 

  4374.                                 E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4375. 

  4375.                                 F = filter E by bloom(name);
    4376. 

  4376.                                 G = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    4377. 

  4377.                                 H = join G by name, F by name using 'repl';
    4378. 

  4378.                                 store H into ':OUTPATH:';",
    4379. 

  4379.                         'notmq' => 1,
    4380. 

  4380.                         'verify_pig_script' => "
    4381. 

  4381.                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name, age:int ,gpa:double);
    4382. 

  4382.                                 B = filter A by name == 'alice allen';
    4383. 

  4383.                                 C = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
    4384. 

  4384.                                 D = join C by name, B by name;
    4385. 

  4385.                                 store D into ':OUTPATH:';",
    4386. 

  4386.                     }
    4387. 

  4387.                 ],
    4388. 

  4388.             },{
    4389. 

  4389.                 'name' => 'UDFContext',
    4390. 

  4390.                 'tests' => [
    4391. 

  4391.                     {
    4392. 

  4392.                         # See PIG-2338
    4393. 

  4393.                         'num' => 1,
    4394. 

  4394.                         'pig' => q?register :FUNCPATH:/testudf.jar
    4395. 

  4395.                                 a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
    4396. 

  4396.                                 b = foreach a generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(a0);
    4397. 

  4397.                                 c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
    4398. 

  4398.                                 d = foreach c generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(c0);
    4399. 

  4399.                                 e = union b, d;
    4400. 

  4400.                                 store e into ':OUTPATH:';?,
    4401. 

  4401.                         'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
    4402. 

  4402.                                 b = foreach a generate '{a0: bytearray}';
    4403. 

  4403.                                 c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
    4404. 

  4404.                                 d = foreach c generate '{c0: chararray}';
    4405. 

  4405.                                 e = union b, d;
    4406. 

  4406.                                 store e into ':OUTPATH:';?,
    4407. 

  4407.                     }
    4408. 

  4408.                 ], 
    4409. 

  4409.                 
    4410. 

  4410.                },{
    4411. 

  4411.                 'name' => 'UDFContextAuto',
    4412. 

  4412.                 'tests' => [
    4413. 

  4413.                     {
    4414. 

  4414.                         # See PIG-2337
    4415. 

  4415.                         'num' => 1,
    4416. 

  4416.                         'pig' => q?register :FUNCPATH:/testudf.jar
    4417. 

  4417.                                 a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
    4418. 

  4418.                                 b = foreach a generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(a0);
    4419. 

  4419.                                 c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
    4420. 

  4420.                                 d = foreach c generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(c0);
    4421. 

  4421.                                 e = union b, d;
    4422. 

  4422.                                 store e into ':OUTPATH:';?,
    4423. 

  4423.                         'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
    4424. 

  4424.                                 b = foreach a generate '{a0: bytearray}';
    4425. 

  4425.                                 c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
    4426. 

  4426.                                 d = foreach c generate '{c0: chararray}';
    4427. 

  4427.                                 e = union b, d;
    4428. 

  4428.                                 store e into ':OUTPATH:';?,
    4429. 

  4429.                     }
    4430. 

  4430.                 ],
    4431. 

  4431.             },{
    4432. 

  4432.                 'name' => 'JsonLoaderStorage',
    4433. 

  4433.                 'tests' => [
    4434. 

  4434.                     {
    4435. 

  4435.                         'num' => 1,
    4436. 

  4436.                         'pig' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4437. 

  4437.                                 store A into ':OUTPATH:.intermediate' using JsonStorage();
    4438. 

  4438.                                 exec
    4439. 

  4439.                                 A = LOAD ':OUTPATH:.intermediate' using JsonLoader();
    4440. 

  4440.                                 store A into ':OUTPATH:';?,
    4441. 

  4441.                         'notmq' => 1,
    4442. 

  4442.                         'verify_pig_script' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int,gpa:double);
    4443. 

  4443.                                 store A into ':OUTPATH:';?,
    4444. 

  4444.                     }, {
    4445. 

  4445.                         'num' => 2,
    4446. 

  4446.                         'pig' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
    4447. 

  4447.                                 store A into ':OUTPATH:.intermediate1' using JsonStorage();
    4448. 

  4448.                                 B = LOAD ':INPATH:/singlefile/votertab10k' AS (name:chararray, age:int, registration:chararray, contributions:double);
    4449. 

  4449.                                 store B into ':OUTPATH:.intermediate2' using JsonStorage();
    4450. 

  4450.                                 exec
    4451. 

  4451.                                 A = LOAD ':OUTPATH:.intermediate1' using JsonLoader();
    4452. 

  4452.                                 B = LOAD ':OUTPATH:.intermediate2' using JsonLoader();
    4453. 

  4453.                                 C = JOIN A by name, B by name;
    4454. 

  4454.                                 store C into ':OUTPATH:';?,
    4455. 

  4455.                         'notmq' => 1,
    4456. 

  4456.                         'verify_pig_script' => q?A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int,gpa:double);
    4457. 

  4457.                                 B = LOAD ':INPATH:/singlefile/votertab10k' AS (name:chararray, age:int, registration:chararray, contributions:double);
    4458. 

  4458.                                 C = JOIN A by name, B by name;
    4459. 

  4459.                                 store C into ':OUTPATH:';?,
    4460. 

  4460.                     }, {
    4461. 

  4461.                         'num' => 3,
    4462. 

  4462.                         'ignore' => 1, # PIG-2594
    4463. 

  4463. 			'pig' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:boolean);
    4464. 

  4464.                                 store a into ':OUTPATH:.intermediate' using JsonStorage();
    4465. 

  4465.                                 exec
    4466. 

  4466.                                 B = LOAD ':OUTPATH:.intermediate' using JsonLoader();
    4467. 

  4467.                                 store B into ':OUTPATH:';\,
    4468. 

  4468.                         'notmq' => 1,
    4469. 

  4469.                         'verify_pig_script' => q\a = load ':INPATH:/singlefile/allscalar10k' using PigStorage() as (name:chararray, age:int, gpa:double, instate:chararray);
    4470. 

  4470. store a into ':OUTPATH:';\,
    4471. 

  4471.                      }
    4472. 

  4472. 

  4473.                 ],
    4474. 

  4474.             },{
    4475. 

  4475.                 'name' => 'STRSPLIT',
    4476. 

  4476.                 'tests' => [
    4477. 

  4477.                     {
    4478. 

  4478.                         # See PIG-2311
    4479. 

  4479.                         'num' => 1,
    4480. 

  4480.                         'pig' => q?a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
    4481. 

  4481.                         		b= filter a by NOT (a0 is null);
    4482. 

  4482.                         		c= foreach b generate STRSPLIT(a0);
    4483. 

  4483.                                 store c into ':OUTPATH:';?,
    4484. 

  4484.                         'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
    4485. 

  4485.                         		b= filter a by NOT (a0 is null);
    4486. 

  4486.                         		b= foreach b generate (chararray)a0 as a0 ;
    4487. 

  4487.                         		c= foreach b generate STRSPLIT(a0);
    4488. 

  4488.                                 store c into ':OUTPATH:';?,
    4489. 

  4489.                     }
    4490. 

  4490.                 ],
    4491. 

  4491.             },
    4492. 

  4492.         {
    4493. 

  4493. 		'name' => 'Tokenize',
    4494. 

  4494. 		'tests' => [
    4495. 

  4495. 			{
    4496. 

  4496.                 'num' => 1,
    4497. 

  4497.                 'pig' => q\
    4498. 

  4498.                         A = LOAD ':INPATH:/singlefile/studenttab10k';
    4499. 

  4499.                         B = foreach A generate TOKENIZE($0);
    4500. 

  4500.                         store B into ':OUTPATH:';\,
    4501. 

  4501.             },
    4502. 

  4502.             {
    4503. 

  4503.                 'num' => 2,
    4504. 

  4504.                 'pig' => q\
    4505. 

  4505.                         A = LOAD ':INPATH:/singlefile/studenttab10k';
    4506. 

  4506.                         B = foreach A generate TOKENIZE($1,'9');
    4507. 

  4507.                         store B into ':OUTPATH:';\,
    4508. 

  4508.                 'verify_pig_script' => q\
    4509. 

  4509.                         A = LOAD ':INPATH:/singlefile/studenttab10k';
    4510. 

  4510.                         -- TOKENIZE has tokens hardcoded so have to replace the '9' with
    4511. 

  4511.                         -- one of the hardcoded tokens
    4512. 

  4512.                         B = foreach A generate TOKENIZE(REPLACE($1, '9', ','));
    4513. 

  4513.                         store B into ':OUTPATH:';\,
    4514. 

  4514. }
    4515. 

  4515.             ]
    4516. 

  4516.         }, {
    4517. 

  4517. 		'name' => 'Realias',
    4518. 

  4518. 		'tests' => [
    4519. 

  4519. 			{
    4520. 

  4520.                 'num' => 1,
    4521. 

  4521.                 'pig' => q\
    4522. 

  4522.                         A = LOAD ':INPATH:/singlefile/studenttab10k';
    4523. 

  4523.                         B = A;
    4524. 

  4524.                         store B into ':OUTPATH:';\,
    4525. 

  4525.                 'verify_pig_script' => q\
    4526. 

  4526.                         A = LOAD ':INPATH:/singlefile/studenttab10k';
    4527. 

  4527.                         store A into ':OUTPATH:';\,
    4528. 

  4528.                     }
    4529. 

  4529.                 ]
    4530. 

  4530.             },
    4531. 

  4531.             {
    4532. 

  4532.  		'name' => 'NestedForEach',
    4533. 

  4533.  		'tests' => [
    4534. 

  4534.  			{
    4535. 

  4535.                  'num' => 1,
    4536. 

  4536.                  'pig' => q\
    4537. 

  4537.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    4538. 

  4538.                         B = group A by name;
    4539. 

  4539.                         C = foreach B {
    4540. 

  4540.                             C1 = foreach A generate UPPER(name), age+1 as age, gpa;
    4541. 

  4541.                             generate C1;
    4542. 

  4542.                         }
    4543. 

  4543.                         D = foreach C generate flatten(C1);
    4544. 

  4544.                         store D into ':OUTPATH:';\,
    4545. 

  4545.                 'verify_pig_script' => q\
    4546. 

  4546.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    4547. 

  4547.                         B = foreach A generate UPPER(name), age+1, gpa;
    4548. 

  4548.                         store B into ':OUTPATH:';\,
    4549. 

  4549.             },
    4550. 

  4550.             {
    4551. 

  4551.                 'num' => 2,
    4552. 

  4552.                 'pig' => q\
    4553. 

  4553.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);;
    4554. 

  4554.                         B = group A by name;
    4555. 

  4555.                         C = foreach B {
    4556. 

  4556.                             C1 = A.age;
    4557. 

  4557.                             C2 = filter C1 by age>=30;
    4558. 

  4558.                             C3 = foreach C2 generate age+1 as age;
    4559. 

  4559.                             C4 = order C3 by age desc;
    4560. 

  4560.                             generate C4;
    4561. 

  4561.                         }
    4562. 

  4562.                         D = foreach C generate flatten(C4);
    4563. 

  4563.                         store D into ':OUTPATH:';\,
    4564. 

  4564.                 'verify_pig_script' => q\
    4565. 

  4565.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    4566. 

  4566.                         B = filter A by age>=30;
    4567. 

  4567.                         C = foreach B generate age+1 as age;
    4568. 

  4568.                         D = order C by age desc;
    4569. 

  4569.                         store D into ':OUTPATH:';\,
    4570. 

  4570.             }
    4571. 

  4571.             ]
    4572. 

  4572.         },
    4573. 

  4573.         {
    4574. 

  4574. 		'name' => 'NestedCross',
    4575. 

  4575. 		'tests' => [
    4576. 

  4576. 			{
    4577. 

  4577.                 'num' => 1,
    4578. 

  4578.                 'pig' => q\
    4579. 

  4579.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    4580. 

  4580.                         B = LOAD ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration, contributions:double);
    4581. 

  4581.                         C = cogroup A by name, B by name;
    4582. 

  4582.                         D = foreach C {
    4583. 

  4583.                             C1 = cross A, B;
    4584. 

  4584.                             generate flatten(C1);
    4585. 

  4585.                         }
    4586. 

  4586.                         store D into ':OUTPATH:';\,
    4587. 

  4587.                 'verify_pig_script' => q\
    4588. 

  4588.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    4589. 

  4589.                         B = LOAD ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration, contributions:double);
    4590. 

  4590.                         C = JOIN A by name, B by name;
    4591. 

  4591.                         store C into ':OUTPATH:';\,
    4592. 

  4592.                         },
    4593. 

  4593. 			{
    4594. 

  4594.                 'num' => 2,
    4595. 

  4595.                 'pig' => q\
    4596. 

  4596.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    4597. 

  4597.                         B = LOAD ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration, contributions:double);
    4598. 

  4598.                         C = cogroup A by name, B by name;
    4599. 

  4599.                         D = foreach C {
    4600. 

  4600.                             C1 = filter A by gpa > 4;
    4601. 

  4601.                             C2 = filter B by contributions > 500;
    4602. 

  4602.                             C3 = cross C1, C2;
    4603. 

  4603.                             C4 = foreach C3 generate CONCAT(CONCAT(gpa, '_'), contributions);
    4604. 

  4604.                             generate flatten(C4);
    4605. 

  4605.                         }
    4606. 

  4606.                         store D into ':OUTPATH:';\,
    4607. 

  4607.                 'verify_pig_script' => q\
    4608. 

  4608.                         A = LOAD ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
    4609. 

  4609.                         B = LOAD ':INPATH:/singlefile/votertab10k' as (name:chararray, age:int, registration, contributions:double);
    4610. 

  4610.                         C = filter A by gpa > 4;
    4611. 

  4611.                         D = filter B by contributions > 500;
    4612. 

  4612.                         E = JOIN C by name, D by name;
    4613. 

  4613.                         F = foreach E generate CONCAT(CONCAT(gpa, '_'), contributions);
    4614. 

  4614.                         store F into ':OUTPATH:';\,
    4615. 

  4615.                         }
    4616. 

  4616.                 ]
    4617. 

  4617.             }
    4618. 

  4618.         ],
    4619. 

  4619.     },
    4620. 

  4620. ;
    4621. 

  4621. 

  4622. 

  4623. 

  4624.