PageRenderTime 48ms CodeModel.GetById 19ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out

#
text | 313 lines | 305 code | 8 blank | 0 comment | 0 complexity | efaa452beddc46c6b9c00b998b976dd6 MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause, JSON, CPL-1.0 
    

  1. PREHOOK: query: CREATE TABLE dest1(key STRING, c1 INT, c2 STRING, C3 INT, c4 INT) STORED AS TEXTFILE
    2. 

  2. PREHOOK: type: CREATETABLE
    3. 

  3. POSTHOOK: query: CREATE TABLE dest1(key STRING, c1 INT, c2 STRING, C3 INT, c4 INT) STORED AS TEXTFILE
    4. 

  4. POSTHOOK: type: CREATETABLE
    5. 

  5. POSTHOOK: Output: default@dest1
    6. 

  6. PREHOOK: query: EXPLAIN EXTENDED
    7. 

  7. FROM srcpart src
    8. 

  8. INSERT OVERWRITE TABLE dest1 
    9. 

  9. SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(DISTINCT src.value) 
    10. 

  10. WHERE src.ds = '2008-04-08'
    11. 

  11. GROUP BY substr(src.key,1,1)
    12. 

  12. PREHOOK: type: QUERY
    13. 

  13. POSTHOOK: query: EXPLAIN EXTENDED
    14. 

  14. FROM srcpart src
    15. 

  15. INSERT OVERWRITE TABLE dest1 
    16. 

  16. SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(DISTINCT src.value) 
    17. 

  17. WHERE src.ds = '2008-04-08'
    18. 

  18. GROUP BY substr(src.key,1,1)
    19. 

  19. POSTHOOK: type: QUERY
    20. 

  20. ABSTRACT SYNTAX TREE:
    21. 

  21.   (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart) src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (. (TOK_TABLE_OR_COL src) key) 1 1)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_FUNCTION substr (. (TOK_TABLE_OR_COL src) value) 5))) (TOK_SELEXPR (TOK_FUNCTION concat (TOK_FUNCTION substr (. (TOK_TABLE_OR_COL src) key) 1 1) (TOK_FUNCTION sum (TOK_FUNCTION substr (. (TOK_TABLE_OR_COL src) value) 5)))) (TOK_SELEXPR (TOK_FUNCTIONDI sum (TOK_FUNCTION substr (. (TOK_TABLE_OR_COL src) value) 5))) (TOK_SELEXPR (TOK_FUNCTIONDI count (. (TOK_TABLE_OR_COL src) value)))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL src) ds) '2008-04-08')) (TOK_GROUPBY (TOK_FUNCTION substr (. (TOK_TABLE_OR_COL src) key) 1 1))))
    22. 

  22. 

  23. STAGE DEPENDENCIES:
    24. 

  24.   Stage-1 is a root stage
    25. 

  25.   Stage-0 depends on stages: Stage-1
    26. 

  26.   Stage-2 depends on stages: Stage-0
    27. 

  27. 

  28. STAGE PLANS:
    29. 

  29.   Stage: Stage-1
    30. 

  30.     Map Reduce
    31. 

  31.       Alias -> Map Operator Tree:
    32. 

  32.         src 
    33. 

  33.           TableScan
    34. 

  34.             alias: src
    35. 

  35.             GatherStats: false
    36. 

  36.             Filter Operator
    37. 

  37.               isSamplingPred: false
    38. 

  38.               predicate:
    39. 

  39.                   expr: (ds = '2008-04-08')
    40. 

  40.                   type: boolean
    41. 

  41.               Select Operator
    42. 

  42.                 expressions:
    43. 

  43.                       expr: key
    44. 

  44.                       type: string
    45. 

  45.                       expr: value
    46. 

  46.                       type: string
    47. 

  47.                 outputColumnNames: key, value
    48. 

  48.                 Group By Operator
    49. 

  49.                   aggregations:
    50. 

  50.                         expr: count(DISTINCT substr(value, 5))
    51. 

  51.                         expr: sum(substr(value, 5))
    52. 

  52.                         expr: sum(DISTINCT substr(value, 5))
    53. 

  53.                         expr: count(DISTINCT value)
    54. 

  54.                   bucketGroup: false
    55. 

  55.                   keys:
    56. 

  56.                         expr: substr(key, 1, 1)
    57. 

  57.                         type: string
    58. 

  58.                         expr: substr(value, 5)
    59. 

  59.                         type: string
    60. 

  60.                         expr: value
    61. 

  61.                         type: string
    62. 

  62.                   mode: hash
    63. 

  63.                   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    64. 

  64.                   Reduce Output Operator
    65. 

  65.                     key expressions:
    66. 

  66.                           expr: _col0
    67. 

  67.                           type: string
    68. 

  68.                           expr: _col1
    69. 

  69.                           type: string
    70. 

  70.                           expr: _col2
    71. 

  71.                           type: string
    72. 

  72.                     sort order: +++
    73. 

  73.                     Map-reduce partition columns:
    74. 

  74.                           expr: _col0
    75. 

  75.                           type: string
    76. 

  76.                     tag: -1
    77. 

  77.                     value expressions:
    78. 

  78.                           expr: _col3
    79. 

  79.                           type: bigint
    80. 

  80.                           expr: _col4
    81. 

  81.                           type: double
    82. 

  82.                           expr: _col5
    83. 

  83.                           type: double
    84. 

  84.                           expr: _col6
    85. 

  85.                           type: bigint
    86. 

  86.       Needs Tagging: false
    87. 

  87.       Path -> Alias:
    88. 

  88.         pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [src]
    89. 

  89.         pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 [src]
    90. 

  90.       Path -> Partition:
    91. 

  91.         pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 
    92. 

  92.           Partition
    93. 

  93.             base file name: hr=11
    94. 

  94.             input format: org.apache.hadoop.mapred.TextInputFormat
    95. 

  95.             output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    96. 

  96.             partition values:
    97. 

  97.               ds 2008-04-08
    98. 

  98.               hr 11
    99. 

  99.             properties:
    100. 

  100.               bucket_count -1
    101. 

  101.               columns key,value
    102. 

  102.               columns.types string:string
    103. 

  103.               file.inputformat org.apache.hadoop.mapred.TextInputFormat
    104. 

  104.               file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    105. 

  105.               location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11
    106. 

  106.               name default.srcpart
    107. 

  107.               partition_columns ds/hr
    108. 

  108.               serialization.ddl struct srcpart { string key, string value}
    109. 

  109.               serialization.format 1
    110. 

  110.               serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    111. 

  111.               transient_lastDdlTime 1297328959
    112. 

  112.             serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    113. 

  113.           
    114. 

  114.               input format: org.apache.hadoop.mapred.TextInputFormat
    115. 

  115.               output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    116. 

  116.               properties:
    117. 

  117.                 bucket_count -1
    118. 

  118.                 columns key,value
    119. 

  119.                 columns.types string:string
    120. 

  120.                 file.inputformat org.apache.hadoop.mapred.TextInputFormat
    121. 

  121.                 file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    122. 

  122.                 location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart
    123. 

  123.                 name default.srcpart
    124. 

  124.                 partition_columns ds/hr
    125. 

  125.                 serialization.ddl struct srcpart { string key, string value}
    126. 

  126.                 serialization.format 1
    127. 

  127.                 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    128. 

  128.                 transient_lastDdlTime 1297328959
    129. 

  129.               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    130. 

  130.               name: default.srcpart
    131. 

  131.             name: default.srcpart
    132. 

  132.         pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 
    133. 

  133.           Partition
    134. 

  134.             base file name: hr=12
    135. 

  135.             input format: org.apache.hadoop.mapred.TextInputFormat
    136. 

  136.             output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    137. 

  137.             partition values:
    138. 

  138.               ds 2008-04-08
    139. 

  139.               hr 12
    140. 

  140.             properties:
    141. 

  141.               bucket_count -1
    142. 

  142.               columns key,value
    143. 

  143.               columns.types string:string
    144. 

  144.               file.inputformat org.apache.hadoop.mapred.TextInputFormat
    145. 

  145.               file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    146. 

  146.               location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12
    147. 

  147.               name default.srcpart
    148. 

  148.               partition_columns ds/hr
    149. 

  149.               serialization.ddl struct srcpart { string key, string value}
    150. 

  150.               serialization.format 1
    151. 

  151.               serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    152. 

  152.               transient_lastDdlTime 1297328959
    153. 

  153.             serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    154. 

  154.           
    155. 

  155.               input format: org.apache.hadoop.mapred.TextInputFormat
    156. 

  156.               output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    157. 

  157.               properties:
    158. 

  158.                 bucket_count -1
    159. 

  159.                 columns key,value
    160. 

  160.                 columns.types string:string
    161. 

  161.                 file.inputformat org.apache.hadoop.mapred.TextInputFormat
    162. 

  162.                 file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    163. 

  163.                 location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart
    164. 

  164.                 name default.srcpart
    165. 

  165.                 partition_columns ds/hr
    166. 

  166.                 serialization.ddl struct srcpart { string key, string value}
    167. 

  167.                 serialization.format 1
    168. 

  168.                 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    169. 

  169.                 transient_lastDdlTime 1297328959
    170. 

  170.               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    171. 

  171.               name: default.srcpart
    172. 

  172.             name: default.srcpart
    173. 

  173.       Reduce Operator Tree:
    174. 

  174.         Group By Operator
    175. 

  175.           aggregations:
    176. 

  176.                 expr: count(DISTINCT KEY._col1:0._col0)
    177. 

  177.                 expr: sum(VALUE._col1)
    178. 

  178.                 expr: sum(DISTINCT KEY._col1:1._col0)
    179. 

  179.                 expr: count(DISTINCT KEY._col1:2._col0)
    180. 

  180.           bucketGroup: false
    181. 

  181.           keys:
    182. 

  182.                 expr: KEY._col0
    183. 

  183.                 type: string
    184. 

  184.           mode: mergepartial
    185. 

  185.           outputColumnNames: _col0, _col1, _col2, _col3, _col4
    186. 

  186.           Select Operator
    187. 

  187.             expressions:
    188. 

  188.                   expr: _col0
    189. 

  189.                   type: string
    190. 

  190.                   expr: _col1
    191. 

  191.                   type: bigint
    192. 

  192.                   expr: concat(_col0, _col2)
    193. 

  193.                   type: string
    194. 

  194.                   expr: _col3
    195. 

  195.                   type: double
    196. 

  196.                   expr: _col4
    197. 

  197.                   type: bigint
    198. 

  198.             outputColumnNames: _col0, _col1, _col2, _col3, _col4
    199. 

  199.             Select Operator
    200. 

  200.               expressions:
    201. 

  201.                     expr: _col0
    202. 

  202.                     type: string
    203. 

  203.                     expr: UDFToInteger(_col1)
    204. 

  204.                     type: int
    205. 

  205.                     expr: _col2
    206. 

  206.                     type: string
    207. 

  207.                     expr: UDFToInteger(_col3)
    208. 

  208.                     type: int
    209. 

  209.                     expr: UDFToInteger(_col4)
    210. 

  210.                     type: int
    211. 

  211.               outputColumnNames: _col0, _col1, _col2, _col3, _col4
    212. 

  212.               File Output Operator
    213. 

  213.                 compressed: false
    214. 

  214.                 GlobalTableId: 1
    215. 

  215.                 directory: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-54-56_939_392063944204159853/-ext-10000
    216. 

  216.                 NumFilesPerFileSink: 1
    217. 

  217.                 Stats Publishing Key Prefix: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-54-56_939_392063944204159853/-ext-10000/
    218. 

  218.                 table:
    219. 

  219.                     input format: org.apache.hadoop.mapred.TextInputFormat
    220. 

  220.                     output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    221. 

  221.                     properties:
    222. 

  222.                       bucket_count -1
    223. 

  223.                       columns key,c1,c2,c3,c4
    224. 

  224.                       columns.types string:int:string:int:int
    225. 

  225.                       file.inputformat org.apache.hadoop.mapred.TextInputFormat
    226. 

  226.                       file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    227. 

  227.                       location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/dest1
    228. 

  228.                       name default.dest1
    229. 

  229.                       serialization.ddl struct dest1 { string key, i32 c1, string c2, i32 c3, i32 c4}
    230. 

  230.                       serialization.format 1
    231. 

  231.                       serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    232. 

  232.                       transient_lastDdlTime 1297331696
    233. 

  233.                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    234. 

  234.                     name: default.dest1
    235. 

  235.                 TotalFiles: 1
    236. 

  236.                 GatherStats: true
    237. 

  237.                 MultiFileSpray: false
    238. 

  238. 

  239.   Stage: Stage-0
    240. 

  240.     Move Operator
    241. 

  241.       tables:
    242. 

  242.           replace: true
    243. 

  243.           source: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-54-56_939_392063944204159853/-ext-10000
    244. 

  244.           table:
    245. 

  245.               input format: org.apache.hadoop.mapred.TextInputFormat
    246. 

  246.               output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    247. 

  247.               properties:
    248. 

  248.                 bucket_count -1
    249. 

  249.                 columns key,c1,c2,c3,c4
    250. 

  250.                 columns.types string:int:string:int:int
    251. 

  251.                 file.inputformat org.apache.hadoop.mapred.TextInputFormat
    252. 

  252.                 file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    253. 

  253.                 location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/dest1
    254. 

  254.                 name default.dest1
    255. 

  255.                 serialization.ddl struct dest1 { string key, i32 c1, string c2, i32 c3, i32 c4}
    256. 

  256.                 serialization.format 1
    257. 

  257.                 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    258. 

  258.                 transient_lastDdlTime 1297331696
    259. 

  259.               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    260. 

  260.               name: default.dest1
    261. 

  261.           tmp directory: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-54-56_939_392063944204159853/-ext-10001
    262. 

  262. 

  263.   Stage: Stage-2
    264. 

  264.     Stats-Aggr Operator
    265. 

  265.       Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-54-56_939_392063944204159853/-ext-10000/
    266. 

  266. 

  267. 

  268. PREHOOK: query: FROM srcpart src
    269. 

  269. INSERT OVERWRITE TABLE dest1 
    270. 

  270. SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(DISTINCT src.value) 
    271. 

  271. WHERE src.ds = '2008-04-08'
    272. 

  272. GROUP BY substr(src.key,1,1)
    273. 

  273. PREHOOK: type: QUERY
    274. 

  274. PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    275. 

  275. PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    276. 

  276. PREHOOK: Output: default@dest1
    277. 

  277. POSTHOOK: query: FROM srcpart src
    278. 

  278. INSERT OVERWRITE TABLE dest1 
    279. 

  279. SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(DISTINCT src.value) 
    280. 

  280. WHERE src.ds = '2008-04-08'
    281. 

  281. GROUP BY substr(src.key,1,1)
    282. 

  282. POSTHOOK: type: QUERY
    283. 

  283. POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    284. 

  284. POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    285. 

  285. POSTHOOK: Output: default@dest1
    286. 

  286. POSTHOOK: Lineage: dest1.c1 EXPRESSION [(srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    287. 

  287. POSTHOOK: Lineage: dest1.c2 EXPRESSION [(srcpart)src.FieldSchema(name:key, type:string, comment:default), (srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    288. 

  288. POSTHOOK: Lineage: dest1.c3 EXPRESSION [(srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    289. 

  289. POSTHOOK: Lineage: dest1.c4 EXPRESSION [(srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    290. 

  290. POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src.FieldSchema(name:key, type:string, comment:default), ]
    291. 

  291. PREHOOK: query: SELECT dest1.* FROM dest1
    292. 

  292. PREHOOK: type: QUERY
    293. 

  293. PREHOOK: Input: default@dest1
    294. 

  294. PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-55-03_112_7822131255872092269/-mr-10000
    295. 

  295. POSTHOOK: query: SELECT dest1.* FROM dest1
    296. 

  296. POSTHOOK: type: QUERY
    297. 

  297. POSTHOOK: Input: default@dest1
    298. 

  298. POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-55-03_112_7822131255872092269/-mr-10000
    299. 

  299. POSTHOOK: Lineage: dest1.c1 EXPRESSION [(srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    300. 

  300. POSTHOOK: Lineage: dest1.c2 EXPRESSION [(srcpart)src.FieldSchema(name:key, type:string, comment:default), (srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    301. 

  301. POSTHOOK: Lineage: dest1.c3 EXPRESSION [(srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    302. 

  302. POSTHOOK: Lineage: dest1.c4 EXPRESSION [(srcpart)src.FieldSchema(name:value, type:string, comment:default), ]
    303. 

  303. POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src.FieldSchema(name:key, type:string, comment:default), ]
    304. 

  304. 0	1	00.0	0	1
    305. 

  305. 1	71	132828.0	10044	71
    306. 

  306. 2	69	251142.0	15780	69
    307. 

  307. 3	62	364008.0	20119	62
    308. 

  308. 4	74	4105526.0	30965	74
    309. 

  309. 5	6	5794.0	278	6
    310. 

  310. 6	5	6796.0	331	5
    311. 

  311. 7	6	71470.0	447	6
    312. 

  312. 8	8	81524.0	595	8
    313. 

  313. 9	7	92094.0	577	7
    314. 

  314.