PageRenderTime 52ms CodeModel.GetById 12ms RepoModel.GetById 0ms app.codeStats 1ms

/tags/release-0.0.0-rc0/hive/external/ql/src/test/results/clientpositive/skewjoin.q.out

#
text | 1770 lines | 1709 code | 61 blank | 0 comment | 0 complexity | a18a8fab6eb72bd91c7c54813fd64a57 MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause, JSON, CPL-1.0

Large files files are truncated, but you can click here to view the full file

  1. PREHOOK: query: CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE
  2. PREHOOK: type: CREATETABLE
  3. POSTHOOK: query: CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE
  4. POSTHOOK: type: CREATETABLE
  5. POSTHOOK: Output: default@T1
  6. PREHOOK: query: CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE
  7. PREHOOK: type: CREATETABLE
  8. POSTHOOK: query: CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE
  9. POSTHOOK: type: CREATETABLE
  10. POSTHOOK: Output: default@T2
  11. PREHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE
  12. PREHOOK: type: CREATETABLE
  13. POSTHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE
  14. POSTHOOK: type: CREATETABLE
  15. POSTHOOK: Output: default@T3
  16. PREHOOK: query: CREATE TABLE T4(key STRING, val STRING) STORED AS TEXTFILE
  17. PREHOOK: type: CREATETABLE
  18. POSTHOOK: query: CREATE TABLE T4(key STRING, val STRING) STORED AS TEXTFILE
  19. POSTHOOK: type: CREATETABLE
  20. POSTHOOK: Output: default@T4
  21. PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE
  22. PREHOOK: type: CREATETABLE
  23. POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE
  24. POSTHOOK: type: CREATETABLE
  25. POSTHOOK: Output: default@dest_j1
  26. PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T1.txt' INTO TABLE T1
  27. PREHOOK: type: LOAD
  28. POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T1.txt' INTO TABLE T1
  29. POSTHOOK: type: LOAD
  30. POSTHOOK: Output: default@t1
  31. PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T2.txt' INTO TABLE T2
  32. PREHOOK: type: LOAD
  33. POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T2.txt' INTO TABLE T2
  34. POSTHOOK: type: LOAD
  35. POSTHOOK: Output: default@t2
  36. PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T3.txt' INTO TABLE T3
  37. PREHOOK: type: LOAD
  38. POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T3.txt' INTO TABLE T3
  39. POSTHOOK: type: LOAD
  40. POSTHOOK: Output: default@t3
  41. PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T1.txt' INTO TABLE T4
  42. PREHOOK: type: LOAD
  43. POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T1.txt' INTO TABLE T4
  44. POSTHOOK: type: LOAD
  45. POSTHOOK: Output: default@t4
  46. PREHOOK: query: EXPLAIN
  47. FROM src src1 JOIN src src2 ON (src1.key = src2.key)
  48. INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
  49. PREHOOK: type: QUERY
  50. POSTHOOK: query: EXPLAIN
  51. FROM src src1 JOIN src src2 ON (src1.key = src2.key)
  52. INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
  53. POSTHOOK: type: QUERY
  54. ABSTRACT SYNTAX TREE:
  55. (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) src1) (TOK_TABREF (TOK_TABNAME src) src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value)))))
  56. STAGE DEPENDENCIES:
  57. Stage-1 is a root stage
  58. Stage-5 depends on stages: Stage-1 , consists of Stage-6
  59. Stage-6
  60. Stage-4 depends on stages: Stage-6
  61. Stage-0 depends on stages: Stage-1, Stage-4
  62. Stage-2 depends on stages: Stage-0
  63. STAGE PLANS:
  64. Stage: Stage-1
  65. Map Reduce
  66. Alias -> Map Operator Tree:
  67. src1
  68. TableScan
  69. alias: src1
  70. Reduce Output Operator
  71. key expressions:
  72. expr: key
  73. type: string
  74. sort order: +
  75. Map-reduce partition columns:
  76. expr: key
  77. type: string
  78. tag: 0
  79. value expressions:
  80. expr: key
  81. type: string
  82. src2
  83. TableScan
  84. alias: src2
  85. Reduce Output Operator
  86. key expressions:
  87. expr: key
  88. type: string
  89. sort order: +
  90. Map-reduce partition columns:
  91. expr: key
  92. type: string
  93. tag: 1
  94. value expressions:
  95. expr: value
  96. type: string
  97. Reduce Operator Tree:
  98. Join Operator
  99. condition map:
  100. Inner Join 0 to 1
  101. condition expressions:
  102. 0 {VALUE._col0}
  103. 1 {VALUE._col1}
  104. handleSkewJoin: true
  105. outputColumnNames: _col0, _col5
  106. Select Operator
  107. expressions:
  108. expr: _col0
  109. type: string
  110. expr: _col5
  111. type: string
  112. outputColumnNames: _col0, _col1
  113. Select Operator
  114. expressions:
  115. expr: UDFToInteger(_col0)
  116. type: int
  117. expr: _col1
  118. type: string
  119. outputColumnNames: _col0, _col1
  120. File Output Operator
  121. compressed: false
  122. GlobalTableId: 1
  123. table:
  124. input format: org.apache.hadoop.mapred.TextInputFormat
  125. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  126. serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
  127. name: default.dest_j1
  128. Stage: Stage-5
  129. Conditional Operator
  130. Stage: Stage-6
  131. Map Reduce Local Work
  132. Alias -> Map Local Tables:
  133. 1
  134. Fetch Operator
  135. limit: -1
  136. Alias -> Map Local Operator Tree:
  137. 1
  138. HashTable Sink Operator
  139. condition expressions:
  140. 0 {0_VALUE_0}
  141. 1 {1_VALUE_0}
  142. handleSkewJoin: false
  143. keys:
  144. 0 [Column[joinkey0]]
  145. 1 [Column[joinkey0]]
  146. Position of Big Table: 0
  147. Stage: Stage-4
  148. Map Reduce
  149. Alias -> Map Operator Tree:
  150. 0
  151. Map Join Operator
  152. condition map:
  153. Inner Join 0 to 1
  154. condition expressions:
  155. 0 {0_VALUE_0}
  156. 1 {1_VALUE_0}
  157. handleSkewJoin: false
  158. keys:
  159. 0 [Column[joinkey0]]
  160. 1 [Column[joinkey0]]
  161. outputColumnNames: _col0, _col5
  162. Position of Big Table: 0
  163. Select Operator
  164. expressions:
  165. expr: _col0
  166. type: string
  167. expr: _col5
  168. type: string
  169. outputColumnNames: _col0, _col1
  170. Select Operator
  171. expressions:
  172. expr: UDFToInteger(_col0)
  173. type: int
  174. expr: _col1
  175. type: string
  176. outputColumnNames: _col0, _col1
  177. File Output Operator
  178. compressed: false
  179. GlobalTableId: 1
  180. table:
  181. input format: org.apache.hadoop.mapred.TextInputFormat
  182. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  183. serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
  184. name: default.dest_j1
  185. Local Work:
  186. Map Reduce Local Work
  187. Stage: Stage-0
  188. Move Operator
  189. tables:
  190. replace: true
  191. table:
  192. input format: org.apache.hadoop.mapred.TextInputFormat
  193. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  194. serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
  195. name: default.dest_j1
  196. Stage: Stage-2
  197. Stats-Aggr Operator
  198. PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
  199. INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
  200. PREHOOK: type: QUERY
  201. PREHOOK: Input: default@src
  202. PREHOOK: Output: default@dest_j1
  203. POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
  204. INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
  205. POSTHOOK: type: QUERY
  206. POSTHOOK: Input: default@src
  207. POSTHOOK: Output: default@dest_j1
  208. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  209. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  210. PREHOOK: query: SELECT sum(hash(key)), sum(hash(value)) FROM dest_j1
  211. PREHOOK: type: QUERY
  212. PREHOOK: Input: default@dest_j1
  213. PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-24_642_2468932754198583760/-mr-10000
  214. POSTHOOK: query: SELECT sum(hash(key)), sum(hash(value)) FROM dest_j1
  215. POSTHOOK: type: QUERY
  216. POSTHOOK: Input: default@dest_j1
  217. POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-24_642_2468932754198583760/-mr-10000
  218. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  219. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  220. 278697 101852390308
  221. PREHOOK: query: EXPLAIN
  222. SELECT /*+ STREAMTABLE(a) */ *
  223. FROM T1 a JOIN T2 b ON a.key = b.key
  224. JOIN T3 c ON b.key = c.key
  225. JOIN T4 d ON c.key = d.key
  226. PREHOOK: type: QUERY
  227. POSTHOOK: query: EXPLAIN
  228. SELECT /*+ STREAMTABLE(a) */ *
  229. FROM T1 a JOIN T2 b ON a.key = b.key
  230. JOIN T3 c ON b.key = c.key
  231. JOIN T4 d ON c.key = d.key
  232. POSTHOOK: type: QUERY
  233. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  234. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  235. ABSTRACT SYNTAX TREE:
  236. (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME T2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME T3) c) (= (. (TOK_TABLE_OR_COL b) key) (. (TOK_TABLE_OR_COL c) key))) (TOK_TABREF (TOK_TABNAME T4) d) (= (. (TOK_TABLE_OR_COL c) key) (. (TOK_TABLE_OR_COL d) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_STREAMTABLE (TOK_HINTARGLIST a))) (TOK_SELEXPR TOK_ALLCOLREF))))
  237. STAGE DEPENDENCIES:
  238. Stage-1 is a root stage
  239. Stage-0 is a root stage
  240. STAGE PLANS:
  241. Stage: Stage-1
  242. Map Reduce
  243. Alias -> Map Operator Tree:
  244. a
  245. TableScan
  246. alias: a
  247. Reduce Output Operator
  248. key expressions:
  249. expr: key
  250. type: string
  251. sort order: +
  252. Map-reduce partition columns:
  253. expr: key
  254. type: string
  255. tag: 3
  256. value expressions:
  257. expr: key
  258. type: string
  259. expr: val
  260. type: string
  261. b
  262. TableScan
  263. alias: b
  264. Reduce Output Operator
  265. key expressions:
  266. expr: key
  267. type: string
  268. sort order: +
  269. Map-reduce partition columns:
  270. expr: key
  271. type: string
  272. tag: 1
  273. value expressions:
  274. expr: key
  275. type: string
  276. expr: val
  277. type: string
  278. c
  279. TableScan
  280. alias: c
  281. Reduce Output Operator
  282. key expressions:
  283. expr: key
  284. type: string
  285. sort order: +
  286. Map-reduce partition columns:
  287. expr: key
  288. type: string
  289. tag: 2
  290. value expressions:
  291. expr: key
  292. type: string
  293. expr: val
  294. type: string
  295. d
  296. TableScan
  297. alias: d
  298. Reduce Output Operator
  299. key expressions:
  300. expr: key
  301. type: string
  302. sort order: +
  303. Map-reduce partition columns:
  304. expr: key
  305. type: string
  306. tag: 0
  307. value expressions:
  308. expr: key
  309. type: string
  310. expr: val
  311. type: string
  312. Reduce Operator Tree:
  313. Join Operator
  314. condition map:
  315. Inner Join 0 to 1
  316. Inner Join 1 to 2
  317. Inner Join 2 to 3
  318. condition expressions:
  319. 0 {VALUE._col0} {VALUE._col1}
  320. 1 {VALUE._col0} {VALUE._col1}
  321. 2 {VALUE._col0} {VALUE._col1}
  322. 3 {VALUE._col0} {VALUE._col1}
  323. handleSkewJoin: false
  324. outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13
  325. Select Operator
  326. expressions:
  327. expr: _col0
  328. type: string
  329. expr: _col1
  330. type: string
  331. expr: _col4
  332. type: string
  333. expr: _col5
  334. type: string
  335. expr: _col8
  336. type: string
  337. expr: _col9
  338. type: string
  339. expr: _col12
  340. type: string
  341. expr: _col13
  342. type: string
  343. outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
  344. File Output Operator
  345. compressed: false
  346. GlobalTableId: 0
  347. table:
  348. input format: org.apache.hadoop.mapred.TextInputFormat
  349. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  350. Stage: Stage-0
  351. Fetch Operator
  352. limit: -1
  353. PREHOOK: query: SELECT /*+ STREAMTABLE(a) */ *
  354. FROM T1 a JOIN T2 b ON a.key = b.key
  355. JOIN T3 c ON b.key = c.key
  356. JOIN T4 d ON c.key = d.key
  357. PREHOOK: type: QUERY
  358. PREHOOK: Input: default@t1
  359. PREHOOK: Input: default@t2
  360. PREHOOK: Input: default@t3
  361. PREHOOK: Input: default@t4
  362. PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-38_293_3748436233439546/-mr-10000
  363. POSTHOOK: query: SELECT /*+ STREAMTABLE(a) */ *
  364. FROM T1 a JOIN T2 b ON a.key = b.key
  365. JOIN T3 c ON b.key = c.key
  366. JOIN T4 d ON c.key = d.key
  367. POSTHOOK: type: QUERY
  368. POSTHOOK: Input: default@t1
  369. POSTHOOK: Input: default@t2
  370. POSTHOOK: Input: default@t3
  371. POSTHOOK: Input: default@t4
  372. POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-38_293_3748436233439546/-mr-10000
  373. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  374. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  375. 2 12 2 22 2 12 2 12
  376. PREHOOK: query: EXPLAIN
  377. SELECT /*+ STREAMTABLE(a,c) */ *
  378. FROM T1 a JOIN T2 b ON a.key = b.key
  379. JOIN T3 c ON b.key = c.key
  380. JOIN T4 d ON c.key = d.key
  381. PREHOOK: type: QUERY
  382. POSTHOOK: query: EXPLAIN
  383. SELECT /*+ STREAMTABLE(a,c) */ *
  384. FROM T1 a JOIN T2 b ON a.key = b.key
  385. JOIN T3 c ON b.key = c.key
  386. JOIN T4 d ON c.key = d.key
  387. POSTHOOK: type: QUERY
  388. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  389. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  390. ABSTRACT SYNTAX TREE:
  391. (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME T2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME T3) c) (= (. (TOK_TABLE_OR_COL b) key) (. (TOK_TABLE_OR_COL c) key))) (TOK_TABREF (TOK_TABNAME T4) d) (= (. (TOK_TABLE_OR_COL c) key) (. (TOK_TABLE_OR_COL d) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_STREAMTABLE (TOK_HINTARGLIST a c))) (TOK_SELEXPR TOK_ALLCOLREF))))
  392. STAGE DEPENDENCIES:
  393. Stage-1 is a root stage
  394. Stage-0 is a root stage
  395. STAGE PLANS:
  396. Stage: Stage-1
  397. Map Reduce
  398. Alias -> Map Operator Tree:
  399. a
  400. TableScan
  401. alias: a
  402. Reduce Output Operator
  403. key expressions:
  404. expr: key
  405. type: string
  406. sort order: +
  407. Map-reduce partition columns:
  408. expr: key
  409. type: string
  410. tag: 3
  411. value expressions:
  412. expr: key
  413. type: string
  414. expr: val
  415. type: string
  416. b
  417. TableScan
  418. alias: b
  419. Reduce Output Operator
  420. key expressions:
  421. expr: key
  422. type: string
  423. sort order: +
  424. Map-reduce partition columns:
  425. expr: key
  426. type: string
  427. tag: 1
  428. value expressions:
  429. expr: key
  430. type: string
  431. expr: val
  432. type: string
  433. c
  434. TableScan
  435. alias: c
  436. Reduce Output Operator
  437. key expressions:
  438. expr: key
  439. type: string
  440. sort order: +
  441. Map-reduce partition columns:
  442. expr: key
  443. type: string
  444. tag: 2
  445. value expressions:
  446. expr: key
  447. type: string
  448. expr: val
  449. type: string
  450. d
  451. TableScan
  452. alias: d
  453. Reduce Output Operator
  454. key expressions:
  455. expr: key
  456. type: string
  457. sort order: +
  458. Map-reduce partition columns:
  459. expr: key
  460. type: string
  461. tag: 0
  462. value expressions:
  463. expr: key
  464. type: string
  465. expr: val
  466. type: string
  467. Reduce Operator Tree:
  468. Join Operator
  469. condition map:
  470. Inner Join 0 to 1
  471. Inner Join 1 to 2
  472. Inner Join 2 to 3
  473. condition expressions:
  474. 0 {VALUE._col0} {VALUE._col1}
  475. 1 {VALUE._col0} {VALUE._col1}
  476. 2 {VALUE._col0} {VALUE._col1}
  477. 3 {VALUE._col0} {VALUE._col1}
  478. handleSkewJoin: false
  479. outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9, _col12, _col13
  480. Select Operator
  481. expressions:
  482. expr: _col0
  483. type: string
  484. expr: _col1
  485. type: string
  486. expr: _col4
  487. type: string
  488. expr: _col5
  489. type: string
  490. expr: _col8
  491. type: string
  492. expr: _col9
  493. type: string
  494. expr: _col12
  495. type: string
  496. expr: _col13
  497. type: string
  498. outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
  499. File Output Operator
  500. compressed: false
  501. GlobalTableId: 0
  502. table:
  503. input format: org.apache.hadoop.mapred.TextInputFormat
  504. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  505. Stage: Stage-0
  506. Fetch Operator
  507. limit: -1
  508. PREHOOK: query: SELECT /*+ STREAMTABLE(a,c) */ *
  509. FROM T1 a JOIN T2 b ON a.key = b.key
  510. JOIN T3 c ON b.key = c.key
  511. JOIN T4 d ON c.key = d.key
  512. PREHOOK: type: QUERY
  513. PREHOOK: Input: default@t1
  514. PREHOOK: Input: default@t2
  515. PREHOOK: Input: default@t3
  516. PREHOOK: Input: default@t4
  517. PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-42_943_1971937621048870984/-mr-10000
  518. POSTHOOK: query: SELECT /*+ STREAMTABLE(a,c) */ *
  519. FROM T1 a JOIN T2 b ON a.key = b.key
  520. JOIN T3 c ON b.key = c.key
  521. JOIN T4 d ON c.key = d.key
  522. POSTHOOK: type: QUERY
  523. POSTHOOK: Input: default@t1
  524. POSTHOOK: Input: default@t2
  525. POSTHOOK: Input: default@t3
  526. POSTHOOK: Input: default@t4
  527. POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-42_943_1971937621048870984/-mr-10000
  528. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  529. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  530. 2 12 2 22 2 12 2 12
  531. PREHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key))
  532. PREHOOK: type: QUERY
  533. POSTHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key))
  534. POSTHOOK: type: QUERY
  535. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  536. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  537. ABSTRACT SYNTAX TREE:
  538. (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME src) c) (= (+ (. (TOK_TABLE_OR_COL c) key) 1) (. (TOK_TABLE_OR_COL a) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_STREAMTABLE (TOK_HINTARGLIST a))) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) key)))) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) val)))) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL c) key)))))))
  539. STAGE DEPENDENCIES:
  540. Stage-1 is a root stage
  541. Stage-2 depends on stages: Stage-1
  542. Stage-0 is a root stage
  543. STAGE PLANS:
  544. Stage: Stage-1
  545. Map Reduce
  546. Alias -> Map Operator Tree:
  547. a
  548. TableScan
  549. alias: a
  550. Reduce Output Operator
  551. key expressions:
  552. expr: UDFToDouble(key)
  553. type: double
  554. sort order: +
  555. Map-reduce partition columns:
  556. expr: UDFToDouble(key)
  557. type: double
  558. tag: 1
  559. value expressions:
  560. expr: key
  561. type: string
  562. expr: val
  563. type: string
  564. c
  565. TableScan
  566. alias: c
  567. Reduce Output Operator
  568. key expressions:
  569. expr: (key + 1)
  570. type: double
  571. sort order: +
  572. Map-reduce partition columns:
  573. expr: (key + 1)
  574. type: double
  575. tag: 0
  576. value expressions:
  577. expr: key
  578. type: string
  579. Reduce Operator Tree:
  580. Join Operator
  581. condition map:
  582. Inner Join 0 to 1
  583. condition expressions:
  584. 0 {VALUE._col0} {VALUE._col1}
  585. 1 {VALUE._col0}
  586. handleSkewJoin: false
  587. outputColumnNames: _col0, _col1, _col4
  588. Select Operator
  589. expressions:
  590. expr: _col0
  591. type: string
  592. expr: _col1
  593. type: string
  594. expr: _col4
  595. type: string
  596. outputColumnNames: _col0, _col1, _col4
  597. Group By Operator
  598. aggregations:
  599. expr: sum(hash(_col0))
  600. expr: sum(hash(_col1))
  601. expr: sum(hash(_col4))
  602. bucketGroup: false
  603. mode: hash
  604. outputColumnNames: _col0, _col1, _col2
  605. File Output Operator
  606. compressed: false
  607. GlobalTableId: 0
  608. table:
  609. input format: org.apache.hadoop.mapred.SequenceFileInputFormat
  610. output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
  611. Stage: Stage-2
  612. Map Reduce
  613. Alias -> Map Operator Tree:
  614. file:/tmp/sdong/hive_2011-02-10_17-09-47_420_5981143518066828337/-mr-10002
  615. Reduce Output Operator
  616. sort order:
  617. tag: -1
  618. value expressions:
  619. expr: _col0
  620. type: bigint
  621. expr: _col1
  622. type: bigint
  623. expr: _col2
  624. type: bigint
  625. Reduce Operator Tree:
  626. Group By Operator
  627. aggregations:
  628. expr: sum(VALUE._col0)
  629. expr: sum(VALUE._col1)
  630. expr: sum(VALUE._col2)
  631. bucketGroup: false
  632. mode: mergepartial
  633. outputColumnNames: _col0, _col1, _col2
  634. Select Operator
  635. expressions:
  636. expr: _col0
  637. type: bigint
  638. expr: _col1
  639. type: bigint
  640. expr: _col2
  641. type: bigint
  642. outputColumnNames: _col0, _col1, _col2
  643. File Output Operator
  644. compressed: false
  645. GlobalTableId: 0
  646. table:
  647. input format: org.apache.hadoop.mapred.TextInputFormat
  648. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  649. Stage: Stage-0
  650. Fetch Operator
  651. limit: -1
  652. PREHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key))
  653. PREHOOK: type: QUERY
  654. PREHOOK: Input: default@src
  655. PREHOOK: Input: default@t1
  656. PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-47_566_240192624436212534/-mr-10000
  657. POSTHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key))
  658. POSTHOOK: type: QUERY
  659. POSTHOOK: Input: default@src
  660. POSTHOOK: Input: default@t1
  661. POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-47_566_240192624436212534/-mr-10000
  662. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  663. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  664. 198 6274 194
  665. PREHOOK: query: EXPLAIN FROM
  666. (SELECT src.* FROM src) x
  667. JOIN
  668. (SELECT src.* FROM src) Y
  669. ON (x.key = Y.key)
  670. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  671. PREHOOK: type: QUERY
  672. POSTHOOK: query: EXPLAIN FROM
  673. (SELECT src.* FROM src) x
  674. JOIN
  675. (SELECT src.* FROM src) Y
  676. ON (x.key = Y.key)
  677. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  678. POSTHOOK: type: QUERY
  679. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  680. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  681. ABSTRACT SYNTAX TREE:
  682. (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL Y) key)))) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL Y) value)))))))
  683. STAGE DEPENDENCIES:
  684. Stage-1 is a root stage
  685. Stage-5 depends on stages: Stage-1 , consists of Stage-6
  686. Stage-6
  687. Stage-4 depends on stages: Stage-6
  688. Stage-2 depends on stages: Stage-1, Stage-4
  689. Stage-0 is a root stage
  690. STAGE PLANS:
  691. Stage: Stage-1
  692. Map Reduce
  693. Alias -> Map Operator Tree:
  694. x:src
  695. TableScan
  696. alias: src
  697. Select Operator
  698. expressions:
  699. expr: key
  700. type: string
  701. outputColumnNames: _col0
  702. Reduce Output Operator
  703. key expressions:
  704. expr: _col0
  705. type: string
  706. sort order: +
  707. Map-reduce partition columns:
  708. expr: _col0
  709. type: string
  710. tag: 0
  711. y:src
  712. TableScan
  713. alias: src
  714. Select Operator
  715. expressions:
  716. expr: key
  717. type: string
  718. expr: value
  719. type: string
  720. outputColumnNames: _col0, _col1
  721. Reduce Output Operator
  722. key expressions:
  723. expr: _col0
  724. type: string
  725. sort order: +
  726. Map-reduce partition columns:
  727. expr: _col0
  728. type: string
  729. tag: 1
  730. value expressions:
  731. expr: _col0
  732. type: string
  733. expr: _col1
  734. type: string
  735. Reduce Operator Tree:
  736. Join Operator
  737. condition map:
  738. Inner Join 0 to 1
  739. condition expressions:
  740. 0
  741. 1 {VALUE._col0} {VALUE._col1}
  742. handleSkewJoin: true
  743. outputColumnNames: _col2, _col3
  744. Select Operator
  745. expressions:
  746. expr: _col2
  747. type: string
  748. expr: _col3
  749. type: string
  750. outputColumnNames: _col2, _col3
  751. Group By Operator
  752. aggregations:
  753. expr: sum(hash(_col2))
  754. expr: sum(hash(_col3))
  755. bucketGroup: false
  756. mode: hash
  757. outputColumnNames: _col0, _col1
  758. File Output Operator
  759. compressed: false
  760. GlobalTableId: 0
  761. table:
  762. input format: org.apache.hadoop.mapred.SequenceFileInputFormat
  763. output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
  764. Stage: Stage-5
  765. Conditional Operator
  766. Stage: Stage-6
  767. Map Reduce Local Work
  768. Alias -> Map Local Tables:
  769. 1
  770. Fetch Operator
  771. limit: -1
  772. Alias -> Map Local Operator Tree:
  773. 1
  774. HashTable Sink Operator
  775. condition expressions:
  776. 0
  777. 1 {1_VALUE_0} {1_VALUE_1}
  778. handleSkewJoin: false
  779. keys:
  780. 0 [Column[joinkey0]]
  781. 1 [Column[joinkey0]]
  782. Position of Big Table: 0
  783. Stage: Stage-4
  784. Map Reduce
  785. Alias -> Map Operator Tree:
  786. 0
  787. Map Join Operator
  788. condition map:
  789. Inner Join 0 to 1
  790. condition expressions:
  791. 0
  792. 1 {1_VALUE_0} {1_VALUE_1}
  793. handleSkewJoin: false
  794. keys:
  795. 0 [Column[joinkey0]]
  796. 1 [Column[joinkey0]]
  797. outputColumnNames: _col2, _col3
  798. Position of Big Table: 0
  799. Select Operator
  800. expressions:
  801. expr: _col2
  802. type: string
  803. expr: _col3
  804. type: string
  805. outputColumnNames: _col2, _col3
  806. Group By Operator
  807. aggregations:
  808. expr: sum(hash(_col2))
  809. expr: sum(hash(_col3))
  810. bucketGroup: false
  811. mode: hash
  812. outputColumnNames: _col0, _col1
  813. File Output Operator
  814. compressed: false
  815. GlobalTableId: 0
  816. table:
  817. input format: org.apache.hadoop.mapred.SequenceFileInputFormat
  818. output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
  819. Local Work:
  820. Map Reduce Local Work
  821. Stage: Stage-2
  822. Map Reduce
  823. Alias -> Map Operator Tree:
  824. file:/tmp/sdong/hive_2011-02-10_17-09-54_587_1570310782941051679/-mr-10002
  825. Reduce Output Operator
  826. sort order:
  827. tag: -1
  828. value expressions:
  829. expr: _col0
  830. type: bigint
  831. expr: _col1
  832. type: bigint
  833. Reduce Operator Tree:
  834. Group By Operator
  835. aggregations:
  836. expr: sum(VALUE._col0)
  837. expr: sum(VALUE._col1)
  838. bucketGroup: false
  839. mode: mergepartial
  840. outputColumnNames: _col0, _col1
  841. Select Operator
  842. expressions:
  843. expr: _col0
  844. type: bigint
  845. expr: _col1
  846. type: bigint
  847. outputColumnNames: _col0, _col1
  848. File Output Operator
  849. compressed: false
  850. GlobalTableId: 0
  851. table:
  852. input format: org.apache.hadoop.mapred.TextInputFormat
  853. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  854. Stage: Stage-0
  855. Fetch Operator
  856. limit: -1
  857. PREHOOK: query: FROM
  858. (SELECT src.* FROM src) x
  859. JOIN
  860. (SELECT src.* FROM src) Y
  861. ON (x.key = Y.key)
  862. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  863. PREHOOK: type: QUERY
  864. PREHOOK: Input: default@src
  865. PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-54_779_5556867573127106668/-mr-10000
  866. POSTHOOK: query: FROM
  867. (SELECT src.* FROM src) x
  868. JOIN
  869. (SELECT src.* FROM src) Y
  870. ON (x.key = Y.key)
  871. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  872. POSTHOOK: type: QUERY
  873. POSTHOOK: Input: default@src
  874. POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-09-54_779_5556867573127106668/-mr-10000
  875. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  876. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  877. 44481300 101852390308
  878. PREHOOK: query: EXPLAIN FROM
  879. (SELECT src.* FROM src) x
  880. JOIN
  881. (SELECT src.* FROM src) Y
  882. ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1)
  883. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  884. PREHOOK: type: QUERY
  885. POSTHOOK: query: EXPLAIN FROM
  886. (SELECT src.* FROM src) x
  887. JOIN
  888. (SELECT src.* FROM src) Y
  889. ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1)
  890. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  891. POSTHOOK: type: QUERY
  892. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  893. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  894. ABSTRACT SYNTAX TREE:
  895. (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))))) Y) (and (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)) (= (TOK_FUNCTION substring (. (TOK_TABLE_OR_COL x) value) 5) (+ (TOK_FUNCTION substring (. (TOK_TABLE_OR_COL y) value) 5) 1))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL Y) key)))) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL Y) value)))))))
  896. STAGE DEPENDENCIES:
  897. Stage-1 is a root stage
  898. Stage-5 depends on stages: Stage-1 , consists of Stage-6
  899. Stage-6
  900. Stage-4 depends on stages: Stage-6
  901. Stage-2 depends on stages: Stage-1, Stage-4
  902. Stage-0 is a root stage
  903. STAGE PLANS:
  904. Stage: Stage-1
  905. Map Reduce
  906. Alias -> Map Operator Tree:
  907. x:src
  908. TableScan
  909. alias: src
  910. Select Operator
  911. expressions:
  912. expr: key
  913. type: string
  914. expr: value
  915. type: string
  916. outputColumnNames: _col0, _col1
  917. Reduce Output Operator
  918. key expressions:
  919. expr: _col0
  920. type: string
  921. expr: UDFToDouble(substring(_col1, 5))
  922. type: double
  923. sort order: ++
  924. Map-reduce partition columns:
  925. expr: _col0
  926. type: string
  927. expr: UDFToDouble(substring(_col1, 5))
  928. type: double
  929. tag: 0
  930. y:src
  931. TableScan
  932. alias: src
  933. Select Operator
  934. expressions:
  935. expr: key
  936. type: string
  937. expr: value
  938. type: string
  939. outputColumnNames: _col0, _col1
  940. Reduce Output Operator
  941. key expressions:
  942. expr: _col0
  943. type: string
  944. expr: (substring(_col1, 5) + 1)
  945. type: double
  946. sort order: ++
  947. Map-reduce partition columns:
  948. expr: _col0
  949. type: string
  950. expr: (substring(_col1, 5) + 1)
  951. type: double
  952. tag: 1
  953. value expressions:
  954. expr: _col0
  955. type: string
  956. expr: _col1
  957. type: string
  958. Reduce Operator Tree:
  959. Join Operator
  960. condition map:
  961. Inner Join 0 to 1
  962. condition expressions:
  963. 0
  964. 1 {VALUE._col0} {VALUE._col1}
  965. handleSkewJoin: true
  966. outputColumnNames: _col2, _col3
  967. Select Operator
  968. expressions:
  969. expr: _col2
  970. type: string
  971. expr: _col3
  972. type: string
  973. outputColumnNames: _col2, _col3
  974. Group By Operator
  975. aggregations:
  976. expr: sum(hash(_col2))
  977. expr: sum(hash(_col3))
  978. bucketGroup: false
  979. mode: hash
  980. outputColumnNames: _col0, _col1
  981. File Output Operator
  982. compressed: false
  983. GlobalTableId: 0
  984. table:
  985. input format: org.apache.hadoop.mapred.SequenceFileInputFormat
  986. output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
  987. Stage: Stage-5
  988. Conditional Operator
  989. Stage: Stage-6
  990. Map Reduce Local Work
  991. Alias -> Map Local Tables:
  992. 1
  993. Fetch Operator
  994. limit: -1
  995. Alias -> Map Local Operator Tree:
  996. 1
  997. HashTable Sink Operator
  998. condition expressions:
  999. 0
  1000. 1 {1_VALUE_0} {1_VALUE_1}
  1001. handleSkewJoin: false
  1002. keys:
  1003. 0 [Column[joinkey0], Column[joinkey1]]
  1004. 1 [Column[joinkey0], Column[joinkey1]]
  1005. Position of Big Table: 0
  1006. Stage: Stage-4
  1007. Map Reduce
  1008. Alias -> Map Operator Tree:
  1009. 0
  1010. Map Join Operator
  1011. condition map:
  1012. Inner Join 0 to 1
  1013. condition expressions:
  1014. 0
  1015. 1 {1_VALUE_0} {1_VALUE_1}
  1016. handleSkewJoin: false
  1017. keys:
  1018. 0 [Column[joinkey0], Column[joinkey1]]
  1019. 1 [Column[joinkey0], Column[joinkey1]]
  1020. outputColumnNames: _col2, _col3
  1021. Position of Big Table: 0
  1022. Select Operator
  1023. expressions:
  1024. expr: _col2
  1025. type: string
  1026. expr: _col3
  1027. type: string
  1028. outputColumnNames: _col2, _col3
  1029. Group By Operator
  1030. aggregations:
  1031. expr: sum(hash(_col2))
  1032. expr: sum(hash(_col3))
  1033. bucketGroup: false
  1034. mode: hash
  1035. outputColumnNames: _col0, _col1
  1036. File Output Operator
  1037. compressed: false
  1038. GlobalTableId: 0
  1039. table:
  1040. input format: org.apache.hadoop.mapred.SequenceFileInputFormat
  1041. output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
  1042. Local Work:
  1043. Map Reduce Local Work
  1044. Stage: Stage-2
  1045. Map Reduce
  1046. Alias -> Map Operator Tree:
  1047. file:/tmp/sdong/hive_2011-02-10_17-10-26_747_2453193727648511264/-mr-10002
  1048. Reduce Output Operator
  1049. sort order:
  1050. tag: -1
  1051. value expressions:
  1052. expr: _col0
  1053. type: bigint
  1054. expr: _col1
  1055. type: bigint
  1056. Reduce Operator Tree:
  1057. Group By Operator
  1058. aggregations:
  1059. expr: sum(VALUE._col0)
  1060. expr: sum(VALUE._col1)
  1061. bucketGroup: false
  1062. mode: mergepartial
  1063. outputColumnNames: _col0, _col1
  1064. Select Operator
  1065. expressions:
  1066. expr: _col0
  1067. type: bigint
  1068. expr: _col1
  1069. type: bigint
  1070. outputColumnNames: _col0, _col1
  1071. File Output Operator
  1072. compressed: false
  1073. GlobalTableId: 0
  1074. table:
  1075. input format: org.apache.hadoop.mapred.TextInputFormat
  1076. output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  1077. Stage: Stage-0
  1078. Fetch Operator
  1079. limit: -1
  1080. PREHOOK: query: FROM
  1081. (SELECT src.* FROM src) x
  1082. JOIN
  1083. (SELECT src.* FROM src) Y
  1084. ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1)
  1085. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  1086. PREHOOK: type: QUERY
  1087. PREHOOK: Input: default@src
  1088. PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-10-26_950_857999915064847230/-mr-10000
  1089. POSTHOOK: query: FROM
  1090. (SELECT src.* FROM src) x
  1091. JOIN
  1092. (SELECT src.* FROM src) Y
  1093. ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1)
  1094. SELECT sum(hash(Y.key)), sum(hash(Y.value))
  1095. POSTHOOK: type: QUERY
  1096. POSTHOOK: Input: default@src
  1097. POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-10-26_950_857999915064847230/-mr-10000
  1098. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  1099. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  1100. NULL NULL
  1101. PREHOOK: query: EXPLAIN
  1102. SELECT sum(hash(src1.c1)), sum(hash(src2.c4))
  1103. FROM
  1104. (SELECT src.key as c1, src.value as c2 from src) src1
  1105. JOIN
  1106. (SELECT src.key as c3, src.value as c4 from src) src2
  1107. ON src1.c1 = src2.c3 AND src1.c1 < 100
  1108. JOIN
  1109. (SELECT src.key as c5, src.value as c6 from src) src3
  1110. ON src1.c1 = src3.c5 AND src3.c5 < 80
  1111. PREHOOK: type: QUERY
  1112. POSTHOOK: query: EXPLAIN
  1113. SELECT sum(hash(src1.c1)), sum(hash(src2.c4))
  1114. FROM
  1115. (SELECT src.key as c1, src.value as c2 from src) src1
  1116. JOIN
  1117. (SELECT src.key as c3, src.value as c4 from src) src2
  1118. ON src1.c1 = src2.c3 AND src1.c1 < 100
  1119. JOIN
  1120. (SELECT src.key as c5, src.value as c6 from src) src3
  1121. ON src1.c1 = src3.c5 AND src3.c5 < 80
  1122. POSTHOOK: type: QUERY
  1123. POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
  1124. POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
  1125. ABSTRACT SYNTAX TREE:
  1126. (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c2)))) src1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c4)))) src2) (AND (= (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c3)) (< (. (TOK_TABLE_OR_COL src1) c1) 100))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c5) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c6)))) src3) (AND (= (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src3) c5)) (< (. (TOK_TABLE_OR_COL src3) c5) 80)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL src1) c1)))) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL src2) c4)))))))
  1127. STAGE DEPENDENCIES:
  1128. Stage-1 is a root stage
  1129. Stage-7 depends on stages: Stage-1 , consists of Stage-8, Stage-9
  1130. Stage-8
  1131. Stage-5 depends on stages: Stage-8
  1132. Stage-2 depends on stages: Stage-1, Stage-5, Stage-6
  1133. Stage-9
  1134. Stage-6 depends on stages: Stage-9
  1135. Stage-0 is a root stage
  1136. STAGE PLANS:
  1137. Stage: Stage-1
  1138. Map Reduce
  1139. Alias -> Map Operator Tree:
  1140. src1:src
  1141. TableScan
  1142. alias: src
  1143. Filter Operator
  1144. predicate:
  1145. expr: (key < 100)
  1146. type: boolean
  1147. Select Operator
  1148. expressions:
  1149. expr: key
  1150. type: string
  1151. outputColumnNames: _col0
  1152. Filter Operator
  1153. predicate:
  1154. expr: (_col0 < 100)
  1155. type: boolean
  1156. Reduce Output Operator
  1157. key expressions:
  1158. expr: _col0
  1159. type: string
  1160. sort order: +
  1161. Map-reduce partition columns:
  1162. expr: _col0
  1163. type: string
  1164. tag: 0
  1165. value expressions:
  1166. expr: _col0
  1167. type: string
  1168. src2:src
  1169. TableScan
  1170. alias: src
  1171. Select Operator
  1172. expressions:
  1173. expr: key
  1174. type: string
  1175. expr: value
  1176. type: string
  1177. outputColumnNames: _col0, _col1
  1178. Reduce Output Operator
  1179. key expressions:
  1180. expr: _col0
  1181. type: string
  1182. sort order: +
  1183. Map-reduce partition columns:
  1184. expr: _col0
  1185. type: string
  1186. tag: 1
  1187. value expressions:
  1188. expr: _col1
  1189. type: string
  1190. src3:src
  1191. TableScan
  1192. alias: src
  1193. Filter Operator
  1194. predicate:
  1195. expr: (key < 80)
  1196. type: boolean
  1197. Select Operator
  1198. expressions:
  1199. expr: key
  1200. type: string
  1201. outputColumnNames: _col0
  1202. Filter Operator
  1203. predicate:
  1204. expr: (_col0 < 80)
  1205. type: boolean
  1206. Reduce Output Operator
  1207. key expressions:
  1208. expr: _col0
  1209. type: string
  1210. sort order: +
  1211. Map-reduce partition columns:
  1212. expr: _col0
  1213. type: string
  1214. tag: 2
  1215. Reduce Operator Tree:
  1216. Join Operator
  1217. condition map:
  1218. Inner Join 0 to 1
  1219. Inner Join 0 to 2
  1220. condition expressions:
  1221. 0 {VALUE._col0}
  1222. 1 {VALUE._col1}
  1223. 2
  1224. handleSkewJoin: true
  1225. outputColumnNames: _col0, _col3
  1226. Select Operator
  1227. expressions:
  1228. expr: _col0
  1229. type: string
  1230. expr: _col3
  1231. type: string
  1232. outputColumnNames: _col0, _col3
  1233. Group By Operator
  1234. aggregations:
  1235. expr: sum(hash(_col0))
  1236. expr: sum(hash(_col3))
  1237. bucketGroup: false
  1238. mode: hash
  1239. outputColumnNames: _col0, _col1
  1240. File Output Operator
  1241. compressed: false
  1242. GlobalTableId: 0
  1243. table:
  1244. input format: org.apache.hadoop.mapred.SequenceFileInputFormat
  1245. output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
  1246. Stage: Stage-7
  1247. Conditional Operator
  1248. Stage: Stage-8
  1249. Map Reduce Local Work
  1250. Alias -> Map Local Tables:
  1251. 1
  1252. Fetch Operator
  1253. limit: -1
  1254. 2
  1255. Fetch Operator
  1256. limit: -1
  1257. Alias -> Map Local Operator Tree:
  1258. 1
  1259. HashTable Sink Operator
  1260. condition expressions:
  1261. 0 {0_VALUE_0}
  1262. 1 {1_VALUE_0}
  1263. 2
  1264. handleSkewJoin: false
  1265. keys:
  1266. 0 [Column[joinkey0]]
  1267. 1 [Column[joinkey0]]
  1268. 2 [Column[joinkey0]]
  1269. Position of Big Table: 0
  1270. 2
  1271. HashTable Sink Operator
  1272. condition expressions:
  1273. 0 {0_VALUE_0}
  1274. 1 {1_VALUE_0}
  1275. 2
  1276. handleSkewJoin: false
  1277. keys:
  1278. 0 [Column[joinkey0]]
  1279. 1 [Column[joinkey0]]
  1280. 2 [Column[joinkey0]]
  1281. Position of Big Table: 0
  1282. Stage: Stage-5
  1283. Map Reduce
  1284. Alias -> Map Operator Tree:
  1285. 0
  1286. Map Join Operator
  1287. condition map:
  1288. Inner Join 0 to 1
  1289. Inner Join 0 to 2
  1290. condition expressions:
  1291. 0 {0_VALUE_0}
  1292. 1 {1_VALUE_0}
  1293. 2
  1294. handleSkewJoin: false
  1295. key

Large files files are truncated, but you can click here to view the full file