PageRenderTime 50ms CodeModel.GetById 13ms RepoModel.GetById 0ms app.codeStats 0ms

/projects/hadoop-1.1.2/docs/hadoop_archives.html

https://gitlab.com/essere.lab.public/qualitas.class-corpus
HTML | 422 lines | 344 code | 13 blank | 65 comment | 0 complexity | adff288318d2cd44c35eacbbcad3c4f5 MD5 | raw file
    

  1. <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
    2. 

  2. <html>
    3. 

  3. <head>
    4. 

  4. <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
    5. 

  5. <meta content="Apache Forrest" name="Generator">
    6. 

  6. <meta name="Forrest-version" content="0.8">
    7. 

  7. <meta name="Forrest-skin-name" content="pelt">
    8. 

  8. <title>Hadoop Archives Guide</title>
    9. 

  9. <link type="text/css" href="skin/basic.css" rel="stylesheet">
    10. 

  10. <link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
    11. 

  11. <link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
    12. 

  12. <link type="text/css" href="skin/profile.css" rel="stylesheet">
    13. 

  13. <script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
    14. 

  14. <link rel="shortcut icon" href="images/favicon.ico">
    15. 

  15. </head>
    16. 

  16. <body onload="init()">
    17. 

  17. <script type="text/javascript">ndeSetTextSize();</script>
    18. 

  18. <div id="top">
    19. 

  19. <!--+
    20. 

  20.     |breadtrail
    21. 

  21.     +-->
    22. 

  22. <div class="breadtrail">
    23. 

  23. <a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/core/">Core</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
    24. 

  24. </div>
    25. 

  25. <!--+
    26. 

  26.     |header
    27. 

  27.     +-->
    28. 

  28. <div class="header">
    29. 

  29. <!--+
    30. 

  30.     |start group logo
    31. 

  31.     +-->
    32. 

  32. <div class="grouplogo">
    33. 

  33. <a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
    34. 

  34. </div>
    35. 

  35. <!--+
    36. 

  36.     |end group logo
    37. 

  37.     +-->
    38. 

  38. <!--+
    39. 

  39.     |start Project Logo
    40. 

  40.     +-->
    41. 

  41. <div class="projectlogo">
    42. 

  42. <a href="http://hadoop.apache.org/core/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo-2.gif" title="Scalable Computing Platform"></a>
    43. 

  43. </div>
    44. 

  44. <!--+
    45. 

  45.     |end Project Logo
    46. 

  46.     +-->
    47. 

  47. <!--+
    48. 

  48.     |start Search
    49. 

  49.     +-->
    50. 

  50. <div class="searchbox">
    51. 

  51. <form action="http://www.google.com/search" method="get" class="roundtopsmall">
    52. 

  52. <input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp; 
    53. 

  53.                     <input name="Search" value="Search" type="submit">
    54. 

  54. </form>
    55. 

  55. </div>
    56. 

  56. <!--+
    57. 

  57.     |end search
    58. 

  58.     +-->
    59. 

  59. <!--+
    60. 

  60.     |start Tabs
    61. 

  61.     +-->
    62. 

  62. <ul id="tabs">
    63. 

  63. <li>
    64. 

  64. <a class="unselected" href="http://hadoop.apache.org/core/">Project</a>
    65. 

  65. </li>
    66. 

  66. <li>
    67. 

  67. <a class="unselected" href="http://wiki.apache.org/hadoop">Wiki</a>
    68. 

  68. </li>
    69. 

  69. <li class="current">
    70. 

  70. <a class="selected" href="index.html">Hadoop 1.1.2 Documentation</a>
    71. 

  71. </li>
    72. 

  72. </ul>
    73. 

  73. <!--+
    74. 

  74.     |end Tabs
    75. 

  75.     +-->
    76. 

  76. </div>
    77. 

  77. </div>
    78. 

  78. <div id="main">
    79. 

  79. <div id="publishedStrip">
    80. 

  80. <!--+
    81. 

  81.     |start Subtabs
    82. 

  82.     +-->
    83. 

  83. <div id="level2tabs"></div>
    84. 

  84. <!--+
    85. 

  85.     |end Endtabs
    86. 

  86.     +-->
    87. 

  87. <script type="text/javascript"><!--
    88. 

  88. document.write("Last Published: " + document.lastModified);
    89. 

  89. //  --></script>
    90. 

  90. </div>
    91. 

  91. <!--+
    92. 

  92.     |breadtrail
    93. 

  93.     +-->
    94. 

  94. <div class="breadtrail">
    95. 

  95. 

  96.              &nbsp;
    97. 

  97.            </div>
    98. 

  98. <!--+
    99. 

  99.     |start Menu, mainarea
    100. 

  100.     +-->
    101. 

  101. <!--+
    102. 

  102.     |start Menu
    103. 

  103.     +-->
    104. 

  104. <div id="menu">
    105. 

  105. <div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Getting Started</div>
    106. 

  106. <div id="menu_1.1" class="menuitemgroup">
    107. 

  107. <div class="menuitem">
    108. 

  108. <a href="index.html">Overview</a>
    109. 

  109. </div>
    110. 

  110. <div class="menuitem">
    111. 

  111. <a href="single_node_setup.html">Single Node Setup</a>
    112. 

  112. </div>
    113. 

  113. <div class="menuitem">
    114. 

  114. <a href="cluster_setup.html">Cluster Setup</a>
    115. 

  115. </div>
    116. 

  116. </div>
    117. 

  117. <div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Guides</div>
    118. 

  118. <div id="menu_1.2" class="menuitemgroup">
    119. 

  119. <div class="menuitem">
    120. 

  120. <a href="HttpAuthentication.html">Authentication for Hadoop HTTP web-consoles</a>
    121. 

  121. </div>
    122. 

  122. </div>
    123. 

  123. <div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">MapReduce</div>
    124. 

  124. <div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
    125. 

  125. <div class="menuitem">
    126. 

  126. <a href="mapred_tutorial.html">MapReduce Tutorial</a>
    127. 

  127. </div>
    128. 

  128. <div class="menuitem">
    129. 

  129. <a href="streaming.html">Hadoop Streaming</a>
    130. 

  130. </div>
    131. 

  131. <div class="menuitem">
    132. 

  132. <a href="commands_manual.html">Hadoop Commands</a>
    133. 

  133. </div>
    134. 

  134. <div class="menuitem">
    135. 

  135. <a href="distcp.html">DistCp</a>
    136. 

  136. </div>
    137. 

  137. <div class="menuitem">
    138. 

  138. <a href="vaidya.html">Vaidya</a>
    139. 

  139. </div>
    140. 

  140. <div class="menupage">
    141. 

  141. <div class="menupagetitle">Hadoop Archives</div>
    142. 

  142. </div>
    143. 

  143. <div class="menuitem">
    144. 

  144. <a href="gridmix.html">Gridmix</a>
    145. 

  145. </div>
    146. 

  146. <div class="menuitem">
    147. 

  147. <a href="rumen.html">Rumen</a>
    148. 

  148. </div>
    149. 

  149. <div class="menuitem">
    150. 

  150. <a href="capacity_scheduler.html">Capacity Scheduler</a>
    151. 

  151. </div>
    152. 

  152. <div class="menuitem">
    153. 

  153. <a href="fair_scheduler.html">Fair Scheduler</a>
    154. 

  154. </div>
    155. 

  155. <div class="menuitem">
    156. 

  156. <a href="hod_scheduler.html">Hod Scheduler</a>
    157. 

  157. </div>
    158. 

  158. </div>
    159. 

  159. <div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">HDFS</div>
    160. 

  160. <div id="menu_1.4" class="menuitemgroup">
    161. 

  161. <div class="menuitem">
    162. 

  162. <a href="hdfs_user_guide.html">HDFS Users </a>
    163. 

  163. </div>
    164. 

  164. <div class="menuitem">
    165. 

  165. <a href="hdfs_design.html">HDFS Architecture</a>
    166. 

  166. </div>
    167. 

  167. <div class="menuitem">
    168. 

  168. <a href="hdfs_permissions_guide.html">Permissions</a>
    169. 

  169. </div>
    170. 

  170. <div class="menuitem">
    171. 

  171. <a href="hdfs_quota_admin_guide.html">Quotas</a>
    172. 

  172. </div>
    173. 

  173. <div class="menuitem">
    174. 

  174. <a href="SLG_user_guide.html">Synthetic Load Generator</a>
    175. 

  175. </div>
    176. 

  176. <div class="menuitem">
    177. 

  177. <a href="webhdfs.html">WebHDFS REST API</a>
    178. 

  178. </div>
    179. 

  179. <div class="menuitem">
    180. 

  180. <a href="libhdfs.html">C API libhdfs</a>
    181. 

  181. </div>
    182. 

  182. </div>
    183. 

  183. <div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Common</div>
    184. 

  184. <div id="menu_1.5" class="menuitemgroup">
    185. 

  185. <div class="menuitem">
    186. 

  186. <a href="deployment_layout.html">Deployment Layout</a>
    187. 

  187. </div>
    188. 

  188. <div class="menuitem">
    189. 

  189. <a href="file_system_shell.html">File System Shell</a>
    190. 

  190. </div>
    191. 

  191. <div class="menuitem">
    192. 

  192. <a href="service_level_auth.html">Service Level Authorization</a>
    193. 

  193. </div>
    194. 

  194. <div class="menuitem">
    195. 

  195. <a href="native_libraries.html">Native Libraries</a>
    196. 

  196. </div>
    197. 

  197. </div>
    198. 

  198. <div onclick="SwitchMenu('menu_1.6', 'skin/')" id="menu_1.6Title" class="menutitle">Miscellaneous</div>
    199. 

  199. <div id="menu_1.6" class="menuitemgroup">
    200. 

  200. <div class="menuitem">
    201. 

  201. <a href="Secure_Impersonation.html">Secure Impersonation</a>
    202. 

  202. </div>
    203. 

  203. <div class="menuitem">
    204. 

  204. <a href="api/index.html">API Docs</a>
    205. 

  205. </div>
    206. 

  206. <div class="menuitem">
    207. 

  207. <a href="jdiff/changes.html">API Changes</a>
    208. 

  208. </div>
    209. 

  209. <div class="menuitem">
    210. 

  210. <a href="http://wiki.apache.org/hadoop/">Wiki</a>
    211. 

  211. </div>
    212. 

  212. <div class="menuitem">
    213. 

  213. <a href="http://wiki.apache.org/hadoop/FAQ">FAQ</a>
    214. 

  214. </div>
    215. 

  215. <div class="menuitem">
    216. 

  216. <a href="releasenotes.html">Release Notes</a>
    217. 

  217. </div>
    218. 

  218. <div class="menuitem">
    219. 

  219. <a href="changes.html">Change Log</a>
    220. 

  220. </div>
    221. 

  221. </div>
    222. 

  222. <div id="credit"></div>
    223. 

  223. <div id="roundbottom">
    224. 

  224. <img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
    225. 

  225. <!--+
    226. 

  226.   |alternative credits
    227. 

  227.   +-->
    228. 

  228. <div id="credit2"></div>
    229. 

  229. </div>
    230. 

  230. <!--+
    231. 

  231.     |end Menu
    232. 

  232.     +-->
    233. 

  233. <!--+
    234. 

  234.     |start content
    235. 

  235.     +-->
    236. 

  236. <div id="content">
    237. 

  237. <div title="Portable Document Format" class="pdflink">
    238. 

  238. <a class="dida" href="hadoop_archives.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
    239. 

  239.         PDF</a>
    240. 

  240. </div>
    241. 

  241. <h1>Hadoop Archives Guide</h1>
    242. 

  242. <div id="minitoc-area">
    243. 

  243. <ul class="minitoc">
    244. 

  244. <li>
    245. 

  245. <a href="#Overview">Overview</a>
    246. 

  246. </li>
    247. 

  247. <li>
    248. 

  248. <a href="#How+to+Create+an+Archive">How to Create an Archive</a>
    249. 

  249. </li>
    250. 

  250. <li>
    251. 

  251. <a href="#How+to+Look+Up+Files+in+Archives">How to Look Up Files in Archives</a>
    252. 

  252. </li>
    253. 

  253. <li>
    254. 

  254. <a href="#Archives+Examples">Archives Examples</a>
    255. 

  255. <ul class="minitoc">
    256. 

  256. <li>
    257. 

  257. <a href="#Creating+an+Archive">Creating an Archive</a>
    258. 

  258. </li>
    259. 

  259. <li>
    260. 

  260. <a href="#Looking+Up+Files"> Looking Up Files</a>
    261. 

  261. </li>
    262. 

  262. </ul>
    263. 

  263. </li>
    264. 

  264. <li>
    265. 

  265. <a href="#Hadoop+Archives+and+MapReduce">Hadoop Archives and MapReduce </a>
    266. 

  266. </li>
    267. 

  267. </ul>
    268. 

  268. </div>
    269. 

  269.         
    270. 

  270. <a name="N1000D"></a><a name="Overview"></a>
    271. 

  271. <h2 class="h3">Overview</h2>
    272. 

  272. <div class="section">
    273. 

  273. <p>
    274. 

  274.         Hadoop archives are special format archives. A Hadoop archive
    275. 

  275.         maps to a file system directory. A Hadoop archive always has a *.har
    276. 

  276.         extension. A Hadoop archive directory contains metadata (in the form 
    277. 

  277.         of _index and _masterindex) and data (part-*) files. The _index file contains
    278. 

  278.         the name of the files that are part of the archive and the location
    279. 

  279.         within the part files. 
    280. 

  280.         </p>
    281. 

  281. </div>
    282. 

  282.         
    283. 

  283.         
    284. 

  284. <a name="N10017"></a><a name="How+to+Create+an+Archive"></a>
    285. 

  285. <h2 class="h3">How to Create an Archive</h2>
    286. 

  286. <div class="section">
    287. 

  287. <p>
    288. 

  288.         
    289. 

  289. <span class="codefrag">Usage: hadoop archive -archiveName name -p &lt;parent&gt; &lt;src&gt;* &lt;dest&gt;</span>
    290. 

  290.         
    291. 

  291. </p>
    292. 

  292. <p>
    293. 

  293.         -archiveName is the name of the archive you would like to create. 
    294. 

  294.         An example would be foo.har. The name should have a *.har extension. 
    295. 

  295.        	The parent argument is to specify the relative path to which the files should be
    296. 

  296.        	archived to. Example would be :
    297. 

  297.         </p>
    298. 

  298. <p>
    299. 

  299. <span class="codefrag"> -p /foo/bar a/b/c e/f/g </span>
    300. 

  300. </p>
    301. 

  301. <p>
    302. 

  302.         Here /foo/bar is the parent path and a/b/c, e/f/g are relative paths to parent. 
    303. 

  303.         Note that this is a Map/Reduce job that creates the archives. You would
    304. 

  304.         need a map reduce cluster to run this. For a detailed example the later sections. </p>
    305. 

  305. <p> If you just want to archive a single directory /foo/bar then you can just use </p>
    306. 

  306. <p>
    307. 

  307. <span class="codefrag"> hadoop archive -archiveName zoo.har -p /foo/bar /outputdir </span>
    308. 

  308. </p>
    309. 

  309. </div>
    310. 

  310.         
    311. 

  311.         
    312. 

  312. <a name="N10033"></a><a name="How+to+Look+Up+Files+in+Archives"></a>
    313. 

  313. <h2 class="h3">How to Look Up Files in Archives</h2>
    314. 

  314. <div class="section">
    315. 

  315. <p>
    316. 

  316.         The archive exposes itself as a file system layer. So all the fs shell
    317. 

  317.         commands in the archives work but with a different URI. Also, note that
    318. 

  318.         archives are immutable. So, rename's, deletes and creates return
    319. 

  319.         an error. URI for Hadoop Archives is 
    320. 

  320.         </p>
    321. 

  321. <p>
    322. 

  322. <span class="codefrag">har://scheme-hostname:port/archivepath/fileinarchive</span>
    323. 

  323. </p>
    324. 

  324. <p>
    325. 

  325.         If no scheme is provided it assumes the underlying filesystem. 
    326. 

  326.         In that case the URI would look like </p>
    327. 

  327. <p>
    328. 

  328. <span class="codefrag">har:///archivepath/fileinarchive</span>
    329. 

  329. </p>
    330. 

  330. </div>
    331. 

  331. 

  332.  		
    333. 

  333. <a name="N10046"></a><a name="Archives+Examples"></a>
    334. 

  334. <h2 class="h3">Archives Examples</h2>
    335. 

  335. <div class="section">
    336. 

  336. <a name="N1004C"></a><a name="Creating+an+Archive"></a>
    337. 

  337. <h3 class="h4">Creating an Archive</h3>
    338. 

  338. <p>
    339. 

  339. <span class="codefrag">hadoop archive -archiveName foo.har -p /user/hadoop dir1 dir2 /user/zoo </span>
    340. 

  340. </p>
    341. 

  341. <p>
    342. 

  342.          The above example is creating an archive using /user/hadoop as the relative archive directory.
    343. 

  343.          The directories /user/hadoop/dir1 and /user/hadoop/dir2 will be 
    344. 

  344.         archived in the following file system directory -- /user/zoo/foo.har. Archiving does not delete the input
    345. 

  345.         files. If you want to delete the input files after creating the archives (to reduce namespace), you
    346. 

  346.         will have to do it on your own. 
    347. 

  347.         </p>
    348. 

  348. <a name="N1005A"></a><a name="Looking+Up+Files"></a>
    349. 

  349. <h3 class="h4"> Looking Up Files</h3>
    350. 

  350. <p> Looking up files in hadoop archives is as easy as doing an ls on the filesystem. After you have
    351. 

  351. 		 archived the directories /user/hadoop/dir1 and /user/hadoop/dir2 as in the example above, to see all
    352. 

  352. 		 the files in the archives you can just run: </p>
    353. 

  353. <p>
    354. 

  354. <span class="codefrag">hadoop dfs -lsr har:///user/zoo/foo.har/</span>
    355. 

  355. </p>
    356. 

  356. <p> To understand the significance of the -p argument, lets go through the above example again. If you just do
    357. 

  357. 		 an ls (not lsr) on the hadoop archive using </p>
    358. 

  358. <p>
    359. 

  359. <span class="codefrag">hadoop dfs -ls har:///user/zoo/foo.har</span>
    360. 

  360. </p>
    361. 

  361. <p>The output should be:</p>
    362. 

  362. <pre class="code">
    363. 

  363. har:///user/zoo/foo.har/dir1
    364. 

  364. har:///user/zoo/foo.har/dir2
    365. 

  365. </pre>
    366. 

  366. <p> As you can recall the archives were created with the following command </p>
    367. 

  367. <p>
    368. 

  368. <span class="codefrag">hadoop archive -archiveName foo.har -p /user/hadoop dir1 dir2 /user/zoo </span>
    369. 

  369. </p>
    370. 

  370. <p> If we were to change the command to: </p>
    371. 

  371. <p>
    372. 

  372. <span class="codefrag">hadoop archive -archiveName foo.har -p /user/  hadoop/dir1 hadoop/dir2 /user/zoo </span>
    373. 

  373. </p>
    374. 

  374. <p> then a ls on the hadoop archive using </p>
    375. 

  375. <p>
    376. 

  376. <span class="codefrag">hadoop dfs -ls har:///user/zoo/foo.har</span>
    377. 

  377. </p>
    378. 

  378. <p>would give you</p>
    379. 

  379. <pre class="code">
    380. 

  380. har:///user/zoo/foo.har/hadoop/dir1
    381. 

  381. har:///user/zoo/foo.har/hadoop/dir2
    382. 

  382. </pre>
    383. 

  383. <p>
    384. 

  384. 		Notice that the archived files have been archived relative to /user/ rather than /user/hadoop.
    385. 

  385. 		</p>
    386. 

  386. </div>
    387. 

  387. 		
    388. 

  388. 		
    389. 

  389. <a name="N10096"></a><a name="Hadoop+Archives+and+MapReduce"></a>
    390. 

  390. <h2 class="h3">Hadoop Archives and MapReduce </h2>
    391. 

  391. <div class="section">
    392. 

  392. <p>Using Hadoop Archives in MapReduce is as easy as specifying a different input filesystem than the default file system.
    393. 

  393. 		If you have a hadoop archive stored in HDFS in /user/zoo/foo.har then for using this archive for MapReduce input, all
    394. 

  394. 		you need to specify the input directory as har:///user/zoo/foo.har. Since Hadoop Archives is exposed as a file system 
    395. 

  395. 		MapReduce will be able to use all the logical input files in Hadoop Archives as input.</p>
    396. 

  396. </div>
    397. 

  397.   
    398. 

  398. </div>
    399. 

  399. <!--+
    400. 

  400.     |end content
    401. 

  401.     +-->
    402. 

  402. <div class="clearboth">&nbsp;</div>
    403. 

  403. </div>
    404. 

  404. <div id="footer">
    405. 

  405. <!--+
    406. 

  406.     |start bottomstrip
    407. 

  407.     +-->
    408. 

  408. <div class="lastmodified">
    409. 

  409. <script type="text/javascript"><!--
    410. 

  410. document.write("Last Published: " + document.lastModified);
    411. 

  411. //  --></script>
    412. 

  412. </div>
    413. 

  413. <div class="copyright">
    414. 

  414.         Copyright &copy;
    415. 

  415.          2008 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
    416. 

  416. </div>
    417. 

  417. <!--+
    418. 

  418.     |end bottomstrip
    419. 

  419.     +-->
    420. 

  420. </div>
    421. 

  421. </body>
    422. 

  422. </html>
    423. 

  423.