PageRenderTime 61ms CodeModel.GetById 21ms RepoModel.GetById 1ms app.codeStats 0ms

/Lib/tarfile.py

http://unladen-swallow.googlecode.com/
Python | 2530 lines | 2419 code | 29 blank | 82 comment | 22 complexity | 0fe24e5b7d3de8de04402c6cfc8ef97d MD5 | raw file
Possible License(s): 0BSD, BSD-3-Clause
  1. #!/usr/bin/env python
  2. # -*- coding: iso-8859-1 -*-
  3. #-------------------------------------------------------------------
  4. # tarfile.py
  5. #-------------------------------------------------------------------
  6. # Copyright (C) 2002 Lars Gustäbel <lars@gustaebel.de>
  7. # All rights reserved.
  8. #
  9. # Permission is hereby granted, free of charge, to any person
  10. # obtaining a copy of this software and associated documentation
  11. # files (the "Software"), to deal in the Software without
  12. # restriction, including without limitation the rights to use,
  13. # copy, modify, merge, publish, distribute, sublicense, and/or sell
  14. # copies of the Software, and to permit persons to whom the
  15. # Software is furnished to do so, subject to the following
  16. # conditions:
  17. #
  18. # The above copyright notice and this permission notice shall be
  19. # included in all copies or substantial portions of the Software.
  20. #
  21. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  22. # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  23. # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  24. # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  25. # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  26. # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  27. # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  28. # OTHER DEALINGS IN THE SOFTWARE.
  29. #
  30. """Read from and write to tar format archives.
  31. """
  32. __version__ = "$Revision: 73770 $"
  33. # $Source$
  34. version = "0.9.0"
  35. __author__ = "Lars Gustäbel (lars@gustaebel.de)"
  36. __date__ = "$Date: 2009-07-02 17:37:21 +0200 (Thu, 02 Jul 2009) $"
  37. __cvsid__ = "$Id: tarfile.py 73770 2009-07-02 15:37:21Z jesus.cea $"
  38. __credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend."
  39. #---------
  40. # Imports
  41. #---------
  42. import sys
  43. import os
  44. import shutil
  45. import stat
  46. import errno
  47. import time
  48. import struct
  49. import copy
  50. import re
  51. import operator
  52. if sys.platform == 'mac':
  53. # This module needs work for MacOS9, especially in the area of pathname
  54. # handling. In many places it is assumed a simple substitution of / by the
  55. # local os.path.sep is good enough to convert pathnames, but this does not
  56. # work with the mac rooted:path:name versus :nonrooted:path:name syntax
  57. raise ImportError, "tarfile does not work for platform==mac"
  58. try:
  59. import grp, pwd
  60. except ImportError:
  61. grp = pwd = None
  62. # from tarfile import *
  63. __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]
  64. #---------------------------------------------------------
  65. # tar constants
  66. #---------------------------------------------------------
  67. NUL = "\0" # the null character
  68. BLOCKSIZE = 512 # length of processing blocks
  69. RECORDSIZE = BLOCKSIZE * 20 # length of records
  70. GNU_MAGIC = "ustar \0" # magic gnu tar string
  71. POSIX_MAGIC = "ustar\x0000" # magic posix tar string
  72. LENGTH_NAME = 100 # maximum length of a filename
  73. LENGTH_LINK = 100 # maximum length of a linkname
  74. LENGTH_PREFIX = 155 # maximum length of the prefix field
  75. REGTYPE = "0" # regular file
  76. AREGTYPE = "\0" # regular file
  77. LNKTYPE = "1" # link (inside tarfile)
  78. SYMTYPE = "2" # symbolic link
  79. CHRTYPE = "3" # character special device
  80. BLKTYPE = "4" # block special device
  81. DIRTYPE = "5" # directory
  82. FIFOTYPE = "6" # fifo special device
  83. CONTTYPE = "7" # contiguous file
  84. GNUTYPE_LONGNAME = "L" # GNU tar longname
  85. GNUTYPE_LONGLINK = "K" # GNU tar longlink
  86. GNUTYPE_SPARSE = "S" # GNU tar sparse file
  87. XHDTYPE = "x" # POSIX.1-2001 extended header
  88. XGLTYPE = "g" # POSIX.1-2001 global header
  89. SOLARIS_XHDTYPE = "X" # Solaris extended header
  90. USTAR_FORMAT = 0 # POSIX.1-1988 (ustar) format
  91. GNU_FORMAT = 1 # GNU tar format
  92. PAX_FORMAT = 2 # POSIX.1-2001 (pax) format
  93. DEFAULT_FORMAT = GNU_FORMAT
  94. #---------------------------------------------------------
  95. # tarfile constants
  96. #---------------------------------------------------------
  97. # File types that tarfile supports:
  98. SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,
  99. SYMTYPE, DIRTYPE, FIFOTYPE,
  100. CONTTYPE, CHRTYPE, BLKTYPE,
  101. GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
  102. GNUTYPE_SPARSE)
  103. # File types that will be treated as a regular file.
  104. REGULAR_TYPES = (REGTYPE, AREGTYPE,
  105. CONTTYPE, GNUTYPE_SPARSE)
  106. # File types that are part of the GNU tar format.
  107. GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
  108. GNUTYPE_SPARSE)
  109. # Fields from a pax header that override a TarInfo attribute.
  110. PAX_FIELDS = ("path", "linkpath", "size", "mtime",
  111. "uid", "gid", "uname", "gname")
  112. # Fields in a pax header that are numbers, all other fields
  113. # are treated as strings.
  114. PAX_NUMBER_FIELDS = {
  115. "atime": float,
  116. "ctime": float,
  117. "mtime": float,
  118. "uid": int,
  119. "gid": int,
  120. "size": int
  121. }
  122. #---------------------------------------------------------
  123. # Bits used in the mode field, values in octal.
  124. #---------------------------------------------------------
  125. S_IFLNK = 0120000 # symbolic link
  126. S_IFREG = 0100000 # regular file
  127. S_IFBLK = 0060000 # block device
  128. S_IFDIR = 0040000 # directory
  129. S_IFCHR = 0020000 # character device
  130. S_IFIFO = 0010000 # fifo
  131. TSUID = 04000 # set UID on execution
  132. TSGID = 02000 # set GID on execution
  133. TSVTX = 01000 # reserved
  134. TUREAD = 0400 # read by owner
  135. TUWRITE = 0200 # write by owner
  136. TUEXEC = 0100 # execute/search by owner
  137. TGREAD = 0040 # read by group
  138. TGWRITE = 0020 # write by group
  139. TGEXEC = 0010 # execute/search by group
  140. TOREAD = 0004 # read by other
  141. TOWRITE = 0002 # write by other
  142. TOEXEC = 0001 # execute/search by other
  143. #---------------------------------------------------------
  144. # initialization
  145. #---------------------------------------------------------
  146. ENCODING = sys.getfilesystemencoding()
  147. if ENCODING is None:
  148. ENCODING = sys.getdefaultencoding()
  149. #---------------------------------------------------------
  150. # Some useful functions
  151. #---------------------------------------------------------
  152. def stn(s, length):
  153. """Convert a python string to a null-terminated string buffer.
  154. """
  155. return s[:length] + (length - len(s)) * NUL
  156. def nts(s):
  157. """Convert a null-terminated string field to a python string.
  158. """
  159. # Use the string up to the first null char.
  160. p = s.find("\0")
  161. if p == -1:
  162. return s
  163. return s[:p]
  164. def nti(s):
  165. """Convert a number field to a python number.
  166. """
  167. # There are two possible encodings for a number field, see
  168. # itn() below.
  169. if s[0] != chr(0200):
  170. try:
  171. n = int(nts(s) or "0", 8)
  172. except ValueError:
  173. raise HeaderError("invalid header")
  174. else:
  175. n = 0L
  176. for i in xrange(len(s) - 1):
  177. n <<= 8
  178. n += ord(s[i + 1])
  179. return n
  180. def itn(n, digits=8, format=DEFAULT_FORMAT):
  181. """Convert a python number to a number field.
  182. """
  183. # POSIX 1003.1-1988 requires numbers to be encoded as a string of
  184. # octal digits followed by a null-byte, this allows values up to
  185. # (8**(digits-1))-1. GNU tar allows storing numbers greater than
  186. # that if necessary. A leading 0200 byte indicates this particular
  187. # encoding, the following digits-1 bytes are a big-endian
  188. # representation. This allows values up to (256**(digits-1))-1.
  189. if 0 <= n < 8 ** (digits - 1):
  190. s = "%0*o" % (digits - 1, n) + NUL
  191. else:
  192. if format != GNU_FORMAT or n >= 256 ** (digits - 1):
  193. raise ValueError("overflow in number field")
  194. if n < 0:
  195. # XXX We mimic GNU tar's behaviour with negative numbers,
  196. # this could raise OverflowError.
  197. n = struct.unpack("L", struct.pack("l", n))[0]
  198. s = ""
  199. for i in xrange(digits - 1):
  200. s = chr(n & 0377) + s
  201. n >>= 8
  202. s = chr(0200) + s
  203. return s
  204. def uts(s, encoding, errors):
  205. """Convert a unicode object to a string.
  206. """
  207. if errors == "utf-8":
  208. # An extra error handler similar to the -o invalid=UTF-8 option
  209. # in POSIX.1-2001. Replace untranslatable characters with their
  210. # UTF-8 representation.
  211. try:
  212. return s.encode(encoding, "strict")
  213. except UnicodeEncodeError:
  214. x = []
  215. for c in s:
  216. try:
  217. x.append(c.encode(encoding, "strict"))
  218. except UnicodeEncodeError:
  219. x.append(c.encode("utf8"))
  220. return "".join(x)
  221. else:
  222. return s.encode(encoding, errors)
  223. def calc_chksums(buf):
  224. """Calculate the checksum for a member's header by summing up all
  225. characters except for the chksum field which is treated as if
  226. it was filled with spaces. According to the GNU tar sources,
  227. some tars (Sun and NeXT) calculate chksum with signed char,
  228. which will be different if there are chars in the buffer with
  229. the high bit set. So we calculate two checksums, unsigned and
  230. signed.
  231. """
  232. unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
  233. signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
  234. return unsigned_chksum, signed_chksum
  235. def copyfileobj(src, dst, length=None):
  236. """Copy length bytes from fileobj src to fileobj dst.
  237. If length is None, copy the entire content.
  238. """
  239. if length == 0:
  240. return
  241. if length is None:
  242. shutil.copyfileobj(src, dst)
  243. return
  244. BUFSIZE = 16 * 1024
  245. blocks, remainder = divmod(length, BUFSIZE)
  246. for b in xrange(blocks):
  247. buf = src.read(BUFSIZE)
  248. if len(buf) < BUFSIZE:
  249. raise IOError("end of file reached")
  250. dst.write(buf)
  251. if remainder != 0:
  252. buf = src.read(remainder)
  253. if len(buf) < remainder:
  254. raise IOError("end of file reached")
  255. dst.write(buf)
  256. return
  257. filemode_table = (
  258. ((S_IFLNK, "l"),
  259. (S_IFREG, "-"),
  260. (S_IFBLK, "b"),
  261. (S_IFDIR, "d"),
  262. (S_IFCHR, "c"),
  263. (S_IFIFO, "p")),
  264. ((TUREAD, "r"),),
  265. ((TUWRITE, "w"),),
  266. ((TUEXEC|TSUID, "s"),
  267. (TSUID, "S"),
  268. (TUEXEC, "x")),
  269. ((TGREAD, "r"),),
  270. ((TGWRITE, "w"),),
  271. ((TGEXEC|TSGID, "s"),
  272. (TSGID, "S"),
  273. (TGEXEC, "x")),
  274. ((TOREAD, "r"),),
  275. ((TOWRITE, "w"),),
  276. ((TOEXEC|TSVTX, "t"),
  277. (TSVTX, "T"),
  278. (TOEXEC, "x"))
  279. )
  280. def filemode(mode):
  281. """Convert a file's mode to a string of the form
  282. -rwxrwxrwx.
  283. Used by TarFile.list()
  284. """
  285. perm = []
  286. for table in filemode_table:
  287. for bit, char in table:
  288. if mode & bit == bit:
  289. perm.append(char)
  290. break
  291. else:
  292. perm.append("-")
  293. return "".join(perm)
  294. if os.sep != "/":
  295. normpath = lambda path: os.path.normpath(path).replace(os.sep, "/")
  296. else:
  297. normpath = os.path.normpath
  298. class TarError(Exception):
  299. """Base exception."""
  300. pass
  301. class ExtractError(TarError):
  302. """General exception for extract errors."""
  303. pass
  304. class ReadError(TarError):
  305. """Exception for unreadble tar archives."""
  306. pass
  307. class CompressionError(TarError):
  308. """Exception for unavailable compression methods."""
  309. pass
  310. class StreamError(TarError):
  311. """Exception for unsupported operations on stream-like TarFiles."""
  312. pass
  313. class HeaderError(TarError):
  314. """Exception for invalid headers."""
  315. pass
  316. #---------------------------
  317. # internal stream interface
  318. #---------------------------
  319. class _LowLevelFile:
  320. """Low-level file object. Supports reading and writing.
  321. It is used instead of a regular file object for streaming
  322. access.
  323. """
  324. def __init__(self, name, mode):
  325. mode = {
  326. "r": os.O_RDONLY,
  327. "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
  328. }[mode]
  329. if hasattr(os, "O_BINARY"):
  330. mode |= os.O_BINARY
  331. self.fd = os.open(name, mode)
  332. def close(self):
  333. os.close(self.fd)
  334. def read(self, size):
  335. return os.read(self.fd, size)
  336. def write(self, s):
  337. os.write(self.fd, s)
  338. class _Stream:
  339. """Class that serves as an adapter between TarFile and
  340. a stream-like object. The stream-like object only
  341. needs to have a read() or write() method and is accessed
  342. blockwise. Use of gzip or bzip2 compression is possible.
  343. A stream-like object could be for example: sys.stdin,
  344. sys.stdout, a socket, a tape device etc.
  345. _Stream is intended to be used only internally.
  346. """
  347. def __init__(self, name, mode, comptype, fileobj, bufsize):
  348. """Construct a _Stream object.
  349. """
  350. self._extfileobj = True
  351. if fileobj is None:
  352. fileobj = _LowLevelFile(name, mode)
  353. self._extfileobj = False
  354. if comptype == '*':
  355. # Enable transparent compression detection for the
  356. # stream interface
  357. fileobj = _StreamProxy(fileobj)
  358. comptype = fileobj.getcomptype()
  359. self.name = name or ""
  360. self.mode = mode
  361. self.comptype = comptype
  362. self.fileobj = fileobj
  363. self.bufsize = bufsize
  364. self.buf = ""
  365. self.pos = 0L
  366. self.closed = False
  367. if comptype == "gz":
  368. try:
  369. import zlib
  370. except ImportError:
  371. raise CompressionError("zlib module is not available")
  372. self.zlib = zlib
  373. self.crc = zlib.crc32("") & 0xffffffffL
  374. if mode == "r":
  375. self._init_read_gz()
  376. else:
  377. self._init_write_gz()
  378. if comptype == "bz2":
  379. try:
  380. import bz2
  381. except ImportError:
  382. raise CompressionError("bz2 module is not available")
  383. if mode == "r":
  384. self.dbuf = ""
  385. self.cmp = bz2.BZ2Decompressor()
  386. else:
  387. self.cmp = bz2.BZ2Compressor()
  388. def __del__(self):
  389. if hasattr(self, "closed") and not self.closed:
  390. self.close()
  391. def _init_write_gz(self):
  392. """Initialize for writing with gzip compression.
  393. """
  394. self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
  395. -self.zlib.MAX_WBITS,
  396. self.zlib.DEF_MEM_LEVEL,
  397. 0)
  398. timestamp = struct.pack("<L", long(time.time()))
  399. self.__write("\037\213\010\010%s\002\377" % timestamp)
  400. if self.name.endswith(".gz"):
  401. self.name = self.name[:-3]
  402. self.__write(self.name + NUL)
  403. def write(self, s):
  404. """Write string s to the stream.
  405. """
  406. if self.comptype == "gz":
  407. self.crc = self.zlib.crc32(s, self.crc) & 0xffffffffL
  408. self.pos += len(s)
  409. if self.comptype != "tar":
  410. s = self.cmp.compress(s)
  411. self.__write(s)
  412. def __write(self, s):
  413. """Write string s to the stream if a whole new block
  414. is ready to be written.
  415. """
  416. self.buf += s
  417. while len(self.buf) > self.bufsize:
  418. self.fileobj.write(self.buf[:self.bufsize])
  419. self.buf = self.buf[self.bufsize:]
  420. def close(self):
  421. """Close the _Stream object. No operation should be
  422. done on it afterwards.
  423. """
  424. if self.closed:
  425. return
  426. if self.mode == "w" and self.comptype != "tar":
  427. self.buf += self.cmp.flush()
  428. if self.mode == "w" and self.buf:
  429. self.fileobj.write(self.buf)
  430. self.buf = ""
  431. if self.comptype == "gz":
  432. # The native zlib crc is an unsigned 32-bit integer, but
  433. # the Python wrapper implicitly casts that to a signed C
  434. # long. So, on a 32-bit box self.crc may "look negative",
  435. # while the same crc on a 64-bit box may "look positive".
  436. # To avoid irksome warnings from the `struct` module, force
  437. # it to look positive on all boxes.
  438. self.fileobj.write(struct.pack("<L", self.crc & 0xffffffffL))
  439. self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFFL))
  440. if not self._extfileobj:
  441. self.fileobj.close()
  442. self.closed = True
  443. def _init_read_gz(self):
  444. """Initialize for reading a gzip compressed fileobj.
  445. """
  446. self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)
  447. self.dbuf = ""
  448. # taken from gzip.GzipFile with some alterations
  449. if self.__read(2) != "\037\213":
  450. raise ReadError("not a gzip file")
  451. if self.__read(1) != "\010":
  452. raise CompressionError("unsupported compression method")
  453. flag = ord(self.__read(1))
  454. self.__read(6)
  455. if flag & 4:
  456. xlen = ord(self.__read(1)) + 256 * ord(self.__read(1))
  457. self.read(xlen)
  458. if flag & 8:
  459. while True:
  460. s = self.__read(1)
  461. if not s or s == NUL:
  462. break
  463. if flag & 16:
  464. while True:
  465. s = self.__read(1)
  466. if not s or s == NUL:
  467. break
  468. if flag & 2:
  469. self.__read(2)
  470. def tell(self):
  471. """Return the stream's file pointer position.
  472. """
  473. return self.pos
  474. def seek(self, pos=0):
  475. """Set the stream's file pointer to pos. Negative seeking
  476. is forbidden.
  477. """
  478. if pos - self.pos >= 0:
  479. blocks, remainder = divmod(pos - self.pos, self.bufsize)
  480. for i in xrange(blocks):
  481. self.read(self.bufsize)
  482. self.read(remainder)
  483. else:
  484. raise StreamError("seeking backwards is not allowed")
  485. return self.pos
  486. def read(self, size=None):
  487. """Return the next size number of bytes from the stream.
  488. If size is not defined, return all bytes of the stream
  489. up to EOF.
  490. """
  491. if size is None:
  492. t = []
  493. while True:
  494. buf = self._read(self.bufsize)
  495. if not buf:
  496. break
  497. t.append(buf)
  498. buf = "".join(t)
  499. else:
  500. buf = self._read(size)
  501. self.pos += len(buf)
  502. return buf
  503. def _read(self, size):
  504. """Return size bytes from the stream.
  505. """
  506. if self.comptype == "tar":
  507. return self.__read(size)
  508. c = len(self.dbuf)
  509. t = [self.dbuf]
  510. while c < size:
  511. buf = self.__read(self.bufsize)
  512. if not buf:
  513. break
  514. try:
  515. buf = self.cmp.decompress(buf)
  516. except IOError:
  517. raise ReadError("invalid compressed data")
  518. t.append(buf)
  519. c += len(buf)
  520. t = "".join(t)
  521. self.dbuf = t[size:]
  522. return t[:size]
  523. def __read(self, size):
  524. """Return size bytes from stream. If internal buffer is empty,
  525. read another block from the stream.
  526. """
  527. c = len(self.buf)
  528. t = [self.buf]
  529. while c < size:
  530. buf = self.fileobj.read(self.bufsize)
  531. if not buf:
  532. break
  533. t.append(buf)
  534. c += len(buf)
  535. t = "".join(t)
  536. self.buf = t[size:]
  537. return t[:size]
  538. # class _Stream
  539. class _StreamProxy(object):
  540. """Small proxy class that enables transparent compression
  541. detection for the Stream interface (mode 'r|*').
  542. """
  543. def __init__(self, fileobj):
  544. self.fileobj = fileobj
  545. self.buf = self.fileobj.read(BLOCKSIZE)
  546. def read(self, size):
  547. self.read = self.fileobj.read
  548. return self.buf
  549. def getcomptype(self):
  550. if self.buf.startswith("\037\213\010"):
  551. return "gz"
  552. if self.buf.startswith("BZh91"):
  553. return "bz2"
  554. return "tar"
  555. def close(self):
  556. self.fileobj.close()
  557. # class StreamProxy
  558. class _BZ2Proxy(object):
  559. """Small proxy class that enables external file object
  560. support for "r:bz2" and "w:bz2" modes. This is actually
  561. a workaround for a limitation in bz2 module's BZ2File
  562. class which (unlike gzip.GzipFile) has no support for
  563. a file object argument.
  564. """
  565. blocksize = 16 * 1024
  566. def __init__(self, fileobj, mode):
  567. self.fileobj = fileobj
  568. self.mode = mode
  569. self.name = getattr(self.fileobj, "name", None)
  570. self.init()
  571. def init(self):
  572. import bz2
  573. self.pos = 0
  574. if self.mode == "r":
  575. self.bz2obj = bz2.BZ2Decompressor()
  576. self.fileobj.seek(0)
  577. self.buf = ""
  578. else:
  579. self.bz2obj = bz2.BZ2Compressor()
  580. def read(self, size):
  581. b = [self.buf]
  582. x = len(self.buf)
  583. while x < size:
  584. raw = self.fileobj.read(self.blocksize)
  585. if not raw:
  586. break
  587. try:
  588. data = self.bz2obj.decompress(raw)
  589. except EOFError:
  590. break
  591. b.append(data)
  592. x += len(data)
  593. self.buf = "".join(b)
  594. buf = self.buf[:size]
  595. self.buf = self.buf[size:]
  596. self.pos += len(buf)
  597. return buf
  598. def seek(self, pos):
  599. if pos < self.pos:
  600. self.init()
  601. self.read(pos - self.pos)
  602. def tell(self):
  603. return self.pos
  604. def write(self, data):
  605. self.pos += len(data)
  606. raw = self.bz2obj.compress(data)
  607. self.fileobj.write(raw)
  608. def close(self):
  609. if self.mode == "w":
  610. raw = self.bz2obj.flush()
  611. self.fileobj.write(raw)
  612. # class _BZ2Proxy
  613. #------------------------
  614. # Extraction file object
  615. #------------------------
  616. class _FileInFile(object):
  617. """A thin wrapper around an existing file object that
  618. provides a part of its data as an individual file
  619. object.
  620. """
  621. def __init__(self, fileobj, offset, size, sparse=None):
  622. self.fileobj = fileobj
  623. self.offset = offset
  624. self.size = size
  625. self.sparse = sparse
  626. self.position = 0
  627. def tell(self):
  628. """Return the current file position.
  629. """
  630. return self.position
  631. def seek(self, position):
  632. """Seek to a position in the file.
  633. """
  634. self.position = position
  635. def read(self, size=None):
  636. """Read data from the file.
  637. """
  638. if size is None:
  639. size = self.size - self.position
  640. else:
  641. size = min(size, self.size - self.position)
  642. if self.sparse is None:
  643. return self.readnormal(size)
  644. else:
  645. return self.readsparse(size)
  646. def readnormal(self, size):
  647. """Read operation for regular files.
  648. """
  649. self.fileobj.seek(self.offset + self.position)
  650. self.position += size
  651. return self.fileobj.read(size)
  652. def readsparse(self, size):
  653. """Read operation for sparse files.
  654. """
  655. data = []
  656. while size > 0:
  657. buf = self.readsparsesection(size)
  658. if not buf:
  659. break
  660. size -= len(buf)
  661. data.append(buf)
  662. return "".join(data)
  663. def readsparsesection(self, size):
  664. """Read a single section of a sparse file.
  665. """
  666. section = self.sparse.find(self.position)
  667. if section is None:
  668. return ""
  669. size = min(size, section.offset + section.size - self.position)
  670. if isinstance(section, _data):
  671. realpos = section.realpos + self.position - section.offset
  672. self.fileobj.seek(self.offset + realpos)
  673. self.position += size
  674. return self.fileobj.read(size)
  675. else:
  676. self.position += size
  677. return NUL * size
  678. #class _FileInFile
  679. class ExFileObject(object):
  680. """File-like object for reading an archive member.
  681. Is returned by TarFile.extractfile().
  682. """
  683. blocksize = 1024
  684. def __init__(self, tarfile, tarinfo):
  685. self.fileobj = _FileInFile(tarfile.fileobj,
  686. tarinfo.offset_data,
  687. tarinfo.size,
  688. getattr(tarinfo, "sparse", None))
  689. self.name = tarinfo.name
  690. self.mode = "r"
  691. self.closed = False
  692. self.size = tarinfo.size
  693. self.position = 0
  694. self.buffer = ""
  695. def read(self, size=None):
  696. """Read at most size bytes from the file. If size is not
  697. present or None, read all data until EOF is reached.
  698. """
  699. if self.closed:
  700. raise ValueError("I/O operation on closed file")
  701. buf = ""
  702. if self.buffer:
  703. if size is None:
  704. buf = self.buffer
  705. self.buffer = ""
  706. else:
  707. buf = self.buffer[:size]
  708. self.buffer = self.buffer[size:]
  709. if size is None:
  710. buf += self.fileobj.read()
  711. else:
  712. buf += self.fileobj.read(size - len(buf))
  713. self.position += len(buf)
  714. return buf
  715. def readline(self, size=-1):
  716. """Read one entire line from the file. If size is present
  717. and non-negative, return a string with at most that
  718. size, which may be an incomplete line.
  719. """
  720. if self.closed:
  721. raise ValueError("I/O operation on closed file")
  722. if "\n" in self.buffer:
  723. pos = self.buffer.find("\n") + 1
  724. else:
  725. buffers = [self.buffer]
  726. while True:
  727. buf = self.fileobj.read(self.blocksize)
  728. buffers.append(buf)
  729. if not buf or "\n" in buf:
  730. self.buffer = "".join(buffers)
  731. pos = self.buffer.find("\n") + 1
  732. if pos == 0:
  733. # no newline found.
  734. pos = len(self.buffer)
  735. break
  736. if size != -1:
  737. pos = min(size, pos)
  738. buf = self.buffer[:pos]
  739. self.buffer = self.buffer[pos:]
  740. self.position += len(buf)
  741. return buf
  742. def readlines(self):
  743. """Return a list with all remaining lines.
  744. """
  745. result = []
  746. while True:
  747. line = self.readline()
  748. if not line: break
  749. result.append(line)
  750. return result
  751. def tell(self):
  752. """Return the current file position.
  753. """
  754. if self.closed:
  755. raise ValueError("I/O operation on closed file")
  756. return self.position
  757. def seek(self, pos, whence=os.SEEK_SET):
  758. """Seek to a position in the file.
  759. """
  760. if self.closed:
  761. raise ValueError("I/O operation on closed file")
  762. if whence == os.SEEK_SET:
  763. self.position = min(max(pos, 0), self.size)
  764. elif whence == os.SEEK_CUR:
  765. if pos < 0:
  766. self.position = max(self.position + pos, 0)
  767. else:
  768. self.position = min(self.position + pos, self.size)
  769. elif whence == os.SEEK_END:
  770. self.position = max(min(self.size + pos, self.size), 0)
  771. else:
  772. raise ValueError("Invalid argument")
  773. self.buffer = ""
  774. self.fileobj.seek(self.position)
  775. def close(self):
  776. """Close the file object.
  777. """
  778. self.closed = True
  779. def __iter__(self):
  780. """Get an iterator over the file's lines.
  781. """
  782. while True:
  783. line = self.readline()
  784. if not line:
  785. break
  786. yield line
  787. #class ExFileObject
  788. #------------------
  789. # Exported Classes
  790. #------------------
  791. class TarInfo(object):
  792. """Informational class which holds the details about an
  793. archive member given by a tar header block.
  794. TarInfo objects are returned by TarFile.getmember(),
  795. TarFile.getmembers() and TarFile.gettarinfo() and are
  796. usually created internally.
  797. """
  798. def __init__(self, name=""):
  799. """Construct a TarInfo object. name is the optional name
  800. of the member.
  801. """
  802. self.name = name # member name
  803. self.mode = 0644 # file permissions
  804. self.uid = 0 # user id
  805. self.gid = 0 # group id
  806. self.size = 0 # file size
  807. self.mtime = 0 # modification time
  808. self.chksum = 0 # header checksum
  809. self.type = REGTYPE # member type
  810. self.linkname = "" # link name
  811. self.uname = "root" # user name
  812. self.gname = "root" # group name
  813. self.devmajor = 0 # device major number
  814. self.devminor = 0 # device minor number
  815. self.offset = 0 # the tar header starts here
  816. self.offset_data = 0 # the file's data starts here
  817. self.pax_headers = {} # pax header information
  818. # In pax headers the "name" and "linkname" field are called
  819. # "path" and "linkpath".
  820. def _getpath(self):
  821. return self.name
  822. def _setpath(self, name):
  823. self.name = name
  824. path = property(_getpath, _setpath)
  825. def _getlinkpath(self):
  826. return self.linkname
  827. def _setlinkpath(self, linkname):
  828. self.linkname = linkname
  829. linkpath = property(_getlinkpath, _setlinkpath)
  830. def __repr__(self):
  831. return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
  832. def get_info(self, encoding, errors):
  833. """Return the TarInfo's attributes as a dictionary.
  834. """
  835. info = {
  836. "name": normpath(self.name),
  837. "mode": self.mode & 07777,
  838. "uid": self.uid,
  839. "gid": self.gid,
  840. "size": self.size,
  841. "mtime": self.mtime,
  842. "chksum": self.chksum,
  843. "type": self.type,
  844. "linkname": normpath(self.linkname) if self.linkname else "",
  845. "uname": self.uname,
  846. "gname": self.gname,
  847. "devmajor": self.devmajor,
  848. "devminor": self.devminor
  849. }
  850. if info["type"] == DIRTYPE and not info["name"].endswith("/"):
  851. info["name"] += "/"
  852. for key in ("name", "linkname", "uname", "gname"):
  853. if type(info[key]) is unicode:
  854. info[key] = info[key].encode(encoding, errors)
  855. return info
  856. def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="strict"):
  857. """Return a tar header as a string of 512 byte blocks.
  858. """
  859. info = self.get_info(encoding, errors)
  860. if format == USTAR_FORMAT:
  861. return self.create_ustar_header(info)
  862. elif format == GNU_FORMAT:
  863. return self.create_gnu_header(info)
  864. elif format == PAX_FORMAT:
  865. return self.create_pax_header(info, encoding, errors)
  866. else:
  867. raise ValueError("invalid format")
  868. def create_ustar_header(self, info):
  869. """Return the object as a ustar header block.
  870. """
  871. info["magic"] = POSIX_MAGIC
  872. if len(info["linkname"]) > LENGTH_LINK:
  873. raise ValueError("linkname is too long")
  874. if len(info["name"]) > LENGTH_NAME:
  875. info["prefix"], info["name"] = self._posix_split_name(info["name"])
  876. return self._create_header(info, USTAR_FORMAT)
  877. def create_gnu_header(self, info):
  878. """Return the object as a GNU header block sequence.
  879. """
  880. info["magic"] = GNU_MAGIC
  881. buf = ""
  882. if len(info["linkname"]) > LENGTH_LINK:
  883. buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK)
  884. if len(info["name"]) > LENGTH_NAME:
  885. buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME)
  886. return buf + self._create_header(info, GNU_FORMAT)
  887. def create_pax_header(self, info, encoding, errors):
  888. """Return the object as a ustar header block. If it cannot be
  889. represented this way, prepend a pax extended header sequence
  890. with supplement information.
  891. """
  892. info["magic"] = POSIX_MAGIC
  893. pax_headers = self.pax_headers.copy()
  894. # Test string fields for values that exceed the field length or cannot
  895. # be represented in ASCII encoding.
  896. for name, hname, length in (
  897. ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK),
  898. ("uname", "uname", 32), ("gname", "gname", 32)):
  899. if hname in pax_headers:
  900. # The pax header has priority.
  901. continue
  902. val = info[name].decode(encoding, errors)
  903. # Try to encode the string as ASCII.
  904. try:
  905. val.encode("ascii")
  906. except UnicodeEncodeError:
  907. pax_headers[hname] = val
  908. continue
  909. if len(info[name]) > length:
  910. pax_headers[hname] = val
  911. # Test number fields for values that exceed the field limit or values
  912. # that like to be stored as float.
  913. for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)):
  914. if name in pax_headers:
  915. # The pax header has priority. Avoid overflow.
  916. info[name] = 0
  917. continue
  918. val = info[name]
  919. if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float):
  920. pax_headers[name] = unicode(val)
  921. info[name] = 0
  922. # Create a pax extended header if necessary.
  923. if pax_headers:
  924. buf = self._create_pax_generic_header(pax_headers)
  925. else:
  926. buf = ""
  927. return buf + self._create_header(info, USTAR_FORMAT)
  928. @classmethod
  929. def create_pax_global_header(cls, pax_headers):
  930. """Return the object as a pax global header block sequence.
  931. """
  932. return cls._create_pax_generic_header(pax_headers, type=XGLTYPE)
  933. def _posix_split_name(self, name):
  934. """Split a name longer than 100 chars into a prefix
  935. and a name part.
  936. """
  937. prefix = name[:LENGTH_PREFIX + 1]
  938. while prefix and prefix[-1] != "/":
  939. prefix = prefix[:-1]
  940. name = name[len(prefix):]
  941. prefix = prefix[:-1]
  942. if not prefix or len(name) > LENGTH_NAME:
  943. raise ValueError("name is too long")
  944. return prefix, name
  945. @staticmethod
  946. def _create_header(info, format):
  947. """Return a header block. info is a dictionary with file
  948. information, format must be one of the *_FORMAT constants.
  949. """
  950. parts = [
  951. stn(info.get("name", ""), 100),
  952. itn(info.get("mode", 0) & 07777, 8, format),
  953. itn(info.get("uid", 0), 8, format),
  954. itn(info.get("gid", 0), 8, format),
  955. itn(info.get("size", 0), 12, format),
  956. itn(info.get("mtime", 0), 12, format),
  957. " ", # checksum field
  958. info.get("type", REGTYPE),
  959. stn(info.get("linkname", ""), 100),
  960. stn(info.get("magic", POSIX_MAGIC), 8),
  961. stn(info.get("uname", "root"), 32),
  962. stn(info.get("gname", "root"), 32),
  963. itn(info.get("devmajor", 0), 8, format),
  964. itn(info.get("devminor", 0), 8, format),
  965. stn(info.get("prefix", ""), 155)
  966. ]
  967. buf = struct.pack("%ds" % BLOCKSIZE, "".join(parts))
  968. chksum = calc_chksums(buf[-BLOCKSIZE:])[0]
  969. buf = buf[:-364] + "%06o\0" % chksum + buf[-357:]
  970. return buf
  971. @staticmethod
  972. def _create_payload(payload):
  973. """Return the string payload filled with zero bytes
  974. up to the next 512 byte border.
  975. """
  976. blocks, remainder = divmod(len(payload), BLOCKSIZE)
  977. if remainder > 0:
  978. payload += (BLOCKSIZE - remainder) * NUL
  979. return payload
  980. @classmethod
  981. def _create_gnu_long_header(cls, name, type):
  982. """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence
  983. for name.
  984. """
  985. name += NUL
  986. info = {}
  987. info["name"] = "././@LongLink"
  988. info["type"] = type
  989. info["size"] = len(name)
  990. info["magic"] = GNU_MAGIC
  991. # create extended header + name blocks.
  992. return cls._create_header(info, USTAR_FORMAT) + \
  993. cls._create_payload(name)
  994. @classmethod
  995. def _create_pax_generic_header(cls, pax_headers, type=XHDTYPE):
  996. """Return a POSIX.1-2001 extended or global header sequence
  997. that contains a list of keyword, value pairs. The values
  998. must be unicode objects.
  999. """
  1000. records = []
  1001. for keyword, value in pax_headers.iteritems():
  1002. keyword = keyword.encode("utf8")
  1003. value = value.encode("utf8")
  1004. l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n'
  1005. n = p = 0
  1006. while True:
  1007. n = l + len(str(p))
  1008. if n == p:
  1009. break
  1010. p = n
  1011. records.append("%d %s=%s\n" % (p, keyword, value))
  1012. records = "".join(records)
  1013. # We use a hardcoded "././@PaxHeader" name like star does
  1014. # instead of the one that POSIX recommends.
  1015. info = {}
  1016. info["name"] = "././@PaxHeader"
  1017. info["type"] = type
  1018. info["size"] = len(records)
  1019. info["magic"] = POSIX_MAGIC
  1020. # Create pax header + record blocks.
  1021. return cls._create_header(info, USTAR_FORMAT) + \
  1022. cls._create_payload(records)
  1023. @classmethod
  1024. def frombuf(cls, buf):
  1025. """Construct a TarInfo object from a 512 byte string buffer.
  1026. """
  1027. if len(buf) != BLOCKSIZE:
  1028. raise HeaderError("truncated header")
  1029. if buf.count(NUL) == BLOCKSIZE:
  1030. raise HeaderError("empty header")
  1031. chksum = nti(buf[148:156])
  1032. if chksum not in calc_chksums(buf):
  1033. raise HeaderError("bad checksum")
  1034. obj = cls()
  1035. obj.buf = buf
  1036. obj.name = nts(buf[0:100])
  1037. obj.mode = nti(buf[100:108])
  1038. obj.uid = nti(buf[108:116])
  1039. obj.gid = nti(buf[116:124])
  1040. obj.size = nti(buf[124:136])
  1041. obj.mtime = nti(buf[136:148])
  1042. obj.chksum = chksum
  1043. obj.type = buf[156:157]
  1044. obj.linkname = nts(buf[157:257])
  1045. obj.uname = nts(buf[265:297])
  1046. obj.gname = nts(buf[297:329])
  1047. obj.devmajor = nti(buf[329:337])
  1048. obj.devminor = nti(buf[337:345])
  1049. prefix = nts(buf[345:500])
  1050. # Old V7 tar format represents a directory as a regular
  1051. # file with a trailing slash.
  1052. if obj.type == AREGTYPE and obj.name.endswith("/"):
  1053. obj.type = DIRTYPE
  1054. # Remove redundant slashes from directories.
  1055. if obj.isdir():
  1056. obj.name = obj.name.rstrip("/")
  1057. # Reconstruct a ustar longname.
  1058. if prefix and obj.type not in GNU_TYPES:
  1059. obj.name = prefix + "/" + obj.name
  1060. return obj
  1061. @classmethod
  1062. def fromtarfile(cls, tarfile):
  1063. """Return the next TarInfo object from TarFile object
  1064. tarfile.
  1065. """
  1066. buf = tarfile.fileobj.read(BLOCKSIZE)
  1067. if not buf:
  1068. return
  1069. obj = cls.frombuf(buf)
  1070. obj.offset = tarfile.fileobj.tell() - BLOCKSIZE
  1071. return obj._proc_member(tarfile)
  1072. #--------------------------------------------------------------------------
  1073. # The following are methods that are called depending on the type of a
  1074. # member. The entry point is _proc_member() which can be overridden in a
  1075. # subclass to add custom _proc_*() methods. A _proc_*() method MUST
  1076. # implement the following
  1077. # operations:
  1078. # 1. Set self.offset_data to the position where the data blocks begin,
  1079. # if there is data that follows.
  1080. # 2. Set tarfile.offset to the position where the next member's header will
  1081. # begin.
  1082. # 3. Return self or another valid TarInfo object.
  1083. def _proc_member(self, tarfile):
  1084. """Choose the right processing method depending on
  1085. the type and call it.
  1086. """
  1087. if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK):
  1088. return self._proc_gnulong(tarfile)
  1089. elif self.type == GNUTYPE_SPARSE:
  1090. return self._proc_sparse(tarfile)
  1091. elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE):
  1092. return self._proc_pax(tarfile)
  1093. else:
  1094. return self._proc_builtin(tarfile)
  1095. def _proc_builtin(self, tarfile):
  1096. """Process a builtin type or an unknown type which
  1097. will be treated as a regular file.
  1098. """
  1099. self.offset_data = tarfile.fileobj.tell()
  1100. offset = self.offset_data
  1101. if self.isreg() or self.type not in SUPPORTED_TYPES:
  1102. # Skip the following data blocks.
  1103. offset += self._block(self.size)
  1104. tarfile.offset = offset
  1105. # Patch the TarInfo object with saved global
  1106. # header information.
  1107. self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors)
  1108. return self
  1109. def _proc_gnulong(self, tarfile):
  1110. """Process the blocks that hold a GNU longname
  1111. or longlink member.
  1112. """
  1113. buf = tarfile.fileobj.read(self._block(self.size))
  1114. # Fetch the next header and process it.
  1115. next = self.fromtarfile(tarfile)
  1116. if next is None:
  1117. raise HeaderError("missing subsequent header")
  1118. # Patch the TarInfo object from the next header with
  1119. # the longname information.
  1120. next.offset = self.offset
  1121. if self.type == GNUTYPE_LONGNAME:
  1122. next.name = nts(buf)
  1123. elif self.type == GNUTYPE_LONGLINK:
  1124. next.linkname = nts(buf)
  1125. return next
  1126. def _proc_sparse(self, tarfile):
  1127. """Process a GNU sparse header plus extra headers.
  1128. """
  1129. buf = self.buf
  1130. sp = _ringbuffer()
  1131. pos = 386
  1132. lastpos = 0L
  1133. realpos = 0L
  1134. # There are 4 possible sparse structs in the
  1135. # first header.
  1136. for i in xrange(4):
  1137. try:
  1138. offset = nti(buf[pos:pos + 12])
  1139. numbytes = nti(buf[pos + 12:pos + 24])
  1140. except ValueError:
  1141. break
  1142. if offset > lastpos:
  1143. sp.append(_hole(lastpos, offset - lastpos))
  1144. sp.append(_data(offset, numbytes, realpos))
  1145. realpos += numbytes
  1146. lastpos = offset + numbytes
  1147. pos += 24
  1148. isextended = ord(buf[482])
  1149. origsize = nti(buf[483:495])
  1150. # If the isextended flag is given,
  1151. # there are extra headers to process.
  1152. while isextended == 1:
  1153. buf = tarfile.fileobj.read(BLOCKSIZE)
  1154. pos = 0
  1155. for i in xrange(21):
  1156. try:
  1157. offset = nti(buf[pos:pos + 12])
  1158. numbytes = nti(buf[pos + 12:pos + 24])
  1159. except ValueError:
  1160. break
  1161. if offset > lastpos:
  1162. sp.append(_hole(lastpos, offset - lastpos))
  1163. sp.append(_data(offset, numbytes, realpos))
  1164. realpos += numbytes
  1165. lastpos = offset + numbytes
  1166. pos += 24
  1167. isextended = ord(buf[504])
  1168. if lastpos < origsize:
  1169. sp.append(_hole(lastpos, origsize - lastpos))
  1170. self.sparse = sp
  1171. self.offset_data = tarfile.fileobj.tell()
  1172. tarfile.offset = self.offset_data + self._block(self.size)
  1173. self.size = origsize
  1174. return self
  1175. def _proc_pax(self, tarfile):
  1176. """Process an extended or global header as described in
  1177. POSIX.1-2001.
  1178. """
  1179. # Read the header information.
  1180. buf = tarfile.fileobj.read(self._block(self.size))
  1181. # A pax header stores supplemental information for either
  1182. # the following file (extended) or all following files
  1183. # (global).
  1184. if self.type == XGLTYPE:
  1185. pax_headers = tarfile.pax_headers
  1186. else:
  1187. pax_headers = tarfile.pax_headers.copy()
  1188. # Parse pax header information. A record looks like that:
  1189. # "%d %s=%s\n" % (length, keyword, value). length is the size
  1190. # of the complete record including the length field itself and
  1191. # the newline. keyword and value are both UTF-8 encoded strings.
  1192. regex = re.compile(r"(\d+) ([^=]+)=", re.U)
  1193. pos = 0
  1194. while True:
  1195. match = regex.match(buf, pos)
  1196. if not match:
  1197. break
  1198. length, keyword = match.groups()
  1199. length = int(length)
  1200. value = buf[match.end(2) + 1:match.start(1) + length - 1]
  1201. keyword = keyword.decode("utf8")
  1202. value = value.decode("utf8")
  1203. pax_headers[keyword] = value
  1204. pos += length
  1205. # Fetch the next header.
  1206. next = self.fromtarfile(tarfile)
  1207. if self.type in (XHDTYPE, SOLARIS_XHDTYPE):
  1208. if next is None:
  1209. raise HeaderError("missing subsequent header")
  1210. # Patch the TarInfo object with the extended header info.
  1211. next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors)
  1212. next.offset = self.offset
  1213. if "size" in pax_headers:
  1214. # If the extended header replaces the size field,
  1215. # we need to recalculate the offset where the next
  1216. # header starts.
  1217. offset = next.offset_data
  1218. if next.isreg() or next.type not in SUPPORTED_TYPES:
  1219. offset += next._block(next.size)
  1220. tarfile.offset = offset
  1221. return next
  1222. def _apply_pax_info(self, pax_headers, encoding, errors):
  1223. """Replace fields with supplemental information from a previous
  1224. pax extended or global header.
  1225. """
  1226. for keyword, value in pax_headers.iteritems():
  1227. if keyword not in PAX_FIELDS:
  1228. continue
  1229. if keyword == "path":
  1230. value = value.rstrip("/")
  1231. if keyword in PAX_NUMBER_FIELDS:
  1232. try:
  1233. value = PAX_NUMBER_FIELDS[keyword](value)
  1234. except ValueError:
  1235. value = 0
  1236. else:
  1237. value = uts(value, encoding, errors)
  1238. setattr(self, keyword, value)
  1239. self.pax_headers = pax_headers.copy()
  1240. def _block(self, count):
  1241. """Round up a byte count by BLOCKSIZE and return it,
  1242. e.g. _block(834) => 1024.
  1243. """
  1244. blocks, remainder = divmod(count, BLOCKSIZE)
  1245. if remainder:
  1246. blocks += 1
  1247. return blocks * BLOCKSIZE
  1248. def isreg(self):
  1249. return self.type in REGULAR_TYPES
  1250. def isfile(self):
  1251. return self.isreg()
  1252. def isdir(self):
  1253. return self.type == DIRTYPE
  1254. def issym(self):
  1255. return self.type == SYMTYPE
  1256. def islnk(self):
  1257. return self.type == LNKTYPE
  1258. def ischr(self):
  1259. return self.type == CHRTYPE
  1260. def isblk(self):
  1261. return self.type == BLKTYPE
  1262. def isfifo(self):
  1263. return self.type == FIFOTYPE
  1264. def issparse(self):
  1265. return self.type == GNUTYPE_SPARSE
  1266. def isdev(self):
  1267. return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)
  1268. # class TarInfo
  1269. class TarFile(object):
  1270. """The TarFile Class provides an interface to tar archives.
  1271. """
  1272. debug = 0 # May be set from 0 (no msgs) to 3 (all msgs)
  1273. dereference = False # If true, add content of linked file to the
  1274. # tar file, else the link.
  1275. ignore_zeros = False # If true, skips empty or invalid blocks and
  1276. # continues processing.
  1277. errorlevel = 0 # If 0, fatal errors only appear in debug
  1278. # messages (if debug >= 0). If > 0, errors
  1279. # are passed to the caller as exceptions.
  1280. format = DEFAULT_FORMAT # The format to use when creating an archive.
  1281. encoding = ENCODING # Encoding for 8-bit character strings.
  1282. errors = None # Error handler for unicode conversion.
  1283. tarinfo = TarInfo # The default TarInfo class to use.
  1284. fileobject = ExFileObject # The default ExFileObject class to use.
  1285. def __init__(self, name=None, mode="r", fileobj=None, format=None,
  1286. tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
  1287. errors=None, pax_headers=None, debug=None, errorlevel=None):
  1288. """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to
  1289. read from an existing archive, 'a' to append data to an existing
  1290. file or 'w' to create a new file overwriting an existing one. `mode'
  1291. defaults to 'r'.
  1292. If `fileobj' is given, it is used for reading or writing data. If it
  1293. can be determined, `mode' is overridden by `fileobj's mode.
  1294. `fileobj' is not closed, when TarFile is closed.
  1295. """
  1296. if len(mode) > 1 or mode not in "raw":
  1297. raise ValueError("mode must be 'r', 'a' or 'w'")
  1298. self.mode = mode
  1299. self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
  1300. if not fileobj:
  1301. if self.mode == "a" and not os.path.exists(name):
  1302. # Create nonexistent files in append mode.
  1303. self.mode = "w"
  1304. self._mode = "wb"
  1305. fileobj = bltn_open(name, self._mode)
  1306. self._extfileobj = False
  1307. else:
  1308. if name is None and hasattr(fileobj, "name"):
  1309. name = fileobj.name
  1310. if hasattr(fileobj, "mode"):
  1311. self._mode = fileobj.mode
  1312. self._extfileobj = True
  1313. self.name = os.path.abspath(name) if name else None
  1314. self.fileobj = fileobj
  1315. # Init attributes.
  1316. if format is not None:
  1317. self.format = format
  1318. if tarinfo is not None:
  1319. self.tarinfo = tarinfo
  1320. if dereference is not None:
  1321. self.dereference = dereference
  1322. if ignore_zeros is not None:
  1323. self.ignore_zeros = ignore_zeros
  1324. if encoding is not None:
  1325. self.encoding = encoding
  1326. if errors is not None:
  1327. self.errors = errors
  1328. elif mode == "r":
  1329. self.errors = "utf-8"
  1330. else:
  1331. self.errors = "strict"
  1332. if pax_headers is not None and self.format == PAX_FORMAT:
  1333. self.pax_headers = pax_headers
  1334. else:
  1335. self.pax_headers = {}
  1336. if debug is not None:
  1337. self.debug = debug
  1338. if errorlevel is not None:
  1339. self.errorlevel = errorlevel
  1340. # Init datastructures.
  1341. self.closed = False
  1342. self.members = [] # list of members as TarInfo objects
  1343. self._loaded = False # flag if all members have been read
  1344. self.offset = self.fileobj.tell()
  1345. # current position in the archive file
  1346. self.inodes = {} # dictionary caching the inodes of
  1347. # archive members already added
  1348. if self.mode == "r":
  1349. self.firstmember = None
  1350. self.firstmember = self.next()
  1351. if self.mode == "a":
  1352. # Move to the end of the archive,
  1353. # before the first empty block.
  1354. self.firstmember = None
  1355. while True:
  1356. if self.next() is None:
  1357. if self.offset > 0:
  1358. self.fileobj.seek(- BLOCKSIZE, 1)
  1359. break
  1360. if self.mode in "aw":
  1361. self._loaded = True
  1362. if self.pax_headers:
  1363. buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy())
  1364. self.fileobj.write(buf)
  1365. self.offset += len(buf)
  1366. def _getposix(self):
  1367. return self.format == USTAR_FORMAT
  1368. def _setposix(self, value):
  1369. import warnings
  1370. warnings.warn("use the format attribute instead", DeprecationWarning,
  1371. 2)
  1372. if value:
  1373. self.format = USTAR_FORMAT
  1374. else:
  1375. self.format = GNU_FORMAT
  1376. posix = property(_getposix, _setposix)
  1377. #--------------------------------------------------------------------------
  1378. # Below are the classmethods which act as alternate constructors to the
  1379. # TarFile class. The open() method is the only one that is needed for
  1380. # public use; it is the "super"-constructor and is able to select an
  1381. # adequate "sub"-constructor for a particular compression using the mapping
  1382. # from OPEN_METH.
  1383. #
  1384. # This concept allows one to subclass TarFile without losing the comfort of
  1385. # the super-constructor. A sub-constructor is registered and made available
  1386. # by adding it to the mapping in OPEN_METH.
  1387. @classmethod
  1388. def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
  1389. """Open a tar archive for reading, writing or appending. Return
  1390. an appropriate TarFile class.
  1391. mode:
  1392. 'r' or 'r:*' open for reading with transparent compression
  1393. 'r:' open for reading exclusively uncompressed
  1394. 'r:gz' open for reading with gzip compression
  1395. 'r:bz2' open for reading with bzip2 compression
  1396. 'a' or 'a:' open for appending, creating the file if necessary
  1397. 'w' or 'w:' open for writing without compression
  1398. 'w:gz' open for writing with gzip compression
  1399. 'w:bz2' open for writing with bzip2 compression
  1400. 'r|*' open a stream of tar blocks with transparent compression
  1401. 'r|' open an uncompressed stream of tar blocks for reading
  1402. 'r|gz' open a gzip compressed stream of tar blocks
  1403. 'r|bz2' open a bzip2 compressed stream of tar blocks
  1404. 'w|' open an uncompressed stream for writing
  1405. 'w|gz' open a gzip compressed stream for writing
  1406. 'w|bz2' open a bzip2 compressed stream for writing
  1407. """
  1408. if not name and not fileobj:
  1409. raise ValueError("nothing to open")
  1410. if mode in ("r", "r:*"):
  1411. # Find out which *open() is appropriate for opening the file.
  1412. for comptype in cls.OPEN_METH:
  1413. func = getattr(cls, cls.OPEN_METH[comptype])
  1414. if fileobj is not None:
  1415. saved_pos = fileobj.tell()
  1416. try:
  1417. return func(name, "r", fileobj, **kwargs)
  1418. except (ReadError, CompressionError), e:
  1419. if fileobj is not None:
  1420. fileobj.seek(saved_pos)
  1421. continue
  1422. raise ReadError("file could not be opened successfully")
  1423. elif ":" in mode:
  1424. filemode, comptype = mode.split(":", 1)
  1425. filemode = filemode or "r"
  1426. comptype = comptype or "tar"
  1427. # Select the *open() function according to
  1428. # given compression.
  1429. if comptype in cls.OPEN_METH:
  1430. func = getattr(cls, cls.OPEN_METH[comptype])
  1431. else:
  1432. raise CompressionError("unknown compression type %r" % comptype)
  1433. return func(name, filemode, fileobj, **kwargs)
  1434. elif "|" in mode:
  1435. filemode, comptype = mode.split("|", 1)
  1436. filemode = filemode or "r"
  1437. comptype = comptype or "tar"
  1438. if filemode not in "rw":
  1439. raise ValueError("mode must be 'r' or 'w'")
  1440. t = cls(name, filemode,
  1441. _Stream(name, filemode, comptype, fileobj, bufsize),
  1442. **kwargs)
  1443. t._extfileobj = False
  1444. return t
  1445. elif mode in "aw":
  1446. return cls.taropen(name, mode, fileobj, **kwargs)
  1447. raise ValueError("undiscernible mode")
  1448. @classmethod
  1449. def taropen(cls, name, mode="r", fileobj=None, **kwargs):
  1450. """Open uncompressed tar archive name for reading or writing.
  1451. """
  1452. if len(mode) > 1 or mode not in "raw":
  1453. raise ValueError("mode must be 'r', 'a' or 'w'")
  1454. return cls(name, mode, fileobj, **kwargs)
  1455. @classmethod
  1456. def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
  1457. """Open gzip compressed tar archive name for reading or writing.
  1458. Appending is not allowed.
  1459. """
  1460. if len(mode) > 1 or mode not in "rw":
  1461. raise ValueError("mode must be 'r' or 'w'")
  1462. try:
  1463. import gzip
  1464. gzip.GzipFile
  1465. except (ImportError, AttributeError):
  1466. raise CompressionError("gzip module is not available")
  1467. if fileobj is None:
  1468. fileobj = bltn_open(name, mode + "b")
  1469. try:
  1470. t = cls.taropen(name, mode,
  1471. gzip.GzipFile(name, mode, compresslevel, fileobj),
  1472. **kwargs)
  1473. except IOError:
  1474. raise ReadError("not a gzip file")
  1475. t._extfileobj = False
  1476. return t
  1477. @classmethod
  1478. def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
  1479. """Open bzip2 compressed tar archive name for reading or writing.
  1480. Appending is not allowed.
  1481. """
  1482. if len(mode) > 1 or mode not in "rw":
  1483. raise ValueError("mode must be 'r' or 'w'.")
  1484. try:
  1485. import bz2
  1486. except ImportError:
  1487. raise CompressionError("bz2 module is not available")
  1488. if fileobj is not None:
  1489. fileobj = _BZ2Proxy(fileobj, mode)
  1490. else:
  1491. fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
  1492. try:
  1493. t = cls.taropen(name, mode, fileobj, **kwargs)
  1494. except IOError:
  1495. raise ReadError("not a bzip2 file")
  1496. t._extfileobj = False
  1497. return t
  1498. # All *open() methods are registered here.
  1499. OPEN_METH = {
  1500. "tar": "taropen", # uncompressed tar
  1501. "gz": "gzopen", # gzip compressed tar
  1502. "bz2": "bz2open" # bzip2 compressed tar
  1503. }
  1504. #--------------------------------------------------------------------------
  1505. # The public methods which TarFile provides:
  1506. def close(self):
  1507. """Close the TarFile. In write-mode, two finishing zero blocks are
  1508. appended to the archive.
  1509. """
  1510. if self.closed:
  1511. return
  1512. if self.mode in "aw":
  1513. self.fileobj.write(NUL * (BLOCKSIZE * 2))
  1514. self.offset += (BLOCKSIZE * 2)
  1515. # fill up the end with zero-blocks
  1516. # (like option -b20 for tar does)
  1517. blocks, remainder = divmod(self.offset, RECORDSIZE)
  1518. if remainder > 0:
  1519. self.fileobj.write(NUL * (RECORDSIZE - remainder))
  1520. if not self._extfileobj:
  1521. self.fileobj.close()
  1522. self.closed = True
  1523. def getmember(self, name):
  1524. """Return a TarInfo object for member `name'. If `name' can not be
  1525. found in the archive, KeyError is raised. If a member occurs more
  1526. than once in the archive, its last occurrence is assumed to be the
  1527. most up-to-date version.
  1528. """
  1529. tarinfo = self._getmember(name)
  1530. if tarinfo is None:
  1531. raise KeyError("filename %r not found" % name)
  1532. return tarinfo
  1533. def getmembers(self):
  1534. """Return the members of the archive as a list of TarInfo objects. The
  1535. list has the same order as the members in the archive.
  1536. """
  1537. self._check()
  1538. if not self._loaded: # if we want to obtain a list of
  1539. self._load() # all members, we first have to
  1540. # scan the whole archive.
  1541. return self.members
  1542. def getnames(self):
  1543. """Return the members of the archive as a list of their names. It has
  1544. the same order as the list returned by getmembers().
  1545. """
  1546. return [tarinfo.name for tarinfo in self.getmembers()]
  1547. def gettarinfo(self, name=None, arcname=None, fileobj=None):
  1548. """Create a TarInfo object for either the file `name' or the file
  1549. object `fileobj' (using os.fstat on its file descriptor). You can
  1550. modify some of the TarInfo's attributes before you add it using
  1551. addfile(). If given, `arcname' specifies an alternative name for the
  1552. file in the archive.
  1553. """
  1554. self._check("aw")
  1555. # When fileobj is given, replace name by
  1556. # fileobj's real name.
  1557. if fileobj is not None:
  1558. name = fileobj.name
  1559. # Building the name of the member in the archive.
  1560. # Backward slashes are converted to forward slashes,
  1561. # Absolute paths are turned to relative paths.
  1562. if arcname is None:
  1563. arcname = name
  1564. arcname = normpath(arcname)
  1565. drv, arcname = os.path.splitdrive(arcname)
  1566. while arcname[0:1] == "/":
  1567. arcname = arcname[1:]
  1568. # Now, fill the TarInfo object with
  1569. # information specific for the file.
  1570. tarinfo = self.tarinfo()
  1571. tarinfo.tarfile = self
  1572. # Use os.stat or os.lstat, depending on platform
  1573. # and if symlinks shall be resolved.
  1574. if fileobj is None:
  1575. if hasattr(os, "lstat") and not self.dereference:
  1576. statres = os.lstat(name)
  1577. else:
  1578. statres = os.stat(name)
  1579. else:
  1580. statres = os.fstat(fileobj.fileno())
  1581. linkname = ""
  1582. stmd = statres.st_mode
  1583. if stat.S_ISREG(stmd):
  1584. inode = (statres.st_ino, statres.st_dev)
  1585. if not self.dereference and statres.st_nlink > 1 and \
  1586. inode in self.inodes and arcname != self.inodes[inode]:
  1587. # Is it a hardlink to an already
  1588. # archived file?
  1589. type = LNKTYPE
  1590. linkname = self.inodes[inode]
  1591. else:
  1592. # The inode is added only if its valid.
  1593. # For win32 it is always 0.
  1594. type = REGTYPE
  1595. if inode[0]:
  1596. self.inodes[inode] = arcname
  1597. elif stat.S_ISDIR(stmd):
  1598. type = DIRTYPE
  1599. elif stat.S_ISFIFO(stmd):
  1600. type = FIFOTYPE
  1601. elif stat.S_ISLNK(stmd):
  1602. type = SYMTYPE
  1603. linkname = os.readlink(name)
  1604. elif stat.S_ISCHR(stmd):
  1605. type = CHRTYPE
  1606. elif stat.S_ISBLK(stmd):
  1607. type = BLKTYPE
  1608. else:
  1609. return None
  1610. # Fill the TarInfo object with all
  1611. # information we can get.
  1612. tarinfo.name = arcname
  1613. tarinfo.mode = stmd
  1614. tarinfo.uid = statres.st_uid
  1615. tarinfo.gid = statres.st_gid
  1616. if stat.S_ISREG(stmd):
  1617. tarinfo.size = statres.st_size
  1618. else:
  1619. tarinfo.size = 0L
  1620. tarinfo.mtime = statres.st_mtime
  1621. tarinfo.type = type
  1622. tarinfo.linkname = linkname
  1623. if pwd:
  1624. try:
  1625. tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
  1626. except KeyError:
  1627. pass
  1628. if grp:
  1629. try:
  1630. tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
  1631. except KeyError:
  1632. pass
  1633. if type in (CHRTYPE, BLKTYPE):
  1634. if hasattr(os, "major") and hasattr(os, "minor"):
  1635. tarinfo.devmajor = os.major(statres.st_rdev)
  1636. tarinfo.devminor = os.minor(statres.st_rdev)
  1637. return tarinfo
  1638. def list(self, verbose=True):
  1639. """Print a table of contents to sys.stdout. If `verbose' is False, only
  1640. the names of the members are printed. If it is True, an `ls -l'-like
  1641. output is produced.
  1642. """
  1643. self._check()
  1644. for tarinfo in self:
  1645. if verbose:
  1646. print filemode(tarinfo.mode),
  1647. print "%s/%s" % (tarinfo.uname or tarinfo.uid,
  1648. tarinfo.gname or tarinfo.gid),
  1649. if tarinfo.ischr() or tarinfo.isblk():
  1650. print "%10s" % ("%d,%d" \
  1651. % (tarinfo.devmajor, tarinfo.devminor)),
  1652. else:
  1653. print "%10d" % tarinfo.size,
  1654. print "%d-%02d-%02d %02d:%02d:%02d" \
  1655. % time.localtime(tarinfo.mtime)[:6],
  1656. print tarinfo.name + ("/" if tarinfo.isdir() else ""),
  1657. if verbose:
  1658. if tarinfo.issym():
  1659. print "->", tarinfo.linkname,
  1660. if tarinfo.islnk():
  1661. print "link to", tarinfo.linkname,
  1662. print
  1663. def add(self, name, arcname=None, recursive=True, exclude=None):
  1664. """Add the file `name' to the archive. `name' may be any type of file
  1665. (directory, fifo, symbolic link, etc.). If given, `arcname'
  1666. specifies an alternative name for the file in the archive.
  1667. Directories are added recursively by default. This can be avoided by
  1668. setting `recursive' to False. `exclude' is a function that should
  1669. return True for each filename to be excluded.
  1670. """
  1671. self._check("aw")
  1672. if arcname is None:
  1673. arcname = name
  1674. # Exclude pathnames.
  1675. if exclude is not None and exclude(name):
  1676. self._dbg(2, "tarfile: Excluded %r" % name)
  1677. return
  1678. # Skip if somebody tries to archive the archive...
  1679. if self.name is not None and os.path.abspath(name) == self.name:
  1680. self._dbg(2, "tarfile: Skipped %r" % name)
  1681. return
  1682. # Special case: The user wants to add the current
  1683. # working directory.
  1684. if name == ".":
  1685. if recursive:
  1686. if arcname == ".":
  1687. arcname = ""
  1688. for f in os.listdir(name):
  1689. self.add(f, os.path.join(arcname, f), recursive, exclude)
  1690. return
  1691. self._dbg(1, name)
  1692. # Create a TarInfo object from the file.
  1693. tarinfo = self.gettarinfo(name, arcname)
  1694. if tarinfo is None:
  1695. self._dbg(1, "tarfile: Unsupported type %r" % name)
  1696. return
  1697. # Append the tar header and data to the archive.
  1698. if tarinfo.isreg():
  1699. f = bltn_open(name, "rb")
  1700. self.addfile(tarinfo, f)
  1701. f.close()
  1702. elif tarinfo.isdir():
  1703. self.addfile(tarinfo)
  1704. if recursive:
  1705. for f in os.listdir(name):
  1706. self.add(os.path.join(name, f), os.path.join(arcname, f), recursive, exclude)
  1707. else:
  1708. self.addfile(tarinfo)
  1709. def addfile(self, tarinfo, fileobj=None):
  1710. """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
  1711. given, tarinfo.size bytes are read from it and added to the archive.
  1712. You can create TarInfo objects using gettarinfo().
  1713. On Windows platforms, `fileobj' should always be opened with mode
  1714. 'rb' to avoid irritation about the file size.
  1715. """
  1716. self._check("aw")
  1717. tarinfo = copy.copy(tarinfo)
  1718. buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
  1719. self.fileobj.write(buf)
  1720. self.offset += len(buf)
  1721. # If there's data to follow, append it.
  1722. if fileobj is not None:
  1723. copyfileobj(fileobj, self.fileobj, tarinfo.size)
  1724. blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
  1725. if remainder > 0:
  1726. self.fileobj.write(NUL * (BLOCKSIZE - remainder))
  1727. blocks += 1
  1728. self.offset += blocks * BLOCKSIZE
  1729. self.members.append(tarinfo)
  1730. def extractall(self, path=".", members=None):
  1731. """Extract all members from the archive to the current working
  1732. directory and set owner, modification time and permissions on
  1733. directories afterwards. `path' specifies a different directory
  1734. to extract to. `members' is optional and must be a subset of the
  1735. list returned by getmembers().
  1736. """
  1737. directories = []
  1738. if members is None:
  1739. members = self
  1740. for tarinfo in members:
  1741. if tarinfo.isdir():
  1742. # Extract directories with a safe mode.
  1743. directories.append(tarinfo)
  1744. tarinfo = copy.copy(tarinfo)
  1745. tarinfo.mode = 0700
  1746. self.extract(tarinfo, path)
  1747. # Reverse sort directories.
  1748. directories.sort(key=operator.attrgetter('name'))
  1749. directories.reverse()
  1750. # Set correct owner, mtime and filemode on directories.
  1751. for tarinfo in directories:
  1752. dirpath = os.path.join(path, tarinfo.name)
  1753. try:
  1754. self.chown(tarinfo, dirpath)
  1755. self.utime(tarinfo, dirpath)
  1756. self.chmod(tarinfo, dirpath)
  1757. except ExtractError, e:
  1758. if self.errorlevel > 1:
  1759. raise
  1760. else:
  1761. self._dbg(1, "tarfile: %s" % e)
  1762. def extract(self, member, path=""):
  1763. """Extract a member from the archive to the current working directory,
  1764. using its full name. Its file information is extracted as accurately
  1765. as possible. `member' may be a filename or a TarInfo object. You can
  1766. specify a different directory using `path'.
  1767. """
  1768. self._check("r")
  1769. if isinstance(member, basestring):
  1770. tarinfo = self.getmember(member)
  1771. else:
  1772. tarinfo = member
  1773. # Prepare the link target for makelink().
  1774. if tarinfo.islnk():
  1775. tarinfo._link_target = os.path.join(path, tarinfo.linkname)
  1776. try:
  1777. self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
  1778. except EnvironmentError, e:
  1779. if self.errorlevel > 0:
  1780. raise
  1781. else:
  1782. if e.filename is None:
  1783. self._dbg(1, "tarfile: %s" % e.strerror)
  1784. else:
  1785. self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
  1786. except ExtractError, e:
  1787. if self.errorlevel > 1:
  1788. raise
  1789. else:
  1790. self._dbg(1, "tarfile: %s" % e)
  1791. def extractfile(self, member):
  1792. """Extract a member from the archive as a file object. `member' may be
  1793. a filename or a TarInfo object. If `member' is a regular file, a
  1794. file-like object is returned. If `member' is a link, a file-like
  1795. object is constructed from the link's target. If `member' is none of
  1796. the above, None is returned.
  1797. The file-like object is read-only and provides the following
  1798. methods: read(), readline(), readlines(), seek() and tell()
  1799. """
  1800. self._check("r")
  1801. if isinstance(member, basestring):
  1802. tarinfo = self.getmember(member)
  1803. else:
  1804. tarinfo = member
  1805. if tarinfo.isreg():
  1806. return self.fileobject(self, tarinfo)
  1807. elif tarinfo.type not in SUPPORTED_TYPES:
  1808. # If a member's type is unknown, it is treated as a
  1809. # regular file.
  1810. return self.fileobject(self, tarinfo)
  1811. elif tarinfo.islnk() or tarinfo.issym():
  1812. if isinstance(self.fileobj, _Stream):
  1813. # A small but ugly workaround for the case that someone tries
  1814. # to extract a (sym)link as a file-object from a non-seekable
  1815. # stream of tar blocks.
  1816. raise StreamError("cannot extract (sym)link as file object")
  1817. else:
  1818. # A (sym)link's file object is its target's file object.
  1819. return self.extractfile(self._getmember(tarinfo.linkname,
  1820. tarinfo))
  1821. else:
  1822. # If there's no data associated with the member (directory, chrdev,
  1823. # blkdev, etc.), return None instead of a file object.
  1824. return None
  1825. def _extract_member(self, tarinfo, targetpath):
  1826. """Extract the TarInfo object tarinfo to a physical
  1827. file called targetpath.
  1828. """
  1829. # Fetch the TarInfo object for the given name
  1830. # and build the destination pathname, replacing
  1831. # forward slashes to platform specific separators.
  1832. if targetpath[-1:] == "/":
  1833. targetpath = targetpath[:-1]
  1834. targetpath = os.path.normpath(targetpath)
  1835. # Create all upper directories.
  1836. upperdirs = os.path.dirname(targetpath)
  1837. if upperdirs and not os.path.exists(upperdirs):
  1838. # Create directories that are not part of the archive with
  1839. # default permissions.
  1840. os.makedirs(upperdirs)
  1841. if tarinfo.islnk() or tarinfo.issym():
  1842. self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname))
  1843. else:
  1844. self._dbg(1, tarinfo.name)
  1845. if tarinfo.isreg():
  1846. self.makefile(tarinfo, targetpath)
  1847. elif tarinfo.isdir():
  1848. self.makedir(tarinfo, targetpath)
  1849. elif tarinfo.isfifo():
  1850. self.makefifo(tarinfo, targetpath)
  1851. elif tarinfo.ischr() or tarinfo.isblk():
  1852. self.makedev(tarinfo, targetpath)
  1853. elif tarinfo.islnk() or tarinfo.issym():
  1854. self.makelink(tarinfo, targetpath)
  1855. elif tarinfo.type not in SUPPORTED_TYPES:
  1856. self.makeunknown(tarinfo, targetpath)
  1857. else:
  1858. self.makefile(tarinfo, targetpath)
  1859. self.chown(tarinfo, targetpath)
  1860. if not tarinfo.issym():
  1861. self.chmod(tarinfo, targetpath)
  1862. self.utime(tarinfo, targetpath)
  1863. #--------------------------------------------------------------------------
  1864. # Below are the different file methods. They are called via
  1865. # _extract_member() when extract() is called. They can be replaced in a
  1866. # subclass to implement other functionality.
  1867. def makedir(self, tarinfo, targetpath):
  1868. """Make a directory called targetpath.
  1869. """
  1870. try:
  1871. # Use a safe mode for the directory, the real mode is set
  1872. # later in _extract_member().
  1873. os.mkdir(targetpath, 0700)
  1874. except EnvironmentError, e:
  1875. if e.errno != errno.EEXIST:
  1876. raise
  1877. def makefile(self, tarinfo, targetpath):
  1878. """Make a file called targetpath.
  1879. """
  1880. source = self.extractfile(tarinfo)
  1881. target = bltn_open(targetpath, "wb")
  1882. copyfileobj(source, target)
  1883. source.close()
  1884. target.close()
  1885. def makeunknown(self, tarinfo, targetpath):
  1886. """Make a file from a TarInfo object with an unknown type
  1887. at targetpath.
  1888. """
  1889. self.makefile(tarinfo, targetpath)
  1890. self._dbg(1, "tarfile: Unknown file type %r, " \
  1891. "extracted as regular file." % tarinfo.type)
  1892. def makefifo(self, tarinfo, targetpath):
  1893. """Make a fifo called targetpath.
  1894. """
  1895. if hasattr(os, "mkfifo"):
  1896. os.mkfifo(targetpath)
  1897. else:
  1898. raise ExtractError("fifo not supported by system")
  1899. def makedev(self, tarinfo, targetpath):
  1900. """Make a character or block device called targetpath.
  1901. """
  1902. if not hasattr(os, "mknod") or not hasattr(os, "makedev"):
  1903. raise ExtractError("special devices not supported by system")
  1904. mode = tarinfo.mode
  1905. if tarinfo.isblk():
  1906. mode |= stat.S_IFBLK
  1907. else:
  1908. mode |= stat.S_IFCHR
  1909. os.mknod(targetpath, mode,
  1910. os.makedev(tarinfo.devmajor, tarinfo.devminor))
  1911. def makelink(self, tarinfo, targetpath):
  1912. """Make a (symbolic) link called targetpath. If it cannot be created
  1913. (platform limitation), we try to make a copy of the referenced file
  1914. instead of a link.
  1915. """
  1916. linkpath = tarinfo.linkname
  1917. try:
  1918. if tarinfo.issym():
  1919. os.symlink(linkpath, targetpath)
  1920. else:
  1921. # See extract().
  1922. os.link(tarinfo._link_target, targetpath)
  1923. except AttributeError:
  1924. if tarinfo.issym():
  1925. linkpath = os.path.join(os.path.dirname(tarinfo.name),
  1926. linkpath)
  1927. linkpath = normpath(linkpath)
  1928. try:
  1929. self._extract_member(self.getmember(linkpath), targetpath)
  1930. except (EnvironmentError, KeyError), e:
  1931. linkpath = os.path.normpath(linkpath)
  1932. try:
  1933. shutil.copy2(linkpath, targetpath)
  1934. except EnvironmentError, e:
  1935. raise IOError("link could not be created")
  1936. def chown(self, tarinfo, targetpath):
  1937. """Set owner of targetpath according to tarinfo.
  1938. """
  1939. if pwd and hasattr(os, "geteuid") and os.geteuid() == 0:
  1940. # We have to be root to do so.
  1941. try:
  1942. g = grp.getgrnam(tarinfo.gname)[2]
  1943. except KeyError:
  1944. try:
  1945. g = grp.getgrgid(tarinfo.gid)[2]
  1946. except KeyError:
  1947. g = os.getgid()
  1948. try:
  1949. u = pwd.getpwnam(tarinfo.uname)[2]
  1950. except KeyError:
  1951. try:
  1952. u = pwd.getpwuid(tarinfo.uid)[2]
  1953. except KeyError:
  1954. u = os.getuid()
  1955. try:
  1956. if tarinfo.issym() and hasattr(os, "lchown"):
  1957. os.lchown(targetpath, u, g)
  1958. else:
  1959. if sys.platform != "os2emx":
  1960. os.chown(targetpath, u, g)
  1961. except EnvironmentError, e:
  1962. raise ExtractError("could not change owner")
  1963. def chmod(self, tarinfo, targetpath):
  1964. """Set file permissions of targetpath according to tarinfo.
  1965. """
  1966. if hasattr(os, 'chmod'):
  1967. try:
  1968. os.chmod(targetpath, tarinfo.mode)
  1969. except EnvironmentError, e:
  1970. raise ExtractError("could not change mode")
  1971. def utime(self, tarinfo, targetpath):
  1972. """Set modification time of targetpath according to tarinfo.
  1973. """
  1974. if not hasattr(os, 'utime'):
  1975. return
  1976. try:
  1977. os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
  1978. except EnvironmentError, e:
  1979. raise ExtractError("could not change modification time")
  1980. #--------------------------------------------------------------------------
  1981. def next(self):
  1982. """Return the next member of the archive as a TarInfo object, when
  1983. TarFile is opened for reading. Return None if there is no more
  1984. available.
  1985. """
  1986. self._check("ra")
  1987. if self.firstmember is not None:
  1988. m = self.firstmember
  1989. self.firstmember = None
  1990. return m
  1991. # Read the next block.
  1992. self.fileobj.seek(self.offset)
  1993. while True:
  1994. try:
  1995. tarinfo = self.tarinfo.fromtarfile(self)
  1996. if tarinfo is None:
  1997. return
  1998. self.members.append(tarinfo)
  1999. except HeaderError, e:
  2000. if self.ignore_zeros:
  2001. self._dbg(2, "0x%X: %s" % (self.offset, e))
  2002. self.offset += BLOCKSIZE
  2003. continue
  2004. else:
  2005. if self.offset == 0:
  2006. raise ReadError(str(e))
  2007. return None
  2008. break
  2009. return tarinfo
  2010. #--------------------------------------------------------------------------
  2011. # Little helper methods:
  2012. def _getmember(self, name, tarinfo=None):
  2013. """Find an archive member by name from bottom to top.
  2014. If tarinfo is given, it is used as the starting point.
  2015. """
  2016. # Ensure that all members have been loaded.
  2017. members = self.getmembers()
  2018. if tarinfo is None:
  2019. end = len(members)
  2020. else:
  2021. end = members.index(tarinfo)
  2022. for i in xrange(end - 1, -1, -1):
  2023. if name == members[i].name:
  2024. return members[i]
  2025. def _load(self):
  2026. """Read through the entire archive file and look for readable
  2027. members.
  2028. """
  2029. while True:
  2030. tarinfo = self.next()
  2031. if tarinfo is None:
  2032. break
  2033. self._loaded = True
  2034. def _check(self, mode=None):
  2035. """Check if TarFile is still open, and if the operation's mode
  2036. corresponds to TarFile's mode.
  2037. """
  2038. if self.closed:
  2039. raise IOError("%s is closed" % self.__class__.__name__)
  2040. if mode is not None and self.mode not in mode:
  2041. raise IOError("bad operation for mode %r" % self.mode)
  2042. def __iter__(self):
  2043. """Provide an iterator object.
  2044. """
  2045. if self._loaded:
  2046. return iter(self.members)
  2047. else:
  2048. return TarIter(self)
  2049. def _dbg(self, level, msg):
  2050. """Write debugging output to sys.stderr.
  2051. """
  2052. if level <= self.debug:
  2053. print >> sys.stderr, msg
  2054. # class TarFile
  2055. class TarIter:
  2056. """Iterator Class.
  2057. for tarinfo in TarFile(...):
  2058. suite...
  2059. """
  2060. def __init__(self, tarfile):
  2061. """Construct a TarIter object.
  2062. """
  2063. self.tarfile = tarfile
  2064. self.index = 0
  2065. def __iter__(self):
  2066. """Return iterator object.
  2067. """
  2068. return self
  2069. def next(self):
  2070. """Return the next item using TarFile's next() method.
  2071. When all members have been read, set TarFile as _loaded.
  2072. """
  2073. # Fix for SF #1100429: Under rare circumstances it can
  2074. # happen that getmembers() is called during iteration,
  2075. # which will cause TarIter to stop prematurely.
  2076. if not self.tarfile._loaded:
  2077. tarinfo = self.tarfile.next()
  2078. if not tarinfo:
  2079. self.tarfile._loaded = True
  2080. raise StopIteration
  2081. else:
  2082. try:
  2083. tarinfo = self.tarfile.members[self.index]
  2084. except IndexError:
  2085. raise StopIteration
  2086. self.index += 1
  2087. return tarinfo
  2088. # Helper classes for sparse file support
  2089. class _section:
  2090. """Base class for _data and _hole.
  2091. """
  2092. def __init__(self, offset, size):
  2093. self.offset = offset
  2094. self.size = size
  2095. def __contains__(self, offset):
  2096. return self.offset <= offset < self.offset + self.size
  2097. class _data(_section):
  2098. """Represent a data section in a sparse file.
  2099. """
  2100. def __init__(self, offset, size, realpos):
  2101. _section.__init__(self, offset, size)
  2102. self.realpos = realpos
  2103. class _hole(_section):
  2104. """Represent a hole section in a sparse file.
  2105. """
  2106. pass
  2107. class _ringbuffer(list):
  2108. """Ringbuffer class which increases performance
  2109. over a regular list.
  2110. """
  2111. def __init__(self):
  2112. self.idx = 0
  2113. def find(self, offset):
  2114. idx = self.idx
  2115. while True:
  2116. item = self[idx]
  2117. if offset in item:
  2118. break
  2119. idx += 1
  2120. if idx == len(self):
  2121. idx = 0
  2122. if idx == self.idx:
  2123. # End of File
  2124. return None
  2125. self.idx = idx
  2126. return item
  2127. #---------------------------------------------
  2128. # zipfile compatible TarFile class
  2129. #---------------------------------------------
  2130. TAR_PLAIN = 0 # zipfile.ZIP_STORED
  2131. TAR_GZIPPED = 8 # zipfile.ZIP_DEFLATED
  2132. class TarFileCompat:
  2133. """TarFile class compatible with standard module zipfile's
  2134. ZipFile class.
  2135. """
  2136. def __init__(self, file, mode="r", compression=TAR_PLAIN):
  2137. from warnings import warnpy3k
  2138. warnpy3k("the TarFileCompat class has been removed in Python 3.0",
  2139. stacklevel=2)
  2140. if compression == TAR_PLAIN:
  2141. self.tarfile = TarFile.taropen(file, mode)
  2142. elif compression == TAR_GZIPPED:
  2143. self.tarfile = TarFile.gzopen(file, mode)
  2144. else:
  2145. raise ValueError("unknown compression constant")
  2146. if mode[0:1] == "r":
  2147. members = self.tarfile.getmembers()
  2148. for m in members:
  2149. m.filename = m.name
  2150. m.file_size = m.size
  2151. m.date_time = time.gmtime(m.mtime)[:6]
  2152. def namelist(self):
  2153. return map(lambda m: m.name, self.infolist())
  2154. def infolist(self):
  2155. return filter(lambda m: m.type in REGULAR_TYPES,
  2156. self.tarfile.getmembers())
  2157. def printdir(self):
  2158. self.tarfile.list()
  2159. def testzip(self):
  2160. return
  2161. def getinfo(self, name):
  2162. return self.tarfile.getmember(name)
  2163. def read(self, name):
  2164. return self.tarfile.extractfile(self.tarfile.getmember(name)).read()
  2165. def write(self, filename, arcname=None, compress_type=None):
  2166. self.tarfile.add(filename, arcname)
  2167. def writestr(self, zinfo, bytes):
  2168. try:
  2169. from cStringIO import StringIO
  2170. except ImportError:
  2171. from StringIO import StringIO
  2172. import calendar
  2173. tinfo = TarInfo(zinfo.filename)
  2174. tinfo.size = len(bytes)
  2175. tinfo.mtime = calendar.timegm(zinfo.date_time)
  2176. self.tarfile.addfile(tinfo, StringIO(bytes))
  2177. def close(self):
  2178. self.tarfile.close()
  2179. #class TarFileCompat
  2180. #--------------------
  2181. # exported functions
  2182. #--------------------
  2183. def is_tarfile(name):
  2184. """Return True if name points to a tar archive that we
  2185. are able to handle, else return False.
  2186. """
  2187. try:
  2188. t = open(name)
  2189. t.close()
  2190. return True
  2191. except TarError:
  2192. return False
  2193. bltn_open = open
  2194. open = TarFile.open