/Lib/test/test_struct.py
Python | 590 lines | 586 code | 4 blank | 0 comment | 8 complexity | d2bceef34c3357b8d1795ed2b69000a6 MD5 | raw file
1import array 2import unittest 3import struct 4import warnings 5warnings.filterwarnings("ignore", "struct integer overflow masking is deprecated", 6 DeprecationWarning) 7 8from functools import wraps 9from test.test_support import TestFailed, verbose, run_unittest 10 11import sys 12ISBIGENDIAN = sys.byteorder == "big" 13IS32BIT = sys.maxsize == 0x7fffffff 14del sys 15 16try: 17 import _struct 18except ImportError: 19 PY_STRUCT_RANGE_CHECKING = 0 20 PY_STRUCT_OVERFLOW_MASKING = 1 21 PY_STRUCT_FLOAT_COERCE = 2 22else: 23 PY_STRUCT_RANGE_CHECKING = getattr(_struct, '_PY_STRUCT_RANGE_CHECKING', 0) 24 PY_STRUCT_OVERFLOW_MASKING = getattr(_struct, '_PY_STRUCT_OVERFLOW_MASKING', 0) 25 PY_STRUCT_FLOAT_COERCE = getattr(_struct, '_PY_STRUCT_FLOAT_COERCE', 0) 26 27def string_reverse(s): 28 return "".join(reversed(s)) 29 30def bigendian_to_native(value): 31 if ISBIGENDIAN: 32 return value 33 else: 34 return string_reverse(value) 35 36def with_warning_restore(func): 37 @wraps(func) 38 def decorator(*args, **kw): 39 with warnings.catch_warnings(): 40 # We need this function to warn every time, so stick an 41 # unqualifed 'always' at the head of the filter list 42 warnings.simplefilter("always") 43 warnings.filterwarnings("error", category=DeprecationWarning) 44 return func(*args, **kw) 45 return decorator 46 47@with_warning_restore 48def deprecated_err(func, *args): 49 try: 50 func(*args) 51 except (struct.error, TypeError): 52 pass 53 except DeprecationWarning: 54 if not PY_STRUCT_OVERFLOW_MASKING: 55 raise TestFailed, "%s%s expected to raise DeprecationWarning" % ( 56 func.__name__, args) 57 else: 58 raise TestFailed, "%s%s did not raise error" % ( 59 func.__name__, args) 60 61 62class StructTest(unittest.TestCase): 63 64 @with_warning_restore 65 def check_float_coerce(self, format, number): 66 # SF bug 1530559. struct.pack raises TypeError where it used to convert. 67 if PY_STRUCT_FLOAT_COERCE == 2: 68 # Test for pre-2.5 struct module 69 packed = struct.pack(format, number) 70 floored = struct.unpack(format, packed)[0] 71 self.assertEqual(floored, int(number), 72 "did not correcly coerce float to int") 73 return 74 try: 75 struct.pack(format, number) 76 except (struct.error, TypeError): 77 if PY_STRUCT_FLOAT_COERCE: 78 self.fail("expected DeprecationWarning for float coerce") 79 except DeprecationWarning: 80 if not PY_STRUCT_FLOAT_COERCE: 81 self.fail("expected to raise struct.error for float coerce") 82 else: 83 self.fail("did not raise error for float coerce") 84 85 def test_isbigendian(self): 86 self.assertEqual((struct.pack('=i', 1)[0] == chr(0)), ISBIGENDIAN) 87 88 def test_consistence(self): 89 self.assertRaises(struct.error, struct.calcsize, 'Z') 90 91 sz = struct.calcsize('i') 92 self.assertEqual(sz * 3, struct.calcsize('iii')) 93 94 fmt = 'cbxxxxxxhhhhiillffd?' 95 fmt3 = '3c3b18x12h6i6l6f3d3?' 96 sz = struct.calcsize(fmt) 97 sz3 = struct.calcsize(fmt3) 98 self.assertEqual(sz * 3, sz3) 99 100 self.assertRaises(struct.error, struct.pack, 'iii', 3) 101 self.assertRaises(struct.error, struct.pack, 'i', 3, 3, 3) 102 self.assertRaises(struct.error, struct.pack, 'i', 'foo') 103 self.assertRaises(struct.error, struct.pack, 'P', 'foo') 104 self.assertRaises(struct.error, struct.unpack, 'd', 'flap') 105 s = struct.pack('ii', 1, 2) 106 self.assertRaises(struct.error, struct.unpack, 'iii', s) 107 self.assertRaises(struct.error, struct.unpack, 'i', s) 108 109 def test_transitiveness(self): 110 c = 'a' 111 b = 1 112 h = 255 113 i = 65535 114 l = 65536 115 f = 3.1415 116 d = 3.1415 117 t = True 118 119 for prefix in ('', '@', '<', '>', '=', '!'): 120 for format in ('xcbhilfd?', 'xcBHILfd?'): 121 format = prefix + format 122 s = struct.pack(format, c, b, h, i, l, f, d, t) 123 cp, bp, hp, ip, lp, fp, dp, tp = struct.unpack(format, s) 124 self.assertEqual(cp, c) 125 self.assertEqual(bp, b) 126 self.assertEqual(hp, h) 127 self.assertEqual(ip, i) 128 self.assertEqual(lp, l) 129 self.assertEqual(int(100 * fp), int(100 * f)) 130 self.assertEqual(int(100 * dp), int(100 * d)) 131 self.assertEqual(tp, t) 132 133 def test_new_features(self): 134 # Test some of the new features in detail 135 # (format, argument, big-endian result, little-endian result, asymmetric) 136 tests = [ 137 ('c', 'a', 'a', 'a', 0), 138 ('xc', 'a', '\0a', '\0a', 0), 139 ('cx', 'a', 'a\0', 'a\0', 0), 140 ('s', 'a', 'a', 'a', 0), 141 ('0s', 'helloworld', '', '', 1), 142 ('1s', 'helloworld', 'h', 'h', 1), 143 ('9s', 'helloworld', 'helloworl', 'helloworl', 1), 144 ('10s', 'helloworld', 'helloworld', 'helloworld', 0), 145 ('11s', 'helloworld', 'helloworld\0', 'helloworld\0', 1), 146 ('20s', 'helloworld', 'helloworld'+10*'\0', 'helloworld'+10*'\0', 1), 147 ('b', 7, '\7', '\7', 0), 148 ('b', -7, '\371', '\371', 0), 149 ('B', 7, '\7', '\7', 0), 150 ('B', 249, '\371', '\371', 0), 151 ('h', 700, '\002\274', '\274\002', 0), 152 ('h', -700, '\375D', 'D\375', 0), 153 ('H', 700, '\002\274', '\274\002', 0), 154 ('H', 0x10000-700, '\375D', 'D\375', 0), 155 ('i', 70000000, '\004,\035\200', '\200\035,\004', 0), 156 ('i', -70000000, '\373\323\342\200', '\200\342\323\373', 0), 157 ('I', 70000000L, '\004,\035\200', '\200\035,\004', 0), 158 ('I', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0), 159 ('l', 70000000, '\004,\035\200', '\200\035,\004', 0), 160 ('l', -70000000, '\373\323\342\200', '\200\342\323\373', 0), 161 ('L', 70000000L, '\004,\035\200', '\200\035,\004', 0), 162 ('L', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0), 163 ('f', 2.0, '@\000\000\000', '\000\000\000@', 0), 164 ('d', 2.0, '@\000\000\000\000\000\000\000', 165 '\000\000\000\000\000\000\000@', 0), 166 ('f', -2.0, '\300\000\000\000', '\000\000\000\300', 0), 167 ('d', -2.0, '\300\000\000\000\000\000\000\000', 168 '\000\000\000\000\000\000\000\300', 0), 169 ('?', 0, '\0', '\0', 0), 170 ('?', 3, '\1', '\1', 1), 171 ('?', True, '\1', '\1', 0), 172 ('?', [], '\0', '\0', 1), 173 ('?', (1,), '\1', '\1', 1), 174 ] 175 176 for fmt, arg, big, lil, asy in tests: 177 for (xfmt, exp) in [('>'+fmt, big), ('!'+fmt, big), ('<'+fmt, lil), 178 ('='+fmt, ISBIGENDIAN and big or lil)]: 179 res = struct.pack(xfmt, arg) 180 self.assertEqual(res, exp) 181 self.assertEqual(struct.calcsize(xfmt), len(res)) 182 rev = struct.unpack(xfmt, res)[0] 183 if rev != arg: 184 self.assert_(asy) 185 186 def test_native_qQ(self): 187 # can't pack -1 as unsigned regardless 188 self.assertRaises((struct.error, TypeError), struct.pack, "Q", -1) 189 # can't pack string as 'q' regardless 190 self.assertRaises(struct.error, struct.pack, "q", "a") 191 # ditto, but 'Q' 192 self.assertRaises(struct.error, struct.pack, "Q", "a") 193 194 try: 195 struct.pack("q", 5) 196 except struct.error: 197 # does not have native q/Q 198 pass 199 else: 200 bytes = struct.calcsize('q') 201 # The expected values here are in big-endian format, primarily 202 # because I'm on a little-endian machine and so this is the 203 # clearest way (for me) to force the code to get exercised. 204 for format, input, expected in ( 205 ('q', -1, '\xff' * bytes), 206 ('q', 0, '\x00' * bytes), 207 ('Q', 0, '\x00' * bytes), 208 ('q', 1L, '\x00' * (bytes-1) + '\x01'), 209 ('Q', (1L << (8*bytes))-1, '\xff' * bytes), 210 ('q', (1L << (8*bytes-1))-1, '\x7f' + '\xff' * (bytes - 1))): 211 got = struct.pack(format, input) 212 native_expected = bigendian_to_native(expected) 213 self.assertEqual(got, native_expected) 214 retrieved = struct.unpack(format, got)[0] 215 self.assertEqual(retrieved, input) 216 217 def test_standard_integers(self): 218 # Standard integer tests (bBhHiIlLqQ). 219 import binascii 220 221 class IntTester(unittest.TestCase): 222 223 # XXX Most std integer modes fail to test for out-of-range. 224 # The "i" and "l" codes appear to range-check OK on 32-bit boxes, but 225 # fail to check correctly on some 64-bit ones (Tru64 Unix + Compaq C 226 # reported by Mark Favas). 227 BUGGY_RANGE_CHECK = "bBhHiIlL" 228 229 def __init__(self, formatpair, bytesize): 230 self.assertEqual(len(formatpair), 2) 231 self.formatpair = formatpair 232 for direction in "<>!=": 233 for code in formatpair: 234 format = direction + code 235 self.assertEqual(struct.calcsize(format), bytesize) 236 self.bytesize = bytesize 237 self.bitsize = bytesize * 8 238 self.signed_code, self.unsigned_code = formatpair 239 self.unsigned_min = 0 240 self.unsigned_max = 2L**self.bitsize - 1 241 self.signed_min = -(2L**(self.bitsize-1)) 242 self.signed_max = 2L**(self.bitsize-1) - 1 243 244 def test_one(self, x, pack=struct.pack, 245 unpack=struct.unpack, 246 unhexlify=binascii.unhexlify): 247 # Try signed. 248 code = self.signed_code 249 if self.signed_min <= x <= self.signed_max: 250 # Try big-endian. 251 expected = long(x) 252 if x < 0: 253 expected += 1L << self.bitsize 254 self.assert_(expected > 0) 255 expected = hex(expected)[2:-1] # chop "0x" and trailing 'L' 256 if len(expected) & 1: 257 expected = "0" + expected 258 expected = unhexlify(expected) 259 expected = "\x00" * (self.bytesize - len(expected)) + expected 260 261 # Pack work? 262 format = ">" + code 263 got = pack(format, x) 264 self.assertEqual(got, expected) 265 266 # Unpack work? 267 retrieved = unpack(format, got)[0] 268 self.assertEqual(x, retrieved) 269 270 # Adding any byte should cause a "too big" error. 271 self.assertRaises((struct.error, TypeError), unpack, format, 272 '\x01' + got) 273 # Try little-endian. 274 format = "<" + code 275 expected = string_reverse(expected) 276 277 # Pack work? 278 got = pack(format, x) 279 self.assertEqual(got, expected) 280 281 # Unpack work? 282 retrieved = unpack(format, got)[0] 283 self.assertEqual(x, retrieved) 284 285 # Adding any byte should cause a "too big" error. 286 self.assertRaises((struct.error, TypeError), unpack, format, 287 '\x01' + got) 288 289 else: 290 # x is out of range -- verify pack realizes that. 291 if not PY_STRUCT_RANGE_CHECKING and code in self.BUGGY_RANGE_CHECK: 292 if verbose: 293 print "Skipping buggy range check for code", code 294 else: 295 deprecated_err(pack, ">" + code, x) 296 deprecated_err(pack, "<" + code, x) 297 298 # Much the same for unsigned. 299 code = self.unsigned_code 300 if self.unsigned_min <= x <= self.unsigned_max: 301 # Try big-endian. 302 format = ">" + code 303 expected = long(x) 304 expected = hex(expected)[2:-1] # chop "0x" and trailing 'L' 305 if len(expected) & 1: 306 expected = "0" + expected 307 expected = unhexlify(expected) 308 expected = "\x00" * (self.bytesize - len(expected)) + expected 309 310 # Pack work? 311 got = pack(format, x) 312 self.assertEqual(got, expected) 313 314 # Unpack work? 315 retrieved = unpack(format, got)[0] 316 self.assertEqual(x, retrieved) 317 318 # Adding any byte should cause a "too big" error. 319 self.assertRaises((struct.error, TypeError), unpack, format, 320 '\x01' + got) 321 322 # Try little-endian. 323 format = "<" + code 324 expected = string_reverse(expected) 325 326 # Pack work? 327 got = pack(format, x) 328 self.assertEqual(got, expected) 329 330 # Unpack work? 331 retrieved = unpack(format, got)[0] 332 self.assertEqual(x, retrieved) 333 334 # Adding any byte should cause a "too big" error. 335 self.assertRaises((struct.error, TypeError), unpack, format, 336 '\x01' + got) 337 338 else: 339 # x is out of range -- verify pack realizes that. 340 if not PY_STRUCT_RANGE_CHECKING and code in self.BUGGY_RANGE_CHECK: 341 if verbose: 342 print "Skipping buggy range check for code", code 343 else: 344 deprecated_err(pack, ">" + code, x) 345 deprecated_err(pack, "<" + code, x) 346 347 def run(self): 348 from random import randrange 349 350 # Create all interesting powers of 2. 351 values = [] 352 for exp in range(self.bitsize + 3): 353 values.append(1L << exp) 354 355 # Add some random values. 356 for i in range(self.bitsize): 357 val = 0L 358 for j in range(self.bytesize): 359 val = (val << 8) | randrange(256) 360 values.append(val) 361 362 # Try all those, and their negations, and +-1 from them. Note 363 # that this tests all power-of-2 boundaries in range, and a few out 364 # of range, plus +-(2**n +- 1). 365 for base in values: 366 for val in -base, base: 367 for incr in -1, 0, 1: 368 x = val + incr 369 try: 370 x = int(x) 371 except OverflowError: 372 pass 373 self.test_one(x) 374 375 # Some error cases. 376 for direction in "<>": 377 for code in self.formatpair: 378 for badobject in "a string", 3+42j, randrange: 379 self.assertRaises((struct.error, TypeError), 380 struct.pack, direction + code, 381 badobject) 382 383 for args in [("bB", 1), 384 ("hH", 2), 385 ("iI", 4), 386 ("lL", 4), 387 ("qQ", 8)]: 388 t = IntTester(*args) 389 t.run() 390 391 def test_p_code(self): 392 # Test p ("Pascal string") code. 393 for code, input, expected, expectedback in [ 394 ('p','abc', '\x00', ''), 395 ('1p', 'abc', '\x00', ''), 396 ('2p', 'abc', '\x01a', 'a'), 397 ('3p', 'abc', '\x02ab', 'ab'), 398 ('4p', 'abc', '\x03abc', 'abc'), 399 ('5p', 'abc', '\x03abc\x00', 'abc'), 400 ('6p', 'abc', '\x03abc\x00\x00', 'abc'), 401 ('1000p', 'x'*1000, '\xff' + 'x'*999, 'x'*255)]: 402 got = struct.pack(code, input) 403 self.assertEqual(got, expected) 404 (got,) = struct.unpack(code, got) 405 self.assertEqual(got, expectedback) 406 407 def test_705836(self): 408 # SF bug 705836. "<f" and ">f" had a severe rounding bug, where a carry 409 # from the low-order discarded bits could propagate into the exponent 410 # field, causing the result to be wrong by a factor of 2. 411 import math 412 413 for base in range(1, 33): 414 # smaller <- largest representable float less than base. 415 delta = 0.5 416 while base - delta / 2.0 != base: 417 delta /= 2.0 418 smaller = base - delta 419 # Packing this rounds away a solid string of trailing 1 bits. 420 packed = struct.pack("<f", smaller) 421 unpacked = struct.unpack("<f", packed)[0] 422 # This failed at base = 2, 4, and 32, with unpacked = 1, 2, and 423 # 16, respectively. 424 self.assertEqual(base, unpacked) 425 bigpacked = struct.pack(">f", smaller) 426 self.assertEqual(bigpacked, string_reverse(packed)) 427 unpacked = struct.unpack(">f", bigpacked)[0] 428 self.assertEqual(base, unpacked) 429 430 # Largest finite IEEE single. 431 big = (1 << 24) - 1 432 big = math.ldexp(big, 127 - 23) 433 packed = struct.pack(">f", big) 434 unpacked = struct.unpack(">f", packed)[0] 435 self.assertEqual(big, unpacked) 436 437 # The same, but tack on a 1 bit so it rounds up to infinity. 438 big = (1 << 25) - 1 439 big = math.ldexp(big, 127 - 24) 440 self.assertRaises(OverflowError, struct.pack, ">f", big) 441 442 if PY_STRUCT_RANGE_CHECKING: 443 def test_1229380(self): 444 # SF bug 1229380. No struct.pack exception for some out of 445 # range integers 446 import sys 447 for endian in ('', '>', '<'): 448 for cls in (int, long): 449 for fmt in ('B', 'H', 'I', 'L'): 450 deprecated_err(struct.pack, endian + fmt, cls(-1)) 451 452 deprecated_err(struct.pack, endian + 'B', cls(300)) 453 deprecated_err(struct.pack, endian + 'H', cls(70000)) 454 455 deprecated_err(struct.pack, endian + 'I', sys.maxint * 4L) 456 deprecated_err(struct.pack, endian + 'L', sys.maxint * 4L) 457 458 def XXXtest_1530559(self): 459 # XXX This is broken: see the bug report 460 # SF bug 1530559. struct.pack raises TypeError where it used to convert. 461 for endian in ('', '>', '<'): 462 for fmt in ('B', 'H', 'I', 'L', 'b', 'h', 'i', 'l'): 463 self.check_float_coerce(endian + fmt, 1.0) 464 self.check_float_coerce(endian + fmt, 1.5) 465 466 def test_issue4228(self): 467 # Packing a long may yield either 32 or 64 bits 468 x = struct.pack('L', -1)[:4] 469 self.assertEqual(x, '\xff'*4) 470 471 def test_unpack_from(self): 472 test_string = 'abcd01234' 473 fmt = '4s' 474 s = struct.Struct(fmt) 475 for cls in (str, buffer): 476 data = cls(test_string) 477 self.assertEqual(s.unpack_from(data), ('abcd',)) 478 self.assertEqual(s.unpack_from(data, 2), ('cd01',)) 479 self.assertEqual(s.unpack_from(data, 4), ('0123',)) 480 for i in xrange(6): 481 self.assertEqual(s.unpack_from(data, i), (data[i:i+4],)) 482 for i in xrange(6, len(test_string) + 1): 483 self.assertRaises(struct.error, s.unpack_from, data, i) 484 for cls in (str, buffer): 485 data = cls(test_string) 486 self.assertEqual(struct.unpack_from(fmt, data), ('abcd',)) 487 self.assertEqual(struct.unpack_from(fmt, data, 2), ('cd01',)) 488 self.assertEqual(struct.unpack_from(fmt, data, 4), ('0123',)) 489 for i in xrange(6): 490 self.assertEqual(struct.unpack_from(fmt, data, i), (data[i:i+4],)) 491 for i in xrange(6, len(test_string) + 1): 492 self.assertRaises(struct.error, struct.unpack_from, fmt, data, i) 493 494 def test_pack_into(self): 495 test_string = 'Reykjavik rocks, eow!' 496 writable_buf = array.array('c', ' '*100) 497 fmt = '21s' 498 s = struct.Struct(fmt) 499 500 # Test without offset 501 s.pack_into(writable_buf, 0, test_string) 502 from_buf = writable_buf.tostring()[:len(test_string)] 503 self.assertEqual(from_buf, test_string) 504 505 # Test with offset. 506 s.pack_into(writable_buf, 10, test_string) 507 from_buf = writable_buf.tostring()[:len(test_string)+10] 508 self.assertEqual(from_buf, test_string[:10] + test_string) 509 510 # Go beyond boundaries. 511 small_buf = array.array('c', ' '*10) 512 self.assertRaises(struct.error, s.pack_into, small_buf, 0, test_string) 513 self.assertRaises(struct.error, s.pack_into, small_buf, 2, test_string) 514 515 # Test bogus offset (issue 3694) 516 sb = small_buf 517 self.assertRaises(TypeError, struct.pack_into, b'1', sb, None) 518 519 def test_pack_into_fn(self): 520 test_string = 'Reykjavik rocks, eow!' 521 writable_buf = array.array('c', ' '*100) 522 fmt = '21s' 523 pack_into = lambda *args: struct.pack_into(fmt, *args) 524 525 # Test without offset. 526 pack_into(writable_buf, 0, test_string) 527 from_buf = writable_buf.tostring()[:len(test_string)] 528 self.assertEqual(from_buf, test_string) 529 530 # Test with offset. 531 pack_into(writable_buf, 10, test_string) 532 from_buf = writable_buf.tostring()[:len(test_string)+10] 533 self.assertEqual(from_buf, test_string[:10] + test_string) 534 535 # Go beyond boundaries. 536 small_buf = array.array('c', ' '*10) 537 self.assertRaises(struct.error, pack_into, small_buf, 0, test_string) 538 self.assertRaises(struct.error, pack_into, small_buf, 2, test_string) 539 540 def test_unpack_with_buffer(self): 541 # SF bug 1563759: struct.unpack doens't support buffer protocol objects 542 data1 = array.array('B', '\x12\x34\x56\x78') 543 data2 = buffer('......\x12\x34\x56\x78......', 6, 4) 544 for data in [data1, data2]: 545 value, = struct.unpack('>I', data) 546 self.assertEqual(value, 0x12345678) 547 548 def test_bool(self): 549 for prefix in tuple("<>!=")+('',): 550 false = (), [], [], '', 0 551 true = [1], 'test', 5, -1, 0xffffffffL+1, 0xffffffff/2 552 553 falseFormat = prefix + '?' * len(false) 554 packedFalse = struct.pack(falseFormat, *false) 555 unpackedFalse = struct.unpack(falseFormat, packedFalse) 556 557 trueFormat = prefix + '?' * len(true) 558 packedTrue = struct.pack(trueFormat, *true) 559 unpackedTrue = struct.unpack(trueFormat, packedTrue) 560 561 self.assertEqual(len(true), len(unpackedTrue)) 562 self.assertEqual(len(false), len(unpackedFalse)) 563 564 for t in unpackedFalse: 565 self.assertFalse(t) 566 for t in unpackedTrue: 567 self.assertTrue(t) 568 569 packed = struct.pack(prefix+'?', 1) 570 571 self.assertEqual(len(packed), struct.calcsize(prefix+'?')) 572 573 if len(packed) != 1: 574 self.assertFalse(prefix, msg='encoded bool is not one byte: %r' 575 %packed) 576 577 for c in '\x01\x7f\xff\x0f\xf0': 578 self.assertTrue(struct.unpack('>?', c)[0]) 579 580 if IS32BIT: 581 def test_crasher(self): 582 self.assertRaises(MemoryError, struct.pack, "357913941c", "a") 583 584 585 586def test_main(): 587 run_unittest(StructTest) 588 589if __name__ == '__main__': 590 test_main()