PageRenderTime 62ms CodeModel.GetById 16ms app.highlight 42ms RepoModel.GetById 1ms app.codeStats 0ms

/boto-2.5.2/tests/integration/s3/test_key.py

#
Python | 352 lines | 313 code | 8 blank | 31 comment | 2 complexity | 70e78a13dfd7079e7362286fc6969017 MD5 | raw file
  1# -*- coding: utf-8 -*-
  2
  3# Copyright (c) 2012 Mitch Garnaat http://garnaat.org/
  4# All rights reserved.
  5#
  6# Permission is hereby granted, free of charge, to any person obtaining a
  7# copy of this software and associated documentation files (the
  8# "Software"), to deal in the Software without restriction, including
  9# without limitation the rights to use, copy, modify, merge, publish, dis-
 10# tribute, sublicense, and/or sell copies of the Software, and to permit
 11# persons to whom the Software is furnished to do so, subject to the fol-
 12# lowing conditions:
 13#
 14# The above copyright notice and this permission notice shall be included
 15# in all copies or substantial portions of the Software.
 16#
 17# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 18# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
 19# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
 20# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 21# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 22# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 23# IN THE SOFTWARE.
 24
 25"""
 26Some unit tests for S3 Key
 27"""
 28
 29import unittest
 30import time
 31import StringIO
 32from boto.s3.connection import S3Connection
 33from boto.s3.key import Key
 34from boto.exception import S3ResponseError
 35
 36class S3KeyTest (unittest.TestCase):
 37    s3 = True
 38
 39    def setUp(self):
 40        self.conn = S3Connection()
 41        self.bucket_name = 'keytest-%d' % int(time.time())
 42        self.bucket = self.conn.create_bucket(self.bucket_name)
 43
 44    def tearDown(self):
 45        for key in self.bucket:
 46            key.delete()
 47        self.bucket.delete()
 48
 49    def test_set_contents_from_file_dataloss(self):
 50        # Create an empty stringio and write to it.
 51        content = "abcde"
 52        sfp = StringIO.StringIO()
 53        sfp.write(content)
 54        # Try set_contents_from_file() without rewinding sfp
 55        k = self.bucket.new_key("k")
 56        try:
 57            k.set_contents_from_file(sfp)
 58            self.fail("forgot to rewind so should fail.")
 59        except AttributeError:
 60            pass
 61        # call with rewind and check if we wrote 5 bytes
 62        k.set_contents_from_file(sfp, rewind=True)
 63        self.assertEqual(k.size, 5)
 64        # check actual contents by getting it.
 65        kn = self.bucket.new_key("k")
 66        ks = kn.get_contents_as_string()
 67        self.assertEqual(ks, content)
 68
 69        # finally, try with a 0 length string
 70        sfp = StringIO.StringIO()
 71        k = self.bucket.new_key("k")
 72        k.set_contents_from_file(sfp)
 73        self.assertEqual(k.size, 0)
 74        # check actual contents by getting it.
 75        kn = self.bucket.new_key("k")
 76        ks = kn.get_contents_as_string()
 77        self.assertEqual(ks, "")
 78        
 79    def test_set_contents_as_file(self):
 80        content="01234567890123456789"
 81        sfp = StringIO.StringIO(content)
 82
 83        # fp is set at 0 for just opened (for read) files. 
 84        # set_contents should write full content to key.
 85        k = self.bucket.new_key("k")
 86        k.set_contents_from_file(sfp)
 87        self.assertEqual(k.size, 20)
 88        kn = self.bucket.new_key("k")
 89        ks = kn.get_contents_as_string()
 90        self.assertEqual(ks, content)
 91
 92        # set fp to 5 and set contents. this should
 93        # set "567890123456789" to the key
 94        sfp.seek(5)
 95        k = self.bucket.new_key("k")
 96        k.set_contents_from_file(sfp)
 97        self.assertEqual(k.size, 15)
 98        kn = self.bucket.new_key("k")
 99        ks = kn.get_contents_as_string()
100        self.assertEqual(ks, content[5:])
101
102        # set fp to 5 and only set 5 bytes. this should
103        # write the value "56789" to the key.
104        sfp.seek(5)
105        k = self.bucket.new_key("k")
106        k.set_contents_from_file(sfp, size=5)
107        self.assertEqual(k.size, 5)
108        self.assertEqual(sfp.tell(), 10)
109        kn = self.bucket.new_key("k")
110        ks = kn.get_contents_as_string()
111        self.assertEqual(ks, content[5:10])
112
113    def test_set_contents_with_md5(self):
114        content="01234567890123456789"
115        sfp = StringIO.StringIO(content)
116
117        # fp is set at 0 for just opened (for read) files. 
118        # set_contents should write full content to key.
119        k = self.bucket.new_key("k")
120        good_md5 = k.compute_md5(sfp)
121        k.set_contents_from_file(sfp, md5=good_md5)
122        kn = self.bucket.new_key("k")
123        ks = kn.get_contents_as_string()
124        self.assertEqual(ks, content)
125
126        # set fp to 5 and only set 5 bytes. this should
127        # write the value "56789" to the key.
128        sfp.seek(5)
129        k = self.bucket.new_key("k")
130        good_md5 = k.compute_md5(sfp, size=5)
131        k.set_contents_from_file(sfp, size=5, md5=good_md5)
132        self.assertEqual(sfp.tell(), 10)
133        kn = self.bucket.new_key("k")
134        ks = kn.get_contents_as_string()
135        self.assertEqual(ks, content[5:10])
136
137        # let's try a wrong md5 by just altering it.
138        k = self.bucket.new_key("k")
139        sfp.seek(0)
140        hexdig, base64 = k.compute_md5(sfp)
141        bad_md5 = (hexdig, base64[3:])
142        try:
143            k.set_contents_from_file(sfp, md5=bad_md5)
144            self.fail("should fail with bad md5")
145        except S3ResponseError:
146            pass
147
148    def test_get_contents_with_md5(self):
149        content="01234567890123456789"
150        sfp = StringIO.StringIO(content)
151
152        k = self.bucket.new_key("k")
153        k.set_contents_from_file(sfp)
154        kn = self.bucket.new_key("k")
155        s = kn.get_contents_as_string()
156        self.assertEqual(kn.md5, k.md5)       
157        self.assertEqual(s, content)
158
159    def test_file_callback(self):
160        def callback(wrote, total):
161            self.my_cb_cnt += 1
162            self.assertNotEqual(wrote, self.my_cb_last, "called twice with same value")
163            self.my_cb_last = wrote
164
165        # Zero bytes written => 1 call
166        self.my_cb_cnt = 0
167        self.my_cb_last = None
168        k = self.bucket.new_key("k")
169        k.BufferSize = 2
170        sfp = StringIO.StringIO("")
171        k.set_contents_from_file(sfp, cb=callback, num_cb=10)
172        self.assertEqual(self.my_cb_cnt, 1)
173        self.assertEqual(self.my_cb_last, 0)
174        sfp.close()
175
176        # Read back zero bytes => 1 call
177        self.my_cb_cnt = 0
178        self.my_cb_last = None
179        s = k.get_contents_as_string(cb=callback)
180        self.assertEqual(self.my_cb_cnt, 1)
181        self.assertEqual(self.my_cb_last, 0)
182
183        content="01234567890123456789"
184        sfp = StringIO.StringIO(content)
185
186        # expect 2 calls due start/finish
187        self.my_cb_cnt = 0
188        self.my_cb_last = None
189        k = self.bucket.new_key("k")
190        k.set_contents_from_file(sfp, cb=callback, num_cb=10)
191        self.assertEqual(self.my_cb_cnt, 2)
192        self.assertEqual(self.my_cb_last, 20)
193
194        # Read back all bytes => 2 calls
195        self.my_cb_cnt = 0
196        self.my_cb_last = None
197        s = k.get_contents_as_string(cb=callback)
198        self.assertEqual(self.my_cb_cnt, 2)
199        self.assertEqual(self.my_cb_last, 20)
200        self.assertEqual(s, content)
201
202        # rewind sfp and try upload again. -1 should call
203        # for every read/write so that should make 11 when bs=2
204        sfp.seek(0)
205        self.my_cb_cnt = 0
206        self.my_cb_last = None
207        k = self.bucket.new_key("k")
208        k.BufferSize = 2
209        k.set_contents_from_file(sfp, cb=callback, num_cb=-1)
210        self.assertEqual(self.my_cb_cnt, 11)
211        self.assertEqual(self.my_cb_last, 20)
212
213        # Read back all bytes => 11 calls
214        self.my_cb_cnt = 0
215        self.my_cb_last = None
216        s = k.get_contents_as_string(cb=callback, num_cb=-1)
217        self.assertEqual(self.my_cb_cnt, 11)
218        self.assertEqual(self.my_cb_last, 20)
219        self.assertEqual(s, content)
220
221        # no more than 1 times => 2 times
222        # last time always 20 bytes
223        sfp.seek(0)
224        self.my_cb_cnt = 0
225        self.my_cb_last = None
226        k = self.bucket.new_key("k")
227        k.BufferSize = 2
228        k.set_contents_from_file(sfp, cb=callback, num_cb=1)
229        self.assertTrue(self.my_cb_cnt <= 2)
230        self.assertEqual(self.my_cb_last, 20)
231
232        # no more than 1 times => 2 times
233        self.my_cb_cnt = 0
234        self.my_cb_last = None
235        s = k.get_contents_as_string(cb=callback, num_cb=1)
236        self.assertTrue(self.my_cb_cnt <= 2)
237        self.assertEqual(self.my_cb_last, 20)
238        self.assertEqual(s, content)
239
240        # no more than 2 times
241        # last time always 20 bytes
242        sfp.seek(0)
243        self.my_cb_cnt = 0
244        self.my_cb_last = None
245        k = self.bucket.new_key("k")
246        k.BufferSize = 2
247        k.set_contents_from_file(sfp, cb=callback, num_cb=2)
248        self.assertTrue(self.my_cb_cnt <= 2)
249        self.assertEqual(self.my_cb_last, 20)
250
251        # no more than 2 times
252        self.my_cb_cnt = 0
253        self.my_cb_last = None
254        s = k.get_contents_as_string(cb=callback, num_cb=2)
255        self.assertTrue(self.my_cb_cnt <= 2)
256        self.assertEqual(self.my_cb_last, 20)
257        self.assertEqual(s, content)
258
259        # no more than 3 times
260        # last time always 20 bytes
261        sfp.seek(0)
262        self.my_cb_cnt = 0
263        self.my_cb_last = None
264        k = self.bucket.new_key("k")
265        k.BufferSize = 2
266        k.set_contents_from_file(sfp, cb=callback, num_cb=3)
267        self.assertTrue(self.my_cb_cnt <= 3)
268        self.assertEqual(self.my_cb_last, 20)
269
270        # no more than 3 times
271        self.my_cb_cnt = 0
272        self.my_cb_last = None
273        s = k.get_contents_as_string(cb=callback, num_cb=3)
274        self.assertTrue(self.my_cb_cnt <= 3)
275        self.assertEqual(self.my_cb_last, 20)
276        self.assertEqual(s, content)
277
278        # no more than 4 times
279        # last time always 20 bytes
280        sfp.seek(0)
281        self.my_cb_cnt = 0
282        self.my_cb_last = None
283        k = self.bucket.new_key("k")
284        k.BufferSize = 2
285        k.set_contents_from_file(sfp, cb=callback, num_cb=4)
286        self.assertTrue(self.my_cb_cnt <= 4)
287        self.assertEqual(self.my_cb_last, 20)
288
289        # no more than 4 times
290        self.my_cb_cnt = 0
291        self.my_cb_last = None
292        s = k.get_contents_as_string(cb=callback, num_cb=4)
293        self.assertTrue(self.my_cb_cnt <= 4)
294        self.assertEqual(self.my_cb_last, 20)
295        self.assertEqual(s, content)
296
297        # no more than 6 times
298        # last time always 20 bytes
299        sfp.seek(0)
300        self.my_cb_cnt = 0
301        self.my_cb_last = None
302        k = self.bucket.new_key("k")
303        k.BufferSize = 2
304        k.set_contents_from_file(sfp, cb=callback, num_cb=6)
305        self.assertTrue(self.my_cb_cnt <= 6)
306        self.assertEqual(self.my_cb_last, 20)
307
308        # no more than 6 times
309        self.my_cb_cnt = 0
310        self.my_cb_last = None
311        s = k.get_contents_as_string(cb=callback, num_cb=6)
312        self.assertTrue(self.my_cb_cnt <= 6)
313        self.assertEqual(self.my_cb_last, 20)
314        self.assertEqual(s, content)
315
316        # no more than 10 times
317        # last time always 20 bytes
318        sfp.seek(0)
319        self.my_cb_cnt = 0 
320        self.my_cb_last = None
321        k = self.bucket.new_key("k")
322        k.BufferSize = 2
323        k.set_contents_from_file(sfp, cb=callback, num_cb=10)
324        self.assertTrue(self.my_cb_cnt <= 10)
325        self.assertEqual(self.my_cb_last, 20)
326
327        # no more than 10 times
328        self.my_cb_cnt = 0
329        self.my_cb_last = None
330        s = k.get_contents_as_string(cb=callback, num_cb=10)
331        self.assertTrue(self.my_cb_cnt <= 10)
332        self.assertEqual(self.my_cb_last, 20)
333        self.assertEqual(s, content)
334
335        # no more than 1000 times
336        # last time always 20 bytes
337        sfp.seek(0)
338        self.my_cb_cnt = 0 
339        self.my_cb_last = None
340        k = self.bucket.new_key("k")
341        k.BufferSize = 2
342        k.set_contents_from_file(sfp, cb=callback, num_cb=1000)
343        self.assertTrue(self.my_cb_cnt <= 1000)
344        self.assertEqual(self.my_cb_last, 20)
345
346        # no more than 1000 times
347        self.my_cb_cnt = 0
348        self.my_cb_last = None
349        s = k.get_contents_as_string(cb=callback, num_cb=1000)
350        self.assertTrue(self.my_cb_cnt <= 1000)
351        self.assertEqual(self.my_cb_last, 20)
352        self.assertEqual(s, content)