/test/test_mem_cache.rb

  1# encoding: utf-8
  2require 'stringio'
  3require 'test/unit'
  4require 'rubygems'
  5begin
  6  gem 'flexmock'
  7  require 'flexmock/test_unit'
  8rescue => e
  9  puts "Some tests require flexmock, please run `gem install flexmock`"
 10end
 11
 12$TESTING = true
 13
 14require File.dirname(__FILE__) + '/../lib/memcache'
 15
 16class MemCache
 17
 18  attr_writer :namespace
 19
 20end
 21
 22class FakeSocket
 23
 24  attr_reader :written, :data
 25
 26  def initialize
 27    @written = StringIO.new
 28    @data = StringIO.new
 29  end
 30
 31  def write(data)
 32    @written.write data
 33  end
 34
 35  def gets
 36    @data.gets
 37  end
 38
 39  def read(arg)
 40    @data.read arg
 41  end
 42
 43end
 44
 45class FakeServer
 46
 47  attr_reader :host, :port, :socket
 48
 49  def initialize(socket = nil)
 50    @closed = false
 51    @host = 'example.com'
 52    @port = 11211
 53    @socket = socket || FakeSocket.new
 54  end
 55
 56  def close
 57    @closed = true
 58  end
 59
 60  def alive?
 61    !@closed
 62  end
 63
 64end
 65
 66class TestMemCache < Test::Unit::TestCase
 67
 68  def setup
 69    @cache = MemCache.new 'localhost:1', :namespace => 'my_namespace'
 70  end
 71
 72  def test_consistent_hashing
 73    flexmock(MemCache::Server).new_instances.should_receive(:alive?).and_return(true)
 74
 75    # Setup a continuum of two servers
 76    @cache.servers = ['mike1', 'mike2', 'mike3']
 77
 78    keys = []
 79    1000.times do |idx|
 80      keys << idx.to_s
 81    end
 82
 83    before_continuum = keys.map {|key| @cache.get_server_for_key(key) }
 84
 85    @cache.servers = ['mike1', 'mike2', 'mike3', 'mike4']
 86
 87    after_continuum = keys.map {|key| @cache.get_server_for_key(key) }
 88
 89    same_count = before_continuum.zip(after_continuum).find_all {|a| a[0].host == a[1].host }.size
 90
 91    # With continuum, we should see about 75% of the keys map to the same server
 92    # With modulo, we would see about 25%.
 93    assert same_count > 700
 94  end
 95
 96  def test_cache_get
 97    server = util_setup_fake_server
 98
 99    assert_equal "\004\b\"\0170123456789",
100                 @cache.cache_get(server, 'my_namespace:key')
101
102    assert_equal "get my_namespace:key\r\n",
103                 server.socket.written.string
104  end
105
106  def test_cache_get_EOF
107    server = util_setup_fake_server
108    server.socket.data.string = ''
109
110    e = assert_raise MemCache::MemCacheError do
111      @cache.cache_get server, 'my_namespace:key'
112    end
113
114    assert_equal "lost connection to example.com:11211", e.message
115  end
116
117  def test_cache_get_bad_state
118    server = FakeServer.new
119
120    # Write two messages to the socket to test failover
121    server.socket.data.write "bogus response\r\nbogus response\r\n"
122    server.socket.data.rewind
123
124    @cache.servers = []
125    @cache.servers << server
126
127    e = assert_raise MemCache::MemCacheError do
128      @cache.cache_get(server, 'my_namespace:key')
129    end
130
131    assert_match /#{Regexp.quote 'unexpected response "bogus response\r\n"'}/, e.message
132
133    assert !server.alive?
134
135    assert_match /get my_namespace:key\r\n/, server.socket.written.string
136  end
137
138  def test_cache_get_miss
139    socket = FakeSocket.new
140    socket.data.write "END\r\n"
141    socket.data.rewind
142    server = FakeServer.new socket
143
144    assert_equal nil, @cache.cache_get(server, 'my_namespace:key')
145
146    assert_equal "get my_namespace:key\r\n",
147                 socket.written.string
148  end
149
150  def test_cache_get_multi
151    server = util_setup_fake_server
152    server.socket.data.write "VALUE foo 0 7\r\n"
153    server.socket.data.write "\004\b\"\bfoo\r\n"
154    server.socket.data.write "VALUE bar 0 7\r\n"
155    server.socket.data.write "\004\b\"\bbar\r\n"
156    server.socket.data.write "END\r\n"
157    server.socket.data.rewind
158
159    result = @cache.cache_get_multi server, 'foo bar baz'
160
161    assert_equal 2, result.length
162    assert_equal "\004\b\"\bfoo", result['foo']
163    assert_equal "\004\b\"\bbar", result['bar']
164  end
165
166  def test_cache_get_multi_EOF
167    server = util_setup_fake_server
168    server.socket.data.string = ''
169
170    e = assert_raise MemCache::MemCacheError do
171      @cache.cache_get_multi server, 'my_namespace:key'
172    end
173
174    assert_equal "lost connection to example.com:11211", e.message
175  end
176
177  def test_cache_get_multi_bad_state
178    server = FakeServer.new
179
180    # Write two messages to the socket to test failover
181    server.socket.data.write "bogus response\r\nbogus response\r\n"
182    server.socket.data.rewind
183
184    @cache.servers = []
185    @cache.servers << server
186
187    e = assert_raise MemCache::MemCacheError do
188      @cache.cache_get_multi server, 'my_namespace:key'
189    end
190
191    assert_match /#{Regexp.quote 'unexpected response "bogus response\r\n"'}/, e.message
192
193    assert !server.alive?
194
195    assert_match /get my_namespace:key\r\n/, server.socket.written.string
196  end
197
198  def test_initialize
199    cache = MemCache.new :namespace => 'my_namespace', :readonly => true
200
201    assert_equal 'my_namespace', cache.namespace
202    assert_equal true, cache.readonly?
203    assert_equal true, cache.servers.empty?
204  end
205
206  def test_initialize_compatible
207    cache = MemCache.new ['localhost:11211', 'localhost:11212'],
208            :namespace => 'my_namespace', :readonly => true
209
210    assert_equal 'my_namespace', cache.namespace
211    assert_equal true, cache.readonly?
212    assert_equal false, cache.servers.empty?
213  end
214
215  def test_initialize_compatible_no_hash
216    cache = MemCache.new ['localhost:11211', 'localhost:11212']
217
218    assert_equal nil, cache.namespace
219    assert_equal false, cache.readonly?
220    assert_equal false, cache.servers.empty?
221  end
222
223  def test_initialize_compatible_one_server
224    cache = MemCache.new 'localhost:11211'
225
226    assert_equal nil, cache.namespace
227    assert_equal false, cache.readonly?
228    assert_equal false, cache.servers.empty?
229  end
230
231  def test_initialize_compatible_bad_arg
232    e = assert_raise ArgumentError do
233      cache = MemCache.new Object.new
234    end
235
236    assert_equal 'first argument must be Array, Hash or String', e.message
237  end
238
239  def test_initialize_multiple_servers
240    cache = MemCache.new %w[localhost:11211 localhost:11212],
241                         :namespace => 'my_namespace', :readonly => true
242
243    assert_equal 'my_namespace', cache.namespace
244    assert_equal true, cache.readonly?
245    assert_equal false, cache.servers.empty?
246    assert !cache.instance_variable_get(:@continuum).empty?
247  end
248
249  def test_initialize_too_many_args
250    assert_raises ArgumentError do
251      MemCache.new 1, 2, 3
252    end
253  end
254
255  def test_decr
256    server = FakeServer.new
257    server.socket.data.write "5\r\n"
258    server.socket.data.rewind
259
260    @cache.servers = []
261    @cache.servers << server
262
263    value = @cache.decr 'key'
264
265    assert_equal "decr my_namespace:key 1\r\n",
266                 @cache.servers.first.socket.written.string
267
268    assert_equal 5, value
269  end
270
271  def test_decr_not_found
272    server = FakeServer.new
273    server.socket.data.write "NOT_FOUND\r\n"
274    server.socket.data.rewind
275
276    @cache.servers = []
277    @cache.servers << server
278
279    value = @cache.decr 'key'
280
281    assert_equal "decr my_namespace:key 1\r\n",
282                 @cache.servers.first.socket.written.string
283
284    assert_equal nil, value
285  end
286
287  def test_decr_space_padding
288    server = FakeServer.new
289    server.socket.data.write "5 \r\n"
290    server.socket.data.rewind
291
292    @cache.servers = []
293    @cache.servers << server
294
295    value = @cache.decr 'key'
296
297    assert_equal "decr my_namespace:key 1\r\n",
298                 @cache.servers.first.socket.written.string
299
300    assert_equal 5, value
301  end
302
303  def test_get
304    util_setup_fake_server
305
306    value = @cache.get 'key'
307
308    assert_equal "get my_namespace:key\r\n",
309                 @cache.servers.first.socket.written.string
310
311    assert_equal '0123456789', value
312  end
313
314  def test_get_bad_key
315    util_setup_fake_server
316    assert_raise ArgumentError do @cache.get 'k y' end
317
318    util_setup_fake_server
319    assert_raise ArgumentError do @cache.get 'k' * 250 end
320  end
321
322  def test_get_cache_get_IOError
323    socket = Object.new
324    def socket.write(arg) raise IOError, 'some io error'; end
325    server = FakeServer.new socket
326
327    @cache.servers = []
328    @cache.servers << server
329
330    e = assert_raise MemCache::MemCacheError do
331      @cache.get 'my_namespace:key'
332    end
333
334    assert_equal 'some io error', e.message
335  end
336
337  def test_get_cache_get_SystemCallError
338    socket = Object.new
339    def socket.write(arg) raise SystemCallError, 'some syscall error'; end
340    server = FakeServer.new socket
341
342    @cache.servers = []
343    @cache.servers << server
344
345    e = assert_raise MemCache::MemCacheError do
346      @cache.get 'my_namespace:key'
347    end
348
349    assert_equal 'unknown error - some syscall error', e.message
350  end
351
352  def test_get_no_connection
353    @cache.servers = 'localhost:1'
354    e = assert_raise MemCache::MemCacheError do
355      @cache.get 'key'
356    end
357
358    assert_match /^No connection to server/, e.message
359  end
360
361  def test_get_no_servers
362    @cache.servers = []
363    e = assert_raise MemCache::MemCacheError do
364      @cache.get 'key'
365    end
366
367    assert_equal 'No active servers', e.message
368  end
369
370  def test_get_multi
371    server = FakeServer.new
372    server.socket.data.write "VALUE my_namespace:key 0 14\r\n"
373    server.socket.data.write "\004\b\"\0170123456789\r\n"
374    server.socket.data.write "VALUE my_namespace:keyb 0 14\r\n"
375    server.socket.data.write "\004\b\"\0179876543210\r\n"
376    server.socket.data.write "END\r\n"
377    server.socket.data.rewind
378
379    @cache.servers = []
380    @cache.servers << server
381
382    values = @cache.get_multi 'key', 'keyb'
383
384    assert_equal "get my_namespace:key my_namespace:keyb\r\n",
385                 server.socket.written.string
386
387    expected = { 'key' => '0123456789', 'keyb' => '9876543210' }
388
389    assert_equal expected.sort, values.sort
390  end
391
392  def test_get_raw
393    server = FakeServer.new
394    server.socket.data.write "VALUE my_namespace:key 0 10\r\n"
395    server.socket.data.write "0123456789\r\n"
396    server.socket.data.write "END\r\n"
397    server.socket.data.rewind
398
399    @cache.servers = []
400    @cache.servers << server
401
402
403    value = @cache.get 'key', true
404
405    assert_equal "get my_namespace:key\r\n",
406                 @cache.servers.first.socket.written.string
407
408    assert_equal '0123456789', value
409  end
410
411  def test_get_server_for_key
412    server = @cache.get_server_for_key 'key'
413    assert_equal 'localhost', server.host
414    assert_equal 1, server.port
415  end
416
417  def test_get_server_for_key_multiple
418    s1 = util_setup_server @cache, 'one.example.com', ''
419    s2 = util_setup_server @cache, 'two.example.com', ''
420    @cache.servers = [s1, s2]
421
422    server = @cache.get_server_for_key 'keya'
423    assert_equal 'two.example.com', server.host
424    server = @cache.get_server_for_key 'keyb'
425    assert_equal 'two.example.com', server.host
426    server = @cache.get_server_for_key 'keyc'
427    assert_equal 'two.example.com', server.host
428    server = @cache.get_server_for_key 'keyd'
429    assert_equal 'one.example.com', server.host
430  end
431
432  def test_get_server_for_key_no_servers
433    @cache.servers = []
434
435    e = assert_raise MemCache::MemCacheError do
436      @cache.get_server_for_key 'key'
437    end
438
439    assert_equal 'No servers available', e.message
440  end
441
442  def test_get_server_for_key_spaces
443    e = assert_raise ArgumentError do
444      @cache.get_server_for_key 'space key'
445    end
446    assert_equal 'illegal character in key "space key"', e.message
447  end
448
449  def test_get_server_for_key_length
450    @cache.get_server_for_key 'x' * 250
451    long_key = 'x' * 251
452    e = assert_raise ArgumentError do
453      @cache.get_server_for_key long_key
454    end
455    assert_equal "key too long #{long_key.inspect}", e.message
456  end
457
458  def test_incr
459    server = FakeServer.new
460    server.socket.data.write "5\r\n"
461    server.socket.data.rewind
462
463    @cache.servers = []
464    @cache.servers << server
465
466    value = @cache.incr 'key'
467
468    assert_equal "incr my_namespace:key 1\r\n",
469                 @cache.servers.first.socket.written.string
470
471    assert_equal 5, value
472  end
473
474  def test_incr_not_found
475    server = FakeServer.new
476    server.socket.data.write "NOT_FOUND\r\n"
477    server.socket.data.rewind
478
479    @cache.servers = []
480    @cache.servers << server
481
482    value = @cache.incr 'key'
483
484    assert_equal "incr my_namespace:key 1\r\n",
485                 @cache.servers.first.socket.written.string
486
487    assert_equal nil, value
488  end
489
490  def test_incr_space_padding
491    server = FakeServer.new
492    server.socket.data.write "5 \r\n"
493    server.socket.data.rewind
494
495    @cache.servers = []
496    @cache.servers << server
497
498    value = @cache.incr 'key'
499
500    assert_equal "incr my_namespace:key 1\r\n",
501                 @cache.servers.first.socket.written.string
502
503    assert_equal 5, value
504  end
505
506  def test_make_cache_key
507    assert_equal 'my_namespace:key', @cache.make_cache_key('key')
508    @cache.namespace = nil
509    assert_equal 'key', @cache.make_cache_key('key')
510  end
511
512  def test_servers
513    server = FakeServer.new
514    @cache.servers = []
515    @cache.servers << server
516    assert_equal [server], @cache.servers
517  end
518
519  def test_servers_equals_type_error
520    e = assert_raise TypeError do
521      @cache.servers = [Object.new]
522    end
523
524    assert_equal 'cannot convert Object into MemCache::Server', e.message
525  end
526
527  def test_set
528    server = FakeServer.new
529    server.socket.data.write "STORED\r\n"
530    server.socket.data.rewind
531    @cache.servers = []
532    @cache.servers << server
533
534    @cache.set 'key', 'value'
535
536    dumped = Marshal.dump('value')
537    expected = "set my_namespace:key 0 0 #{dumped.length}\r\n#{dumped}\r\n"
538#    expected = "set my_namespace:key 0 0 9\r\n\004\b\"\nvalue\r\n"
539    assert_equal expected, server.socket.written.string
540  end
541
542  def test_set_expiry
543    server = FakeServer.new
544    server.socket.data.write "STORED\r\n"
545    server.socket.data.rewind
546    @cache.servers = []
547    @cache.servers << server
548
549    @cache.set 'key', 'value', 5
550
551    dumped = Marshal.dump('value')
552    expected = "set my_namespace:key 0 5 #{dumped.length}\r\n#{dumped}\r\n"
553    assert_equal expected, server.socket.written.string
554  end
555
556  def test_set_raw
557    server = FakeServer.new
558    server.socket.data.write "STORED\r\n"
559    server.socket.data.rewind
560    @cache.servers = []
561    @cache.servers << server
562
563    @cache.set 'key', 'value', 0, true
564
565    expected = "set my_namespace:key 0 0 5\r\nvalue\r\n"
566    assert_equal expected, server.socket.written.string
567  end
568
569  def test_set_readonly
570    cache = MemCache.new :readonly => true
571
572    e = assert_raise MemCache::MemCacheError do
573      cache.set 'key', 'value'
574    end
575
576    assert_equal 'Update of readonly cache', e.message
577  end
578
579  def test_set_too_big
580    server = FakeServer.new
581
582    # Write two messages to the socket to test failover
583    server.socket.data.write "SERVER_ERROR\r\nSERVER_ERROR object too large for cache\r\n"
584    server.socket.data.rewind
585
586    @cache.servers = []
587    @cache.servers << server
588
589    e = assert_raise MemCache::MemCacheError do
590      @cache.set 'key', 'v'
591    end
592
593    assert_match /object too large for cache/, e.message
594  end
595
596  def test_add
597    server = FakeServer.new
598    server.socket.data.write "STORED\r\n"
599    server.socket.data.rewind
600    @cache.servers = []
601    @cache.servers << server
602
603    @cache.add 'key', 'value'
604    
605    dumped = Marshal.dump('value')
606
607    expected = "add my_namespace:key 0 0 #{dumped.length}\r\n#{dumped}\r\n"
608    assert_equal expected, server.socket.written.string
609  end
610
611  def test_add_exists
612    server = FakeServer.new
613    server.socket.data.write "NOT_STORED\r\n"
614    server.socket.data.rewind
615    @cache.servers = []
616    @cache.servers << server
617
618    @cache.add 'key', 'value'
619
620    dumped = Marshal.dump('value')
621    expected = "add my_namespace:key 0 0 #{dumped.length}\r\n#{dumped}\r\n"
622    assert_equal expected, server.socket.written.string
623  end
624
625  def test_add_expiry
626    server = FakeServer.new
627    server.socket.data.write "STORED\r\n"
628    server.socket.data.rewind
629    @cache.servers = []
630    @cache.servers << server
631
632    @cache.add 'key', 'value', 5
633
634    dumped = Marshal.dump('value')
635    expected = "add my_namespace:key 0 5 #{dumped.length}\r\n#{dumped}\r\n"
636    assert_equal expected, server.socket.written.string
637  end
638
639  def test_add_raw
640    server = FakeServer.new
641    server.socket.data.write "STORED\r\n"
642    server.socket.data.rewind
643    @cache.servers = []
644    @cache.servers << server
645
646    @cache.add 'key', 'value', 0, true
647
648    expected = "add my_namespace:key 0 0 5\r\nvalue\r\n"
649    assert_equal expected, server.socket.written.string
650  end
651
652  def test_add_readonly
653    cache = MemCache.new :readonly => true
654
655    e = assert_raise MemCache::MemCacheError do
656      cache.add 'key', 'value'
657    end
658
659    assert_equal 'Update of readonly cache', e.message
660  end
661
662  def test_delete
663    server = FakeServer.new
664    @cache.servers = []
665    @cache.servers << server
666    
667    @cache.delete 'key'
668    
669    expected = "delete my_namespace:key 0\r\n"
670    assert_equal expected, server.socket.written.string
671  end
672
673  def test_delete_with_expiry
674    server = FakeServer.new
675    @cache.servers = []
676    @cache.servers << server
677    
678    @cache.delete 'key', 300
679    
680    expected = "delete my_namespace:key 300\r\n"
681    assert_equal expected, server.socket.written.string
682  end
683
684  def test_flush_all
685    @cache.servers = []
686    3.times { @cache.servers << FakeServer.new }
687
688    @cache.flush_all
689
690    expected = "flush_all\r\n"
691    @cache.servers.each do |server|
692      assert_equal expected, server.socket.written.string
693    end
694  end
695
696  def test_flush_all_failure
697    socket = FakeSocket.new
698
699    # Write two messages to the socket to test failover
700    socket.data.write "ERROR\r\nERROR\r\n"
701    socket.data.rewind
702
703    server = FakeServer.new socket
704
705    @cache.servers = []
706    @cache.servers << server
707
708    assert_raise MemCache::MemCacheError do
709      @cache.flush_all
710    end
711
712    assert_match /flush_all\r\n/, socket.written.string
713  end
714
715  def test_stats
716    socket = FakeSocket.new
717    socket.data.write "STAT pid 20188\r\nSTAT total_items 32\r\nSTAT version 1.2.3\r\nSTAT rusage_user 1:300\r\nSTAT dummy ok\r\nEND\r\n"
718    socket.data.rewind
719    server = FakeServer.new socket
720    def server.host() 'localhost'; end
721    def server.port() 11211; end
722
723    @cache.servers = []
724    @cache.servers << server
725
726    expected = {
727      'localhost:11211' => {
728        'pid' => 20188, 'total_items' => 32, 'version' => '1.2.3',
729        'rusage_user' => 1.0003, 'dummy' => 'ok'
730      }
731    }
732    assert_equal expected, @cache.stats
733
734    assert_equal "stats\r\n", socket.written.string
735  end
736
737  def test_basic_threaded_operations_should_work
738    cache = MemCache.new :multithread => true,
739                         :namespace => 'my_namespace',
740                         :readonly => false
741
742    server = FakeServer.new
743    server.socket.data.write "STORED\r\n"
744    server.socket.data.rewind
745
746    cache.servers = []
747    cache.servers << server
748
749    assert cache.multithread
750
751    assert_nothing_raised do
752      cache.set "test", "test value"
753    end
754
755    # TODO Fails in 1.9
756    assert_match /set my_namespace:test.*\r\n.*test value.*\r\n/, server.socket.written.string
757  end
758
759  def test_basic_unthreaded_operations_should_work
760    cache = MemCache.new :multithread => false,
761                         :namespace => 'my_namespace',
762                         :readonly => false
763
764    server = FakeServer.new
765    server.socket.data.write "STORED\r\n"
766    server.socket.data.rewind
767
768    cache.servers = []
769    cache.servers << server
770
771    assert !cache.multithread
772
773    assert_nothing_raised do
774      cache.set "test", "test value"
775    end
776
777    # TODO Fails in 1.9
778    assert_match /set my_namespace:test.*\r\n.*test value\r\n/, server.socket.written.string
779  end
780
781  def util_setup_fake_server
782    server = FakeServer.new
783    server.socket.data.write "VALUE my_namespace:key 0 14\r\n"
784    server.socket.data.write "\004\b\"\0170123456789\r\n"
785    server.socket.data.write "END\r\n"
786    server.socket.data.rewind
787
788    @cache.servers = []
789    @cache.servers << server
790
791    return server
792  end
793
794  def util_setup_server(memcache, host, responses)
795    server = MemCache::Server.new memcache, host
796    server.instance_variable_set :@sock, StringIO.new(responses)
797
798    @cache.servers = []
799    @cache.servers << server
800
801    return server
802  end
803
804end
805