PageRenderTime 49ms CodeModel.GetById 1ms app.highlight 40ms RepoModel.GetById 1ms app.codeStats 1ms

/test/tests.php

https://github.com/ksmythe/scribe
PHP | 364 lines | 320 code | 28 blank | 16 comment | 5 complexity | ccb285aa344086ca86a28d8d8ddaa167 MD5 | raw file
  1<?php
  2//  Copyright (c) 2007-2008 Facebook
  3//
  4//  Licensed under the Apache License, Version 2.0 (the "License");
  5//  you may not use this file except in compliance with the License.
  6//  You may obtain a copy of the License at
  7//
  8//      http://www.apache.org/licenses/LICENSE-2.0
  9//
 10//  Unless required by applicable law or agreed to in writing, software
 11//  distributed under the License is distributed on an "AS IS" BASIS,
 12//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13//  See the License for the specific language governing permissions and
 14//  limitations under the License.
 15//
 16// See accompanying file LICENSE or visit the Scribe site at:
 17// http://developers.facebook.com/scribe/
 18
 19$GLOBALS['THRIFT_ROOT'] = '/usr/local/thrift/php/thrift';
 20$GLOBALS['SCRIBE_ROOT'] = '/usr/local/thrift/php/thrift/packages';
 21
 22include_once $GLOBALS['SCRIBE_ROOT'].'/scribe.php';
 23include_once $GLOBALS['THRIFT_ROOT'].'/protocol/TBinaryProtocol.php';
 24include_once $GLOBALS['THRIFT_ROOT'].'/transport/TFramedTransport.php';
 25include_once $GLOBALS['THRIFT_ROOT'].'/transport/TSocketPool.php';
 26
 27function simple_test() {
 28  $messages = array();
 29  $msg = new LogEntry;
 30  $msg->category = 'scribe_test';
 31  $msg->message = "this is a message\n";
 32  $messages []= $msg;
 33  $msg2 = new LogEntry;
 34  $msg2->category = 'scribe_test';
 35  $msg2->message = "and a binary" . chr(0) . chr(1) . " message\n";
 36  $messages []= $msg2;
 37  $msg3 = new LogEntry;
 38  $msg3->category = 'buckettest';
 39  $msg3->message = '99	a key-value message with a non-printable delimiter\n';
 40  $messages []= $msg3;
 41  $msg4 = new LogEntry;
 42  $msg4->category = 'buckettest';
 43  $msg4->message = '99	a different message in the same bucket\n';
 44  $messages []= $msg4;
 45  $msg5 = new LogEntry;
 46  $msg5->category = 'buckettest';
 47  $msg5->message = '98	a different bucket\n';
 48  $messages []= $msg5;
 49
 50  $scribe_client = create_scribe_client();
 51  $ret = scribe_Log_test($messages, $scribe_client);
 52
 53  print "Log returned: " . $ret . "\n";
 54}
 55
 56
 57function bucket_test() {
 58  $messages = array();
 59  $msg = new LogEntry;
 60  $msg->category = 'scribe_test';
 61  $msg->message = "this is a message\n";
 62  $messages []= $msg;
 63  $msg2 = new LogEntry;
 64  $msg2->category = 'scribe_test';
 65  $msg2->message = "and a binary" . chr(0) . chr(1) . " message\n";
 66  $messages []= $msg2;
 67  $msg3 = new LogEntry;
 68  $msg3->category = 'buckettest2';
 69  $msg3->message = '99' . chr(1) . 'a key-value message with a non-printable delimiter\n';
 70  $messages []= $msg3;
 71  $msg4 = new LogEntry;
 72  $msg4->category = 'buckettest2';
 73  $msg4->message = '99' . chr(1) . 'a different message in the same bucket\n';
 74  $messages []= $msg4;
 75  $msg5 = new LogEntry;
 76  $msg5->category = 'buckettest2';
 77  $msg5->message = '98' . chr(1) . 'a different bucket\n';
 78  $messages []= $msg5;
 79  $msg6 = new LogEntry;
 80  $msg6->category = 'buckettest2';
 81  $msg6->message = '97' . chr(1) . 'a different bucket\n';
 82  $messages []= $msg6;
 83  $msg7 = new LogEntry;
 84  $msg7->category = 'buckettest2';
 85  $msg7->message = '96' . chr(1) . 'a different bucket\n';
 86  $messages []= $msg7;
 87
 88  $scribe_client = create_scribe_client();
 89  $ret = scribe_Log_test($messages, $scribe_client);
 90
 91  print "Log returned: " . $ret . "\n";
 92}
 93
 94
 95function strange_input_test() {
 96  $messages = array();
 97  $msg = new LogEntry;
 98  $msg->category = '%xa\n\ndfsdfjfdsjlkasjlkjerl%slkjasdf%dlkjasdlkjf\\\\\\\\\\\\\adskljasdl;kjsg[pa;lksdkjmdkjfkjfkjkdjfslkkjlaasdfasdfasdfasdfasdfasdflkhjlaksjdlkfjalksjdflkjasdflkjsdaflkjsdflkjadsflkjjsadflkkjsdflkjjsdfalkjfdsakljfsdaljk';
 99  $msg->message = '%xlasdlkfjalskjlkjasdklg\\\\\\\/////\\\/\\//\\;klf;klds;klfsdaflsk;kl;lk;sfkl;sdf%d<><>><><<>>l;kadsl;kadsl;k;klkl;fsdal;ksdfa;klsaf;lsdfl;kfsdl;ksdflksdaf;lkfds;lkfsd;lksdafl;kf;klsflk;sdf;klsdfka;lskdl;fkls;dfalk;fsdl;ksfadkl;sfdlk;sfadlk;fsld;kasflkad;klfsad;lksdfal;ksfda;lksdaflk;sdfal;kl;sdfakl;sdaf;klsdfa;klsdafk;lsdfakl;sdafkl;sdfak;lasdfkl;sdaflk;sdaflk;sdafkl;sadf;lksdafl;ksdaf;klsdafl;ksdafl;ksdfak;lsdafkl;sdfl;kdsfakl;sdaf;lkdsafk;lsdfkl;sdfakl;fdsa;klfsdk;lasfdk;lfsdakl;sdfak;lfsdakl;sdfakl;sfdak;lsdfaklfdsakl;sdfak;lsfdak;lsfdakl;sfdakl;sdfak;lsdfak;lsdfak;lsdfak;lsfdakl;sdfak;lsdfakl;sdfak;lsdfakl;sdfk;lfds;alkadfsk;lsdfak;ldfsak;lsdfa;klsfdakl;sdfak;lsdfakl;sdfak;lsdfak;lsdf;klsdfa;klsdfak;lsdfak;lfsdakl;sdfakl;fdsak;lsdfak;lsdfakl;sdfak;lsdfak;lsdfa;klsdfa;klsfdk;alsfadkl;sdfakl;sdfkl;fdsakl;sfdal;ksdfak;las;lkfsda;lksdfak;lsdfak;lsdfakl;kfds;lk;l;sdf;klfdsl;ksdfal;ksdl;kds;lksdkl;dsfal;kdsfak;ldsfa;kldfasl;kdfasl;kdafsl;kadfskl;dfsa;kldfsak;ldfsakl;dfask;ldfsak;ldfsal;kdfsakl;dfsak;ldfsak;k;dlfsa;lkadsf;kladsf;lkdfsa;lkdsaf;k;lkdfsakl;dfsal;kdfsa;kldsaf;lkdfsa;kldasf;kldfas;kldsaf;klasdfk;lsadf;klsdafk;ldsaf;lkasdfk;ldfas;kladfs;kldfaskl;dfsa;kldfsakl;dfsalk;dfska;kladsfk;ladfs;kladsfkl;adssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss\n\n\nssssssssssssssssssssssssssssssf';
100  $messages []= $msg;
101  $msg2 = new LogEntry;
102  $msg2->category = 'scribe_test';
103  $msg2->message = '\n';
104  $messages []= $msg2;
105  $msg3 = new LogEntry;
106  $msg3->category = 'scribe_test';
107  $msg3->message = '';
108  $messages [] = $msg3;
109
110  $scribe_client = create_scribe_client();
111  $ret = scribe_Log_test($messages, $scribe_client);
112
113  print "Log returned: " . $ret . "\n";
114}
115
116function stress_test($category, $client_name, $rate, $total, $msg_per_call,
117                     $avg_size, $num_categories) {
118
119  $send_interval = $msg_per_call/$rate;
120
121  $random = generate_random($avg_size * 2);
122
123  $msgs_since_send = 0;
124  $messages = array();
125  $last_send_time = microtime(true);
126
127  $scribe_client = create_scribe_client();
128
129  for($i = 0; $i < $total; ++$i) {
130
131    $entry = new LogEntry;
132    $entry->category = $category;
133
134    if ($num_categories > 1) {
135      $entry->category .= rand(1, $num_categories);
136    }
137
138    $entry->message = make_message($client_name, $avg_size, $i, $random);
139    $messages []= $entry;
140
141    ++$msgs_since_send;
142
143    if ($msgs_since_send >= $msg_per_call) {
144
145      $msgs_since_send = 0;
146      $ret = scribe_Log_test($messages, $scribe_client);
147      $messages = array();
148
149      $now = microtime(true);
150      $wait = $last_send_time + $send_interval - $now;
151      $last_send_time = $now;
152      if ($wait > 0) {
153        usleep($wait * 1000000);
154      }
155    }
156  }
157}
158
159function many_connections_test($category, $client_name, $num_connections, $rate,
160                               $total, $msg_per_call, $avg_size) {
161
162  if ($num_connections < 1) {
163    print("you can't run the test with $num_connections connections\n");
164  }
165
166  if ($rate % $num_connections != 0 ||
167      $total % $num_connections != 0 ||
168      ($total/$num_connections) % $msg_per_call != 0) {
169    print("Arguments don't divide evenly, so number of messages won't be accurate\n");
170  }
171
172  $rate_per_conn = $rate/$num_connections;
173  $send_interval = $msg_per_call/$rate_per_conn;
174
175  // open all the connections. This bypasses the normal client library because
176  // we want to open a lot of connections simultaneously, which is something
177  // a real client would never need to do.
178  //
179  $server_ips = array('localhost');
180  $port = 1463;
181  $socks = array();
182  $trans = array();
183  $prots = array();
184  $scribes = array();
185  $opened = array();
186
187  try {
188    for ($i = 0; $i < $num_connections; ++$i) {
189      $socks[$i] = new TSocketPool($server_ips, $port);
190      $socks[$i]->setDebug(1);
191      $socks[$i]->setSendTimeout(2500);
192      $socks[$i]->setRecvTimeout(2500);
193      $socks[$i]->setNumRetries(1);
194      $socks[$i]->setRetryInterval(30);
195      $socks[$i]->setRandomize(true);
196      $socks[$i]->setMaxConsecutiveFailures(2);
197      $socks[$i]->setAlwaysTryLast(false);
198      $trans[$i] = new TFramedTransport($socks[$i], 1024, 1024);
199      $prots[$i] = new TBinaryProtocol($trans[$i]);
200      $scribes[$i] = new scribeClient($prots[$i]);
201      $trans[$i]->open();
202      $opened[$i] = true;
203    }
204  } catch (Exception $x) {
205    print "exception opening connection $i";
206    // Bummer too bad so sad
207    return;
208  }
209
210
211  // Send the messages, a few from each connection every loop
212  //
213  $random = generate_random($avg_size * 2);
214  $last_send_time = microtime(true);
215
216  $i = 0;
217  while ($i < $total) {
218
219    $messages = array();
220    for ($conn = 0; $conn < $num_connections; ++$conn) {
221      for ($j = 0; $j < $msg_per_call; ++$j) {
222
223        $entry = new LogEntry;
224        $entry->category = $category;
225        $entry->message = make_message($client_name, $avg_size, $i, $random);
226        $messages []= $entry;
227        ++$i;
228      }
229     $result = $scribes[$conn]->Log($messages);
230     if ($result != ResultCode::OK) {
231       print "Warning: Log returned $result \n";
232     }
233
234     $messages = array();
235    }
236    $now = microtime(true);
237    $wait = $last_send_time + $send_interval - $now;
238    $last_send_time = $now;
239    if ($wait > 0) {
240      usleep($wait * 1000000);
241    }
242  }
243
244  // Close the connections
245  //
246  for ($i = 0; $i < $num_connections; ++$i) {
247    if ($opened[$i]) {
248      try {
249        $trans[$i]->close();
250      } catch (Exception $x) {
251        print "exception closing connection $i";
252        // Ignore close errors
253      }
254    }
255  }
256}
257
258function generate_random($size) {
259  $random = 'qwertyiopui%sfmsg;lmad;lkh[pwermflvps/w]slkbb;k,mtjasdlkjfaslkjdfflkjasdfkljaslkdjgfacebookqweetewatab';
260  while ($random_len = strlen($random) < $size) {
261    $random .= $random;
262  }
263  return $random;
264}
265
266function make_message($client_name, $avg_size, $sequence, $random) {
267
268  $padding_size = $avg_size - strlen($client_name) - 10;
269  if ($padding_size > 0) {
270    // TODO: we could make this fancier with pseudo-random sizes
271    $padding_size = $sequence % ($padding_size * 2);
272  } else {
273    $padding_size = 0;
274  }
275
276  $message = $client_name . '-' . $sequence;
277  if ($padding_size) {
278    $message .= '-' . $padding_size . '-';
279    $message .= substr($random, 0, $padding_size);
280  }
281
282  $message .= "\n";
283  return $message;
284}
285
286function create_scribe_client() {
287  try {
288    // Set up the socket connections
289    $scribe_servers = array('localhost');
290    $scribe_ports = array(1463);
291    print "creating socket pool\n";
292    $sock = new TSocketPool($scribe_servers, $scribe_ports);
293    $sock->setDebug(0);
294    $sock->setSendTimeout(1000);
295    $sock->setRecvTimeout(2500);
296    $sock->setNumRetries(1);
297    $sock->setRandomize(false);
298    $sock->setAlwaysTryLast(true);
299    $trans = new TFramedTransport($sock);
300    $prot = new TBinaryProtocol($trans);
301
302    // Create the client
303    print "creating scribe client\n";
304    $scribe_client = new scribeClient($prot);
305
306    // Open the transport (we rely on PHP to close it at script termination)
307    print "opening transport\n";
308    $trans->open();
309
310  } catch (Exception $x) {
311    print "Unable to create global scribe client, received exception: $x \n";
312    return null;
313  }
314
315  return $scribe_client;
316}
317
318function scribe_Log_test($messages, $scribe_client) {
319  try {
320    $result = $scribe_client->Log($messages);
321
322    if ($result != ResultCode::OK) {
323      print "Warning: Log returned $result \n";
324    }
325
326    return $result;
327  } catch (Exception $x) {
328    print "Scribe client failed logging " . count($messages) .
329      " messages with exception: $x \n";
330  }
331}
332
333function super_stress_test($categories, $client_name, $rate, $total,
334                           $msg_per_call, $avg_size, $category_multiplier) {
335  $pids = array();
336
337  // Fork a new process for every category
338  foreach ($categories as $category) {
339    $pid = pcntl_fork();
340
341    if($pid == -1) {
342      print "Error: Could not fork\n";
343      return;
344    }
345    else if($pid == 0) {
346      // In child process
347      print "Sending messages for category $category...\n";
348      stress_test($category, $client_name, $rate, $total,
349                  $msg_per_call, $avg_size, $category_multiplier);
350      print "Done sending messages for category $category.\n";
351      Exit(0);
352    } else {
353      // In parent process
354      $pids[] = $pid;
355    }
356  }
357
358  // have parent wait for all children
359  foreach ($pids as $pid) {
360    pcntl_waitpid($pid, $status);
361  }
362}
363
364?>