/tests/scripts/mesh_test.cpp
C++ | 733 lines | 651 code | 49 blank | 33 comment | 96 complexity | eb855bd411bfeeebd20bda9c0ff5b543 MD5 | raw file
1/* This file is part of VoltDB. 2 * Copyright (C) 2008-2014 VoltDB Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining 5 * a copy of this software and associated documentation files (the 6 * "Software"), to deal in the Software without restriction, including 7 * without limitation the rights to use, copy, modify, merge, publish, 8 * distribute, sublicense, and/or sell copies of the Software, and to 9 * permit persons to whom the Software is furnished to do so, subject to 10 * the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be 13 * included in all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 18 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24#include <iostream> 25#include <string> 26#include <vector> 27#include <boost/unordered_map.hpp> 28#include <boost/unordered_set.hpp> 29#include <set> 30#include <deque> 31#include <boost/bind.hpp> 32#include <boost/asio.hpp> 33#include <queue> 34#include <limits> 35 36using namespace std; 37using boost::asio::ip::tcp; 38 39class QueueEntry { 40public: 41 QueueEntry(tcp::socket *nsocket, int64_t nrequestId) : socket(nsocket), requestId(nrequestId) {} 42 tcp::socket *socket; 43 int64_t requestId; 44}; 45 46class CompareQueueEntry { 47public: 48 bool operator()(const QueueEntry& lhs, const QueueEntry& rhs) const { 49 if (lhs.requestId > rhs.requestId) { 50 return true; 51 } 52 return false; 53 } 54}; 55 56class WriteStuff { 57public: 58 WriteStuff(size_t nlength, size_t noffset, int64_t *ncounter) : length(nlength), offset(noffset), counter(ncounter) {} 59 size_t length; 60 size_t offset; 61 int64_t *counter; 62}; 63 64class Server 65 { 66public: 67 Server(boost::asio::io_service &io_service, std::vector<std::string> servers) : 68 nextRequestId(0), 69 requestsReceived(0), 70 responsesSent(0), 71 meshRequestsReceived(0), 72 meshRequestsSent(0), 73 meshResponsesSent(0), 74 meshResponsesReceived(0), 75 bytesWritten(0), 76 bytesRead(0), 77 bytesWrittenLastTime(0), 78 bytesReadLastTime(0), 79 serverAcceptor(io_service, tcp::endpoint(tcp::v4(), 21413)), 80 clientAcceptor(io_service, tcp::endpoint(tcp::v4(), 21412)) { 81 tcp::socket *socket = new tcp::socket(io_service); 82 serverAcceptor.async_accept( *socket, 83 boost::bind( 84 &Server::handleServerAccept, 85 this, 86 boost::asio::placeholders::error, 87 &serverAcceptor, 88 socket)); 89 90 91 socket = new tcp::socket(io_service); 92 clientAcceptor.async_accept( *socket, 93 boost::bind( 94 &Server::handleClientAccept, 95 this, 96 boost::asio::placeholders::error, 97 &clientAcceptor, 98 socket)); 99 100 tcp::resolver resolver(io_service); 101 for (std::size_t ii = 0; ii < servers.size(); ii++) { 102 tcp::resolver::query query(servers[ii], "21413"); 103 tcp::resolver::iterator endpoint_iterator = resolver.resolve(query); 104 tcp::socket* socket = new tcp::socket(io_service); 105 tcp::endpoint endpoint = *endpoint_iterator; 106 socket->async_connect( endpoint, 107 boost::bind( &Server::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket)); 108 } 109 110 boost::asio::deadline_timer *timer = new boost::asio::deadline_timer(io_service); 111 timer->expires_from_now(boost::posix_time::seconds(10)); 112 timer->async_wait(boost::bind(&Server::timerCallback, this, boost::asio::placeholders::error, timer)); 113 } 114 115 void timerCallback(const boost::system::error_code& error, boost::asio::deadline_timer *timer) { 116 if (error) { 117 std::cout << error.message() << std::endl; 118 } 119 int64_t bytesReadThisTime = bytesRead - bytesReadLastTime; 120 bytesReadLastTime = bytesRead; 121 int64_t bytesWrittenThisTime = bytesWritten - bytesWrittenLastTime; 122 bytesWrittenLastTime = bytesWritten; 123 int64_t mbReadThisTime = bytesReadThisTime / (1024 * 1024); 124 int64_t mbWrittenThisTime = bytesWrittenThisTime / (1024 * 1024); 125 double mbReadPerSec = mbReadThisTime / 10.0; 126 double mbWrittenPerSec = mbWrittenThisTime / 10.0; 127 std::cout << "Megabytes/sec In/Out " << mbReadPerSec << "/" << mbWrittenPerSec << std::endl; 128 std::cout << "Requests received " << requestsReceived << " Responses sent " << responsesSent << " MRequests received " 129 << meshRequestsReceived << " MRequests sent " << meshRequestsSent << " MResponses sent " << meshResponsesSent << 130 " MResponses received " << meshResponsesReceived << " Outstanding requests " << (requestsReceived - responsesSent) << std::endl; 131 timer->expires_from_now(boost::posix_time::seconds(10)); 132 timer->async_wait(boost::bind(&Server::timerCallback, this, boost::asio::placeholders::error, timer)); 133 } 134 135 void handleConnect(const boost::system::error_code& error, 136 tcp::resolver::iterator endpoint_iterator, tcp::socket* socket) { 137 if (error) { 138 std::cout << error.message() << std::endl; 139 socket->close(); 140 ++endpoint_iterator; 141 if (endpoint_iterator != tcp::resolver::iterator()) { 142 tcp::endpoint endpoint = *endpoint_iterator; 143 socket->async_connect(endpoint, 144 boost::bind( &Server::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket)); 145 } 146 } else { 147 std::cout << "Connected to server " << socket->remote_endpoint().address().to_string() << std::endl; 148 boost::asio::ip::tcp::no_delay option(true); 149 socket->set_option(option); 150 boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true); 151 socket->io_control(nonblocking); 152 serverSockets.push_back(socket); 153 socketAsyncWritePending[socket] = false; 154 char *buffer = acquireBuffer(); 155 requestIdsResponded[socket] = new boost::unordered_set<int64_t>(); 156 requestIdsRequested[socket] = new boost::unordered_set<int64_t>(); 157 socketAsyncWrites[socket] = new std::deque<std::pair<char*, WriteStuff> >(); 158 boost::asio::async_read( *socket, boost::asio::buffer( buffer, 4), boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, true)); 159 } 160 } 161 162 void handleServerAccept( 163 const boost::system::error_code& error, 164 tcp::acceptor *acceptor, 165 tcp::socket* socket) { 166 if (error) { 167 std::cout << error.message() << std::endl; 168 return; 169 } 170 if (!error) { 171 std::cout << "Accepted from server " << socket->remote_endpoint().address().to_string() << std::endl; 172 socketAsyncWritePending[socket] = false; 173 serverSockets.push_back(socket); 174 tcp::socket *newsocket = new tcp::socket(acceptor->io_service()); 175 acceptor->async_accept(*newsocket, boost::bind(&Server::handleServerAccept, this, boost::asio::placeholders::error, acceptor, newsocket)); 176 boost::asio::ip::tcp::no_delay option(true); 177 socket->set_option(option); 178 boost::asio::ip::tcp::socket::receive_buffer_size receiveSize(262144); 179 boost::asio::ip::tcp::socket::send_buffer_size sendSize(262144); 180 socket->set_option(receiveSize); 181 socket->set_option(sendSize); 182 char *buffer = acquireBuffer(); 183 boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true); 184 socket->io_control(nonblocking); 185 requestIdsResponded[socket] = new boost::unordered_set<int64_t>(); 186 requestIdsRequested[socket] = new boost::unordered_set<int64_t>(); 187 socketAsyncWrites[socket] = new std::deque<std::pair<char*, WriteStuff> >(); 188 boost::asio::async_read( *socket, boost::asio::buffer( buffer, 4), boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, true)); 189 } 190 } 191 192 void handleServerRead(const boost::system::error_code& error, 193 size_t bytes_transferred, tcp::socket* socket, char *buffer, bool lengthOrMessage) { 194 if (error) { 195 std::cout << error.message() << std::endl; 196 releaseBuffer(buffer); 197 return; 198 } 199 200 bytesRead += bytes_transferred; 201 202 size_t nextLength = 0; 203 size_t nextBufferOffset; 204 while (true) { 205 nextBufferOffset = 0; 206 if (lengthOrMessage) { 207 nextLength = *reinterpret_cast<int32_t*>(buffer); 208 nextBufferOffset = 4; 209 lengthOrMessage = false; 210 if (nextLength <= 0 || nextLength > 1500) { 211 std::cout << "Next length from server was " << nextLength << std::endl; 212 exit(-1); 213 } 214 } else { 215 nextLength = 4; 216 lengthOrMessage = true; 217 if (buffer[4] == 0) { 218 //request 219 meshRequestsReceived++; 220 int64_t requestId = *reinterpret_cast<int64_t*>(&buffer[5]); 221 addQueueEntry(socket, requestId); 222 } else if (buffer[4] == 1) { 223 //response 224 meshResponsesReceived++; 225 int32_t responseLength = *reinterpret_cast<int32_t*>(buffer); 226 int64_t requestId = *reinterpret_cast<int64_t*>(&buffer[5]); 227 char *copy = acquireBuffer(); 228 ::memcpy( copy, buffer, responseLength + 4); 229 tcp::socket *clientSocket = requestIdToClient[requestId]; 230 requestIdToClient.erase(requestId); 231// if (!requestIdsResponded[socket]->insert(requestId).second) { 232// std::cout << "Request ID " << requestId << " has already been responded to" << std::endl; 233// } 234 235 size_t written = 0; 236 bool asyncWritePending = socketAsyncWritePending[clientSocket]; 237 try { 238 if (!asyncWritePending) { 239 written = clientSocket->write_some(boost::asio::buffer(copy, responseLength + 4)); 240 } 241 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 242 } 243 244 bytesWritten += written; 245 if (written != static_cast<size_t>(responseLength + 4)) { 246 size_t length = responseLength + 4 - written; 247 if (asyncWritePending) { 248 socketAsyncWrites[clientSocket]->push_back(std::pair<char*, WriteStuff>(copy, WriteStuff(length, written, &responsesSent))); 249 } else { 250 boost::asio::async_write( *clientSocket, boost::asio::buffer(©[written], length), 251 boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error, 252 boost::asio::placeholders::bytes_transferred, clientSocket, copy, &responsesSent)); 253 socketAsyncWritePending[clientSocket] = true; 254 } 255 } else { 256 releaseBuffer(copy); 257 responsesSent++; 258 backpressureCheck(); 259 } 260 } 261 } 262 263 try { 264 size_t read = socket->read_some(boost::asio::buffer( &buffer[nextBufferOffset], nextLength)); 265 bytesRead += read; 266 if (read < nextLength) { 267 boost::asio::async_read( *socket, boost::asio::buffer(&buffer[nextBufferOffset + read], nextLength - read), 268 boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage)); 269 drainPriorityQueue(); 270 return; 271 } 272 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 273 boost::asio::async_read( *socket, boost::asio::buffer(&buffer[nextBufferOffset], nextLength), 274 boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage)); 275 drainPriorityQueue(); 276 return; 277 } 278 } 279 } 280 281 void handleClientRead(const boost::system::error_code& error, 282 size_t bytes_transferred, tcp::socket* socket, char *buffer, bool lengthOrMessage) { 283 if (error) { 284 std::cout << error.message() << std::endl; 285 releaseBuffer(buffer); 286 return; 287 } 288 bytesRead += bytes_transferred; 289 290 const size_t maxRead = 16384; 291 size_t totalClientRead = 0; 292 size_t nextLength = 0; 293 int outstandingRequests = 0; 294 while (true) { 295 if (lengthOrMessage) { 296 nextLength = *reinterpret_cast<int32_t*>(buffer); 297 lengthOrMessage = false; 298 if (nextLength <= 0 || nextLength > 1500) { 299 std::cout << "Next length from client was " << nextLength << std::endl; 300 exit(-1); 301 } 302 } else { 303 nextLength = 4; 304 lengthOrMessage = true; 305 requestsReceived++; 306 int64_t requestId = nextRequestId++; 307 char *request = acquireBuffer(); 308 *reinterpret_cast<int32_t*>(request) = 60; 309 *reinterpret_cast<int8_t*>(&request[4]) = 0;//request 310 *reinterpret_cast<int64_t*>(&request[5]) = requestId; 311 requestIdToClient[requestId] = socket; 312 313 size_t serverIndex = rand() % serverSockets.size(); 314 tcp::socket *serverSocket = serverSockets[serverIndex]; 315 316 size_t written = 0; 317 bool asyncWritePending = socketAsyncWritePending[serverSocket]; 318 try { 319 if ( !asyncWritePending ) { 320 written = serverSocket->write_some(boost::asio::buffer(request, 64)); 321 } 322 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 323 } 324 325 bytesWritten += written; 326 if (written < 64) { 327 size_t length = 64 - written; 328 if (asyncWritePending) { 329 socketAsyncWrites[serverSocket]->push_back(std::pair<char*, WriteStuff>(request, WriteStuff(length, written, &meshRequestsSent))); 330 } else { 331 boost::asio::async_write( *serverSocket, boost::asio::buffer(&request[written], length), 332 boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error, 333 boost::asio::placeholders::bytes_transferred, serverSocket, request, &meshRequestsSent)); 334 socketAsyncWritePending[serverSocket] = true; 335 } 336 } else { 337 releaseBuffer(request); 338 meshRequestsSent++; 339 } 340 } 341 342 outstandingRequests = requestsReceived - responsesSent; 343 if (outstandingRequests > 15000 && lengthOrMessage) { 344 releaseBuffer(buffer); 345 backpressureSockets.insert(socket); 346 return; 347 } else { 348 try { 349 size_t read = 0; 350 if (lengthOrMessage || totalClientRead < maxRead) { 351 read = socket->read_some(boost::asio::buffer(buffer, nextLength)); 352 bytesRead += read; 353 totalClientRead += read; 354 } 355 if (read < nextLength) { 356 boost::asio::async_read( *socket, boost::asio::buffer(&buffer[read], nextLength - read), 357 boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage)); 358 return; 359 } 360 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 361 boost::asio::async_read( *socket, boost::asio::buffer(buffer, nextLength), 362 boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage)); 363 return; 364 } 365 } 366 } 367 } 368 369 void handleWriteCompletion(const boost::system::error_code& error, 370 size_t bytes_transferred, tcp::socket* socket, char *buffer, int64_t *counter) { 371 if (error) { 372 std::cout << error.message() << std::endl; 373 releaseBuffer(buffer); 374 return; 375 } 376 bytesWritten += bytes_transferred; 377 *counter = *counter + 1; 378 releaseBuffer(buffer); 379 socketAsyncWritePending[socket] = false; 380 backpressureCheck(); 381 std::deque<std::pair<char*, WriteStuff> > *pendingWrites = socketAsyncWrites[socket]; 382 while (!pendingWrites->empty()) { 383 std::pair<char*, WriteStuff> pendingWrite = pendingWrites->front(); 384 pendingWrites->pop_front(); 385 char * pendingBuffer = pendingWrite.first; 386 size_t writeLength = pendingWrite.second.length; 387 size_t writeOffset = pendingWrite.second.offset; 388 int64_t *counter = pendingWrite.second.counter; 389 size_t written = 0; 390 try { 391 written = socket->write_some(boost::asio::buffer( &pendingBuffer[writeOffset], writeLength)); 392 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 393 } 394 395 if (written < writeLength) { 396 size_t length = writeLength - written; 397 boost::asio::async_write( *socket, boost::asio::buffer(&pendingBuffer[writeOffset + written], length), 398 boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error, 399 boost::asio::placeholders::bytes_transferred, socket, pendingBuffer, counter)); 400 socketAsyncWritePending[socket] = true; 401 break; 402 } else { 403 releaseBuffer(pendingBuffer); 404 *counter = *counter + 1; 405 } 406 } 407 } 408 409 /* 410 * Check if there is no more backpressure and reactivate reads for client sockets. 411 */ 412 void backpressureCheck() { 413 int outstandingRequests = requestsReceived - responsesSent; 414 if (outstandingRequests < 10000) { 415 if (backpressureSockets.size() > 0) { 416 for (boost::unordered_set<tcp::socket*>::iterator i = backpressureSockets.begin(); 417 i != backpressureSockets.end(); i++) { 418 tcp::socket *clientSocket = *i; 419 char *newbuffer = acquireBuffer(); 420 boost::asio::async_read( *clientSocket, boost::asio::buffer( newbuffer, 4), boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, clientSocket, newbuffer, true)); 421 } 422 backpressureSockets.clear(); 423 } 424 } 425 } 426 427 void handleClientAccept( 428 const boost::system::error_code& error, 429 tcp::acceptor *acceptor, 430 tcp::socket* socket) { 431 if (error) { 432 std::cout << error.message() << std::endl; 433 return; 434 } 435 if (!error) { 436 std::cout << "Accepted from client " << socket->remote_endpoint().address().to_string() << std::endl; 437 clientSockets.push_back(socket); 438 tcp::socket *newsocket = new tcp::socket(acceptor->io_service()); 439 acceptor->async_accept(*newsocket, boost::bind(&Server::handleClientAccept, this, boost::asio::placeholders::error, acceptor, newsocket)); 440 char *buffer = acquireBuffer(); 441 boost::asio::ip::tcp::socket::receive_buffer_size receiveSize(262144); 442 boost::asio::ip::tcp::socket::send_buffer_size sendSize(262144); 443 boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true); 444 socket->io_control(nonblocking); 445 socket->set_option(receiveSize); 446 socket->set_option(sendSize); 447 socketAsyncWritePending[socket] = false; 448 socketAsyncWrites[socket] = new std::deque<std::pair<char*, WriteStuff> >(); 449 boost::asio::async_read( *socket, boost::asio::buffer( buffer, 4), boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, true)); 450 } 451 } 452 453 char* acquireBuffer() { 454 if (buffers.empty()) { 455 char * buffer = new char[1500]; 456 ::memset(buffer, 0, 1500); 457 return buffer; 458 } else { 459 char *buffer = buffers.back(); 460 buffers.pop_back(); 461 return buffer; 462 } 463 } 464 465 void releaseBuffer(char *buffer) { 466 buffers.push_back(buffer); 467 } 468 469 void addQueueEntry(tcp::socket *socket, int64_t requestId) { 470 std::map<tcp::socket*, int64_t>::iterator iter = lastSafeRequestId.find(socket); 471 if (iter == lastSafeRequestId.end()) { 472 lastSafeRequestId[socket] = requestId; 473 } else if (iter->second < requestId) { 474 iter->second = requestId; 475 } 476 queue.push(QueueEntry(socket, requestId)); 477 } 478 479 void sendResponse(QueueEntry entry) { 480 tcp::socket *socket = entry.socket; 481 482 char *response = acquireBuffer(); 483 int responseLength = 256 + (rand() % 600); 484 ::memset(response, 0, responseLength + 4); 485 *reinterpret_cast<int32_t*>(response) = responseLength; 486 *reinterpret_cast<int8_t*>(&response[4]) = 1;//response 487 *reinterpret_cast<int64_t*>(&response[5]) = entry.requestId; 488// if (!requestIdsRequested[socket]->insert(*reinterpret_cast<int64_t*>(&response[5])).second) { 489// std::cout << "Request ID " << *reinterpret_cast<int64_t*>(&response[5]) << " has already been requested" << std::endl; 490// } 491 size_t written = 0; 492 bool asyncWritePending = socketAsyncWritePending[socket]; 493 try { 494 if (!asyncWritePending) { 495 written = socket->write_some(boost::asio::buffer(response, responseLength + 4)); 496 } 497 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 498 } 499 500 bytesWritten += written; 501 if (written != static_cast<size_t>(responseLength + 4)) { 502 size_t length = responseLength + 4 - written; 503 if (asyncWritePending) { 504 socketAsyncWrites[socket]->push_back(std::pair<char*, WriteStuff>(response, WriteStuff(length, written, &meshResponsesSent))); 505 } else { 506 boost::asio::async_write( *socket, boost::asio::buffer(&response[written], length), 507 boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error, 508 boost::asio::placeholders::bytes_transferred, socket, response, &meshResponsesSent)); 509 socketAsyncWritePending[socket] = true; 510 } 511 } else { 512 releaseBuffer(response); 513 meshResponsesSent++; 514 } 515 } 516 517 void drainPriorityQueue() { 518 int64_t minRequestId = std::numeric_limits<int64_t>::max(); 519 for (std::map<tcp::socket*, int64_t>::iterator iter = lastSafeRequestId.begin(); 520 iter != lastSafeRequestId.end(); 521 iter++) { 522 if (iter->second < minRequestId) { 523 minRequestId = iter->second; 524 } 525 } 526 while (!queue.empty() && queue.top().requestId <= minRequestId) { 527 QueueEntry entry = queue.top(); 528 queue.pop(); 529 sendResponse(entry); 530 } 531 } 532 533 std::deque<char*> buffers; 534 int64_t nextRequestId; 535 int64_t requestsReceived; 536 int64_t responsesSent; 537 int64_t meshRequestsReceived; 538 int64_t meshRequestsSent; 539 int64_t meshResponsesSent; 540 int64_t meshResponsesReceived; 541 int64_t bytesWritten; 542 int64_t bytesRead; 543 int64_t bytesWrittenLastTime; 544 int64_t bytesReadLastTime; 545 boost::unordered_set<tcp::socket*> backpressureSockets; 546 tcp::acceptor serverAcceptor; 547 tcp::acceptor clientAcceptor; 548 std::vector<tcp::socket*> serverSockets; 549 std::vector<tcp::socket*> clientSockets; 550 boost::unordered_map<tcp::socket*, boost::unordered_set<int64_t>*> requestIdsResponded; 551 boost::unordered_map<tcp::socket*, boost::unordered_set<int64_t>*> requestIdsRequested; 552 boost::unordered_map<int64_t, tcp::socket*> requestIdToClient; 553 boost::unordered_map<tcp::socket*, bool> socketAsyncWritePending; 554 boost::unordered_map<tcp::socket*, std::deque<std::pair<char*, WriteStuff> >* > socketAsyncWrites; 555 std::map<tcp::socket*, int64_t> lastSafeRequestId; 556 std::priority_queue< QueueEntry, std::vector<QueueEntry>, CompareQueueEntry> queue; 557}; 558 559class Client { 560public: 561 Client( boost::asio::io_service &io_service, std::vector<std::string> servers) : 562 requestsSent(0), 563 responsesReceived(0), 564 responsesReceivedLastTime(0), 565 bytesWritten(0), 566 bytesRead(0), 567 bytesWrittenLastTime(0), 568 bytesReadLastTime(0) { 569 tcp::resolver resolver(io_service); 570 for (std::size_t ii = 0; ii < servers.size(); ii++) { 571 tcp::resolver::query query(servers[ii], "21412"); 572 tcp::resolver::iterator endpoint_iterator = resolver.resolve(query); 573 tcp::socket* socket = new tcp::socket(io_service); 574 tcp::endpoint endpoint = *endpoint_iterator; 575 socket->async_connect( endpoint, 576 boost::bind( &Client::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket)); 577 } 578 boost::asio::deadline_timer *timer = new boost::asio::deadline_timer(io_service); 579 timer->expires_from_now(boost::posix_time::seconds(10)); 580 timer->async_wait(boost::bind(&Client::timerCallback, this, boost::asio::placeholders::error, timer)); 581 } 582 583 void timerCallback(const boost::system::error_code& error, boost::asio::deadline_timer *timer) { 584 if (error) { 585 std::cout << error.message() << std::endl; 586 } 587 int64_t bytesReadThisTime = bytesRead - bytesReadLastTime; 588 bytesReadLastTime = bytesRead; 589 int64_t bytesWrittenThisTime = bytesWritten - bytesWrittenLastTime; 590 bytesWrittenLastTime = bytesWritten; 591 int64_t mbReadThisTime = bytesReadThisTime / (1024 * 1024); 592 int64_t mbWrittenThisTime = bytesWrittenThisTime / (1024 * 1024); 593 double mbReadPerSec = mbReadThisTime / 10.0; 594 double mbWrittenPerSec = mbWrittenThisTime / 10.0; 595 int64_t responsesReceivedThisTime = responsesReceived - responsesReceivedLastTime; 596 responsesReceivedLastTime = responsesReceived; 597 double requestsPerSec = responsesReceivedThisTime / 10.0; 598 std::cout << "Requests/sec " << requestsPerSec << " Requests sent " << requestsSent << " Responses received " << responsesReceived << " Megabytes/sec In/Out " << mbReadPerSec << "/" << mbWrittenPerSec << std::endl; 599 timer->expires_from_now(boost::posix_time::seconds(10)); 600 timer->async_wait(boost::bind(&Client::timerCallback, this, boost::asio::placeholders::error, timer)); 601 } 602 603 void handleConnect(const boost::system::error_code& error, 604 tcp::resolver::iterator endpoint_iterator, tcp::socket* socket) { 605 if (error) { 606 std::cout << error.message() << std::endl; 607 socket->close(); 608 ++endpoint_iterator; 609 if (endpoint_iterator != tcp::resolver::iterator()) { 610 tcp::endpoint endpoint = *endpoint_iterator; 611 socket->async_connect(endpoint, 612 boost::bind( &Client::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket)); 613 } 614 } else { 615 std::cout << "Connected to " << socket->remote_endpoint().address().to_string() << std::endl; 616 char *recBuffer = new char[1500]; 617 ::memset(recBuffer, 0, 1500); 618 char *sendBuffer = new char[1500]; 619 ::memset(sendBuffer, 0, 1500); 620 *reinterpret_cast<int32_t*>(sendBuffer) = 60; 621 boost::asio::ip::tcp::socket::receive_buffer_size receiveSize(262144); 622 boost::asio::ip::tcp::socket::send_buffer_size sendSize(262144); 623 socket->set_option(receiveSize); 624 socket->set_option(sendSize); 625 boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true); 626 socket->io_control(nonblocking); 627 boost::asio::async_write( *socket, boost::asio::buffer(sendBuffer, 64), 628 boost::bind(&Client::handleWriteCompletion, this, boost::asio::placeholders::error, 629 boost::asio::placeholders::bytes_transferred, socket, sendBuffer, 64)); 630 boost::asio::async_read( *socket, boost::asio::buffer(recBuffer, 4), 631 boost::bind(&Client::handleReadCompletion, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, recBuffer, true)); 632 } 633 } 634 635 void handleWriteCompletion(const boost::system::error_code& error, 636 size_t bytes_transferred, tcp::socket* socket, char *buffer, size_t expectedWrite) { 637 if(error) { 638 std::cout << error.message() << std::endl; 639 return; 640 } 641 642 if (bytes_transferred != expectedWrite) { 643 std::cout << "Bytes transferred was not equal to expected write in client" << std::endl; 644 } 645 646 bytesWritten += bytes_transferred; 647 requestsSent++; 648 size_t written; 649 while (true) { 650 written = 0; 651 try { 652 written = socket->write_some(boost::asio::buffer(buffer, 64)); 653 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 654 } 655 bytesWritten += written; 656 if (written < 64) { 657 break; 658 } 659 requestsSent++; 660 } 661 boost::asio::async_write( *socket, boost::asio::buffer(&buffer[written], 64 - written), 662 boost::bind(&Client::handleWriteCompletion, this, boost::asio::placeholders::error, 663 boost::asio::placeholders::bytes_transferred, socket, buffer, 64 - written)); 664 } 665 666 void handleReadCompletion(const boost::system::error_code& error, 667 size_t bytes_transferred, tcp::socket* socket, char *buffer, bool lengthOrMessage) { 668 if (error) { 669 std::cout << error.message() << std::endl; 670 return; 671 } 672 673 bytesRead += bytes_transferred; 674 size_t currentLength = 0; 675 676 while (true) { 677 if (lengthOrMessage) { 678 currentLength = *reinterpret_cast<int32_t*>(buffer); 679 lengthOrMessage = false; 680 if (currentLength <= 0 || currentLength > 1500) { 681 std::cout << "Next length at client was " << currentLength << std::endl; 682 exit(-1); 683 } 684 } else { 685 responsesReceived++; 686 currentLength = 4; 687 lengthOrMessage = true; 688 } 689 690 try { 691 size_t read = socket->read_some(boost::asio::buffer(buffer, currentLength)); 692 bytesRead += read; 693 if (read < currentLength) { 694 boost::asio::async_read( *socket, boost::asio::buffer(&buffer[read], currentLength - read), 695 boost::bind(&Client::handleReadCompletion, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage)); 696 return; 697 } 698 } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) { 699 boost::asio::async_read( *socket, boost::asio::buffer(buffer, currentLength), 700 boost::bind(&Client::handleReadCompletion, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage)); 701 return; 702 } 703 } 704 } 705 706 int64_t requestsSent; 707 int64_t responsesReceived; 708 int64_t responsesReceivedLastTime; 709 int64_t bytesWritten; 710 int64_t bytesRead; 711 int64_t bytesWrittenLastTime; 712 int64_t bytesReadLastTime; 713}; 714 715int main(int argc, char **argv) { 716 boost::asio::io_service io_service; 717 if (std::string(argv[1]) == "server") { 718 std::vector<std::string> machines; 719 for (int ii = 2; ii < argc; ii++) { 720 machines.push_back(std::string(argv[ii])); 721 } 722 Server server( io_service, machines); 723 io_service.run(); 724 } else if (std::string(argv[1]) == "client") { 725 std::vector<std::string> machines; 726 for (int ii = 2; ii < argc; ii++) { 727 machines.push_back( std::string(argv[ii])); 728 } 729 Client client( io_service, machines); 730 io_service.run(); 731 } 732 return 0; 733}