/tests/scripts/mesh_test.cpp

  1/* This file is part of VoltDB.
  2 * Copyright (C) 2008-2014 VoltDB Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining
  5 * a copy of this software and associated documentation files (the
  6 * "Software"), to deal in the Software without restriction, including
  7 * without limitation the rights to use, copy, modify, merge, publish,
  8 * distribute, sublicense, and/or sell copies of the Software, and to
  9 * permit persons to whom the Software is furnished to do so, subject to
 10 * the following conditions:
 11 *
 12 * The above copyright notice and this permission notice shall be
 13 * included in all copies or substantial portions of the Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 18 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 21 * OTHER DEALINGS IN THE SOFTWARE.
 22 */
 23
 24#include <iostream>
 25#include <string>
 26#include <vector>
 27#include <boost/unordered_map.hpp>
 28#include <boost/unordered_set.hpp>
 29#include <set>
 30#include <deque>
 31#include <boost/bind.hpp>
 32#include <boost/asio.hpp>
 33#include <queue>
 34#include <limits>
 35
 36using namespace std;
 37using boost::asio::ip::tcp;
 38
 39class QueueEntry {
 40public:
 41    QueueEntry(tcp::socket *nsocket, int64_t nrequestId) : socket(nsocket), requestId(nrequestId) {}
 42    tcp::socket *socket;
 43    int64_t requestId;
 44};
 45
 46class CompareQueueEntry {
 47public:
 48    bool operator()(const QueueEntry& lhs, const QueueEntry& rhs) const {
 49        if (lhs.requestId > rhs.requestId) {
 50            return true;
 51        }
 52        return false;
 53    }
 54};
 55
 56class WriteStuff {
 57public:
 58    WriteStuff(size_t nlength, size_t noffset, int64_t *ncounter) : length(nlength), offset(noffset), counter(ncounter) {}
 59    size_t length;
 60    size_t offset;
 61    int64_t *counter;
 62};
 63
 64class Server
 65 {
 66public:
 67    Server(boost::asio::io_service &io_service, std::vector<std::string> servers) :
 68        nextRequestId(0),
 69        requestsReceived(0),
 70        responsesSent(0),
 71        meshRequestsReceived(0),
 72        meshRequestsSent(0),
 73        meshResponsesSent(0),
 74        meshResponsesReceived(0),
 75         bytesWritten(0),
 76         bytesRead(0),
 77         bytesWrittenLastTime(0),
 78         bytesReadLastTime(0),
 79        serverAcceptor(io_service, tcp::endpoint(tcp::v4(), 21413)),
 80        clientAcceptor(io_service, tcp::endpoint(tcp::v4(), 21412)) {
 81        tcp::socket *socket = new tcp::socket(io_service);
 82        serverAcceptor.async_accept( *socket,
 83                boost::bind(
 84                        &Server::handleServerAccept,
 85                        this,
 86                        boost::asio::placeholders::error,
 87                        &serverAcceptor,
 88                        socket));
 89
 90
 91        socket = new tcp::socket(io_service);
 92        clientAcceptor.async_accept( *socket,
 93                boost::bind(
 94                        &Server::handleClientAccept,
 95                        this,
 96                        boost::asio::placeholders::error,
 97                        &clientAcceptor,
 98                        socket));
 99
100        tcp::resolver resolver(io_service);
101        for (std::size_t ii = 0; ii < servers.size(); ii++) {
102            tcp::resolver::query query(servers[ii], "21413");
103            tcp::resolver::iterator endpoint_iterator = resolver.resolve(query);
104            tcp::socket* socket = new tcp::socket(io_service);
105            tcp::endpoint endpoint = *endpoint_iterator;
106            socket->async_connect( endpoint,
107                      boost::bind( &Server::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket));
108        }
109
110        boost::asio::deadline_timer *timer = new boost::asio::deadline_timer(io_service);
111        timer->expires_from_now(boost::posix_time::seconds(10));
112        timer->async_wait(boost::bind(&Server::timerCallback, this, boost::asio::placeholders::error, timer));
113    }
114
115    void timerCallback(const boost::system::error_code& error, boost::asio::deadline_timer *timer) {
116        if (error) {
117            std::cout << error.message() << std::endl;
118        }
119        int64_t bytesReadThisTime = bytesRead - bytesReadLastTime;
120        bytesReadLastTime = bytesRead;
121        int64_t bytesWrittenThisTime = bytesWritten - bytesWrittenLastTime;
122        bytesWrittenLastTime = bytesWritten;
123        int64_t mbReadThisTime = bytesReadThisTime / (1024 * 1024);
124        int64_t mbWrittenThisTime = bytesWrittenThisTime / (1024 * 1024);
125        double mbReadPerSec = mbReadThisTime / 10.0;
126        double mbWrittenPerSec = mbWrittenThisTime / 10.0;
127        std::cout << "Megabytes/sec In/Out " << mbReadPerSec << "/" << mbWrittenPerSec << std::endl;
128        std::cout << "Requests received " << requestsReceived << " Responses sent " << responsesSent << " MRequests received "
129                << meshRequestsReceived << " MRequests sent " << meshRequestsSent << " MResponses sent " << meshResponsesSent <<
130                " MResponses received " << meshResponsesReceived << " Outstanding requests " << (requestsReceived - responsesSent) << std::endl;
131        timer->expires_from_now(boost::posix_time::seconds(10));
132        timer->async_wait(boost::bind(&Server::timerCallback, this, boost::asio::placeholders::error, timer));
133    }
134
135    void handleConnect(const boost::system::error_code& error,
136            tcp::resolver::iterator endpoint_iterator, tcp::socket* socket) {
137        if (error) {
138            std::cout << error.message() << std::endl;
139            socket->close();
140            ++endpoint_iterator;
141            if (endpoint_iterator != tcp::resolver::iterator()) {
142                tcp::endpoint endpoint = *endpoint_iterator;
143                socket->async_connect(endpoint,
144                        boost::bind( &Server::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket));
145            }
146        } else {
147            std::cout << "Connected to server " << socket->remote_endpoint().address().to_string() << std::endl;
148            boost::asio::ip::tcp::no_delay option(true);
149            socket->set_option(option);
150            boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true);
151            socket->io_control(nonblocking);
152            serverSockets.push_back(socket);
153            socketAsyncWritePending[socket] = false;
154            char *buffer = acquireBuffer();
155            requestIdsResponded[socket] = new boost::unordered_set<int64_t>();
156            requestIdsRequested[socket] = new boost::unordered_set<int64_t>();
157            socketAsyncWrites[socket] = new std::deque<std::pair<char*, WriteStuff> >();
158            boost::asio::async_read( *socket, boost::asio::buffer( buffer, 4), boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, true));
159        }
160    }
161
162    void handleServerAccept(
163            const boost::system::error_code& error,
164            tcp::acceptor *acceptor,
165            tcp::socket* socket) {
166        if (error) {
167            std::cout << error.message() << std::endl;
168            return;
169        }
170        if (!error) {
171            std::cout << "Accepted from server " << socket->remote_endpoint().address().to_string() << std::endl;
172            socketAsyncWritePending[socket] = false;
173            serverSockets.push_back(socket);
174            tcp::socket *newsocket = new tcp::socket(acceptor->io_service());
175            acceptor->async_accept(*newsocket, boost::bind(&Server::handleServerAccept, this, boost::asio::placeholders::error, acceptor, newsocket));
176            boost::asio::ip::tcp::no_delay option(true);
177            socket->set_option(option);
178            boost::asio::ip::tcp::socket::receive_buffer_size receiveSize(262144);
179            boost::asio::ip::tcp::socket::send_buffer_size sendSize(262144);
180            socket->set_option(receiveSize);
181            socket->set_option(sendSize);
182            char *buffer = acquireBuffer();
183            boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true);
184            socket->io_control(nonblocking);
185            requestIdsResponded[socket] = new boost::unordered_set<int64_t>();
186            requestIdsRequested[socket] = new boost::unordered_set<int64_t>();
187            socketAsyncWrites[socket] = new std::deque<std::pair<char*, WriteStuff> >();
188            boost::asio::async_read( *socket, boost::asio::buffer( buffer, 4), boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, true));
189        }
190    }
191
192    void handleServerRead(const boost::system::error_code& error,
193            size_t bytes_transferred, tcp::socket* socket, char *buffer, bool lengthOrMessage) {
194        if (error) {
195            std::cout << error.message() << std::endl;
196            releaseBuffer(buffer);
197            return;
198        }
199
200        bytesRead += bytes_transferred;
201
202        size_t nextLength = 0;
203        size_t nextBufferOffset;
204        while (true) {
205            nextBufferOffset = 0;
206            if (lengthOrMessage) {
207                nextLength = *reinterpret_cast<int32_t*>(buffer);
208                nextBufferOffset = 4;
209                lengthOrMessage = false;
210                if (nextLength <= 0 || nextLength > 1500) {
211                    std::cout << "Next length from server was " << nextLength << std::endl;
212                    exit(-1);
213                }
214            } else {
215                nextLength = 4;
216                lengthOrMessage = true;
217                if (buffer[4] == 0) {
218                    //request
219                    meshRequestsReceived++;
220                    int64_t requestId = *reinterpret_cast<int64_t*>(&buffer[5]);
221                    addQueueEntry(socket, requestId);
222                } else if (buffer[4] == 1) {
223                    //response
224                    meshResponsesReceived++;
225                    int32_t responseLength = *reinterpret_cast<int32_t*>(buffer);
226                    int64_t requestId = *reinterpret_cast<int64_t*>(&buffer[5]);
227                    char *copy = acquireBuffer();
228                    ::memcpy( copy, buffer, responseLength + 4);
229                    tcp::socket *clientSocket = requestIdToClient[requestId];
230                    requestIdToClient.erase(requestId);
231//                    if (!requestIdsResponded[socket]->insert(requestId).second) {
232//                        std::cout << "Request ID " << requestId << " has already been responded to" << std::endl;
233//                    }
234
235                    size_t written = 0;
236                    bool asyncWritePending = socketAsyncWritePending[clientSocket];
237                    try {
238                        if (!asyncWritePending) {
239                            written = clientSocket->write_some(boost::asio::buffer(copy, responseLength + 4));
240                        }
241                    } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
242                    }
243
244                    bytesWritten += written;
245                    if (written != static_cast<size_t>(responseLength + 4)) {
246                        size_t length = responseLength + 4 - written;
247                        if (asyncWritePending) {
248                            socketAsyncWrites[clientSocket]->push_back(std::pair<char*, WriteStuff>(copy, WriteStuff(length, written, &responsesSent)));
249                        } else {
250                            boost::asio::async_write( *clientSocket, boost::asio::buffer(&copy[written], length),
251                                    boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error,
252                                            boost::asio::placeholders::bytes_transferred, clientSocket, copy, &responsesSent));
253                            socketAsyncWritePending[clientSocket] = true;
254                        }
255                    } else {
256                        releaseBuffer(copy);
257                        responsesSent++;
258                        backpressureCheck();
259                    }
260                }
261            }
262
263            try {
264                size_t read = socket->read_some(boost::asio::buffer( &buffer[nextBufferOffset], nextLength));
265                bytesRead += read;
266                if (read < nextLength) {
267                    boost::asio::async_read( *socket, boost::asio::buffer(&buffer[nextBufferOffset + read], nextLength - read),
268                            boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage));
269                    drainPriorityQueue();
270                    return;
271                }
272            } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
273                boost::asio::async_read( *socket, boost::asio::buffer(&buffer[nextBufferOffset], nextLength),
274                        boost::bind(&Server::handleServerRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage));
275                drainPriorityQueue();
276                return;
277            }
278        }
279    }
280
281    void handleClientRead(const boost::system::error_code& error,
282            size_t bytes_transferred, tcp::socket* socket, char *buffer, bool lengthOrMessage) {
283        if (error) {
284            std::cout << error.message() << std::endl;
285            releaseBuffer(buffer);
286            return;
287        }
288        bytesRead += bytes_transferred;
289
290        const size_t maxRead = 16384;
291        size_t totalClientRead = 0;
292        size_t nextLength = 0;
293        int outstandingRequests = 0;
294        while (true) {
295            if (lengthOrMessage) {
296                nextLength = *reinterpret_cast<int32_t*>(buffer);
297                lengthOrMessage = false;
298                if (nextLength <= 0 || nextLength > 1500) {
299                    std::cout << "Next length from client was " << nextLength << std::endl;
300                    exit(-1);
301                }
302            } else {
303                nextLength = 4;
304                lengthOrMessage = true;
305                requestsReceived++;
306                int64_t requestId = nextRequestId++;
307                char *request = acquireBuffer();
308                *reinterpret_cast<int32_t*>(request) = 60;
309                *reinterpret_cast<int8_t*>(&request[4]) = 0;//request
310                *reinterpret_cast<int64_t*>(&request[5]) = requestId;
311                requestIdToClient[requestId] = socket;
312
313                size_t serverIndex = rand() % serverSockets.size();
314                tcp::socket *serverSocket = serverSockets[serverIndex];
315
316                size_t written = 0;
317                bool asyncWritePending = socketAsyncWritePending[serverSocket];
318                try {
319                    if ( !asyncWritePending ) {
320                        written = serverSocket->write_some(boost::asio::buffer(request, 64));
321                    }
322                } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
323                }
324
325                bytesWritten += written;
326                if (written < 64) {
327                    size_t length = 64 - written;
328                    if (asyncWritePending) {
329                        socketAsyncWrites[serverSocket]->push_back(std::pair<char*, WriteStuff>(request, WriteStuff(length, written, &meshRequestsSent)));
330                    } else {
331                        boost::asio::async_write( *serverSocket, boost::asio::buffer(&request[written], length),
332                                boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error,
333                                        boost::asio::placeholders::bytes_transferred, serverSocket, request, &meshRequestsSent));
334                        socketAsyncWritePending[serverSocket] = true;
335                    }
336                } else {
337                    releaseBuffer(request);
338                    meshRequestsSent++;
339                }
340            }
341
342            outstandingRequests = requestsReceived - responsesSent;
343            if (outstandingRequests > 15000 && lengthOrMessage) {
344                releaseBuffer(buffer);
345                backpressureSockets.insert(socket);
346                return;
347            } else {
348                try {
349                    size_t read = 0;
350                    if (lengthOrMessage || totalClientRead < maxRead) {
351                        read = socket->read_some(boost::asio::buffer(buffer, nextLength));
352                        bytesRead += read;
353                        totalClientRead += read;
354                    }
355                    if (read < nextLength) {
356                        boost::asio::async_read( *socket, boost::asio::buffer(&buffer[read], nextLength - read),
357                                                            boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage));
358                        return;
359                    }
360                } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
361                    boost::asio::async_read( *socket, boost::asio::buffer(buffer, nextLength),
362                                                        boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage));
363                    return;
364                }
365            }
366        }
367    }
368
369    void handleWriteCompletion(const boost::system::error_code& error,
370            size_t bytes_transferred, tcp::socket* socket, char *buffer, int64_t *counter) {
371        if (error) {
372            std::cout << error.message() << std::endl;
373            releaseBuffer(buffer);
374            return;
375        }
376        bytesWritten += bytes_transferred;
377        *counter = *counter + 1;
378        releaseBuffer(buffer);
379        socketAsyncWritePending[socket] = false;
380        backpressureCheck();
381        std::deque<std::pair<char*, WriteStuff> > *pendingWrites = socketAsyncWrites[socket];
382        while (!pendingWrites->empty()) {
383            std::pair<char*, WriteStuff> pendingWrite = pendingWrites->front();
384            pendingWrites->pop_front();
385            char * pendingBuffer = pendingWrite.first;
386            size_t writeLength = pendingWrite.second.length;
387            size_t writeOffset = pendingWrite.second.offset;
388            int64_t *counter = pendingWrite.second.counter;
389            size_t written = 0;
390            try {
391                written = socket->write_some(boost::asio::buffer( &pendingBuffer[writeOffset], writeLength));
392            } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
393            }
394
395            if (written < writeLength) {
396                size_t length = writeLength - written;
397                boost::asio::async_write( *socket, boost::asio::buffer(&pendingBuffer[writeOffset + written], length),
398                        boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error,
399                                boost::asio::placeholders::bytes_transferred, socket, pendingBuffer, counter));
400                socketAsyncWritePending[socket] = true;
401                break;
402            } else {
403                releaseBuffer(pendingBuffer);
404                *counter = *counter + 1;
405            }
406        }
407    }
408
409    /*
410     * Check if there is no more backpressure and reactivate reads for client sockets.
411     */
412    void backpressureCheck() {
413        int outstandingRequests = requestsReceived - responsesSent;
414        if (outstandingRequests < 10000) {
415            if (backpressureSockets.size() > 0) {
416                for (boost::unordered_set<tcp::socket*>::iterator i = backpressureSockets.begin();
417                        i != backpressureSockets.end(); i++) {
418                    tcp::socket *clientSocket = *i;
419                    char *newbuffer = acquireBuffer();
420                    boost::asio::async_read( *clientSocket, boost::asio::buffer( newbuffer, 4), boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, clientSocket, newbuffer, true));
421                }
422                backpressureSockets.clear();
423            }
424        }
425    }
426
427    void handleClientAccept(
428            const boost::system::error_code& error,
429            tcp::acceptor *acceptor,
430            tcp::socket* socket) {
431        if (error) {
432            std::cout << error.message() << std::endl;
433            return;
434        }
435        if (!error) {
436            std::cout << "Accepted from client " << socket->remote_endpoint().address().to_string() << std::endl;
437            clientSockets.push_back(socket);
438            tcp::socket *newsocket = new tcp::socket(acceptor->io_service());
439            acceptor->async_accept(*newsocket, boost::bind(&Server::handleClientAccept, this, boost::asio::placeholders::error, acceptor, newsocket));
440            char *buffer = acquireBuffer();
441            boost::asio::ip::tcp::socket::receive_buffer_size receiveSize(262144);
442            boost::asio::ip::tcp::socket::send_buffer_size sendSize(262144);
443            boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true);
444            socket->io_control(nonblocking);
445            socket->set_option(receiveSize);
446            socket->set_option(sendSize);
447            socketAsyncWritePending[socket] = false;
448            socketAsyncWrites[socket] = new std::deque<std::pair<char*, WriteStuff> >();
449            boost::asio::async_read( *socket, boost::asio::buffer( buffer, 4), boost::bind(&Server::handleClientRead, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, true));
450        }
451    }
452
453    char* acquireBuffer() {
454        if (buffers.empty()) {
455            char * buffer = new char[1500];
456            ::memset(buffer, 0, 1500);
457            return buffer;
458        } else {
459            char *buffer = buffers.back();
460            buffers.pop_back();
461            return buffer;
462        }
463    }
464
465    void releaseBuffer(char *buffer) {
466        buffers.push_back(buffer);
467    }
468
469    void addQueueEntry(tcp::socket *socket, int64_t requestId) {
470        std::map<tcp::socket*, int64_t>::iterator iter = lastSafeRequestId.find(socket);
471        if (iter == lastSafeRequestId.end()) {
472            lastSafeRequestId[socket] = requestId;
473        } else if (iter->second < requestId) {
474            iter->second = requestId;
475        }
476        queue.push(QueueEntry(socket, requestId));
477    }
478
479    void sendResponse(QueueEntry entry) {
480        tcp::socket *socket = entry.socket;
481
482        char *response = acquireBuffer();
483        int responseLength = 256 + (rand() % 600);
484        ::memset(response, 0, responseLength + 4);
485        *reinterpret_cast<int32_t*>(response) = responseLength;
486        *reinterpret_cast<int8_t*>(&response[4]) = 1;//response
487        *reinterpret_cast<int64_t*>(&response[5]) = entry.requestId;
488//                    if (!requestIdsRequested[socket]->insert(*reinterpret_cast<int64_t*>(&response[5])).second) {
489//                        std::cout << "Request ID " << *reinterpret_cast<int64_t*>(&response[5]) << " has already been requested" << std::endl;
490//                    }
491        size_t written = 0;
492        bool asyncWritePending = socketAsyncWritePending[socket];
493        try {
494            if (!asyncWritePending) {
495                written = socket->write_some(boost::asio::buffer(response, responseLength + 4));
496            }
497        } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
498        }
499
500        bytesWritten += written;
501        if (written != static_cast<size_t>(responseLength + 4)) {
502            size_t length = responseLength + 4 - written;
503            if (asyncWritePending) {
504                socketAsyncWrites[socket]->push_back(std::pair<char*, WriteStuff>(response, WriteStuff(length, written, &meshResponsesSent)));
505            } else {
506                boost::asio::async_write( *socket, boost::asio::buffer(&response[written], length),
507                        boost::bind(&Server::handleWriteCompletion, this, boost::asio::placeholders::error,
508                                boost::asio::placeholders::bytes_transferred, socket, response, &meshResponsesSent));
509                socketAsyncWritePending[socket] = true;
510            }
511        } else {
512            releaseBuffer(response);
513            meshResponsesSent++;
514        }
515    }
516
517    void drainPriorityQueue() {
518        int64_t minRequestId = std::numeric_limits<int64_t>::max();
519        for (std::map<tcp::socket*, int64_t>::iterator iter = lastSafeRequestId.begin();
520                iter != lastSafeRequestId.end();
521                iter++) {
522            if (iter->second < minRequestId) {
523                minRequestId = iter->second;
524            }
525        }
526        while (!queue.empty() && queue.top().requestId <= minRequestId) {
527            QueueEntry entry = queue.top();
528            queue.pop();
529            sendResponse(entry);
530        }
531    }
532
533    std::deque<char*> buffers;
534    int64_t nextRequestId;
535    int64_t requestsReceived;
536    int64_t responsesSent;
537    int64_t meshRequestsReceived;
538    int64_t meshRequestsSent;
539    int64_t meshResponsesSent;
540    int64_t meshResponsesReceived;
541    int64_t bytesWritten;
542    int64_t bytesRead;
543    int64_t bytesWrittenLastTime;
544    int64_t bytesReadLastTime;
545    boost::unordered_set<tcp::socket*> backpressureSockets;
546    tcp::acceptor serverAcceptor;
547    tcp::acceptor clientAcceptor;
548    std::vector<tcp::socket*> serverSockets;
549    std::vector<tcp::socket*> clientSockets;
550    boost::unordered_map<tcp::socket*, boost::unordered_set<int64_t>*> requestIdsResponded;
551    boost::unordered_map<tcp::socket*, boost::unordered_set<int64_t>*> requestIdsRequested;
552    boost::unordered_map<int64_t, tcp::socket*> requestIdToClient;
553    boost::unordered_map<tcp::socket*, bool> socketAsyncWritePending;
554    boost::unordered_map<tcp::socket*, std::deque<std::pair<char*, WriteStuff> >* > socketAsyncWrites;
555    std::map<tcp::socket*, int64_t> lastSafeRequestId;
556    std::priority_queue< QueueEntry, std::vector<QueueEntry>, CompareQueueEntry> queue;
557};
558
559class Client {
560public:
561    Client( boost::asio::io_service &io_service, std::vector<std::string> servers) :
562        requestsSent(0),
563        responsesReceived(0),
564        responsesReceivedLastTime(0),
565     bytesWritten(0),
566     bytesRead(0),
567     bytesWrittenLastTime(0),
568     bytesReadLastTime(0) {
569        tcp::resolver resolver(io_service);
570        for (std::size_t ii = 0; ii < servers.size(); ii++) {
571            tcp::resolver::query query(servers[ii], "21412");
572            tcp::resolver::iterator endpoint_iterator = resolver.resolve(query);
573            tcp::socket* socket = new tcp::socket(io_service);
574            tcp::endpoint endpoint = *endpoint_iterator;
575            socket->async_connect( endpoint,
576                      boost::bind( &Client::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket));
577        }
578        boost::asio::deadline_timer *timer = new boost::asio::deadline_timer(io_service);
579        timer->expires_from_now(boost::posix_time::seconds(10));
580        timer->async_wait(boost::bind(&Client::timerCallback, this, boost::asio::placeholders::error, timer));
581    }
582
583    void timerCallback(const boost::system::error_code& error, boost::asio::deadline_timer *timer) {
584        if (error) {
585            std::cout << error.message() << std::endl;
586        }
587        int64_t bytesReadThisTime = bytesRead - bytesReadLastTime;
588        bytesReadLastTime = bytesRead;
589        int64_t bytesWrittenThisTime = bytesWritten - bytesWrittenLastTime;
590        bytesWrittenLastTime = bytesWritten;
591        int64_t mbReadThisTime = bytesReadThisTime / (1024 * 1024);
592        int64_t mbWrittenThisTime = bytesWrittenThisTime / (1024 * 1024);
593        double mbReadPerSec = mbReadThisTime / 10.0;
594        double mbWrittenPerSec = mbWrittenThisTime / 10.0;
595        int64_t responsesReceivedThisTime = responsesReceived - responsesReceivedLastTime;
596        responsesReceivedLastTime = responsesReceived;
597        double requestsPerSec = responsesReceivedThisTime / 10.0;
598        std::cout << "Requests/sec " <<  requestsPerSec << " Requests sent " << requestsSent << " Responses received " << responsesReceived << " Megabytes/sec In/Out " << mbReadPerSec << "/" << mbWrittenPerSec << std::endl;
599        timer->expires_from_now(boost::posix_time::seconds(10));
600        timer->async_wait(boost::bind(&Client::timerCallback, this, boost::asio::placeholders::error, timer));
601    }
602
603    void handleConnect(const boost::system::error_code& error,
604            tcp::resolver::iterator endpoint_iterator, tcp::socket* socket) {
605        if (error) {
606            std::cout << error.message() << std::endl;
607            socket->close();
608            ++endpoint_iterator;
609            if (endpoint_iterator != tcp::resolver::iterator()) {
610                tcp::endpoint endpoint = *endpoint_iterator;
611                socket->async_connect(endpoint,
612                        boost::bind( &Client::handleConnect, this, boost::asio::placeholders::error, endpoint_iterator, socket));
613            }
614        } else {
615            std::cout << "Connected to " << socket->remote_endpoint().address().to_string() << std::endl;
616            char *recBuffer = new char[1500];
617            ::memset(recBuffer, 0, 1500);
618            char *sendBuffer = new char[1500];
619            ::memset(sendBuffer, 0, 1500);
620            *reinterpret_cast<int32_t*>(sendBuffer) = 60;
621            boost::asio::ip::tcp::socket::receive_buffer_size receiveSize(262144);
622            boost::asio::ip::tcp::socket::send_buffer_size sendSize(262144);
623            socket->set_option(receiveSize);
624            socket->set_option(sendSize);
625            boost::asio::ip::tcp::socket::non_blocking_io nonblocking(true);
626            socket->io_control(nonblocking);
627            boost::asio::async_write( *socket, boost::asio::buffer(sendBuffer, 64),
628                    boost::bind(&Client::handleWriteCompletion, this, boost::asio::placeholders::error,
629                            boost::asio::placeholders::bytes_transferred, socket, sendBuffer, 64));
630            boost::asio::async_read( *socket, boost::asio::buffer(recBuffer, 4),
631                    boost::bind(&Client::handleReadCompletion, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, recBuffer, true));
632        }
633    }
634
635    void handleWriteCompletion(const boost::system::error_code& error,
636            size_t bytes_transferred, tcp::socket* socket, char *buffer, size_t expectedWrite) {
637        if(error) {
638            std::cout << error.message() << std::endl;
639            return;
640        }
641
642        if (bytes_transferred != expectedWrite) {
643            std::cout << "Bytes transferred was not equal to expected write in client" << std::endl;
644        }
645
646        bytesWritten += bytes_transferred;
647        requestsSent++;
648        size_t written;
649        while (true) {
650            written = 0;
651            try {
652                written = socket->write_some(boost::asio::buffer(buffer, 64));
653            } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
654            }
655            bytesWritten += written;
656            if (written < 64) {
657                break;
658            }
659            requestsSent++;
660        }
661        boost::asio::async_write( *socket, boost::asio::buffer(&buffer[written], 64 - written),
662                boost::bind(&Client::handleWriteCompletion, this, boost::asio::placeholders::error,
663                        boost::asio::placeholders::bytes_transferred, socket, buffer, 64 - written));
664    }
665
666    void handleReadCompletion(const boost::system::error_code& error,
667            size_t bytes_transferred, tcp::socket* socket, char *buffer, bool lengthOrMessage) {
668        if (error) {
669            std::cout << error.message() << std::endl;
670            return;
671        }
672
673        bytesRead += bytes_transferred;
674        size_t currentLength = 0;
675
676        while (true) {
677            if (lengthOrMessage) {
678                currentLength = *reinterpret_cast<int32_t*>(buffer);
679                lengthOrMessage = false;
680                if (currentLength <= 0 || currentLength > 1500) {
681                    std::cout << "Next length at client was " << currentLength << std::endl;
682                    exit(-1);
683                }
684            } else {
685                responsesReceived++;
686                currentLength = 4;
687                lengthOrMessage = true;
688            }
689
690            try {
691                size_t read = socket->read_some(boost::asio::buffer(buffer, currentLength));
692                bytesRead += read;
693                if (read < currentLength) {
694                    boost::asio::async_read( *socket, boost::asio::buffer(&buffer[read], currentLength - read),
695                                        boost::bind(&Client::handleReadCompletion, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage));
696                    return;
697                }
698            } catch (boost::exception_detail::clone_impl<boost::exception_detail::error_info_injector<boost::system::system_error> >) {
699                boost::asio::async_read( *socket, boost::asio::buffer(buffer, currentLength),
700                                    boost::bind(&Client::handleReadCompletion, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, socket, buffer, lengthOrMessage));
701                return;
702            }
703        }
704    }
705
706    int64_t requestsSent;
707    int64_t responsesReceived;
708    int64_t responsesReceivedLastTime;
709    int64_t bytesWritten;
710    int64_t bytesRead;
711    int64_t bytesWrittenLastTime;
712    int64_t bytesReadLastTime;
713};
714
715int main(int argc, char **argv) {
716    boost::asio::io_service io_service;
717    if (std::string(argv[1]) == "server") {
718        std::vector<std::string> machines;
719        for (int ii = 2; ii < argc; ii++) {
720            machines.push_back(std::string(argv[ii]));
721        }
722        Server server( io_service, machines);
723        io_service.run();
724    } else if (std::string(argv[1]) == "client") {
725        std::vector<std::string> machines;
726        for (int ii = 2; ii < argc; ii++) {
727            machines.push_back( std::string(argv[ii]));
728        }
729        Client client( io_service, machines);
730        io_service.run();
731    }
732    return 0;
733}