/java/org/apache/tomcat/util/net/Nio2Endpoint.java

   1/*
   2 *  Licensed to the Apache Software Foundation (ASF) under one or more
   3 *  contributor license agreements.  See the NOTICE file distributed with
   4 *  this work for additional information regarding copyright ownership.
   5 *  The ASF licenses this file to You under the Apache License, Version 2.0
   6 *  (the "License"); you may not use this file except in compliance with
   7 *  the License.  You may obtain a copy of the License at
   8 *
   9 *      http://www.apache.org/licenses/LICENSE-2.0
  10 *
  11 *  Unless required by applicable law or agreed to in writing, software
  12 *  distributed under the License is distributed on an "AS IS" BASIS,
  13 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 *  See the License for the specific language governing permissions and
  15 *  limitations under the License.
  16 */
  17
  18package org.apache.tomcat.util.net;
  19
  20import java.io.File;
  21import java.io.IOException;
  22import java.net.InetSocketAddress;
  23import java.net.SocketAddress;
  24import java.net.StandardSocketOptions;
  25import java.nio.ByteBuffer;
  26import java.nio.channels.AsynchronousChannelGroup;
  27import java.nio.channels.AsynchronousServerSocketChannel;
  28import java.nio.channels.AsynchronousSocketChannel;
  29import java.nio.channels.ClosedChannelException;
  30import java.nio.channels.CompletionHandler;
  31import java.nio.channels.FileChannel;
  32import java.nio.file.StandardOpenOption;
  33import java.util.concurrent.Executor;
  34import java.util.concurrent.ExecutorService;
  35import java.util.concurrent.RejectedExecutionException;
  36import java.util.concurrent.TimeUnit;
  37
  38import javax.net.ssl.KeyManager;
  39import javax.net.ssl.SSLContext;
  40import javax.net.ssl.SSLEngine;
  41import javax.net.ssl.SSLSessionContext;
  42import javax.net.ssl.X509KeyManager;
  43
  44import org.apache.juli.logging.Log;
  45import org.apache.juli.logging.LogFactory;
  46import org.apache.tomcat.util.ExceptionUtils;
  47import org.apache.tomcat.util.collections.SynchronizedStack;
  48import org.apache.tomcat.util.net.AbstractEndpoint.Handler.SocketState;
  49import org.apache.tomcat.util.net.SecureNio2Channel.ApplicationBufferHandler;
  50import org.apache.tomcat.util.net.jsse.NioX509KeyManager;
  51
  52/**
  53 * NIO2 endpoint.
  54 */
  55public class Nio2Endpoint extends AbstractEndpoint<Nio2Channel> {
  56
  57
  58    // -------------------------------------------------------------- Constants
  59
  60
  61    private static final Log log = LogFactory.getLog(Nio2Endpoint.class);
  62
  63
  64    // ----------------------------------------------------------------- Fields
  65
  66    /**
  67     * Server socket "pointer".
  68     */
  69    private AsynchronousServerSocketChannel serverSock = null;
  70
  71    /**
  72     * use send file
  73     */
  74    private boolean useSendfile = true;
  75
  76    /**
  77     * The size of the OOM parachute.
  78     */
  79    private int oomParachute = 1024*1024;
  80
  81    /**
  82     * Allows detecting if a completion handler completes inline.
  83     */
  84    private static ThreadLocal<Boolean> inlineCompletion = new ThreadLocal<>();
  85
  86    /**
  87     * Thread group associated with the server socket.
  88     */
  89    private AsynchronousChannelGroup threadGroup = null;
  90
  91    private volatile boolean allClosed;
  92
  93    /**
  94     * The oom parachute, when an OOM error happens,
  95     * will release the data, giving the JVM instantly
  96     * a chunk of data to be able to recover with.
  97     */
  98    private byte[] oomParachuteData = null;
  99
 100    /**
 101     * Make sure this string has already been allocated
 102     */
 103    private static final String oomParachuteMsg =
 104        "SEVERE:Memory usage is low, parachute is non existent, your system may start failing.";
 105
 106    /**
 107     * Keep track of OOM warning messages.
 108     */
 109    private long lastParachuteCheck = System.currentTimeMillis();
 110
 111    /**
 112     * Cache for SocketProcessor objects
 113     */
 114    private SynchronizedStack<SocketProcessor> processorCache;
 115
 116    /**
 117     * Cache for socket wrapper objects
 118     */
 119    private SynchronizedStack<Nio2SocketWrapper> socketWrapperCache;
 120
 121    /**
 122     * Bytebuffer cache, each channel holds a set of buffers (two, except for SSL holds four)
 123     */
 124    private SynchronizedStack<Nio2Channel> nioChannels;
 125
 126
 127    // ------------------------------------------------------------- Properties
 128
 129
 130    /**
 131     * Use the object caches to reduce GC at the expense of additional memory use.
 132     */
 133    private boolean useCaches = false;
 134    public void setUseCaches(boolean useCaches) { this.useCaches = useCaches; }
 135    public boolean getUseCaches() { return useCaches; }
 136
 137
 138    /**
 139     * Priority of the poller threads.
 140     */
 141    private int pollerThreadPriority = Thread.NORM_PRIORITY;
 142    public void setPollerThreadPriority(int pollerThreadPriority) { this.pollerThreadPriority = pollerThreadPriority; }
 143    public int getPollerThreadPriority() { return pollerThreadPriority; }
 144
 145
 146    /**
 147     * Handling of accepted sockets.
 148     */
 149    private Handler handler = null;
 150    public void setHandler(Handler handler ) { this.handler = handler; }
 151    public Handler getHandler() { return handler; }
 152
 153
 154    /**
 155     * Allow comet request handling.
 156     */
 157    private boolean useComet = true;
 158    public void setUseComet(boolean useComet) { this.useComet = useComet; }
 159    @Override
 160    public boolean getUseComet() { return useComet; }
 161    @Override
 162    public boolean getUseCometTimeout() { return getUseComet(); }
 163    @Override
 164    public boolean getUsePolling() { return true; } // Always supported
 165
 166    public void setSocketProperties(SocketProperties socketProperties) {
 167        this.socketProperties = socketProperties;
 168    }
 169
 170    public void setUseSendfile(boolean useSendfile) {
 171        this.useSendfile = useSendfile;
 172    }
 173
 174    /**
 175     * Is deferAccept supported?
 176     */
 177    @Override
 178    public boolean getDeferAccept() {
 179        // Not supported
 180        return false;
 181    }
 182
 183    public void setOomParachute(int oomParachute) {
 184        this.oomParachute = oomParachute;
 185    }
 186
 187    public void setOomParachuteData(byte[] oomParachuteData) {
 188        this.oomParachuteData = oomParachuteData;
 189    }
 190
 191
 192    private SSLContext sslContext = null;
 193    public SSLContext getSSLContext() { return sslContext;}
 194    public void setSSLContext(SSLContext c) { sslContext = c;}
 195    private String[] enabledCiphers;
 196    private String[] enabledProtocols;
 197
 198    /**
 199     * Port in use.
 200     */
 201    @Override
 202    public int getLocalPort() {
 203        AsynchronousServerSocketChannel ssc = serverSock;
 204        if (ssc == null) {
 205            return -1;
 206        } else {
 207            try {
 208                SocketAddress sa = ssc.getLocalAddress();
 209                if (sa != null && sa instanceof InetSocketAddress) {
 210                    return ((InetSocketAddress) sa).getPort();
 211                } else {
 212                    return -1;
 213                }
 214            } catch (IOException e) {
 215                return -1;
 216            }
 217        }
 218    }
 219
 220
 221    @Override
 222    public String[] getCiphersUsed() {
 223        return enabledCiphers;
 224    }
 225
 226
 227    // --------------------------------------------------------- OOM Parachute Methods
 228
 229    protected void checkParachute() {
 230        boolean para = reclaimParachute(false);
 231        if (!para && (System.currentTimeMillis()-lastParachuteCheck)>10000) {
 232            try {
 233                log.fatal(oomParachuteMsg);
 234            }catch (Throwable t) {
 235                ExceptionUtils.handleThrowable(t);
 236                System.err.println(oomParachuteMsg);
 237            }
 238            lastParachuteCheck = System.currentTimeMillis();
 239        }
 240    }
 241
 242    protected boolean reclaimParachute(boolean force) {
 243        if ( oomParachuteData != null ) return true;
 244        if ( oomParachute > 0 && ( force || (Runtime.getRuntime().freeMemory() > (oomParachute*2))) )
 245            oomParachuteData = new byte[oomParachute];
 246        return oomParachuteData != null;
 247    }
 248
 249    protected void releaseCaches() {
 250        if (useCaches) {
 251            this.socketWrapperCache.clear();
 252            this.nioChannels.clear();
 253            this.processorCache.clear();
 254        }
 255        if ( handler != null ) handler.recycle();
 256
 257    }
 258
 259    // --------------------------------------------------------- Public Methods
 260
 261    /**
 262     * Number of keepalive sockets.
 263     */
 264    public int getKeepAliveCount() {
 265        // For this connector, only the overall connection count is relevant
 266        return -1;
 267    }
 268
 269
 270    // ----------------------------------------------- Public Lifecycle Methods
 271
 272
 273    /**
 274     * Initialize the endpoint.
 275     */
 276    @Override
 277    public void bind() throws Exception {
 278
 279        // Create worker collection
 280        if ( getExecutor() == null ) {
 281            createExecutor();
 282        }
 283        if (getExecutor() instanceof ExecutorService) {
 284            threadGroup = AsynchronousChannelGroup.withThreadPool((ExecutorService) getExecutor());
 285        }
 286        // AsynchronousChannelGroup currently needs exclusive access to its executor service
 287        if (!internalExecutor) {
 288            log.warn(sm.getString("endpoint.nio2.exclusiveExecutor"));
 289        }
 290
 291        serverSock = AsynchronousServerSocketChannel.open(threadGroup);
 292        socketProperties.setProperties(serverSock);
 293        InetSocketAddress addr = (getAddress()!=null?new InetSocketAddress(getAddress(),getPort()):new InetSocketAddress(getPort()));
 294        serverSock.bind(addr,getBacklog());
 295
 296        // Initialize thread count defaults for acceptor, poller
 297        if (acceptorThreadCount == 0) {
 298            // NIO2 does not allow any form of IO concurrency
 299            acceptorThreadCount = 1;
 300        }
 301
 302        // Initialize SSL if needed
 303        if (isSSLEnabled()) {
 304            SSLUtil sslUtil = handler.getSslImplementation().getSSLUtil(this);
 305
 306            sslContext = sslUtil.createSSLContext();
 307            sslContext.init(wrap(sslUtil.getKeyManagers()),
 308                    sslUtil.getTrustManagers(), null);
 309
 310            SSLSessionContext sessionContext =
 311                sslContext.getServerSessionContext();
 312            if (sessionContext != null) {
 313                sslUtil.configureSessionContext(sessionContext);
 314            }
 315            // Determine which cipher suites and protocols to enable
 316            enabledCiphers = sslUtil.getEnableableCiphers(sslContext);
 317            enabledProtocols = sslUtil.getEnableableProtocols(sslContext);
 318        }
 319
 320        if (oomParachute>0) reclaimParachute(true);
 321    }
 322
 323    public KeyManager[] wrap(KeyManager[] managers) {
 324        if (managers==null) return null;
 325        KeyManager[] result = new KeyManager[managers.length];
 326        for (int i=0; i<result.length; i++) {
 327            if (managers[i] instanceof X509KeyManager && getKeyAlias()!=null) {
 328                result[i] = new NioX509KeyManager((X509KeyManager)managers[i],getKeyAlias());
 329            } else {
 330                result[i] = managers[i];
 331            }
 332        }
 333        return result;
 334    }
 335
 336
 337    /**
 338     * Start the NIO endpoint, creating acceptor, poller threads.
 339     */
 340    @Override
 341    public void startInternal() throws Exception {
 342
 343        if (!running) {
 344            allClosed = false;
 345            running = true;
 346            paused = false;
 347
 348            // Create worker collection
 349            if ( getExecutor() == null ) {
 350                createExecutor();
 351            }
 352
 353            if (useCaches) {
 354                processorCache = new SynchronizedStack<>(SynchronizedStack.DEFAULT_SIZE,
 355                        socketProperties.getProcessorCache());
 356                socketWrapperCache = new SynchronizedStack<>(SynchronizedStack.DEFAULT_SIZE,
 357                        socketProperties.getSocketWrapperCache());
 358                nioChannels = new SynchronizedStack<>(SynchronizedStack.DEFAULT_SIZE,
 359                        socketProperties.getBufferPool());
 360            }
 361
 362            initializeConnectionLatch();
 363            startAcceptorThreads();
 364
 365            setAsyncTimeout(new AsyncTimeout());
 366            Thread timeoutThread = new Thread(getAsyncTimeout(), getName() + "-AsyncTimeout");
 367            timeoutThread.setPriority(threadPriority);
 368            timeoutThread.setDaemon(true);
 369            timeoutThread.start();
 370        }
 371    }
 372
 373
 374    /**
 375     * Stop the endpoint. This will cause all processing threads to stop.
 376     */
 377    @Override
 378    public void stopInternal() {
 379        releaseConnectionLatch();
 380        if (!paused) {
 381            pause();
 382        }
 383        if (running) {
 384            running = false;
 385            getAsyncTimeout().stop();
 386            unlockAccept();
 387        }
 388        // Use the executor to avoid binding the main thread if something bad
 389        // occurs and unbind will also wait for a bit for it to complete
 390        getExecutor().execute(new Runnable() {
 391            @Override
 392            public void run() {
 393                // Timeout any pending async request
 394                for (SocketWrapper<Nio2Channel> socket : waitingRequests) {
 395                    processSocket(socket, SocketStatus.TIMEOUT, false);
 396                }
 397                // Then close all active connections if any remains
 398                try {
 399                    handler.closeAll();
 400                } catch (Throwable t) {
 401                    ExceptionUtils.handleThrowable(t);
 402                } finally {
 403                    allClosed = true;
 404                }
 405            }
 406        });
 407        if (useCaches) {
 408            socketWrapperCache.clear();
 409            nioChannels.clear();
 410            processorCache.clear();
 411        }
 412    }
 413
 414
 415    /**
 416     * Deallocate NIO memory pools, and close server socket.
 417     */
 418    @Override
 419    public void unbind() throws Exception {
 420        if (running) {
 421            stop();
 422        }
 423        // Close server socket
 424        serverSock.close();
 425        serverSock = null;
 426        sslContext = null;
 427        // Unlike other connectors, the thread pool is tied to the server socket
 428        shutdownExecutor();
 429        releaseCaches();
 430    }
 431
 432
 433    @Override
 434    public void shutdownExecutor() {
 435        if (threadGroup != null && internalExecutor) {
 436            try {
 437                long timeout = getExecutorTerminationTimeoutMillis();
 438                while (timeout > 0 && !allClosed) {
 439                    timeout -= 100;
 440                    Thread.sleep(100);
 441                }
 442                threadGroup.shutdownNow();
 443                if (timeout > 0) {
 444                    threadGroup.awaitTermination(timeout, TimeUnit.MILLISECONDS);
 445                }
 446            } catch (IOException e) {
 447                getLog().warn(sm.getString("endpoint.warn.executorShutdown", getName()), e);
 448            } catch (InterruptedException e) {
 449                // Ignore
 450            }
 451            if (!threadGroup.isTerminated()) {
 452                getLog().warn(sm.getString("endpoint.warn.executorShutdown", getName()));
 453            }
 454            threadGroup = null;
 455        }
 456        // Mostly to cleanup references
 457        super.shutdownExecutor();
 458    }
 459
 460
 461    // ------------------------------------------------------ Protected Methods
 462
 463
 464    public int getWriteBufSize() {
 465        return socketProperties.getTxBufSize();
 466    }
 467
 468    public int getReadBufSize() {
 469        return socketProperties.getRxBufSize();
 470    }
 471
 472    @Override
 473    public boolean getUseSendfile() {
 474        return useSendfile;
 475    }
 476
 477    public int getOomParachute() {
 478        return oomParachute;
 479    }
 480
 481    public byte[] getOomParachuteData() {
 482        return oomParachuteData;
 483    }
 484
 485
 486    @Override
 487    protected AbstractEndpoint.Acceptor createAcceptor() {
 488        return new Acceptor();
 489    }
 490
 491
 492    /**
 493     * Process the specified connection.
 494     */
 495    protected boolean setSocketOptions(AsynchronousSocketChannel socket) {
 496        // Process the connection
 497        try {
 498            socketProperties.setProperties(socket);
 499
 500            Nio2Channel channel = (useCaches) ? nioChannels.pop() : null;
 501            if (channel == null) {
 502                // SSL setup
 503                if (sslContext != null) {
 504                    SSLEngine engine = createSSLEngine();
 505                    int appBufferSize = engine.getSession().getApplicationBufferSize();
 506                    NioBufferHandler bufhandler = new NioBufferHandler(
 507                            Math.max(appBufferSize, socketProperties.getAppReadBufSize()),
 508                            Math.max(appBufferSize, socketProperties.getAppWriteBufSize()),
 509                            socketProperties.getDirectBuffer());
 510                    channel = new SecureNio2Channel(engine, bufhandler, this);
 511                } else {
 512                    NioBufferHandler bufhandler = new NioBufferHandler(
 513                            socketProperties.getAppReadBufSize(),
 514                            socketProperties.getAppWriteBufSize(),
 515                            socketProperties.getDirectBuffer());
 516                    channel = new Nio2Channel(bufhandler);
 517                }
 518            } else {
 519                if (sslContext != null) {
 520                    SSLEngine engine = createSSLEngine();
 521                    ((SecureNio2Channel) channel).setSSLEngine(engine);
 522                }
 523            }
 524            Nio2SocketWrapper socketWrapper = (useCaches) ? socketWrapperCache.pop() : null;
 525            if (socketWrapper == null) {
 526                socketWrapper = new Nio2SocketWrapper(channel);
 527            }
 528            channel.reset(socket, socketWrapper);
 529            socketWrapper.reset(channel, getSocketProperties().getSoTimeout());
 530            socketWrapper.setKeepAliveLeft(Nio2Endpoint.this.getMaxKeepAliveRequests());
 531            socketWrapper.setSecure(isSSLEnabled());
 532            if (sslContext != null) {
 533                // Use the regular processing, as the first handshake needs to be done there
 534                processSocket(socketWrapper, SocketStatus.OPEN_READ, true);
 535            } else {
 536                // Wait until some bytes are available to start the real processing
 537                awaitBytes(socketWrapper);
 538            }
 539        } catch (Throwable t) {
 540            ExceptionUtils.handleThrowable(t);
 541            try {
 542                log.error("",t);
 543            } catch (Throwable tt) {
 544                ExceptionUtils.handleThrowable(t);
 545            }
 546            // Tell to close the socket
 547            return false;
 548        }
 549        return true;
 550    }
 551
 552    protected SSLEngine createSSLEngine() {
 553        SSLEngine engine = sslContext.createSSLEngine();
 554        if ("false".equals(getClientAuth())) {
 555            engine.setNeedClientAuth(false);
 556            engine.setWantClientAuth(false);
 557        } else if ("true".equals(getClientAuth()) || "yes".equals(getClientAuth())){
 558            engine.setNeedClientAuth(true);
 559        } else if ("want".equals(getClientAuth())) {
 560            engine.setWantClientAuth(true);
 561        }
 562        engine.setUseClientMode(false);
 563        engine.setEnabledCipherSuites(enabledCiphers);
 564        engine.setEnabledProtocols(enabledProtocols);
 565
 566        handler.onCreateSSLEngine(engine);
 567        return engine;
 568    }
 569
 570
 571    /**
 572     * Returns true if a worker thread is available for processing.
 573     * @return boolean
 574     */
 575    protected boolean isWorkerAvailable() {
 576        return true;
 577    }
 578
 579
 580    @Override
 581    public void processSocket(SocketWrapper<Nio2Channel> socketWrapper,
 582            SocketStatus socketStatus, boolean dispatch) {
 583        processSocket0(socketWrapper, socketStatus, dispatch);
 584    }
 585
 586    protected boolean processSocket0(SocketWrapper<Nio2Channel> socketWrapper, SocketStatus status, boolean dispatch) {
 587        try {
 588            SocketProcessor sc = (useCaches) ? processorCache.pop() : null;
 589            if (sc == null) {
 590                sc = new SocketProcessor(socketWrapper, status);
 591            } else {
 592                sc.reset(socketWrapper, status);
 593            }
 594            Executor executor = getExecutor();
 595            if (dispatch && executor != null) {
 596                executor.execute(sc);
 597            } else {
 598                sc.run();
 599            }
 600        } catch (RejectedExecutionException ree) {
 601            log.debug(sm.getString("endpoint.executor.fail", socketWrapper), ree);
 602            return false;
 603        } catch (Throwable t) {
 604            ExceptionUtils.handleThrowable(t);
 605            // This means we got an OOM or similar creating a thread, or that
 606            // the pool and its queue are full
 607            log.error(sm.getString("endpoint.process.fail"), t);
 608            return false;
 609        }
 610        return true;
 611    }
 612
 613    public void closeSocket(SocketWrapper<Nio2Channel> socket, SocketStatus status) {
 614        if (socket == null) {
 615            return;
 616        }
 617        try {
 618            if (socket.isComet() && status != null) {
 619                socket.setComet(false);//to avoid a loop
 620                if (status == SocketStatus.TIMEOUT) {
 621                    if (processSocket0(socket, status, true)) {
 622                        return; // don't close on comet timeout
 623                    }
 624                } else {
 625                    // Don't dispatch if the lines below are canceling the key
 626                    processSocket0(socket, status, false);
 627                }
 628            }
 629            handler.release(socket);
 630            try {
 631                if (socket.getSocket() != null) {
 632                    socket.getSocket().close(true);
 633                }
 634            } catch (Exception e){
 635                if (log.isDebugEnabled()) {
 636                    log.debug(sm.getString(
 637                            "endpoint.debug.socketCloseFail"), e);
 638                }
 639            }
 640            Nio2SocketWrapper nio2Socket = (Nio2SocketWrapper) socket;
 641            try {
 642                if (nio2Socket.getSendfileData() != null
 643                        && nio2Socket.getSendfileData().fchannel != null
 644                        && nio2Socket.getSendfileData().fchannel.isOpen()) {
 645                    nio2Socket.getSendfileData().fchannel.close();
 646                }
 647            } catch (Exception ignore) {
 648            }
 649            nio2Socket.reset(null, -1);
 650            countDownConnection();
 651        } catch (Throwable e) {
 652            ExceptionUtils.handleThrowable(e);
 653            if (log.isDebugEnabled()) log.error("",e);
 654        }
 655    }
 656
 657    @Override
 658    protected Log getLog() {
 659        return log;
 660    }
 661
 662
 663    // --------------------------------------------------- Acceptor Inner Class
 664
 665    /**
 666     * With NIO2, the main acceptor thread only initiates the initial accept
 667     * but periodically checks that the connector is still accepting (if not
 668     * it will attempt to start again). It is also responsible for periodic
 669     * checks of async timeouts, rather than use a dedicated thread for that.
 670     */
 671    protected class Acceptor extends AbstractEndpoint.Acceptor {
 672
 673        @Override
 674        public void run() {
 675
 676            int errorDelay = 0;
 677
 678            // Loop until we receive a shutdown command
 679            while (running) {
 680
 681                // Loop if endpoint is paused
 682                while (paused && running) {
 683                    state = AcceptorState.PAUSED;
 684                    try {
 685                        Thread.sleep(50);
 686                    } catch (InterruptedException e) {
 687                        // Ignore
 688                    }
 689                }
 690
 691                if (!running) {
 692                    break;
 693                }
 694                state = AcceptorState.RUNNING;
 695
 696                try {
 697                    //if we have reached max connections, wait
 698                    countUpOrAwaitConnection();
 699
 700                    AsynchronousSocketChannel socket = null;
 701                    try {
 702                        // Accept the next incoming connection from the server
 703                        // socket
 704                        socket = serverSock.accept().get();
 705                    } catch (Exception e) {
 706                        countDownConnection();
 707                        if (running) {
 708                            // Introduce delay if necessary
 709                            errorDelay = handleExceptionWithDelay(errorDelay);
 710                            // re-throw
 711                            throw e;
 712                        } else {
 713                            break;
 714                        }
 715                    }
 716                    // Successful accept, reset the error delay
 717                    errorDelay = 0;
 718
 719                    // Configure the socket
 720                    if (running && !paused) {
 721                        // Hand this socket off to an appropriate processor
 722                        if (!setSocketOptions(socket)) {
 723                            countDownConnection();
 724                            closeSocket(socket);
 725                        }
 726                    } else {
 727                        countDownConnection();
 728                        // Close socket right away
 729                        closeSocket(socket);
 730                    }
 731                } catch (Throwable t) {
 732                    ExceptionUtils.handleThrowable(t);
 733                    log.error(sm.getString("endpoint.accept.fail"), t);
 734                }
 735            }
 736            state = AcceptorState.ENDED;
 737        }
 738
 739    }
 740
 741
 742    private void closeSocket(AsynchronousSocketChannel socket) {
 743        try {
 744            socket.close();
 745        } catch (IOException ioe) {
 746            if (log.isDebugEnabled()) {
 747                log.debug("", ioe);
 748            }
 749        }
 750    }
 751
 752
 753    public static class Nio2SocketWrapper extends SocketWrapper<Nio2Channel> {
 754
 755        private SendfileData sendfileData = null;
 756        private boolean upgradeInit = false;
 757
 758        public Nio2SocketWrapper(Nio2Channel channel) {
 759            super(channel);
 760        }
 761
 762        @Override
 763        public void reset(Nio2Channel channel, long soTimeout) {
 764            super.reset(channel, soTimeout);
 765            upgradeInit = false;
 766            sendfileData = null;
 767        }
 768
 769        @Override
 770        public long getTimeout() {
 771            long timeout = super.getTimeout();
 772            return (timeout > 0) ? timeout : Long.MAX_VALUE;
 773        }
 774
 775        @Override
 776        public void setUpgraded(boolean upgraded) {
 777            if (upgraded && !isUpgraded()) {
 778                upgradeInit = true;
 779            }
 780            super.setUpgraded(upgraded);
 781        }
 782
 783        public boolean isUpgradeInit() {
 784            boolean value = upgradeInit;
 785            upgradeInit = false;
 786            return value;
 787        }
 788
 789        public void setSendfileData(SendfileData sf) { this.sendfileData = sf; }
 790        public SendfileData getSendfileData() { return this.sendfileData; }
 791
 792    }
 793
 794    // ------------------------------------------------ Application Buffer Handler
 795    public static class NioBufferHandler implements ApplicationBufferHandler {
 796        private ByteBuffer readbuf = null;
 797        private ByteBuffer writebuf = null;
 798
 799        public NioBufferHandler(int readsize, int writesize, boolean direct) {
 800            if ( direct ) {
 801                readbuf = ByteBuffer.allocateDirect(readsize);
 802                writebuf = ByteBuffer.allocateDirect(writesize);
 803            }else {
 804                readbuf = ByteBuffer.allocate(readsize);
 805                writebuf = ByteBuffer.allocate(writesize);
 806            }
 807        }
 808
 809        @Override
 810        public ByteBuffer getReadBuffer() {return readbuf;}
 811        @Override
 812        public ByteBuffer getWriteBuffer() {return writebuf;}
 813
 814    }
 815
 816    // ------------------------------------------------ Handler Inner Interface
 817
 818
 819    /**
 820     * Bare bones interface used for socket processing. Per thread data is to be
 821     * stored in the ThreadWithAttributes extra folders, or alternately in
 822     * thread local fields.
 823     */
 824    public interface Handler extends AbstractEndpoint.Handler {
 825        public SocketState process(SocketWrapper<Nio2Channel> socket,
 826                SocketStatus status);
 827        public void release(SocketWrapper<Nio2Channel> socket);
 828        public void closeAll();
 829        public SSLImplementation getSslImplementation();
 830        public void onCreateSSLEngine(SSLEngine engine);
 831    }
 832
 833    /**
 834     * The completion handler used for asynchronous read operations
 835     */
 836    private CompletionHandler<Integer, SocketWrapper<Nio2Channel>> awaitBytes
 837            = new CompletionHandler<Integer, SocketWrapper<Nio2Channel>>() {
 838
 839        @Override
 840        public synchronized void completed(Integer nBytes, SocketWrapper<Nio2Channel> attachment) {
 841            if (nBytes.intValue() < 0) {
 842                failed(new ClosedChannelException(), attachment);
 843                return;
 844            }
 845            processSocket0(attachment, SocketStatus.OPEN_READ, true);
 846        }
 847
 848        @Override
 849        public void failed(Throwable exc, SocketWrapper<Nio2Channel> attachment) {
 850            processSocket0(attachment, SocketStatus.DISCONNECT, true);
 851        }
 852    };
 853
 854    public void addTimeout(SocketWrapper<Nio2Channel> socket) {
 855        waitingRequests.add(socket);
 856    }
 857
 858    public boolean removeTimeout(SocketWrapper<Nio2Channel> socket) {
 859        return waitingRequests.remove(socket);
 860    }
 861
 862    public static void startInline() {
 863        inlineCompletion.set(Boolean.TRUE);
 864    }
 865
 866    public static void endInline() {
 867        inlineCompletion.set(Boolean.FALSE);
 868    }
 869
 870    public static boolean isInline() {
 871        Boolean flag = inlineCompletion.get();
 872        if (flag == null) {
 873            return false;
 874        } else {
 875            return flag.booleanValue();
 876        }
 877    }
 878
 879    public void awaitBytes(SocketWrapper<Nio2Channel> socket) {
 880        if (socket == null || socket.getSocket() == null) {
 881            return;
 882        }
 883        ByteBuffer byteBuffer = socket.getSocket().getBufHandler().getReadBuffer();
 884        byteBuffer.clear();
 885        socket.getSocket().read(byteBuffer, socket.getTimeout(),
 886               TimeUnit.MILLISECONDS, socket, awaitBytes);
 887    }
 888
 889    public boolean processSendfile(final Nio2SocketWrapper socket) {
 890
 891        // Configure the send file data
 892        SendfileData data = socket.getSendfileData();
 893        if (data.fchannel == null || !data.fchannel.isOpen()) {
 894            java.nio.file.Path path = new File(data.fileName).toPath();
 895            try {
 896                data.fchannel = java.nio.channels.FileChannel
 897                        .open(path, StandardOpenOption.READ).position(data.pos);
 898            } catch (IOException e) {
 899                closeSocket(socket, SocketStatus.ERROR);
 900                return false;
 901            }
 902        }
 903
 904        final ByteBuffer buffer;
 905        if (!socketProperties.getDirectBuffer() && sslContext == null) {
 906            // If not using SSL and direct buffers are not used, the
 907            // idea of sendfile is to avoid memory copies, so allocate a
 908            // direct buffer
 909            int bufferSize;
 910            try {
 911                Integer bufferSizeInteger = socket.getSocket().getIOChannel().getOption(StandardSocketOptions.SO_SNDBUF);
 912                if (bufferSizeInteger != null) {
 913                    bufferSize = bufferSizeInteger.intValue();
 914                } else {
 915                    bufferSize = 8192;
 916                }
 917            } catch (IOException e) {
 918                bufferSize = 8192;
 919            }
 920            buffer = ByteBuffer.allocateDirect(bufferSize);
 921        } else {
 922            buffer = socket.getSocket().getBufHandler().getWriteBuffer();
 923        }
 924        int nr = -1;
 925        try {
 926            nr = data.fchannel.read(buffer);
 927        } catch (IOException e1) {
 928            closeSocket(socket, SocketStatus.ERROR);
 929            return false;
 930        }
 931
 932        if (nr >= 0) {
 933            buffer.flip();
 934            socket.getSocket().write(buffer, data, new CompletionHandler<Integer, SendfileData>() {
 935
 936                @Override
 937                public void completed(Integer nw, SendfileData attachment) {
 938                    if (nw.intValue() < 0) { // Reach the end of stream
 939                        closeSocket(socket, SocketStatus.DISCONNECT);
 940                        try {
 941                            attachment.fchannel.close();
 942                        } catch (IOException e) {
 943                            // Ignore
 944                        }
 945                        return;
 946                    }
 947
 948                    attachment.pos += nw.intValue();
 949                    attachment.length -= nw.intValue();
 950
 951                    if (attachment.length <= 0) {
 952                        socket.setSendfileData(null);
 953                        try {
 954                            attachment.fchannel.close();
 955                        } catch (IOException e) {
 956                            // Ignore
 957                        }
 958                        if (attachment.keepAlive) {
 959                            awaitBytes(socket);
 960                        } else {
 961                            closeSocket(socket, SocketStatus.DISCONNECT);
 962                        }
 963                        return;
 964                    }
 965
 966                    boolean ok = true;
 967
 968                    if (!buffer.hasRemaining()) {
 969                        // This means that all data in the buffer has
 970                        // been
 971                        // written => Empty the buffer and read again
 972                        buffer.clear();
 973                        try {
 974                            if (attachment.fchannel.read(buffer) >= 0) {
 975                                buffer.flip();
 976                                if (attachment.length < buffer.remaining()) {
 977                                    buffer.limit(buffer.limit() - buffer.remaining() + (int) attachment.length);
 978                                }
 979                            } else {
 980                                // Reach the EOF
 981                                ok = false;
 982                            }
 983                        } catch (Throwable th) {
 984                            ExceptionUtils.handleThrowable(th);
 985                            if (log.isDebugEnabled()) {
 986                                log.debug(sm.getString("endpoint.sendfile.error"), th);
 987                            }
 988                            ok = false;
 989                        }
 990                    }
 991
 992                    if (ok) {
 993                        socket.getSocket().write(buffer, attachment, this);
 994                    } else {
 995                        try {
 996                            attachment.fchannel.close();
 997                        } catch (IOException e) {
 998                            // Ignore
 999                        }
1000                        closeSocket(socket, SocketStatus.ERROR);
1001                    }
1002                }
1003
1004                @Override
1005                public void failed(Throwable exc, SendfileData attachment) {
1006                    // Closing channels
1007                    closeSocket(socket, SocketStatus.ERROR);
1008                    try {
1009                        attachment.fchannel.close();
1010                    } catch (IOException e) {
1011                        // Ignore
1012                    }
1013                }
1014            });
1015            return true;
1016        } else {
1017            return false;
1018        }
1019    }
1020
1021    // ---------------------------------------------- SocketProcessor Inner Class
1022    /**
1023     * This class is the equivalent of the Worker, but will simply use in an
1024     * external Executor thread pool.
1025     */
1026    protected class SocketProcessor implements Runnable {
1027
1028        private SocketWrapper<Nio2Channel> socket = null;
1029        private SocketStatus status = null;
1030
1031        public SocketProcessor(SocketWrapper<Nio2Channel> socket, SocketStatus status) {
1032            reset(socket,status);
1033        }
1034
1035        public void reset(SocketWrapper<Nio2Channel> socket, SocketStatus status) {
1036            this.socket = socket;
1037            this.status = status;
1038        }
1039
1040        @Override
1041        public void run() {
1042            // Upgraded connections need to allow multiple threads to access the
1043            // connection at the same time to enable blocking IO to be used when
1044            // NIO has been configured
1045            if (socket.isUpgraded() &&
1046                    SocketStatus.OPEN_WRITE == status) {
1047                synchronized (socket.getWriteThreadLock()) {
1048                    doRun();
1049                }
1050            } else {
1051                synchronized (socket) {
1052                    doRun();
1053                }
1054            }
1055        }
1056
1057        private void doRun() {
1058            boolean launch = false;
1059            try {
1060                int handshake = -1;
1061
1062                try {
1063                    if (socket.getSocket() != null) {
1064                        // For STOP there is no point trying to handshake as the
1065                        // Poller has been stopped.
1066                        if (socket.getSocket().isHandshakeComplete() ||
1067                                status == SocketStatus.STOP) {
1068                            handshake = 0;
1069                        } else {
1070                            handshake = socket.getSocket().handshake();
1071                            // The handshake process reads/writes from/to the
1072                            // socket. status may therefore be OPEN_WRITE once
1073                            // the handshake completes. However, the handshake
1074                            // happens when the socket is opened so the status
1075                            // must always be OPEN_READ after it completes. It
1076                            // is OK to always set this as it is only used if
1077                            // the handshake completes.
1078                            status = SocketStatus.OPEN_READ;
1079                        }
1080                    }
1081                } catch (IOException x) {
1082                    handshake = -1;
1083                    if (log.isDebugEnabled()) {
1084                        log.debug(sm.getString("endpoint.err.handshake"), x);
1085                    }
1086                }
1087                if (handshake == 0) {
1088                    SocketState state = SocketState.OPEN;
1089                    // Process the request from this socket
1090                    if (status == null) {
1091                        state = handler.process(socket, SocketStatus.OPEN_READ);
1092                    } else {
1093                        state = handler.process(socket, status);
1094                    }
1095                    if (state == SocketState.CLOSED) {
1096                        // Close socket and pool
1097                        socket.setComet(false);
1098                        closeSocket(socket, SocketStatus.ERROR);
1099                        if (useCaches && running && !paused) {
1100                            nioChannels.push(socket.getSocket());
1101                        }
1102                        if (useCaches && running && !paused && socket != null) {
1103                            socketWrapperCache.push((Nio2SocketWrapper) socket);
1104                        }
1105                    } else if (state == SocketState.UPGRADING) {
1106                        socket.setKeptAlive(true);
1107                        socket.access();
1108                        launch = true;
1109                    }
1110                } else if (handshake == -1 ) {
1111                    closeSocket(socket, SocketStatus.DISCONNECT);
1112                    if (useCaches && running && !paused) {
1113                        nioChannels.push(socket.getSocket());
1114                    }
1115                    if (useCaches && running && !paused && socket != null) {
1116                        socketWrapperCache.push(((Nio2SocketWrapper) socket));
1117                    }
1118                }
1119            } catch (OutOfMemoryError oom) {
1120                try {
1121                    oomParachuteData = null;
1122                    log.error("", oom);
1123                    closeSocket(socket, SocketStatus.ERROR);
1124                    releaseCaches();
1125                } catch (Throwable oomt) {
1126                    try {
1127                        System.err.println(oomParachuteMsg);
1128                        oomt.printStackTrace();
1129                    } catch (Throwable letsHopeWeDontGetHere){
1130                        ExceptionUtils.handleThrowable(letsHopeWeDontGetHere);
1131                    }
1132                }
1133            } catch (VirtualMachineError vme) {
1134                ExceptionUtils.handleThrowable(vme);
1135            } catch (Throwable t) {
1136                log.error(sm.getString("endpoint.processing.fail"), t);
1137                if (socket != null) {
1138                    closeSocket(socket, SocketStatus.ERROR);
1139                }
1140            } finally {
1141                if (launch) {
1142                    try {
1143                        getExecutor().execute(new SocketProcessor(socket, SocketStatus.OPEN_READ));
1144                    } catch (NullPointerException npe) {
1145                        if (running) {
1146                            log.error(sm.getString("endpoint.launch.fail"),
1147                                    npe);
1148                        }
1149                    }
1150                }
1151                socket = null;
1152                status = null;
1153                //return to cache
1154                if (useCaches && running && !paused) {
1155                    processorCache.push(this);
1156                }
1157            }
1158        }
1159    }
1160
1161    // ----------------------------------------------- SendfileData Inner Class
1162    /**
1163     * SendfileData class.
1164     */
1165    public static class SendfileData {
1166        // File
1167        public String fileName;
1168        public FileChannel fchannel;
1169        public long pos;
1170        public long length;
1171        // KeepAlive flag
1172        public boolean keepAlive;
1173    }
1174}