1#include "memcached.h"23#ifdef TLS45#include "tls.h"6#include <string.h>7#include <sysexits.h>8#include <sys/param.h>9#include <openssl/ssl.h>10#include <openssl/err.h>1112/* constant session ID context for application-level SSL session scoping.13 * used in server-side SSL session caching, when enabled. */14#define SESSION_ID_CONTEXT "memcached"1516#ifndef MAXPATHLEN17#define MAXPATHLEN 409618#endif1920static ssize_t ssl_read(conn *c, void *buf, size_t count);21static ssize_t ssl_sendmsg(conn *c, struct msghdr *msg, int flags);22static ssize_t ssl_write(conn *c, void *buf, size_t count);23static void print_ssl_error(char *buff, size_t len);24static void ssl_callback(const SSL *s, int where, int ret);25static int ssl_new_session_callback(SSL *s, SSL_SESSION *sess);2627static pthread_mutex_t ssl_ctx_lock = PTHREAD_MUTEX_INITIALIZER;2829const unsigned ERROR_MSG_SIZE = 64;30const size_t SSL_ERROR_MSG_SIZE = 256;3132static void SSL_LOCK(void) {33    pthread_mutex_lock(&(ssl_ctx_lock));34}3536static void SSL_UNLOCK(void) {37    pthread_mutex_unlock(&(ssl_ctx_lock));38}3940void *ssl_accept(conn *c, int sfd, bool *fail) {41    SSL *ssl = NULL;42    if (c->ssl_enabled) {43        assert(IS_TCP(c->transport) && settings.ssl_enabled);4445        if (settings.ssl_ctx == NULL) {46            if (settings.verbose) {47                fprintf(stderr, "SSL context is not initialized\n");48            }49            *fail = true;50            return NULL;51        }52        SSL_LOCK();53        ssl = SSL_new(settings.ssl_ctx);54        SSL_UNLOCK();55        if (ssl == NULL) {56            if (settings.verbose) {57                fprintf(stderr, "Failed to created the SSL object\n");58            }59            *fail = true;60            ERR_clear_error();61            return NULL;62        }63        SSL_set_fd(ssl, sfd);6465        if (c->ssl_enabled == MC_SSL_ENABLED_NOPEER) {66            // Don't enforce peer certs for this socket.67            SSL_set_verify(ssl, SSL_VERIFY_NONE, NULL);68        } else if (c->ssl_enabled == MC_SSL_ENABLED_PEER) {69            // Force peer validation for this socket.70            SSL_set_verify(ssl, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);71        }7273        ERR_clear_error();74        int ret = SSL_accept(ssl);75        if (ret <= 0) {76            int err = SSL_get_error(ssl, ret);77            if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) {78                // we're actually fine, let the worker thread continue.79                ERR_clear_error();80            } else {81                // TODO: ship full error to log stream? conn events?82                // SYSCALL specifically means we need to check errno/strerror.83                // Else we need to look at the main error stack.84                if (err == SSL_ERROR_SYSCALL) {85                    LOGGER_LOG(NULL, LOG_CONNEVENTS, LOGGER_CONNECTION_TLSERROR,86                            NULL, c->sfd, strerror(errno));87                } else {88                    char ssl_err[SSL_ERROR_MSG_SIZE];89                    ssl_err[0] = '\0';90                    // OpenSSL internal error. One or more, but lets only care about91                    // the top error for now.92                    print_ssl_error(ssl_err, SSL_ERROR_MSG_SIZE);93                    LOGGER_LOG(NULL, LOG_CONNEVENTS, LOGGER_CONNECTION_TLSERROR,94                            NULL, c->sfd, ssl_err);95                }96                ERR_clear_error();97                SSL_free(ssl);98                STATS_LOCK();99                stats.ssl_handshake_errors++;100                STATS_UNLOCK();101                *fail = true;102                return NULL;103            }104        }105    }106107    return ssl;108}109110/*111 * Note on setting errno in the follow functions:112 * We either have to refactor callers of read/write/sendmsg to take an error113 * flag to find out if we're in an EAGAIN state, or we ensure the errno is set114 * properly before returning from our TLS call. We do this because it's115 * _possible_ for OpenSSL to do something weird and land with an errno that116 * doesn't match the WANT_READ|WRITE state.117 *118 * Also: we _might_ have to communicate from these calls if we need to wait on119 * reads or write. Since I haven't yet proved that's even possible I'll save120 * that for a future refactor.121 */122123// TODO: add int offset, and find the nth NID here.124// or different function that accepts a string, then does etc?125// Caller _must immediately_ use the string and not store the pointer.126const unsigned char *ssl_get_peer_cn(conn *c, int *len) {127    if (!c->ssl) {128        return NULL;129    }130131    // can't use get0 to avoid getting a reference since that requires 3.0.0+132    X509 *cert = SSL_get_peer_certificate(c->ssl);133    if (cert == NULL) {134        return NULL;135    }136    X509_NAME *name = X509_get_subject_name(cert);137    if (name == NULL) {138        X509_free(cert);139        return NULL;140    }141142    int r = X509_NAME_get_index_by_NID(name, NID_commonName, -1);143    if (r == -1) {144        X509_free(cert);145        return NULL;146    }147    ASN1_STRING *asn1 = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(name, r));148149    if (asn1 == NULL) {150        X509_free(cert);151        return NULL;152    }153    *len = ASN1_STRING_length(asn1);154    X509_free(cert);155    return ASN1_STRING_get0_data(asn1);156}157158/*159 * Reads decrypted data from the underlying BIO read buffers,160 * which reads from the socket.161 */162static ssize_t ssl_read(conn *c, void *buf, size_t count) {163    assert (c != NULL);164    /* TODO : document the state machine interactions for SSL_read with165        non-blocking sockets/ SSL re-negotiations166    */167168    ssize_t ret = SSL_read(c->ssl, buf, count);169    if (ret <= 0) {170        int err = SSL_get_error(c->ssl, ret);171        if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) {172            errno = EAGAIN;173        } else if (err == SSL_ERROR_ZERO_RETURN) {174            // TLS session is closed... let the caller move this along.175            return 0;176        } else if (err == SSL_ERROR_SYSCALL) {177            // need to rely on errno to find out what happened178            LOGGER_LOG(c->thread->l, LOG_CONNEVENTS, LOGGER_CONNECTION_TLSERROR,179                    NULL, c->sfd, strerror(errno));180        } else if (ret != 0) {181            char ssl_err[SSL_ERROR_MSG_SIZE];182            // OpenSSL internal error. One or more, but lets only care about183            // the top error for now.184            print_ssl_error(ssl_err, SSL_ERROR_MSG_SIZE);185            LOGGER_LOG(c->thread->l, LOG_CONNEVENTS, LOGGER_CONNECTION_TLSERROR,186                    NULL, c->sfd, ssl_err);187            STATS_LOCK();188            stats.ssl_proto_errors++;189            STATS_UNLOCK();190        }191        ERR_clear_error();192    }193194    return ret;195}196197/*198 * Writes data to the underlying BIO write buffers,199 * which encrypt and write them to the socket.200 */201static ssize_t ssl_write(conn *c, void *buf, size_t count) {202    assert (c != NULL);203204    ssize_t ret = SSL_write(c->ssl, buf, count);205    if (ret <= 0) {206        int err = SSL_get_error(c->ssl, ret);207        if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) {208            errno = EAGAIN;209        } else if (err == SSL_ERROR_ZERO_RETURN) {210            // TLS session is closed... let the caller move this along.211            return 0;212        } else if (err == SSL_ERROR_SYSCALL) {213            // need to rely on errno to find out what happened214            LOGGER_LOG(c->thread->l, LOG_CONNEVENTS, LOGGER_CONNECTION_TLSERROR,215                    NULL, c->sfd, strerror(errno));216        } else if (ret != 0) {217            char ssl_err[SSL_ERROR_MSG_SIZE];218            // OpenSSL internal error. One or more, but lets only care about219            // the top error for now.220            print_ssl_error(ssl_err, SSL_ERROR_MSG_SIZE);221            LOGGER_LOG(c->thread->l, LOG_CONNEVENTS, LOGGER_CONNECTION_TLSERROR,222                    NULL, c->sfd, ssl_err);223            STATS_LOCK();224            stats.ssl_proto_errors++;225            STATS_UNLOCK();226        }227        ERR_clear_error();228    }229    return ret;230}231232/*233 * SSL sendmsg implementation. Perform a SSL_write.234 */235static ssize_t ssl_sendmsg(conn *c, struct msghdr *msg, int flags) {236    assert (c != NULL);237    size_t buf_remain = settings.ssl_wbuf_size;238    size_t bytes = 0;239    size_t to_copy;240    int i;241242    // ssl_wbuf is pointing to the buffer allocated in the worker thread.243    assert(c->ssl_wbuf);244    // TODO: allocate a fix buffer in crawler/logger if they start using245    // the sendmsg method. Also, set c->ssl_wbuf  when the side thread246    // start owning the connection and reset the pointer in247    // conn_worker_readd.248    // Currently this connection would not be served by a different thread249    // than the one it's assigned.250    assert(pthread_equal(c->thread->thread_id, pthread_self()) != 0);251252    char *bp = c->ssl_wbuf;253    for (i = 0; i < msg->msg_iovlen; i++) {254        size_t len = msg->msg_iov[i].iov_len;255        to_copy = len < buf_remain ? len : buf_remain;256257        memcpy(bp + bytes, (void*)msg->msg_iov[i].iov_base, to_copy);258        buf_remain -= to_copy;259        bytes += to_copy;260        if (buf_remain == 0)261            break;262    }263    /* TODO : document the state machine interactions for SSL_write with264        non-blocking sockets/ SSL re-negotiations265    */266    return ssl_write(c, c->ssl_wbuf, bytes);267}268269/*270 * Prints an SSL error into the buff, if there's any.271 */272static void print_ssl_error(char *buff, size_t len) {273    unsigned long err;274    if ((err = ERR_get_error()) != 0) {275        ERR_error_string_n(err, buff, len);276    }277}278279/*280 * Loads server certificates to the SSL context and validate them.281 * @return whether certificates are successfully loaded and verified or not.282 * @param error_msg contains the error when unsuccessful.283 */284static bool load_server_certificates(char **errmsg) {285    bool success = false;286287    const size_t CRLF_NULLCHAR_LEN = 3;288    char *error_msg = malloc(MAXPATHLEN + ERROR_MSG_SIZE +289        SSL_ERROR_MSG_SIZE);290    size_t errmax = MAXPATHLEN + ERROR_MSG_SIZE + SSL_ERROR_MSG_SIZE -291        CRLF_NULLCHAR_LEN;292293    if (error_msg == NULL) {294        *errmsg = NULL;295        return false;296    }297298    if (settings.ssl_ctx == NULL) {299        snprintf(error_msg, errmax, "Error TLS not enabled\r\n");300        *errmsg = error_msg;301        return false;302    }303304    char *ssl_err_msg = malloc(SSL_ERROR_MSG_SIZE);305    if (ssl_err_msg == NULL) {306        free(error_msg);307        *errmsg = NULL;308        return false;309    }310    bzero(ssl_err_msg, SSL_ERROR_MSG_SIZE);311    size_t err_msg_size = 0;312313    SSL_LOCK();314    if (!SSL_CTX_use_certificate_chain_file(settings.ssl_ctx,315        settings.ssl_chain_cert)) {316        print_ssl_error(ssl_err_msg, SSL_ERROR_MSG_SIZE);317        err_msg_size = snprintf(error_msg, errmax, "Error loading the certificate chain: "318            "%s : %s", settings.ssl_chain_cert, ssl_err_msg);319    } else if (!SSL_CTX_use_PrivateKey_file(settings.ssl_ctx, settings.ssl_key,320                                        settings.ssl_keyformat)) {321        print_ssl_error(ssl_err_msg, SSL_ERROR_MSG_SIZE);322        err_msg_size = snprintf(error_msg, errmax, "Error loading the key: %s : %s",323            settings.ssl_key, ssl_err_msg);324    } else if (!SSL_CTX_check_private_key(settings.ssl_ctx)) {325        print_ssl_error(ssl_err_msg, SSL_ERROR_MSG_SIZE);326        err_msg_size = snprintf(error_msg, errmax, "Error validating the certificate: %s",327            ssl_err_msg);328    } else if (settings.ssl_ca_cert) {329        if (!SSL_CTX_load_verify_locations(settings.ssl_ctx,330          settings.ssl_ca_cert, NULL)) {331            print_ssl_error(ssl_err_msg, SSL_ERROR_MSG_SIZE);332            err_msg_size = snprintf(error_msg, errmax,333              "Error loading the CA certificate: %s : %s",334              settings.ssl_ca_cert, ssl_err_msg);335        } else {336            SSL_CTX_set_client_CA_list(settings.ssl_ctx,337              SSL_load_client_CA_file(settings.ssl_ca_cert));338            success = true;339        }340    } else {341        success = true;342    }343    SSL_UNLOCK();344    free(ssl_err_msg);345    if (success) {346        settings.ssl_last_cert_refresh_time = current_time;347        free(error_msg);348    } else {349        *errmsg = error_msg;350        error_msg += (err_msg_size >= errmax ? errmax - 1: err_msg_size);351        snprintf(error_msg, CRLF_NULLCHAR_LEN, "\r\n");352        // Print if there are more errors and drain the queue.353        ERR_print_errors_fp(stderr);354    }355    return success;356}357358void ssl_conn_close(void *ssl_in) {359    SSL *ssl = ssl_in;360    SSL_shutdown(ssl);361    SSL_free(ssl);362}363364int ssl_pending(void *ssl_in) {365    SSL *ssl = ssl_in;366    return SSL_pending(ssl);367}368369void ssl_init_conn(conn *c, void *ssl_in) {370    if (ssl_in) {371        SSL *ssl = ssl_in;372        c->ssl = (SSL*)ssl;373        c->read = ssl_read;374        c->sendmsg = ssl_sendmsg;375        c->write = ssl_write;376        SSL_set_info_callback(c->ssl, ssl_callback);377    }378}379380void ssl_init_settings(void) {381    settings.ssl_enabled = false;382    settings.ssl_ctx = NULL;383    settings.ssl_chain_cert = NULL;384    settings.ssl_key = NULL;385    settings.ssl_verify_mode = SSL_VERIFY_NONE;386    settings.ssl_keyformat = SSL_FILETYPE_PEM;387    settings.ssl_ciphers = NULL;388    settings.ssl_ca_cert = NULL;389    settings.ssl_last_cert_refresh_time = current_time;390    settings.ssl_wbuf_size = 16 * 1024; // default is 16KB (SSL max frame size is 17KB)391    settings.ssl_session_cache = false;392    settings.ssl_kernel_tls = false;393    settings.ssl_min_version = TLS1_2_VERSION;394}395396/*397 * Verify SSL settings and initiates the SSL context.398 */399int ssl_init(void) {400    assert(settings.ssl_enabled);401402    OPENSSL_init_ssl(0, NULL);403404    // SSL context for the process. All connections will share one405    // process level context.406    settings.ssl_ctx = SSL_CTX_new(TLS_server_method());407408    SSL_CTX_set_min_proto_version(settings.ssl_ctx, settings.ssl_min_version);409410    // The server certificate, private key and validations.411    char *error_msg;412    if (!load_server_certificates(&error_msg)) {413        fprintf(stderr, "%s", error_msg);414        free(error_msg);415        exit(EX_USAGE);416    }417418    // The verification mode of client certificate, default is SSL_VERIFY_PEER.419    SSL_CTX_set_verify(settings.ssl_ctx, settings.ssl_verify_mode, NULL);420    if (settings.ssl_ciphers && !SSL_CTX_set_cipher_list(settings.ssl_ctx,421                                                    settings.ssl_ciphers)) {422        fprintf(stderr, "Error setting the provided cipher(s): %s\n",423                settings.ssl_ciphers);424        exit(EX_USAGE);425    }426427    // Optional session caching; default disabled.428    if (settings.ssl_session_cache) {429        SSL_CTX_sess_set_new_cb(settings.ssl_ctx, ssl_new_session_callback);430        SSL_CTX_set_session_cache_mode(settings.ssl_ctx, SSL_SESS_CACHE_SERVER);431        SSL_CTX_set_session_id_context(settings.ssl_ctx,432                                       (const unsigned char *) SESSION_ID_CONTEXT,433                                       strlen(SESSION_ID_CONTEXT));434    } else {435        SSL_CTX_set_session_cache_mode(settings.ssl_ctx, SSL_SESS_CACHE_OFF);436    }437438    // Optional kernel TLS offload; default disabled.439    if (settings.ssl_kernel_tls) {440#if defined(SSL_OP_ENABLE_KTLS)441        SSL_CTX_set_options(settings.ssl_ctx, SSL_OP_ENABLE_KTLS);442#else443        fprintf(stderr, "Kernel TLS offload is not available\n");444        exit(EX_USAGE);445#endif446    }447448#ifdef SSL_OP_NO_RENEGOTIATION449    // Disable TLS re-negotiation if SSL_OP_NO_RENEGOTIATION is defined for450    // openssl 1.1.0h or above451    SSL_CTX_set_options(settings.ssl_ctx, SSL_OP_NO_RENEGOTIATION);452#endif453454    // Release TLS read/write buffers of idle connections455    SSL_CTX_set_mode(settings.ssl_ctx, SSL_MODE_RELEASE_BUFFERS);456457    return 0;458}459460/*461 * This method is registered with each SSL connection and abort the SSL session462 * if a client initiates a renegotiation for openssl versions before 1.1.0h.463 * For openssl 1.1.0h and above, TLS re-negotiation is disabled by setting the464 * SSL_OP_NO_RENEGOTIATION option in SSL_CTX_set_options.465 */466void ssl_callback(const SSL *s, int where, int ret) {467    // useful for debugging.468    // fprintf(stderr, "WHERE: %d RET: %d CODE: %s LONG: %s\n", where, ret, SSL_state_string(s), SSL_state_string_long(s));469#ifndef SSL_OP_NO_RENEGOTIATION470    SSL* ssl = (SSL*)s;471    if (SSL_in_before(ssl)) {472        fprintf(stderr, "%d: SSL renegotiation is not supported, "473                "closing the connection\n", SSL_get_fd(ssl));474        SSL_set_shutdown(ssl, SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN);475        return;476    }477#endif478}479480/*481 * This method is invoked with every new successfully negotiated SSL session,482 * when server-side session caching is enabled. Note that this method is not483 * invoked when a session is reused.484 */485int ssl_new_session_callback(SSL *s, SSL_SESSION *sess) {486    STATS_LOCK();487    stats.ssl_new_sessions++;488    STATS_UNLOCK();489490    return 0;491}492493bool refresh_certs(char **errmsg) {494    return load_server_certificates(errmsg);495}496497void ssl_help(void) {498    printf("   - ssl_chain_cert:      certificate chain file in PEM format\n"499           "   - ssl_key:             private key, if not part of the -ssl_chain_cert\n"500           "   - ssl_keyformat:       private key format (PEM, DER or ENGINE) (default: PEM)\n");501    printf("   - ssl_verify_mode:     peer certificate verification mode, default is 0(None).\n"502           "                          valid values are 0(None), 1(Request), 2(Require)\n"503           "                          or 3(Once)\n");504    printf("   - ssl_ciphers:         specify cipher list to be used\n"505           "   - ssl_ca_cert:         PEM format file of acceptable client CA's\n"506           "   - ssl_wbuf_size:       size in kilobytes of per-connection SSL output buffer\n"507           "                          (default: %u)\n", settings.ssl_wbuf_size / (1 << 10));508    printf("   - ssl_session_cache:   enable server-side SSL session cache, to support session\n"509           "                          resumption\n"510           "   - ssl_kernel_tls:      enable kernel TLS offload\n"511           "   - ssl_min_version:     minimum protocol version to accept (default: %s)\n",512           ssl_proto_text(settings.ssl_min_version));513#if defined(TLS1_3_VERSION)514    printf("                          valid values are 0(%s), 1(%s), 2(%s), or 3(%s).\n",515           ssl_proto_text(TLS1_VERSION), ssl_proto_text(TLS1_1_VERSION),516           ssl_proto_text(TLS1_2_VERSION), ssl_proto_text(TLS1_3_VERSION));517#else518    printf("                          valid values are 0(%s), 1(%s), or 2(%s).\n",519           ssl_proto_text(TLS1_VERSION), ssl_proto_text(TLS1_1_VERSION),520           ssl_proto_text(TLS1_2_VERSION));521#endif522    verify_default("ssl_keyformat", settings.ssl_keyformat == SSL_FILETYPE_PEM);523    verify_default("ssl_verify_mode", settings.ssl_verify_mode == SSL_VERIFY_NONE);524    verify_default("ssl_min_version", settings.ssl_min_version == TLS1_2_VERSION);525}526527const char *ssl_proto_text(int version) {528    switch (version) {529        case TLS1_VERSION:530            return "tlsv1.0";531        case TLS1_1_VERSION:532            return "tlsv1.1";533        case TLS1_2_VERSION:534            return "tlsv1.2";535#if defined(TLS1_3_VERSION)536        case TLS1_3_VERSION:537            return "tlsv1.3";538#endif539        default:540            return "unknown";541    }542}543544// TODO: would be nice to pull the entire set of startup option parsing into545// here like we do with extstore. To save time I'm only pulling subsection546// that require openssl headers to start.547bool ssl_set_verify_mode(int verify) {548    switch(verify) {549        case 0:550            settings.ssl_verify_mode = SSL_VERIFY_NONE;551            break;552        case 1:553            settings.ssl_verify_mode = SSL_VERIFY_PEER;554            break;555        case 2:556            settings.ssl_verify_mode = SSL_VERIFY_PEER |557                                        SSL_VERIFY_FAIL_IF_NO_PEER_CERT;558            break;559        case 3:560            settings.ssl_verify_mode = SSL_VERIFY_PEER |561                                        SSL_VERIFY_FAIL_IF_NO_PEER_CERT |562                                        SSL_VERIFY_CLIENT_ONCE;563            break;564        default:565            return false;566    }567    return true;568}569570bool ssl_set_min_version(int version) {571    switch (version) {572        case 0:573            settings.ssl_min_version = TLS1_VERSION;574            break;575        case 1:576            settings.ssl_min_version = TLS1_1_VERSION;577            break;578        case 2:579            settings.ssl_min_version = TLS1_2_VERSION;580            break;581#if defined(TLS1_3_VERSION)582        case 3:583            settings.ssl_min_version = TLS1_3_VERSION;584            break;585#endif586        default:587            return false;588    }589    return true;590}591592#endif // ifdef TLS