1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */23#include "proxy.h"4#include "proxy_tls.h"5#ifdef PROXY_TLS6#include <openssl/ssl.h>7#include <openssl/err.h>89/* Notes on ERR_clear_error() and friends:10 * - Errors from SSL calls leave errors on a thread-local "error stack"11 * - If an error is received from an SSL call, the stack needs to be inspected12 *   and cleared.13 * - The error stack _must_ be clear before any SSL_get_error() calls, as it14 *   may return garbage.15 * - There may be _multiple_ errors queued after one SSL call, so just16 *   checking the top level does not clear it.17 * - ERR_clear_error() is not "free", so we would prefer to avoid calling it18 *   before hotpath calls. Thus, we should ensure it's called _after_ any19 *   hotpath call that receives any kind of error.20 * - We should also call it _before_ any non-hotpath SSL calls (such as21 *   SSL_connect()) for defense against bugs in our code or OpenSSL.22 */2324int mcp_tls_init(proxy_ctx_t *ctx) {25    if (ctx->tls_ctx) {26        return MCP_TLS_OK;27    }2829    // TODO: check for OpenSSL 1.1+ ? should be elsewhere in the code.30    SSL_CTX *tctx = SSL_CTX_new(TLS_client_method());31    if (tctx == NULL) {32        return MCP_TLS_ERR;33    }3435    // TODO: make configurable like main cache server36    SSL_CTX_set_min_proto_version(tctx, TLS1_3_VERSION);37    // reduce memory consumption of idle backends.38    SSL_CTX_set_mode(tctx, SSL_MODE_RELEASE_BUFFERS);3940    ctx->tls_ctx = tctx;41    return 0;42}4344int mcp_tls_backend_init(proxy_ctx_t *ctx, struct mcp_backendconn_s *be) {45    if (!be->be_parent->tunables.use_tls) {46        return MCP_TLS_OK;47    }4849    SSL *ssl = SSL_new(ctx->tls_ctx);50    if (ssl == NULL) {51        return MCP_TLS_ERR;52    }5354    be->ssl = ssl;55    // SSL_set_fd() will free a pre-existing BIO and allocate a new one56    // so we set any file descriptor at connect time instead.5758    return MCP_TLS_OK;59}6061int mcp_tls_shutdown(struct mcp_backendconn_s *be) {62    if (!be->ssl) {63        return MCP_TLS_OK;64    }6566    // TODO: This may need to be called multiple times to "properly" shutdown67    // a session. However we only ever call this when a backend is dead or not68    // in used anymore. Unclear if checking for WANT_READ|WRITE is worth69    // doing.70    SSL_shutdown(be->ssl);7172    return MCP_TLS_OK;73}7475int mcp_tls_cleanup(struct mcp_backendconn_s *be) {76    if (!be->ssl) {77        return MCP_TLS_OK;78    }7980    SSL_free(be->ssl);81    be->ssl = NULL;82    return MCP_TLS_OK;83}8485// Contrary to the name of this function, the underlying tcp socket must86// already be connected.87int mcp_tls_connect(struct mcp_backendconn_s *be) {88    // TODO: check return code. can fail if BIO fails to alloc.89    SSL_set_fd(be->ssl, mcmc_fd(be->client));9091    // TODO:92    // if the backend is changing TLS version or some similar issue, we will93    // be unable to reconnect as the SSL object "Caches" some information94    // about the previous request (why doesn't clear work then???)95    // This will normally be fine, but we should detect severe errors here and96    // decide if we should free and re-alloc the SSL object.97    // Allocating the SSL object can be pretty slow, so we should at least98    // attempt to do not do this.99    // Related: https://github.com/openssl/openssl/issues/20286100    SSL_clear(be->ssl);101    ERR_clear_error();102    int n = SSL_connect(be->ssl);103    int ret = MCP_TLS_OK;104    // TODO: complete error handling.105    if (n == 1) {106        // Successfully established and handshake complete.107        return ret;108    }109110    int err = SSL_get_error(be->ssl, n);111    if (n == 0) {112        // Not successsful, but shut down normally.113        ERR_clear_error();114        ret = MCP_TLS_ERR;115    } else if (n < 0) {116        // Not successful. Check for temporary error.117        if (err == SSL_ERROR_WANT_READ ||118            err == SSL_ERROR_WANT_WRITE) {119            ret = MCP_TLS_OK;120        } else {121            ret = MCP_TLS_ERR;122        }123        ERR_clear_error();124    }125126    return ret;127}128129int mcp_tls_handshake(struct mcp_backendconn_s *be) {130    if (SSL_is_init_finished(be->ssl)) {131        return MCP_TLS_OK;132    }133134    // Non hot path, so clear errors before running.135    ERR_clear_error();136    int n = SSL_do_handshake(be->ssl);137    if (n == 1) {138        return MCP_TLS_OK;139    }140141    int err = SSL_get_error(be->ssl, n);142    // TODO: realistically we'll only ever get WANT_READ here, since OpenSSL143    // is handling the fd and it will have written a small number of bytes.144    // leaving this note just in case.145    if (err == SSL_ERROR_WANT_READ ||146        err == SSL_ERROR_WANT_WRITE) {147        // So far as I can tell there would be an error on the queue here.148        ERR_clear_error();149        return MCP_TLS_NEEDIO;150    } else {151        // TODO: can get the full error message and give to the caller to log152        // to proxyevents?153        ERR_clear_error();154        return MCP_TLS_ERR;155    }156}157158int mcp_tls_send_validate(struct mcp_backendconn_s *be) {159    const char *str = "version\r\n";160    const size_t len = strlen(str);161162    // Non hot path, clear errors.163    ERR_clear_error();164    int n = SSL_write(be->ssl, str, len);165166    // TODO: more detailed error checking.167    if (n < 0 || n != len) {168        ERR_clear_error();169        return MCP_TLS_ERR;170    }171172    return MCP_TLS_OK;173}174175int mcp_tls_read(struct mcp_backendconn_s *be) {176    int n = SSL_read(be->ssl, be->rbuf + be->rbufused, READ_BUFFER_SIZE - be->rbufused);177178    if (n < 0) {179        int err = SSL_get_error(be->ssl, n);180        if (err == SSL_ERROR_WANT_WRITE ||181            err == SSL_ERROR_WANT_READ) {182            ERR_clear_error();183            return MCP_TLS_NEEDIO;184        } else {185            // TODO: log detailed error.186            ERR_clear_error();187            return MCP_TLS_ERR;188        }189    } else {190        be->rbufused += n;191        return n;192    }193194    return 0;195}196197// TODO: option.198#define TLS_WBUF_SIZE 16 * 1024199200// We cache the temporary write buffer on the be's event thread.201// This is actually required when retrying ops (WANT_WRITE/etc) unless202// MOVING_BUFFERS flag is set in OpenSSL.203int mcp_tls_writev(struct mcp_backendconn_s *be, int iovcnt) {204    proxy_event_thread_t *et = be->event_thread;205    // TODO: move this to event thread init to remove branch and move error206    // handling to startup time.207    // Actually we won't know if TLS is in use until a backend shows up and208    // tries to write... so I'm not sure where to move this. TLS compiled in209    // but not used would waste memory.210    // Maybe can at least mark it unlikely()?211    if (et->tls_wbuf_size == 0) {212        et->tls_wbuf_size = TLS_WBUF_SIZE;213        et->tls_wbuf = malloc(et->tls_wbuf_size);214        if (et->tls_wbuf == NULL) {215            return MCP_TLS_ERR;216        }217    }218    size_t remain = et->tls_wbuf_size;219    char *b = et->tls_wbuf;220221    // OpenSSL has no writev or TCP_CORK equivalent, so we have to pre-mempcy222    // the iov's here.223    for (int i = 0; i < iovcnt; i++) {224        size_t len = be->write_iovs[i].iov_len;225        size_t to_copy = len < remain ? len : remain;226227        memcpy(b, (char *)be->write_iovs[i].iov_base, to_copy);228        remain -= to_copy;229        b += to_copy;230        if (remain == 0)231            break;232    }233234    int n = SSL_write(be->ssl, et->tls_wbuf, b - et->tls_wbuf);235    if (n < 0) {236        int err = SSL_get_error(be->ssl, n);237        if (err == SSL_ERROR_WANT_WRITE ||238            err == SSL_ERROR_WANT_READ) {239            ERR_clear_error();240            return MCP_TLS_NEEDIO;241        }242        ERR_clear_error();243        return MCP_TLS_ERR;244    }245246    return n;247}248249#else // PROXY_TLS250251int mcp_tls_writev(struct mcp_backendconn_s *be, int iovcnt) {252    (void)be;253    (void)iovcnt;254    return 0;255}256257#endif // PROXY_TLS