X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=src%2Fsrc%2Ftransports%2Fsmtp.c;h=c6099f960b2c1b008e58ac677fe7a2998c9c8e42;hb=5a8f5d724bbfd81cb2b89540e395359aaedc6c17;hp=b8073b0695745ba0fe23983040e348a6668e2872;hpb=c58803fd11185cbcf673e99cebf8397c9194357a;p=user%2Fhenk%2Fcode%2Fexim.git diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index b8073b069..c6099f960 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -3,6 +3,7 @@ *************************************************/ /* Copyright (c) University of Cambridge 1995 - 2018 */ +/* Copyright (c) The Exim Maintainers 2020 */ /* See the file NOTICE for conditions of use and distribution. */ #include "../exim.h" @@ -18,195 +19,127 @@ over TCP/IP. The options must be in alphabetic order (note that "_" comes before the lower case letters). Some live in the transport_instance block so as to be publicly visible; these are flagged with opt_public. */ +#define LOFF(field) OPT_OFF(smtp_transport_options_block, field) + optionlist smtp_transport_options[] = { { "*expand_multi_domain", opt_stringptr | opt_hidden | opt_public, - (void *)offsetof(transport_instance, expand_multi_domain) }, + OPT_OFF(transport_instance, expand_multi_domain) }, { "*expand_retry_include_ip_address", opt_stringptr | opt_hidden, - (void *)(offsetof(smtp_transport_options_block, expand_retry_include_ip_address)) }, + LOFF(expand_retry_include_ip_address) }, { "address_retry_include_sender", opt_bool, - (void *)offsetof(smtp_transport_options_block, address_retry_include_sender) }, - { "allow_localhost", opt_bool, - (void *)offsetof(smtp_transport_options_block, allow_localhost) }, + LOFF(address_retry_include_sender) }, + { "allow_localhost", opt_bool, LOFF(allow_localhost) }, #ifdef EXPERIMENTAL_ARC - { "arc_sign", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, arc_sign) }, -#endif - { "authenticated_sender", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, authenticated_sender) }, - { "authenticated_sender_force", opt_bool, - (void *)offsetof(smtp_transport_options_block, authenticated_sender_force) }, - { "command_timeout", opt_time, - (void *)offsetof(smtp_transport_options_block, command_timeout) }, - { "connect_timeout", opt_time, - (void *)offsetof(smtp_transport_options_block, connect_timeout) }, + { "arc_sign", opt_stringptr, LOFF(arc_sign) }, +#endif + { "authenticated_sender", opt_stringptr, LOFF(authenticated_sender) }, + { "authenticated_sender_force", opt_bool, LOFF(authenticated_sender_force) }, + { "command_timeout", opt_time, LOFF(command_timeout) }, + { "connect_timeout", opt_time, LOFF(connect_timeout) }, { "connection_max_messages", opt_int | opt_public, - (void *)offsetof(transport_instance, connection_max_messages) }, + OPT_OFF(transport_instance, connection_max_messages) }, # ifdef SUPPORT_DANE - { "dane_require_tls_ciphers", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dane_require_tls_ciphers) }, + { "dane_require_tls_ciphers", opt_stringptr, LOFF(dane_require_tls_ciphers) }, # endif - { "data_timeout", opt_time, - (void *)offsetof(smtp_transport_options_block, data_timeout) }, - { "delay_after_cutoff", opt_bool, - (void *)offsetof(smtp_transport_options_block, delay_after_cutoff) }, + { "data_timeout", opt_time, LOFF(data_timeout) }, + { "delay_after_cutoff", opt_bool, LOFF(delay_after_cutoff) }, #ifndef DISABLE_DKIM - { "dkim_canon", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_canon) }, - { "dkim_domain", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_domain) }, - { "dkim_hash", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_hash) }, - { "dkim_identity", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_identity) }, - { "dkim_private_key", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_private_key) }, - { "dkim_selector", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_selector) }, - { "dkim_sign_headers", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_sign_headers) }, - { "dkim_strict", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_strict) }, - { "dkim_timestamps", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dkim.dkim_timestamps) }, -#endif - { "dns_qualify_single", opt_bool, - (void *)offsetof(smtp_transport_options_block, dns_qualify_single) }, - { "dns_search_parents", opt_bool, - (void *)offsetof(smtp_transport_options_block, dns_search_parents) }, - { "dnssec_request_domains", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dnssec.request) }, - { "dnssec_require_domains", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dnssec.require) }, - { "dscp", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, dscp) }, - { "fallback_hosts", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, fallback_hosts) }, - { "final_timeout", opt_time, - (void *)offsetof(smtp_transport_options_block, final_timeout) }, - { "gethostbyname", opt_bool, - (void *)offsetof(smtp_transport_options_block, gethostbyname) }, - { "helo_data", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, helo_data) }, - { "hosts", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts) }, - { "hosts_avoid_esmtp", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_avoid_esmtp) }, - { "hosts_avoid_pipelining", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_avoid_pipelining) }, + { "dkim_canon", opt_stringptr, LOFF(dkim.dkim_canon) }, + { "dkim_domain", opt_stringptr, LOFF(dkim.dkim_domain) }, + { "dkim_hash", opt_stringptr, LOFF(dkim.dkim_hash) }, + { "dkim_identity", opt_stringptr, LOFF(dkim.dkim_identity) }, + { "dkim_private_key", opt_stringptr, LOFF(dkim.dkim_private_key) }, + { "dkim_selector", opt_stringptr, LOFF(dkim.dkim_selector) }, + { "dkim_sign_headers", opt_stringptr, LOFF(dkim.dkim_sign_headers) }, + { "dkim_strict", opt_stringptr, LOFF(dkim.dkim_strict) }, + { "dkim_timestamps", opt_stringptr, LOFF(dkim.dkim_timestamps) }, +#endif + { "dns_qualify_single", opt_bool, LOFF(dns_qualify_single) }, + { "dns_search_parents", opt_bool, LOFF(dns_search_parents) }, + { "dnssec_request_domains", opt_stringptr, LOFF(dnssec.request) }, + { "dnssec_require_domains", opt_stringptr, LOFF(dnssec.require) }, + { "dscp", opt_stringptr, LOFF(dscp) }, + { "fallback_hosts", opt_stringptr, LOFF(fallback_hosts) }, + { "final_timeout", opt_time, LOFF(final_timeout) }, + { "gethostbyname", opt_bool, LOFF(gethostbyname) }, + { "helo_data", opt_stringptr, LOFF(helo_data) }, + { "hosts", opt_stringptr, LOFF(hosts) }, + { "hosts_avoid_esmtp", opt_stringptr, LOFF(hosts_avoid_esmtp) }, + { "hosts_avoid_pipelining", opt_stringptr, LOFF(hosts_avoid_pipelining) }, #ifndef DISABLE_TLS - { "hosts_avoid_tls", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_avoid_tls) }, + { "hosts_avoid_tls", opt_stringptr, LOFF(hosts_avoid_tls) }, #endif - { "hosts_max_try", opt_int, - (void *)offsetof(smtp_transport_options_block, hosts_max_try) }, - { "hosts_max_try_hardlimit", opt_int, - (void *)offsetof(smtp_transport_options_block, hosts_max_try_hardlimit) }, + { "hosts_max_try", opt_int, LOFF(hosts_max_try) }, + { "hosts_max_try_hardlimit", opt_int, LOFF(hosts_max_try_hardlimit) }, #ifndef DISABLE_TLS - { "hosts_nopass_tls", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_nopass_tls) }, - { "hosts_noproxy_tls", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_noproxy_tls) }, + { "hosts_nopass_tls", opt_stringptr, LOFF(hosts_nopass_tls) }, + { "hosts_noproxy_tls", opt_stringptr, LOFF(hosts_noproxy_tls) }, #endif - { "hosts_override", opt_bool, - (void *)offsetof(smtp_transport_options_block, hosts_override) }, + { "hosts_override", opt_bool, LOFF(hosts_override) }, #ifndef DISABLE_PIPE_CONNECT - { "hosts_pipe_connect", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_pipe_connect) }, + { "hosts_pipe_connect", opt_stringptr, LOFF(hosts_pipe_connect) }, #endif - { "hosts_randomize", opt_bool, - (void *)offsetof(smtp_transport_options_block, hosts_randomize) }, + { "hosts_randomize", opt_bool, LOFF(hosts_randomize) }, #if !defined(DISABLE_TLS) && !defined(DISABLE_OCSP) - { "hosts_request_ocsp", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_request_ocsp) }, + { "hosts_request_ocsp", opt_stringptr, LOFF(hosts_request_ocsp) }, #endif - { "hosts_require_auth", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_require_auth) }, + { "hosts_require_auth", opt_stringptr, LOFF(hosts_require_auth) }, #ifndef DISABLE_TLS # ifdef SUPPORT_DANE - { "hosts_require_dane", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_require_dane) }, + { "hosts_require_dane", opt_stringptr, LOFF(hosts_require_dane) }, # endif # ifndef DISABLE_OCSP - { "hosts_require_ocsp", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_require_ocsp) }, + { "hosts_require_ocsp", opt_stringptr, LOFF(hosts_require_ocsp) }, # endif - { "hosts_require_tls", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_require_tls) }, + { "hosts_require_tls", opt_stringptr, LOFF(hosts_require_tls) }, #endif - { "hosts_try_auth", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_try_auth) }, - { "hosts_try_chunking", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_try_chunking) }, + { "hosts_try_auth", opt_stringptr, LOFF(hosts_try_auth) }, + { "hosts_try_chunking", opt_stringptr, LOFF(hosts_try_chunking) }, #ifdef SUPPORT_DANE - { "hosts_try_dane", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_try_dane) }, + { "hosts_try_dane", opt_stringptr, LOFF(hosts_try_dane) }, #endif - { "hosts_try_fastopen", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_try_fastopen) }, + { "hosts_try_fastopen", opt_stringptr, LOFF(hosts_try_fastopen) }, #ifndef DISABLE_PRDR - { "hosts_try_prdr", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_try_prdr) }, + { "hosts_try_prdr", opt_stringptr, LOFF(hosts_try_prdr) }, #endif #ifndef DISABLE_TLS - { "hosts_verify_avoid_tls", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, hosts_verify_avoid_tls) }, -#endif - { "interface", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, interface) }, - { "keepalive", opt_bool, - (void *)offsetof(smtp_transport_options_block, keepalive) }, - { "lmtp_ignore_quota", opt_bool, - (void *)offsetof(smtp_transport_options_block, lmtp_ignore_quota) }, + { "hosts_verify_avoid_tls", opt_stringptr, LOFF(hosts_verify_avoid_tls) }, +#endif + { "interface", opt_stringptr, LOFF(interface) }, + { "keepalive", opt_bool, LOFF(keepalive) }, + { "lmtp_ignore_quota", opt_bool, LOFF(lmtp_ignore_quota) }, { "max_rcpt", opt_int | opt_public, - (void *)offsetof(transport_instance, max_addresses) }, + OPT_OFF(transport_instance, max_addresses) }, + { "message_linelength_limit", opt_int, LOFF(message_linelength_limit) }, { "multi_domain", opt_expand_bool | opt_public, - (void *)offsetof(transport_instance, multi_domain) }, - { "port", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, port) }, - { "protocol", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, protocol) }, - { "retry_include_ip_address", opt_expand_bool, - (void *)offsetof(smtp_transport_options_block, retry_include_ip_address) }, - { "serialize_hosts", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, serialize_hosts) }, - { "size_addition", opt_int, - (void *)offsetof(smtp_transport_options_block, size_addition) }, + OPT_OFF(transport_instance, multi_domain) }, + { "port", opt_stringptr, LOFF(port) }, + { "protocol", opt_stringptr, LOFF(protocol) }, + { "retry_include_ip_address", opt_expand_bool, LOFF(retry_include_ip_address) }, + { "serialize_hosts", opt_stringptr, LOFF(serialize_hosts) }, + { "size_addition", opt_int, LOFF(size_addition) }, #ifdef SUPPORT_SOCKS - { "socks_proxy", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, socks_proxy) }, + { "socks_proxy", opt_stringptr, LOFF(socks_proxy) }, #endif #ifndef DISABLE_TLS - { "tls_certificate", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_certificate) }, - { "tls_crl", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_crl) }, - { "tls_dh_min_bits", opt_int, - (void *)offsetof(smtp_transport_options_block, tls_dh_min_bits) }, - { "tls_privatekey", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_privatekey) }, - { "tls_require_ciphers", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_require_ciphers) }, -# ifdef EXPERIMENTAL_TLS_RESUME - { "tls_resumption_hosts", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_resumption_hosts) }, + { "tls_certificate", opt_stringptr, LOFF(tls_certificate) }, + { "tls_crl", opt_stringptr, LOFF(tls_crl) }, + { "tls_dh_min_bits", opt_int, LOFF(tls_dh_min_bits) }, + { "tls_privatekey", opt_stringptr, LOFF(tls_privatekey) }, + { "tls_require_ciphers", opt_stringptr, LOFF(tls_require_ciphers) }, +# ifndef DISABLE_TLS_RESUME + { "tls_resumption_hosts", opt_stringptr, LOFF(tls_resumption_hosts) }, # endif - { "tls_sni", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_sni) }, - { "tls_tempfail_tryclear", opt_bool, - (void *)offsetof(smtp_transport_options_block, tls_tempfail_tryclear) }, - { "tls_try_verify_hosts", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_try_verify_hosts) }, - { "tls_verify_cert_hostnames", opt_stringptr, - (void *)offsetof(smtp_transport_options_block,tls_verify_cert_hostnames)}, - { "tls_verify_certificates", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) }, - { "tls_verify_hosts", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) }, + { "tls_sni", opt_stringptr, LOFF(tls_sni) }, + { "tls_tempfail_tryclear", opt_bool, LOFF(tls_tempfail_tryclear) }, + { "tls_try_verify_hosts", opt_stringptr, LOFF(tls_try_verify_hosts) }, + { "tls_verify_cert_hostnames", opt_stringptr, LOFF(tls_verify_cert_hostnames)}, + { "tls_verify_certificates", opt_stringptr, LOFF(tls_verify_certificates) }, + { "tls_verify_hosts", opt_stringptr, LOFF(tls_verify_hosts) }, #endif #ifdef SUPPORT_I18N - { "utf8_downconvert", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, utf8_downconvert) }, + { "utf8_downconvert", opt_stringptr, LOFF(utf8_downconvert) }, #endif }; @@ -230,23 +163,12 @@ void smtp_transport_closedown(transport_instance *tblock) {} /* Default private options block for the smtp transport. */ smtp_transport_options_block smtp_transport_option_defaults = { - .hosts = NULL, - .fallback_hosts = NULL, - .hostlist = NULL, - .fallback_hostlist = NULL, + /* All non-mentioned elements 0/NULL/FALSE */ .helo_data = US"$primary_hostname", - .interface = NULL, - .port = NULL, .protocol = US"smtp", - .dscp = NULL, - .serialize_hosts = NULL, - .hosts_try_auth = NULL, - .hosts_require_auth = NULL, .hosts_try_chunking = US"*", #ifdef SUPPORT_DANE .hosts_try_dane = US"*", - .hosts_require_dane = NULL, - .dane_require_tls_ciphers = NULL, #endif .hosts_try_fastopen = US"*", #ifndef DISABLE_PRDR @@ -254,19 +176,6 @@ smtp_transport_options_block smtp_transport_option_defaults = { #endif #ifndef DISABLE_OCSP .hosts_request_ocsp = US"*", /* hosts_request_ocsp (except under DANE; tls_client_start()) */ - .hosts_require_ocsp = NULL, -#endif - .hosts_require_tls = NULL, - .hosts_avoid_tls = NULL, - .hosts_verify_avoid_tls = NULL, - .hosts_avoid_pipelining = NULL, -#ifndef DISABLE_PIPE_CONNECT - .hosts_pipe_connect = NULL, -#endif - .hosts_avoid_esmtp = NULL, -#ifndef DISABLE_TLS - .hosts_nopass_tls = NULL, - .hosts_noproxy_tls = NULL, #endif .command_timeout = 5*60, .connect_timeout = 5*60, @@ -275,62 +184,28 @@ smtp_transport_options_block smtp_transport_option_defaults = { .size_addition = 1024, .hosts_max_try = 5, .hosts_max_try_hardlimit = 50, + .message_linelength_limit = 998, .address_retry_include_sender = TRUE, - .allow_localhost = FALSE, - .authenticated_sender_force = FALSE, - .gethostbyname = FALSE, .dns_qualify_single = TRUE, - .dns_search_parents = FALSE, .dnssec = { .request= US"*", .require=NULL }, .delay_after_cutoff = TRUE, - .hosts_override = FALSE, - .hosts_randomize = FALSE, .keepalive = TRUE, - .lmtp_ignore_quota = FALSE, - .expand_retry_include_ip_address = NULL, .retry_include_ip_address = TRUE, -#ifdef SUPPORT_SOCKS - .socks_proxy = NULL, -#endif #ifndef DISABLE_TLS - .tls_certificate = NULL, - .tls_crl = NULL, - .tls_privatekey = NULL, - .tls_require_ciphers = NULL, - .tls_sni = NULL, +# if defined(SUPPORT_SYSDEFAULT_CABUNDLE) || !defined(USE_GNUTLS) .tls_verify_certificates = US"system", +# endif .tls_dh_min_bits = EXIM_CLIENT_DH_DEFAULT_MIN_BITS, .tls_tempfail_tryclear = TRUE, -# ifdef EXPERIMENTAL_TLS_RESUME - .tls_resumption_hosts = NULL, -# endif - .tls_verify_hosts = NULL, .tls_try_verify_hosts = US"*", .tls_verify_cert_hostnames = US"*", #endif #ifdef SUPPORT_I18N - .utf8_downconvert = NULL, + .utf8_downconvert = US"-1", #endif #ifndef DISABLE_DKIM .dkim = - {.dkim_domain = NULL, - .dkim_identity = NULL, - .dkim_private_key = NULL, - .dkim_selector = NULL, - .dkim_canon = NULL, - .dkim_sign_headers = NULL, - .dkim_strict = NULL, - .dkim_hash = US"sha256", - .dkim_timestamps = NULL, - .dot_stuffed = FALSE, - .force_bodyhash = FALSE, -# ifdef EXPERIMENTAL_ARC - .arc_signspec = NULL, -# endif - }, -# ifdef EXPERIMENTAL_ARC - .arc_sign = NULL, -# endif + { .dkim_hash = US"sha256", }, #endif }; @@ -430,10 +305,6 @@ smtp_transport_setup(transport_instance *tblock, address_item *addrlist, { smtp_transport_options_block *ob = SOB tblock->options_block; -errmsg = errmsg; /* Keep picky compilers happy */ -uid = uid; -gid = gid; - /* Pass back options if required. This interface is getting very messy. */ if (tf) @@ -509,7 +380,7 @@ if (ob->command_timeout <= 0 || ob->data_timeout <= 0 || /* If hosts_override is set and there are local hosts, set the global flag that stops verify from showing router hosts. */ -if (ob->hosts_override && ob->hosts != NULL) tblock->overrides_hosts = TRUE; +if (ob->hosts_override && ob->hosts) tblock->overrides_hosts = TRUE; /* If there are any fallback hosts listed, build a chain of host items for them, but do not do any lookups at this time. */ @@ -537,6 +408,7 @@ Arguments: host if set, mark addrs as having used this host smtp_greeting from peer helo_response from peer + start points to timestamp of delivery start If errno_value has the special value ERRNO_CONNECTTIMEOUT, ETIMEDOUT is put in the errno field, and RTEF_CTOUT is ORed into the more_errno field, to indicate @@ -547,23 +419,29 @@ Returns: nothing static void set_errno(address_item *addrlist, int errno_value, uschar *msg, int rc, - BOOL pass_message, host_item * host + BOOL pass_message, host_item * host, #ifdef EXPERIMENTAL_DSN_INFO - , const uschar * smtp_greeting, const uschar * helo_response + const uschar * smtp_greeting, const uschar * helo_response, #endif + struct timeval * start ) { int orvalue = 0; +struct timeval deliver_time; + if (errno_value == ERRNO_CONNECTTIMEOUT) { errno_value = ETIMEDOUT; orvalue = RTEF_CTOUT; } +timesince(&deliver_time, start); + for (address_item * addr = addrlist; addr; addr = addr->next) if (addr->transport_return >= PENDING) { addr->basic_errno = errno_value; addr->more_errno |= orvalue; + addr->delivery_time = deliver_time; if (msg) { addr->message = msg; @@ -587,14 +465,14 @@ for (address_item * addr = addrlist; addr; addr = addr->next) } static void -set_errno_nohost(address_item *addrlist, int errno_value, uschar *msg, int rc, - BOOL pass_message) +set_errno_nohost(address_item * addrlist, int errno_value, uschar * msg, int rc, + BOOL pass_message, struct timeval * start) { -set_errno(addrlist, errno_value, msg, rc, pass_message, NULL +set_errno(addrlist, errno_value, msg, rc, pass_message, NULL, #ifdef EXPERIMENTAL_DSN_INFO - , NULL, NULL + NULL, NULL, #endif - ); + start); } @@ -766,12 +644,13 @@ deliver_msglog("%s H=%s [%s] %s\n", tod_stamp(tod_log), Arguments: addr the address item containing error information host the current host + evstr the event Returns: nothing */ static void -deferred_event_raise(address_item *addr, host_item *host) +deferred_event_raise(address_item * addr, host_item * host, uschar * evstr) { uschar * action = addr->transport->event_action; const uschar * save_domain; @@ -793,7 +672,7 @@ transport_name = addr->transport->name; deliver_domain = addr->domain; deliver_localpart = addr->local_part; -(void) event_raise(action, US"msg:host:defer", +(void) event_raise(action, evstr, addr->message ? addr->basic_errno > 0 ? string_sprintf("%s: %s", addr->message, strerror(addr->basic_errno)) @@ -833,7 +712,17 @@ return count; static BOOL smtp_reap_banner(smtp_context * sx) { -BOOL good_response = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), +BOOL good_response; +#if defined(__linux__) && defined(TCP_QUICKACK) + { /* Hack to get QUICKACK disabled; has to be right after 3whs, and has to on->off */ + int sock = sx->cctx.sock; + struct pollfd p = {.fd = sock, .events = POLLOUT}; + int rc = poll(&p, 1, 1000); + (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &on, sizeof(on)); + (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off)); + } +#endif +good_response = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', (SOB sx->conn_args.ob)->command_timeout); #ifdef EXPERIMENTAL_DSN_INFO sx->smtp_greeting = string_copy(sx->buffer); @@ -926,7 +815,7 @@ else uschar * ehlo_resp_key = ehlo_cache_key(sx); dbdata_ehlo_resp * er; - if (!(er = dbfn_read(dbm_file, ehlo_resp_key))) + if (!(er = dbfn_read_enforce_length(dbm_file, ehlo_resp_key, sizeof(dbdata_ehlo_resp)))) { DEBUG(D_transport) debug_printf("no ehlo-resp record\n"); } else if (time(NULL) - er->time_stamp > retry_data_expire) { @@ -940,8 +829,9 @@ else sx->ehlo_resp = er->data; dbfn_close(dbm_file); DEBUG(D_transport) debug_printf( - "EHLO response bits from cache: cleartext 0x%04x crypted 0x%04x\n", - er->data.cleartext_features, er->data.crypted_features); + "EHLO response bits from cache: cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x\n", + er->data.cleartext_features, er->data.cleartext_auths, + er->data.crypted_features, er->data.crypted_auths); return TRUE; } dbfn_close(dbm_file); @@ -971,11 +861,9 @@ names = string_copyn(expand_nstring[1], expand_nlength[1]); for (au = auths, authnum = 0; au; au = au->next, authnum++) if (au->client) { const uschar * list = names; - int sep = ' '; - uschar name[32]; - - while (string_nextinlist(&list, &sep, name, sizeof(name))) - if (strcmpic(au->public_name, name) == 0) + uschar * s; + for (int sep = ' '; s = string_nextinlist(&list, &sep, NULL, 0); ) + if (strcmpic(au->public_name, s) == 0) { authbits |= BIT(authnum); break; } } @@ -1081,7 +969,7 @@ fail: (void) smtp_discard_responses(sx, sx->conn_args.ob, *countp); return rc; } -#endif +#endif /*!DISABLE_PIPE_CONNECT*/ /************************************************* @@ -1148,6 +1036,7 @@ if (sx->pending_MAIL) { DEBUG(D_transport) debug_printf("%s expect mail\n", __FUNCTION__); count--; + sx->pending_MAIL = sx->RCPT_452 = FALSE; if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', ob->command_timeout)) { @@ -1193,7 +1082,7 @@ while (count-- > 0) /* The address was accepted */ addr->host_used = sx->conn_args.host; - DEBUG(D_transport) debug_printf("%s expect rcpt\n", __FUNCTION__); + DEBUG(D_transport) debug_printf("%s expect rcpt for %s\n", __FUNCTION__, addr->address); if (smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', ob->command_timeout)) { @@ -1224,7 +1113,7 @@ while (count-- > 0) { uschar *message = string_sprintf("SMTP timeout after RCPT TO:<%s>", transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes)); - set_errno_nohost(sx->first_addr, ETIMEDOUT, message, DEFER, FALSE); + set_errno_nohost(sx->first_addr, ETIMEDOUT, message, DEFER, FALSE, &sx->delivery_start); retry_add_item(addr, addr->address_retry_key, 0); update_waiting = FALSE; return -1; @@ -1282,41 +1171,60 @@ while (count-- > 0) event_defer_errno = addr->more_errno; msg_event_raise(US"msg:rcpt:host:defer", addr); #endif + /* If a 452 and we've had at least one 2xx or 5xx, set next_addr to the + start point for another MAIL command. */ - /* Log temporary errors if there are more hosts to be tried. - If not, log this last one in the == line. */ - - if (sx->conn_args.host->next) - if (LOGGING(outgoing_port)) - log_write(0, LOG_MAIN, "H=%s [%s]:%d %s", sx->conn_args.host->name, - sx->conn_args.host->address, - sx->port == PORT_NONE ? 25 : sx->port, addr->message); - else - log_write(0, LOG_MAIN, "H=%s [%s]: %s", sx->conn_args.host->name, - sx->conn_args.host->address, addr->message); + if (addr->more_errno >> 8 == 52 && yield & 3) + { + if (!sx->RCPT_452) /* initialised at MAIL-ack above */ + { + DEBUG(D_transport) + debug_printf("%s: seen first 452 too-many-rcpts\n", __FUNCTION__); + sx->RCPT_452 = TRUE; + sx->next_addr = addr; + } + addr->transport_return = PENDING_DEFER; + addr->basic_errno = 0; + } + else + { + /* Log temporary errors if there are more hosts to be tried. + If not, log this last one in the == line. */ + + if (sx->conn_args.host->next) + if (LOGGING(outgoing_port)) + log_write(0, LOG_MAIN, "H=%s [%s]:%d %s", sx->conn_args.host->name, + sx->conn_args.host->address, + sx->port == PORT_NONE ? 25 : sx->port, addr->message); + else + log_write(0, LOG_MAIN, "H=%s [%s]: %s", sx->conn_args.host->name, + sx->conn_args.host->address, addr->message); #ifndef DISABLE_EVENT - else - msg_event_raise(US"msg:rcpt:defer", addr); + else + msg_event_raise(US"msg:rcpt:defer", addr); #endif - /* Do not put this message on the list of those waiting for specific - hosts, as otherwise it is likely to be tried too often. */ + /* Do not put this message on the list of those waiting for specific + hosts, as otherwise it is likely to be tried too often. */ - update_waiting = FALSE; + update_waiting = FALSE; - /* Add a retry item for the address so that it doesn't get tried again - too soon. If address_retry_include_sender is true, add the sender address - to the retry key. */ + /* Add a retry item for the address so that it doesn't get tried again + too soon. If address_retry_include_sender is true, add the sender address + to the retry key. */ - retry_add_item(addr, - ob->address_retry_include_sender - ? string_sprintf("%s:<%s>", addr->address_retry_key, sender_address) - : addr->address_retry_key, - 0); + retry_add_item(addr, + ob->address_retry_include_sender + ? string_sprintf("%s:<%s>", addr->address_retry_key, sender_address) + : addr->address_retry_key, + 0); + } } } } + if (count && !(addr = addr->next)) + return -2; } /* Loop for next RCPT response */ /* Update where to start at for the next block of responses, unless we @@ -1425,7 +1333,7 @@ switch(rc) case ERROR: set_errno_nohost(sx->addrlist, ERRNO_AUTHPROB, string_copy(sx->buffer), - DEFER, FALSE); + DEFER, FALSE, &sx->delivery_start); return ERROR; } return OK; @@ -1592,9 +1500,12 @@ if ( sx->esmtp if (require_auth == OK && !f.smtp_authenticated) { +#ifndef DISABLE_PIPE_CONNECT + invalidate_ehlo_cache_entry(sx); +#endif set_errno_nohost(sx->addrlist, ERRNO_AUTHFAIL, string_sprintf("authentication required but %s", fail_reason), DEFER, - FALSE); + FALSE, &sx->delivery_start); return DEFER; } @@ -1605,9 +1516,9 @@ return OK; /* Construct AUTH appendix string for MAIL TO */ /* Arguments - buffer to build string + sx context for smtp connection + p point in sx->buffer to build string addrlist chain of potential addresses to deliver - ob transport options Globals f.smtp_authenticated client_authenticated_sender @@ -1615,29 +1526,31 @@ Return True on error, otherwise buffer has (possibly empty) terminated string */ static BOOL -smtp_mail_auth_str(uschar *buffer, unsigned bufsize, address_item *addrlist, - smtp_transport_options_block *ob) +smtp_mail_auth_str(smtp_context * sx, uschar * p, address_item * addrlist) { +smtp_transport_options_block * ob = sx->conn_args.ob; uschar * local_authenticated_sender = authenticated_sender; #ifdef notdef - debug_printf("smtp_mail_auth_str: as<%s> os<%s> SA<%s>\n", authenticated_sender, ob->authenticated_sender, f.smtp_authenticated?"Y":"N"); + debug_printf("smtp_mail_auth_str: as<%s> os<%s> SA<%s>\n", + authenticated_sender, ob->authenticated_sender, f.smtp_authenticated?"Y":"N"); #endif if (ob->authenticated_sender) { - uschar *new = expand_string(ob->authenticated_sender); + uschar * new = expand_string(ob->authenticated_sender); if (!new) { if (!f.expand_string_forcedfail) { uschar *message = string_sprintf("failed to expand " "authenticated_sender: %s", expand_string_message); - set_errno_nohost(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE); + set_errno_nohost(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start); return TRUE; } } - else if (*new) local_authenticated_sender = new; + else if (*new) + local_authenticated_sender = new; } /* Add the authenticated sender address if present */ @@ -1645,13 +1558,13 @@ if (ob->authenticated_sender) if ( (f.smtp_authenticated || ob->authenticated_sender_force) && local_authenticated_sender) { - string_format_nt(buffer, bufsize, " AUTH=%s", + string_format_nt(p, sizeof(sx->buffer) - (p-sx->buffer), " AUTH=%s", auth_xtextencode(local_authenticated_sender, Ustrlen(local_authenticated_sender))); client_authenticated_sender = string_copy(local_authenticated_sender); } else - *buffer= 0; + *p = 0; return FALSE; } @@ -1660,8 +1573,8 @@ return FALSE; typedef struct smtp_compare_s { - uschar *current_sender_address; - struct transport_instance *tblock; + uschar * current_sender_address; + struct transport_instance * tblock; } smtp_compare_t; @@ -1724,8 +1637,9 @@ uschar * message_local_identity, current_local_identity = smtp_local_identity(s_compare->current_sender_address, s_compare->tblock); -if (!(new_sender_address = deliver_get_sender_address(message_id))) - return 0; +if (!(new_sender_address = spool_sender_from_msgid(message_id))) + return FALSE; + message_local_identity = smtp_local_identity(new_sender_address, s_compare->tblock); @@ -1949,39 +1863,39 @@ sx->conn_args.ob = ob; sx->lmtp = strcmpic(ob->protocol, US"lmtp") == 0; sx->smtps = strcmpic(ob->protocol, US"smtps") == 0; -sx->ok = FALSE; +/* sx->ok = FALSE; */ sx->send_rset = TRUE; sx->send_quit = TRUE; sx->setting_up = TRUE; sx->esmtp = TRUE; -sx->esmtp_sent = FALSE; +/* sx->esmtp_sent = FALSE; */ #ifdef SUPPORT_I18N -sx->utf8_needed = FALSE; +/* sx->utf8_needed = FALSE; */ #endif sx->dsn_all_lasthop = TRUE; #ifdef SUPPORT_DANE -sx->conn_args.dane = FALSE; +/* sx->conn_args.dane = FALSE; */ sx->dane_required = verify_check_given_host(CUSS &ob->hosts_require_dane, sx->conn_args.host) == OK; #endif #ifndef DISABLE_PIPE_CONNECT -sx->early_pipe_active = sx->early_pipe_ok = FALSE; -sx->ehlo_resp.cleartext_features = sx->ehlo_resp.crypted_features = 0; -sx->pending_BANNER = sx->pending_EHLO = FALSE; +/* sx->early_pipe_active = sx->early_pipe_ok = FALSE; */ +/* sx->ehlo_resp.cleartext_features = sx->ehlo_resp.crypted_features = 0; */ +/* sx->pending_BANNER = sx->pending_EHLO = sx->pending_MAIL = FALSE; */ #endif if ((sx->max_rcpt = sx->conn_args.tblock->max_addresses) == 0) sx->max_rcpt = 999999; -sx->peer_offered = 0; -sx->avoid_option = 0; +/* sx->peer_offered = 0; */ +/* sx->avoid_option = 0; */ sx->igquotstr = US""; if (!sx->helo_data) sx->helo_data = ob->helo_data; #ifdef EXPERIMENTAL_DSN_INFO -sx->smtp_greeting = NULL; -sx->helo_response = NULL; +/* sx->smtp_greeting = NULL; */ +/* sx->helo_response = NULL; */ #endif smtp_command = US"initial connection"; -sx->buffer[0] = '\0'; +/* sx->buffer[0] = '\0'; */ /* Set up the buffer for reading SMTP response packets. */ @@ -1995,9 +1909,9 @@ sx->inblock.ptrend = sx->inbuffer; sx->outblock.buffer = sx->outbuffer; sx->outblock.buffersize = sizeof(sx->outbuffer); sx->outblock.ptr = sx->outbuffer; -sx->outblock.cmd_count = 0; -sx->outblock.authenticating = FALSE; -sx->outblock.conn_args = NULL; +/* sx->outblock.cmd_count = 0; */ +/* sx->outblock.authenticating = FALSE; */ +/* sx->outblock.conn_args = NULL; */ /* Reset the parameters of a TLS session. */ @@ -2010,7 +1924,7 @@ tls_out.peerdn = NULL; tls_out.sni = NULL; #endif tls_out.ocsp = OCSP_NOT_REQ; -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME tls_out.resumption = 0; #endif tls_out.ver = NULL; @@ -2026,10 +1940,83 @@ tls_modify_variables(&tls_out); if (sx->smtps) { set_errno_nohost(sx->addrlist, ERRNO_TLSFAILURE, US"TLS support not available", - DEFER, FALSE); + DEFER, FALSE, &sx->delivery_start); return ERROR; } -#endif +#else + +/* If we have a proxied TLS connection, check usability for this message */ + +if (continue_hostname && continue_proxy_cipher) + { + int rc; + const uschar * sni = US""; + +# ifdef SUPPORT_DANE + /* Check if the message will be DANE-verified; if so force its SNI */ + + tls_out.dane_verified = FALSE; + smtp_port_for_connect(sx->conn_args.host, sx->port); + if ( sx->conn_args.host->dnssec == DS_YES + && ( sx->dane_required + || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK + ) ) + switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) + { + case OK: sx->conn_args.dane = TRUE; + ob->tls_tempfail_tryclear = FALSE; /* force TLS */ + ob->tls_sni = sx->first_addr->domain; /* force SNI */ + break; + case FAIL_FORCED: break; + default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, + string_sprintf("DANE error: tlsa lookup %s", + rc_to_string(rc)), + rc, FALSE, &sx->delivery_start); +# ifndef DISABLE_EVENT + (void) event_raise(sx->conn_args.tblock->event_action, + US"dane:fail", sx->dane_required + ? US"dane-required" : US"dnssec-invalid"); +# endif + return rc; + } +# endif + + /* If the SNI or the DANE status required for the new message differs from the + existing conn drop the connection to force a new one. */ + + if (ob->tls_sni && !(sni = expand_cstring(ob->tls_sni))) + log_write(0, LOG_MAIN|LOG_PANIC, + "<%s>: failed to expand transport's tls_sni value: %s", + sx->addrlist->address, expand_string_message); + +# ifdef SUPPORT_DANE + if ( (continue_proxy_sni ? (Ustrcmp(continue_proxy_sni, sni) == 0) : !*sni) + && continue_proxy_dane == sx->conn_args.dane) + { + tls_out.sni = US sni; + if ((tls_out.dane_verified = continue_proxy_dane)) + sx->conn_args.host->dnssec = DS_YES; + } +# else + if ((continue_proxy_sni ? (Ustrcmp(continue_proxy_sni, sni) == 0) : !*sni)) + tls_out.sni = US sni; +# endif + else + { + DEBUG(D_transport) + debug_printf("Closing proxied-TLS connection due to SNI mismatch\n"); + + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> QUIT\n"); + write(0, "QUIT\r\n", 6); + close(0); + continue_hostname = continue_proxy_cipher = NULL; + f.continue_more = FALSE; + continue_sequence = 1; /* Unfortunately, this process cannot affect success log + which is done by delivery proc. Would have to pass this + back through reporting pipe. */ + } + } +#endif /*!DISABLE_TLS*/ /* Make a connection to the host if this isn't a continued delivery, and handle the initial interaction and HELO/EHLO/LHLO. Connect timeout errors are handled @@ -2059,13 +2046,14 @@ if (!continue_hostname) switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) { case OK: sx->conn_args.dane = TRUE; - ob->tls_tempfail_tryclear = FALSE; + ob->tls_tempfail_tryclear = FALSE; /* force TLS */ + ob->tls_sni = sx->first_addr->domain; /* force SNI */ break; case FAIL_FORCED: break; default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, string_sprintf("DANE error: tlsa lookup %s", rc_to_string(rc)), - rc, FALSE); + rc, FALSE, &sx->delivery_start); # ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, US"dane:fail", sx->dane_required @@ -2078,7 +2066,7 @@ if (!continue_hostname) { set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, string_sprintf("DANE error: %s lookup not DNSSEC", sx->conn_args.host->name), - FAIL, FALSE); + FAIL, FALSE, &sx->delivery_start); # ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, US"dane:fail", US"dane-required"); @@ -2119,20 +2107,30 @@ if (!continue_hostname) else DEBUG(D_transport) debug_printf("helo needs $sending_ip_address\n"); +PIPE_CONNECT_RETRY: if (sx->early_pipe_active) sx->outblock.conn_args = &sx->conn_args; else #endif { - if ((sx->cctx.sock = smtp_connect(&sx->conn_args, NULL)) < 0) + blob lazy_conn = {.data = NULL}; + /* For TLS-connect, a TFO lazy-connect is useful since the Client Hello + can go on the TCP SYN. */ + + if ((sx->cctx.sock = smtp_connect(&sx->conn_args, + sx->smtps ? &lazy_conn : NULL)) < 0) { set_errno_nohost(sx->addrlist, errno == ETIMEDOUT ? ERRNO_CONNECTTIMEOUT : errno, sx->verify ? US strerror(errno) : NULL, - DEFER, FALSE); + DEFER, FALSE, &sx->delivery_start); sx->send_quit = FALSE; return DEFER; } +#ifdef TCP_QUICKACK + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, TCP_QUICKACK, US &off, + sizeof(off)); +#endif } /* Expand the greeting message while waiting for the initial response. (Makes sense if helo_data contains ${lookup dnsdb ...} stuff). The expansion is @@ -2178,10 +2176,6 @@ will be? Somehow I doubt it. */ else #endif { -#ifdef TCP_QUICKACK - (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, TCP_QUICKACK, US &off, - sizeof(off)); -#endif if (!smtp_reap_banner(sx)) goto RESPONSE_FAILED; } @@ -2196,7 +2190,7 @@ will be? Somehow I doubt it. */ { set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, string_sprintf("deferred by smtp:connect event expansion: %s", s), - DEFER, FALSE); + DEFER, FALSE, &sx->delivery_start); yield = DEFER; goto SEND_QUIT; } @@ -2210,7 +2204,7 @@ will be? Somehow I doubt it. */ { message = string_sprintf("failed to expand helo_data: %s", expand_string_message); - set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE); + set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start); yield = DEFER; goto SEND_QUIT; } @@ -2417,6 +2411,7 @@ else } sx->inblock.cctx = sx->outblock.cctx = &sx->cctx; smtp_command = big_buffer; + sx->peer_offered = smtp_peer_options; sx->helo_data = NULL; /* ensure we re-expand ob->helo_data */ /* For a continued connection with TLS being proxied for us, or a @@ -2427,7 +2422,6 @@ else && cutthrough.is_tls) ) { - sx->peer_offered = smtp_peer_options; sx->pipelining_used = pipelining_active = !!(smtp_peer_options & OPTION_PIPE); HDEBUG(D_transport) debug_printf("continued connection, %s TLS\n", continue_proxy_cipher ? "proxied" : "verify conn with"); @@ -2465,7 +2459,10 @@ if ( smtp_peer_options & OPTION_TLS { HDEBUG(D_transport) debug_printf("failed reaping pipelined cmd responses\n"); - goto RESPONSE_FAILED; + close(sx->cctx.sock); + sx->cctx.sock = -1; + sx->early_pipe_active = FALSE; + goto PIPE_CONNECT_RETRY; } #endif @@ -2499,9 +2496,7 @@ if ( smtp_peer_options & OPTION_TLS /* TLS negotiation failed; give an error. From outside, this function may be called again to try in clear on a new connection, if the options permit it for this host. */ -#ifdef USE_GNUTLS - GNUTLS_CONN_FAILED: -#endif + TLS_CONN_FAILED: DEBUG(D_tls) debug_printf("TLS session fail: %s\n", tls_errstr); # ifdef SUPPORT_DANE @@ -2522,8 +2517,25 @@ if ( smtp_peer_options & OPTION_TLS sx->send_quit = FALSE; goto TLS_FAILED; } + sx->send_tlsclose = TRUE; - /* TLS session is set up */ + /* TLS session is set up. Check the inblock fill level. If there is + content then as we have not yet done a tls read it must have arrived before + the TLS handshake, in-clear. That violates the sync requirement of the + STARTTLS RFC, so fail. */ + + if (sx->inblock.ptr != sx->inblock.ptrend) + { + DEBUG(D_tls) + { + int i = sx->inblock.ptrend - sx->inblock.ptr; + debug_printf("unused data in input buffer after ack for STARTTLS:\n" + "'%.*s'%s\n", + i > 100 ? 100 : i, sx->inblock.ptr, i > 100 ? "..." : ""); + } + tls_errstr = US"synch error before connect"; + goto TLS_CONN_FAILED; + } smtp_peer_options_wrap = smtp_peer_options; for (address_item * addr = sx->addrlist; addr; addr = addr->next) @@ -2557,7 +2569,7 @@ if (tls_out.active.sock >= 0) { uschar *message = string_sprintf("failed to expand helo_data: %s", expand_string_message); - set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE); + set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start); yield = DEFER; goto SEND_QUIT; } @@ -2628,7 +2640,7 @@ if (tls_out.active.sock >= 0) Can it do that, with all the flexibility we need? */ tls_errstr = US"error on first read"; - goto GNUTLS_CONN_FAILED; + goto TLS_CONN_FAILED; } #else goto RESPONSE_FAILED; @@ -2667,7 +2679,7 @@ so its response needs to be analyzed. If TLS is not active and this is a continued session down a previously-used socket, we haven't just done EHLO, so we skip this. */ -if (continue_hostname == NULL +if ( !continue_hostname #ifndef DISABLE_TLS || tls_out.active.sock >= 0 #endif @@ -2799,7 +2811,7 @@ if (sx->addrlist->prop.utf8_msg) { message = string_sprintf("failed to expand utf8_downconvert: %s", expand_string_message); - set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE); + set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start); yield = DEFER; goto SEND_QUIT; } @@ -2859,7 +2871,7 @@ return OK; set_errno_nohost(sx->addrlist, errno == ETIMEDOUT ? ERRNO_CONNECTTIMEOUT : errno, sx->verify ? US strerror(errno) : NULL, - DEFER, FALSE); + DEFER, FALSE, &sx->delivery_start); sx->send_quit = FALSE; return DEFER; } @@ -2923,11 +2935,11 @@ FAILED: #endif ? FAIL : DEFER, pass_message, - errno == ECONNREFUSED ? NULL : sx->conn_args.host + errno == ECONNREFUSED ? NULL : sx->conn_args.host, #ifdef EXPERIMENTAL_DSN_INFO - , sx->smtp_greeting, sx->helo_response + sx->smtp_greeting, sx->helo_response, #endif - ); + &sx->delivery_start); } @@ -2939,7 +2951,11 @@ if (sx->send_quit) #ifndef DISABLE_TLS if (sx->cctx.tls_ctx) { - tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); + if (sx->send_tlsclose) + { + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); + sx->send_tlsclose = FALSE; + } sx->cctx.tls_ctx = NULL; } #endif @@ -3069,10 +3085,7 @@ Other expansion failures are serious. An empty result is ignored, but there is otherwise no check - this feature is expected to be used with LMTP and other cases where non-standard addresses (e.g. without domains) might be required. */ -if (smtp_mail_auth_str(p, sizeof(sx->buffer) - (p-sx->buffer), addrlist, sx->conn_args.ob)) - return ERROR; - -return OK; +return smtp_mail_auth_str(sx, p, addrlist) ? ERROR : OK; } @@ -3126,7 +3139,7 @@ int smtp_write_mail_and_rcpt_cmds(smtp_context * sx, int * yield) { address_item * addr; -int address_count; +int address_count, pipe_limit; int rc; if (build_mailcmd_options(sx, sx->first_addr) != OK) @@ -3158,7 +3171,7 @@ sx->pending_MAIL = TRUE; /* The block starts with MAIL */ { if (s = string_address_utf8_to_alabel(s, &errstr), errstr) { - set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE); + set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, &sx->delivery_start); *yield = ERROR; return -4; } @@ -3209,11 +3222,11 @@ that max_rcpt will be large, so all addresses will be done at once. For verify we flush the pipeline after any (the only) rcpt address. */ -for (addr = sx->first_addr, address_count = 0; +for (addr = sx->first_addr, address_count = 0, pipe_limit = 100; addr && address_count < sx->max_rcpt; addr = addr->next) if (addr->transport_return == PENDING_DEFER) { - int count; + int cmds_sent; BOOL no_flush; uschar * rcpt_addr; @@ -3221,7 +3234,10 @@ for (addr = sx->first_addr, address_count = 0; ? dsn_support_yes : dsn_support_no; address_count++; - no_flush = pipelining_active && !sx->verify + if (pipe_limit-- <= 0) + { no_flush = FALSE; pipe_limit = 100; } + else + no_flush = pipelining_active && !sx->verify && (!mua_wrapper || addr->next && address_count < sx->max_rcpt); build_rcptcmd_options(sx, addr); @@ -3244,13 +3260,13 @@ for (addr = sx->first_addr, address_count = 0; } #endif - count = smtp_write_command(sx, no_flush ? SCMD_BUFFER : SCMD_FLUSH, + cmds_sent = smtp_write_command(sx, no_flush ? SCMD_BUFFER : SCMD_FLUSH, "RCPT TO:<%s>%s%s\r\n", rcpt_addr, sx->igquotstr, sx->buffer); - if (count < 0) return -5; - if (count > 0) + if (cmds_sent < 0) return -5; + if (cmds_sent > 0) { - switch(sync_responses(sx, count, 0)) + switch(sync_responses(sx, cmds_sent, 0)) { case 3: sx->ok = TRUE; /* 2xx & 5xx => OK & progress made */ case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */ @@ -3260,6 +3276,17 @@ for (addr = sx->first_addr, address_count = 0; if (!sx->lmtp) /* can't tell about progress yet */ sx->completed_addr = TRUE; case 0: /* No 2xx or 5xx, but no probs */ + /* If any RCPT got a 452 response then next_addr has been updated + for restarting with a new MAIL on the same connection. Send no more + RCPTs for this MAIL. */ + + if (sx->RCPT_452) + { + DEBUG(D_transport) debug_printf("seen 452 too-many-rcpts\n"); + sx->RCPT_452 = FALSE; + /* sx->next_addr has been reset for fast_retry */ + return 0; + } break; case -1: return -3; /* Timeout on RCPT */ @@ -3271,7 +3298,6 @@ for (addr = sx->first_addr, address_count = 0; case -5: return -1; /* TLS first-read error */ #endif } - sx->pending_MAIL = FALSE; /* Dealt with MAIL */ } } /* Loop for next address */ @@ -3308,15 +3334,12 @@ smtp_proxy_tls(void * ct_ctx, uschar * buf, size_t bsize, int * pfd, fd_set rfds, efds; int max_fd = MAX(pfd[0], tls_out.active.sock) + 1; int rc, i; +BOOL send_tls_shutdown = TRUE; close(pfd[1]); -if ((rc = fork())) - { - DEBUG(D_transport) debug_printf("proxy-proc final-pid %d\n", rc); +if ((rc = exim_fork(US"tls-proxy"))) _exit(rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS); - } -testharness_pause_ms(100); /* let parent debug out */ set_process_info("proxying TLS connection for continued transport"); FD_ZERO(&rfds); FD_SET(tls_out.active.sock, &rfds); @@ -3345,19 +3368,23 @@ for (int fd_bits = 3; fd_bits; ) goto done; } + /* For errors where not readable, bomb out */ + if (FD_ISSET(tls_out.active.sock, &efds) || FD_ISSET(pfd[0], &efds)) { DEBUG(D_transport) debug_printf("select: exceptional cond on %s fd\n", FD_ISSET(pfd[0], &efds) ? "proxy" : "tls"); - goto done; + if (!(FD_ISSET(tls_out.active.sock, &rfds) || FD_ISSET(pfd[0], &rfds))) + goto done; + DEBUG(D_transport) debug_printf("- but also readable; no exit yet\n"); } } while (rc < 0 || !(FD_ISSET(tls_out.active.sock, &rfds) || FD_ISSET(pfd[0], &rfds))); /* handle inbound data */ if (FD_ISSET(tls_out.active.sock, &rfds)) - if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0) - { + if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0) /* Expect -1 for EOF; */ + { /* that reaps the TLS Close Notify record */ fd_bits &= ~1; FD_CLR(tls_out.active.sock, &rfds); shutdown(pfd[0], SHUT_WR); @@ -3368,16 +3395,21 @@ for (int fd_bits = 3; fd_bits; ) for (int nbytes = 0; rc - nbytes > 0; nbytes += i) if ((i = write(pfd[0], buf + nbytes, rc - nbytes)) < 0) goto done; } - else if (fd_bits & 1) - FD_SET(tls_out.active.sock, &rfds); - /* handle outbound data */ + /* Handle outbound data. We cannot combine payload and the TLS-close + due to the limitations of the (pipe) channel feeding us. */ if (FD_ISSET(pfd[0], &rfds)) if ((rc = read(pfd[0], buf, bsize)) <= 0) { - fd_bits = 0; - tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT); - ct_ctx = NULL; + fd_bits &= ~2; + FD_CLR(pfd[0], &rfds); + +# ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */ + (void) setsockopt(tls_out.active.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_shutdown_wr(ct_ctx); + send_tls_shutdown = FALSE; + shutdown(tls_out.active.sock, SHUT_WR); } else { @@ -3385,13 +3417,16 @@ for (int fd_bits = 3; fd_bits; ) if ((i = tls_write(ct_ctx, buf + nbytes, rc - nbytes, FALSE)) < 0) goto done; } - else if (fd_bits & 2) - FD_SET(pfd[0], &rfds); + + if (fd_bits & 1) FD_SET(tls_out.active.sock, &rfds); + if (fd_bits & 2) FD_SET(pfd[0], &rfds); } done: + if (send_tls_shutdown) tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT); + ct_ctx = NULL; testharness_pause_ms(100); /* let logging complete */ - exim_exit(0, US"TLS proxy"); + exim_exit(EXIT_SUCCESS); } #endif @@ -3402,8 +3437,9 @@ done: /* If continue_hostname is not null, we get here only when continuing to deliver down an existing channel. The channel was passed as the standard -input. TLS is never active on a passed channel; the previous process always -closes it down before passing the connection on. +input. TLS is never active on a passed channel; the previous process either +closes it down before passing the connection on, or inserts a TLS-proxy +process and passes on a cleartext conection. Otherwise, we have to make a connection to the remote host, and do the initial protocol exchange. @@ -3439,6 +3475,9 @@ Returns: OK - the connection was made and the delivery attempted; and there was a problem setting it up; OR helo_data or add_headers or authenticated_sender is specified for this transport, and the string failed to expand + + For all non-OK returns the first addr of the list carries the + time taken for the attempt. */ static int @@ -3450,18 +3489,19 @@ smtp_transport_options_block * ob = SOB tblock->options_block; int yield = OK; int save_errno; int rc; -struct timeval start_delivery_time; BOOL pass_message = FALSE; uschar *message = NULL; uschar new_message_id[MESSAGE_ID_LENGTH + 1]; - smtp_context * sx = store_get(sizeof(*sx), TRUE); /* tainted, for the data buffers */ +#ifdef SUPPORT_DANE +BOOL dane_held; +#endif +BOOL tcw_done = FALSE, tcw = FALSE; -gettimeofday(&start_delivery_time, NULL); -suppress_tls = suppress_tls; /* stop compiler warning when no TLS support */ *message_defer = FALSE; +memset(sx, 0, sizeof(*sx)); sx->addrlist = addrlist; sx->conn_args.host = host; sx->conn_args.host_af = host_af, @@ -3469,17 +3509,47 @@ sx->port = defport; sx->conn_args.interface = interface; sx->helo_data = NULL; sx->conn_args.tblock = tblock; -sx->verify = FALSE; +/* sx->verify = FALSE; */ +gettimeofday(&sx->delivery_start, NULL); sx->sync_addr = sx->first_addr = addrlist; -/* Get the channel set up ready for a message (MAIL FROM being the next -SMTP command to send */ +#ifdef SUPPORT_DANE +DANE_DOMAINS: +dane_held = FALSE; +#endif + +/* Get the channel set up ready for a message, MAIL FROM being the next +SMTP command to send. */ if ((rc = smtp_setup_conn(sx, suppress_tls)) != OK) - return rc; + { + timesince(&addrlist->delivery_time, &sx->delivery_start); + yield = rc; + goto TIDYUP; + } + +#ifdef SUPPORT_DANE +/* If the connection used DANE, ignore for now any addresses with incompatible +domains. The SNI has to be the domain. Arrange a whole new TCP conn later, +just in case only TLS isn't enough. */ + +if (sx->conn_args.dane) + { + const uschar * dane_domain = sx->first_addr->domain; + + for (address_item * a = sx->first_addr->next; a; a = a->next) + if ( a->transport_return == PENDING_DEFER + && Ustrcmp(dane_domain, a->domain) != 0) + { + DEBUG(D_transport) debug_printf("DANE: holding %s for later\n", a->domain); + dane_held = TRUE; + a->transport_return = DANE; + } + } +#endif /* If there is a filter command specified for this transport, we can now -set it up. This cannot be done until the identify of the host is known. */ +set it up. This cannot be done until the identity of the host is known. */ if (tblock->filter_command) { @@ -3493,7 +3563,7 @@ if (tblock->filter_command) string_sprintf("%.50s transport", tblock->name), NULL)) { set_errno_nohost(addrlist->next, addrlist->basic_errno, addrlist->message, DEFER, - FALSE); + FALSE, &sx->delivery_start); yield = ERROR; goto SEND_QUIT; } @@ -3537,10 +3607,9 @@ always has a sequence number greater than one. */ if (continue_hostname && continue_sequence == 1) { - sx->peer_offered = smtp_peer_options; - sx->pending_MAIL = FALSE; + /* sx->pending_MAIL = FALSE; */ sx->ok = TRUE; - sx->next_addr = NULL; + /* sx->next_addr = NULL; */ for (address_item * addr = addrlist; addr; addr = addr->next) addr->transport_return = PENDING_OK; @@ -3572,7 +3641,7 @@ else { /*XXX could we find a better errno than 0 here? */ set_errno_nohost(addrlist, 0, a->message, FAIL, - testflag(a, af_pass_message)); + testflag(a, af_pass_message), &sx->delivery_start); sx->ok = FALSE; break; } @@ -3718,6 +3787,46 @@ else report_time_since(&t0, US"dkim_exim_sign_init (delta)"); # endif } +#endif + + /* See if we can pipeline QUIT. Reasons not to are + - pipelining not active + - not ok to send quit + - errors in amtp transation responses + - more addrs to send for this message or this host + - this message was being retried + - more messages for this host + If we can, we want the message-write to not flush (the tail end of) its data out. */ + + if ( sx->pipelining_used + && (sx->ok && sx->completed_addr || sx->peer_offered & OPTION_CHUNKING) + && sx->send_quit + && !(sx->first_addr || f.continue_more) + && f.deliver_firsttime + ) + { + smtp_compare_t t_compare = + {.tblock = tblock, .current_sender_address = sender_address}; + + tcw_done = TRUE; + tcw = +#ifndef DISABLE_TLS + ( tls_out.active.sock < 0 && !continue_proxy_cipher + || verify_check_given_host(CUSS &ob->hosts_nopass_tls, host) != OK + ) + && +#endif + transport_check_waiting(tblock->name, host->name, + tblock->connection_max_messages, new_message_id, + (oicf)smtp_are_same_identities, (void*)&t_compare); + if (!tcw) + { + HDEBUG(D_transport) debug_printf("will pipeline QUIT\n"); + tctx.options |= topt_no_flush; + } + } + +#ifndef DISABLE_DKIM sx->ok = dkim_transport_write_message(&tctx, &ob->dkim, CUSS &message); #else sx->ok = transport_write_message(&tctx, 0); @@ -3746,6 +3855,48 @@ else smtp_command = US"end of data"; + /* If we can pipeline a QUIT with the data them send it now. If a new message + for this host appeared in the queue while data was being sent, we will not see + it and it will have to wait for a queue run. If there was one but another + thread took it, we might attempt to send it - but locking of spoolfiles will + detect that. Use _MORE to get QUIT in FIN segment. */ + + if (tcw_done && !tcw) + { + /*XXX jgh 2021/03/10 google et. al screwup. G, at least, sends TCP FIN in response to TLS + close-notify. Under TLS 1.3, violating RFC. + However, TLS 1.2 does not have half-close semantics. */ + + if ( sx->cctx.tls_ctx +#if 0 && !defined(DISABLE_TLS) + && Ustrcmp(tls_out.ver, "TLS1.3") != 0 +#endif + || !f.deliver_firsttime + ) + { /* Send QUIT now and not later */ + (void)smtp_write_command(sx, SCMD_FLUSH, "QUIT\r\n"); + sx->send_quit = FALSE; + } + else + { /* add QUIT to the output buffer */ + (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n"); + sx->send_quit = FALSE; /* avoid sending it later */ + +#ifndef DISABLE_TLS + if (sx->cctx.tls_ctx) /* need to send TLS Cloe Notify */ + { +# ifdef EXIM_TCP_CORK /* Use _CORK to get Close Notify in FIN segment */ + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_shutdown_wr(sx->cctx.tls_ctx); + sx->send_tlsclose = FALSE; /* avoid later repeat */ + } +#endif + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(shutdown)>>\n"); + shutdown(sx->cctx.sock, SHUT_WR); /* flush output buffer, with TCP FIN */ + } + } + if (sx->peer_offered & OPTION_CHUNKING && sx->cmd_count > 1) { /* Reap any outstanding MAIL & RCPT commands, but not a DATA-go-ahead */ @@ -3825,7 +3976,7 @@ else int len; uschar * conf = NULL; - timesince(&delivery_time, &start_delivery_time); + timesince(&delivery_time, &sx->delivery_start); sx->send_rset = FALSE; pipelining_active = FALSE; @@ -3838,15 +3989,16 @@ else !sx->lmtp ) { - const uschar *s = string_printing(sx->buffer); + const uschar * s = string_printing(sx->buffer); /* deconst cast ok here as string_printing was checked to have alloc'n'copied */ - conf = (s == sx->buffer)? US string_copy(s) : US s; + conf = s == sx->buffer ? US string_copy(s) : US s; } /* Process all transported addresses - for LMTP or PRDR, read a status for - each one. */ + each one. We used to drop out at first_addr, until someone returned a 452 + followed by a 250... and we screwed up the accepted addresses. */ - for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) + for (address_item * addr = addrlist; addr; addr = addr->next) { if (addr->transport_return != PENDING_OK) continue; @@ -3900,9 +4052,8 @@ else actual host that was used. */ addr->transport_return = OK; - addr->more_errno = delivery_time.tv_sec; - addr->delivery_usec = delivery_time.tv_usec; addr->host_used = host; + addr->delivery_time = delivery_time; addr->special_action = flag; addr->message = conf; @@ -3945,53 +4096,53 @@ else } #ifndef DISABLE_PRDR - if (sx->prdr_active) - { - const uschar * overall_message; + if (sx->prdr_active) + { + const uschar * overall_message; - /* PRDR - get the final, overall response. For any non-success - upgrade all the address statuses. */ + /* PRDR - get the final, overall response. For any non-success + upgrade all the address statuses. */ - sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', - ob->final_timeout); - if (!sx->ok) + sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', + ob->final_timeout); + if (!sx->ok) + { + if(errno == 0 && sx->buffer[0] == '4') { - if(errno == 0 && sx->buffer[0] == '4') - { - errno = ERRNO_DATA4XX; - addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8; - } - for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) - if (sx->buffer[0] == '5' || addr->transport_return == OK) - addr->transport_return = PENDING_OK; /* allow set_errno action */ - goto RESPONSE_FAILED; + errno = ERRNO_DATA4XX; + addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8; } + for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) + if (sx->buffer[0] == '5' || addr->transport_return == OK) + addr->transport_return = PENDING_OK; /* allow set_errno action */ + goto RESPONSE_FAILED; + } - /* Append the overall response to the individual PRDR response for logging - and update the journal, or setup retry. */ + /* Append the overall response to the individual PRDR response for logging + and update the journal, or setup retry. */ - overall_message = string_printing(sx->buffer); - for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) - if (addr->transport_return == OK) - addr->message = string_sprintf("%s\\n%s", addr->message, overall_message); + overall_message = string_printing(sx->buffer); + for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) + if (addr->transport_return == OK) + addr->message = string_sprintf("%s\\n%s", addr->message, overall_message); - for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) - if (addr->transport_return == OK) - { - if (testflag(addr, af_homonym)) - sprintf(CS sx->buffer, "%.500s/%s\n", addr->unique + 3, tblock->name); - else - sprintf(CS sx->buffer, "%.500s\n", addr->unique); + for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) + if (addr->transport_return == OK) + { + if (testflag(addr, af_homonym)) + sprintf(CS sx->buffer, "%.500s/%s\n", addr->unique + 3, tblock->name); + else + sprintf(CS sx->buffer, "%.500s\n", addr->unique); - DEBUG(D_deliver) debug_printf("journalling(PRDR) %s\n", sx->buffer); - len = Ustrlen(CS sx->buffer); - if (write(journal_fd, sx->buffer, len) != len) - log_write(0, LOG_MAIN|LOG_PANIC, "failed to write journal for " - "%s: %s", sx->buffer, strerror(errno)); - } - else if (addr->transport_return == DEFER) - retry_add_item(addr, addr->address_retry_key, -2); - } + DEBUG(D_deliver) debug_printf("journalling(PRDR) %s\n", sx->buffer); + len = Ustrlen(CS sx->buffer); + if (write(journal_fd, sx->buffer, len) != len) + log_write(0, LOG_MAIN|LOG_PANIC, "failed to write journal for " + "%s: %s", sx->buffer, strerror(errno)); + } + else if (addr->transport_return == DEFER) + retry_add_item(addr, addr->address_retry_key, -2); + } #endif /* Ensure the journal file is pushed out to disk. */ @@ -4020,7 +4171,8 @@ if (!sx->ok) { save_errno = errno; message = NULL; - sx->send_quit = check_response(host, &save_errno, addrlist->more_errno, + /* Clear send_quit flag if needed. Do not set. */ + sx->send_quit &= check_response(host, &save_errno, addrlist->more_errno, sx->buffer, &code, &message, &pass_message); goto FAILED; } @@ -4105,8 +4257,15 @@ if (!sx->ok) *message_defer = TRUE; } +#ifdef TIOCOUTQ + DEBUG(D_transport) if (sx->cctx.sock >= 0) + { + int n; + if (ioctl(sx->cctx.sock, TIOCOUTQ, &n) == 0) + debug_printf("%d bytes remain in socket output buffer\n", n); + } +#endif } - /* Otherwise, we have an I/O error or a timeout other than after MAIL or ".", or some other transportation error. We defer all addresses and yield DEFER, except for the case of failed add_headers expansion, or a transport @@ -4132,14 +4291,13 @@ if (!sx->ok) } } - set_errno(addrlist, save_errno, set_message, set_rc, pass_message, host + set_errno(addrlist, save_errno, set_message, set_rc, pass_message, host, #ifdef EXPERIMENTAL_DSN_INFO - , sx->smtp_greeting, sx->helo_response + sx->smtp_greeting, sx->helo_response, #endif - ); + &sx->delivery_start); } - /* If all has gone well, send_quit will be set TRUE, implying we can end the SMTP session tidily. However, if there were too many addresses to send in one message (indicated by first_addr being non-NULL) we want to carry on with the @@ -4175,14 +4333,12 @@ DEBUG(D_transport) if (sx->completed_addr && sx->ok && sx->send_quit) { - BOOL more; - smtp_compare_t t_compare; + smtp_compare_t t_compare = + {.tblock = tblock, .current_sender_address = sender_address}; - t_compare.tblock = tblock; - t_compare.current_sender_address = sender_address; - - if ( sx->first_addr != NULL - || f.continue_more + if ( sx->first_addr /* more addrs for this message */ + || f.continue_more /* more addrs for continued-host */ + || tcw_done && tcw /* more messages for host */ || ( #ifndef DISABLE_TLS ( tls_out.active.sock < 0 && !continue_proxy_cipher @@ -4191,7 +4347,7 @@ if (sx->completed_addr && sx->ok && sx->send_quit) && #endif transport_check_waiting(tblock->name, host->name, - tblock->connection_max_messages, new_message_id, &more, + tblock->connection_max_messages, new_message_id, (oicf)smtp_are_same_identities, (void*)&t_compare) ) ) { @@ -4227,10 +4383,10 @@ if (sx->completed_addr && sx->ok && sx->send_quit) #endif int socket_fd = sx->cctx.sock; - - if (sx->first_addr != NULL) /* More addresses still to be sent */ - { /* in this run of the transport */ - continue_sequence++; /* Causes * in logging */ + if (sx->first_addr) /* More addresses still to be sent */ + { /* for this message */ + continue_sequence++; /* Causes * in logging */ + pipelining_active = sx->pipelining_used; /* was cleared at DATA */ goto SEND_MESSAGE; } @@ -4251,7 +4407,9 @@ if (sx->completed_addr && sx->ok && sx->send_quit) the socket on. */ tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT); + sx->send_tlsclose = FALSE; sx->cctx.tls_ctx = NULL; + tls_out.active.sock = -1; smtp_peer_options = smtp_peer_options_wrap; sx->ok = !sx->smtps && smtp_write_command(sx, SCMD_FLUSH, "EHLO %s\r\n", sx->helo_data) @@ -4260,7 +4418,7 @@ if (sx->completed_addr && sx->ok && sx->send_quit) '2', ob->command_timeout); if (sx->ok && f.continue_more) - return yield; /* More addresses for another run */ + goto TIDYUP; /* More addresses for another run */ } else { @@ -4271,16 +4429,16 @@ if (sx->completed_addr && sx->ok && sx->send_quit) socket_fd = pfd[1]; else set_errno(sx->first_addr, errno, US"internal allocation problem", - DEFER, FALSE, host + DEFER, FALSE, host, # ifdef EXPERIMENTAL_DSN_INFO - , sx->smtp_greeting, sx->helo_response + sx->smtp_greeting, sx->helo_response, # endif - ); + &sx->delivery_start); } else #endif if (f.continue_more) - return yield; /* More addresses for another run */ + goto TIDYUP; /* More addresses for another run */ /* If the socket is successfully passed, we mustn't send QUIT (or indeed anything!) from here. */ @@ -4301,10 +4459,9 @@ propagate it from the initial #ifndef DISABLE_TLS if (tls_out.active.sock >= 0) { - int pid = fork(); + int pid = exim_fork(US"tls-proxy-interproc"); if (pid == 0) /* child; fork again to disconnect totally */ { - testharness_pause_ms(100); /* let parent debug out */ /* does not return */ smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), pfd, ob->command_timeout); @@ -4312,7 +4469,6 @@ propagate it from the initial if (pid > 0) /* parent */ { - DEBUG(D_transport) debug_printf("proxy-proc inter-pid %d\n", pid); close(pfd[0]); /* tidy the inter-proc to disconn the proxy proc */ waitpid(pid, NULL, 0); @@ -4322,7 +4478,7 @@ propagate it from the initial sx->cctx.sock = -1; continue_transport = NULL; continue_hostname = NULL; - return yield; + goto TIDYUP; } log_write(0, LOG_PANIC_DIE, "fork failed"); } @@ -4332,45 +4488,35 @@ propagate it from the initial /* If RSET failed and there are addresses left, they get deferred. */ else - set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host + set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host, #ifdef EXPERIMENTAL_DSN_INFO - , sx->smtp_greeting, sx->helo_response + sx->smtp_greeting, sx->helo_response, #endif - ); + &sx->delivery_start); } } /* End off tidily with QUIT unless the connection has died or the socket has -been passed to another process. There has been discussion on the net about what -to do after sending QUIT. The wording of the RFC suggests that it is necessary -to wait for a response, but on the other hand, there isn't anything one can do -with an error response, other than log it. Exim used to do that. However, -further discussion suggested that it is positively advantageous not to wait for -the response, but to close the session immediately. This is supposed to move -the TCP/IP TIME_WAIT state from the server to the client, thereby removing some -load from the server. (Hosts that are both servers and clients may not see much -difference, of course.) Further discussion indicated that this was safe to do -on Unix systems which have decent implementations of TCP/IP that leave the -connection around for a while (TIME_WAIT) after the application has gone away. -This enables the response sent by the server to be properly ACKed rather than -timed out, as can happen on broken TCP/IP implementations on other OS. - -This change is being made on 31-Jul-98. After over a year of trouble-free -operation, the old commented-out code was removed on 17-Sep-99. */ +been passed to another process. */ SEND_QUIT: -#ifdef TCP_CORK -(void) setsockopt(sx->cctx.sock, IPPROTO_TCP, TCP_CORK, US &on, sizeof(on)); +if (sx->send_quit) + { /* Use _MORE to get QUIT in FIN segment */ + (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n"); +#ifndef DISABLE_TLS + if (sx->cctx.tls_ctx) + { +# ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */ + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_shutdown_wr(sx->cctx.tls_ctx); + sx->send_tlsclose = FALSE; + } #endif -if (sx->send_quit) (void)smtp_write_command(sx, SCMD_FLUSH, "QUIT\r\n"); + } END_OFF: -#ifndef DISABLE_TLS -tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); -sx->cctx.tls_ctx = NULL; -#endif - /* Close the socket, and return the appropriate value, first setting works because the NULL setting is passed back to the calling process, and remote_max_parallel is forced to 1 when delivering over an existing connection, @@ -4381,25 +4527,96 @@ writing RSET might have failed, or there may be other addresses whose hosts are specified in the transports, and therefore not visible at top level, in which case continue_more won't get set. */ -HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); if (sx->send_quit) { + /* This flushes data queued in the socket, being the QUIT and any TLS Close, + sending them along with the client FIN flag. Us (we hope) sending FIN first + means we (client) take the TIME_WAIT state, so the server (which likely has a + higher connection rate) does not have to. */ + + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(shutdown)>>\n"); shutdown(sx->cctx.sock, SHUT_WR); + } + +if (sx->send_quit || tcw_done && !tcw) + { + /* Wait for (we hope) ack of our QUIT, and a server FIN. Discard any data + received, then discard the socket. Any packet received after then, or receive + data still in the socket, will get a RST - hence the pause/drain. */ + + /* Reap the response to QUIT, timing out after one second */ + (void) smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', 1); +#ifndef DISABLE_TLS + if (sx->cctx.tls_ctx) + { + int n; + + /* Reap the TLS Close Notify from the server, timing out after one second */ + sigalrm_seen = FALSE; + ALARM(1); + do + n = tls_read(sx->cctx.tls_ctx, sx->inbuffer, sizeof(sx->inbuffer)); + while (!sigalrm_seen && n > 0); + ALARM_CLR(0); + +# ifdef EXIM_TCP_CORK + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT); + sx->cctx.tls_ctx = NULL; + } +#endif millisleep(20); - testharness_pause_ms(200); if (fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0) - for (int i = 16; read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer)) > 0 && i > 0;) - i--; /* drain socket */ + for (int i = 16, n; /* drain socket */ + (n = read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer))) > 0 && i > 0; + i--) HDEBUG(D_transport|D_acl|D_v) + { + int m = MIN(n, 64); + debug_printf_indent(" SMTP(drain %d bytes)<< %.*s\n", n, m, sx->inbuffer); + for (m = 0; m < n; m++) + debug_printf("0x%02x\n", sx->inbuffer[m]); + } } +HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); (void)close(sx->cctx.sock); #ifndef DISABLE_EVENT (void) event_raise(tblock->event_action, US"tcp:close", NULL); #endif +#ifdef SUPPORT_DANE +if (dane_held) + { + sx->first_addr = NULL; + for (address_item * a = sx->addrlist->next; a; a = a->next) + if (a->transport_return == DANE) + { + a->transport_return = PENDING_DEFER; + if (!sx->first_addr) + { + /* Remember the new start-point in the addrlist, for smtp_setup_conn() + to get the domain string for SNI */ + + sx->first_addr = a; + DEBUG(D_transport) debug_printf("DANE: go-around for %s\n", a->domain); + } + } + goto DANE_DOMAINS; + } +#endif + continue_transport = NULL; continue_hostname = NULL; return yield; + +TIDYUP: +#ifdef SUPPORT_DANE +if (dane_held) for (address_item * a = sx->addrlist->next; a; a = a->next) + if (a->transport_return == DANE) + a->transport_return = PENDING_DEFER; +#endif +return yield; } @@ -4552,6 +4769,22 @@ DEBUG(D_transport) cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0); } +/* Check the restrictions on line length */ + +if (max_received_linelength > ob->message_linelength_limit) + { + struct timeval now; + gettimeofday(&now, NULL); + + for (address_item * addr = addrlist; addr; addr = addr->next) + if (addr->transport_return == DEFER) + addr->transport_return = PENDING_DEFER; + + set_errno_nohost(addrlist, ERRNO_SMTPFORMAT, + US"message has lines too long for transport", FAIL, TRUE, &now); + goto END_TRANSPORT; + } + /* Set the flag requesting that these hosts be added to the waiting database if the delivery fails temporarily or if we are running with queue_smtp or a 2-stage queue run. This gets unset for certain @@ -4589,7 +4822,7 @@ if (!hostlist || (ob->hosts_override && ob->hosts)) { uschar *s = ob->hosts; - if (Ustrchr(s, '$') != NULL) + if (Ustrchr(s, '$')) { if (!(expanded_hosts = expand_string(s))) { @@ -4892,15 +5125,19 @@ retry_non_continued: were not in it. We don't want to hold up all SMTP deliveries! Except when doing a two-stage queue run, don't do this if forcing. */ - if ((!f.deliver_force || f.queue_2stage) && (f.queue_smtp || - match_isinlist(addrlist->domain, - (const uschar **)&queue_smtp_domains, 0, - &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)) + if ( (!f.deliver_force || f.queue_2stage) + && ( f.queue_smtp + || match_isinlist(addrlist->domain, + CUSS &queue_smtp_domains, 0, + &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK) + ) { + DEBUG(D_transport) debug_printf("first-pass routing only\n"); expired = FALSE; for (address_item * addr = addrlist; addr; addr = addr->next) if (addr->transport_return == DEFER) - addr->message = US"domain matches queue_smtp_domains, or -odqs set"; + addr->message = US"first-pass only routing due to -odqs, " + "queue_smtp_domains or control=queue"; continue; /* With next host */ } @@ -4933,7 +5170,7 @@ retry_non_continued: because connections to the same host from a different interface should be treated separately. */ - host_af = Ustrchr(host->address, ':') == NULL ? AF_INET : AF_INET6; + host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET; { uschar * s = ob->interface; if (s && *s) @@ -5061,7 +5298,9 @@ retry_non_continued: if (f.dont_deliver) { - set_errno_nohost(addrlist, 0, NULL, OK, FALSE); + struct timeval now; + gettimeofday(&now, NULL); + set_errno_nohost(addrlist, 0, NULL, OK, FALSE, &now); for (address_item * addr = addrlist; addr; addr = addr->next) { addr->host_used = host; @@ -5149,7 +5388,7 @@ retry_non_continued: #ifndef DISABLE_EVENT if (rc == DEFER) - deferred_event_raise(first_addr, host); + deferred_event_raise(first_addr, host, US"msg:host:defer"); #endif /* If STARTTLS was accepted, but there was a failure in setting up the @@ -5177,11 +5416,23 @@ retry_non_continued: if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL) write_logs(host, first_addr->message, first_addr->basic_errno); # ifndef DISABLE_EVENT - if (rc == DEFER) - deferred_event_raise(first_addr, host); + if (rc == DEFER) + deferred_event_raise(first_addr, host, US"msg:host:defer"); # endif } #endif /*DISABLE_TLS*/ + +#ifndef DISABLE_EVENT + /* If the last host gave a defer raise a per-message event */ + + if ( !( nexthost + && unexpired_hosts_tried < ob->hosts_max_try + && total_hosts_tried < ob->hosts_max_try_hardlimit + ) + && (message_defer || rc == DEFER) + ) + deferred_event_raise(first_addr, host, US"msg:defer"); +#endif } /* Delivery attempt finished */