1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
5 /* Copyright (c) University of Cambridge 1995 - 2018 */
6 /* Copyright (c) The Exim Maintainers 2020 - 2021 */
7 /* See the file NOTICE for conditions of use and distribution. */
12 #if defined(SUPPORT_DANE) && defined(DISABLE_TLS)
13 # error TLS is required for DANE
17 /* Options specific to the smtp transport. This transport also supports LMTP
18 over TCP/IP. The options must be in alphabetic order (note that "_" comes
19 before the lower case letters). Some live in the transport_instance block so as
20 to be publicly visible; these are flagged with opt_public. */
22 #define LOFF(field) OPT_OFF(smtp_transport_options_block, field)
24 optionlist smtp_transport_options[] = {
25 { "*expand_multi_domain", opt_stringptr | opt_hidden | opt_public,
26 OPT_OFF(transport_instance, expand_multi_domain) },
27 { "*expand_retry_include_ip_address", opt_stringptr | opt_hidden,
28 LOFF(expand_retry_include_ip_address) },
30 { "address_retry_include_sender", opt_bool,
31 LOFF(address_retry_include_sender) },
32 { "allow_localhost", opt_bool, LOFF(allow_localhost) },
33 #ifdef EXPERIMENTAL_ARC
34 { "arc_sign", opt_stringptr, LOFF(arc_sign) },
36 { "authenticated_sender", opt_stringptr, LOFF(authenticated_sender) },
37 { "authenticated_sender_force", opt_bool, LOFF(authenticated_sender_force) },
38 { "command_timeout", opt_time, LOFF(command_timeout) },
39 { "connect_timeout", opt_time, LOFF(connect_timeout) },
40 { "connection_max_messages", opt_int | opt_public,
41 OPT_OFF(transport_instance, connection_max_messages) },
43 { "dane_require_tls_ciphers", opt_stringptr, LOFF(dane_require_tls_ciphers) },
45 { "data_timeout", opt_time, LOFF(data_timeout) },
46 { "delay_after_cutoff", opt_bool, LOFF(delay_after_cutoff) },
48 { "dkim_canon", opt_stringptr, LOFF(dkim.dkim_canon) },
49 { "dkim_domain", opt_stringptr, LOFF(dkim.dkim_domain) },
50 { "dkim_hash", opt_stringptr, LOFF(dkim.dkim_hash) },
51 { "dkim_identity", opt_stringptr, LOFF(dkim.dkim_identity) },
52 { "dkim_private_key", opt_stringptr, LOFF(dkim.dkim_private_key) },
53 { "dkim_selector", opt_stringptr, LOFF(dkim.dkim_selector) },
54 { "dkim_sign_headers", opt_stringptr, LOFF(dkim.dkim_sign_headers) },
55 { "dkim_strict", opt_stringptr, LOFF(dkim.dkim_strict) },
56 { "dkim_timestamps", opt_stringptr, LOFF(dkim.dkim_timestamps) },
58 { "dns_qualify_single", opt_bool, LOFF(dns_qualify_single) },
59 { "dns_search_parents", opt_bool, LOFF(dns_search_parents) },
60 { "dnssec_request_domains", opt_stringptr, LOFF(dnssec.request) },
61 { "dnssec_require_domains", opt_stringptr, LOFF(dnssec.require) },
62 { "dscp", opt_stringptr, LOFF(dscp) },
63 { "fallback_hosts", opt_stringptr, LOFF(fallback_hosts) },
64 { "final_timeout", opt_time, LOFF(final_timeout) },
65 { "gethostbyname", opt_bool, LOFF(gethostbyname) },
66 { "helo_data", opt_stringptr, LOFF(helo_data) },
67 { "hosts", opt_stringptr, LOFF(hosts) },
68 { "hosts_avoid_esmtp", opt_stringptr, LOFF(hosts_avoid_esmtp) },
69 { "hosts_avoid_pipelining", opt_stringptr, LOFF(hosts_avoid_pipelining) },
71 { "hosts_avoid_tls", opt_stringptr, LOFF(hosts_avoid_tls) },
73 { "hosts_max_try", opt_int, LOFF(hosts_max_try) },
74 { "hosts_max_try_hardlimit", opt_int, LOFF(hosts_max_try_hardlimit) },
76 { "hosts_nopass_tls", opt_stringptr, LOFF(hosts_nopass_tls) },
77 { "hosts_noproxy_tls", opt_stringptr, LOFF(hosts_noproxy_tls) },
79 { "hosts_override", opt_bool, LOFF(hosts_override) },
80 #ifndef DISABLE_PIPE_CONNECT
81 { "hosts_pipe_connect", opt_stringptr, LOFF(hosts_pipe_connect) },
83 { "hosts_randomize", opt_bool, LOFF(hosts_randomize) },
84 #if !defined(DISABLE_TLS) && !defined(DISABLE_OCSP)
85 { "hosts_request_ocsp", opt_stringptr, LOFF(hosts_request_ocsp) },
87 { "hosts_require_alpn", opt_stringptr, LOFF(hosts_require_alpn) },
88 { "hosts_require_auth", opt_stringptr, LOFF(hosts_require_auth) },
91 { "hosts_require_dane", opt_stringptr, LOFF(hosts_require_dane) },
94 { "hosts_require_ocsp", opt_stringptr, LOFF(hosts_require_ocsp) },
96 { "hosts_require_tls", opt_stringptr, LOFF(hosts_require_tls) },
98 { "hosts_try_auth", opt_stringptr, LOFF(hosts_try_auth) },
99 { "hosts_try_chunking", opt_stringptr, LOFF(hosts_try_chunking) },
101 { "hosts_try_dane", opt_stringptr, LOFF(hosts_try_dane) },
103 { "hosts_try_fastopen", opt_stringptr, LOFF(hosts_try_fastopen) },
105 { "hosts_try_prdr", opt_stringptr, LOFF(hosts_try_prdr) },
108 { "hosts_verify_avoid_tls", opt_stringptr, LOFF(hosts_verify_avoid_tls) },
110 { "interface", opt_stringptr, LOFF(interface) },
111 { "keepalive", opt_bool, LOFF(keepalive) },
112 { "lmtp_ignore_quota", opt_bool, LOFF(lmtp_ignore_quota) },
113 { "max_rcpt", opt_int | opt_public,
114 OPT_OFF(transport_instance, max_addresses) },
115 { "message_linelength_limit", opt_int, LOFF(message_linelength_limit) },
116 { "multi_domain", opt_expand_bool | opt_public,
117 OPT_OFF(transport_instance, multi_domain) },
118 { "port", opt_stringptr, LOFF(port) },
119 { "protocol", opt_stringptr, LOFF(protocol) },
120 { "retry_include_ip_address", opt_expand_bool, LOFF(retry_include_ip_address) },
121 { "serialize_hosts", opt_stringptr, LOFF(serialize_hosts) },
122 { "size_addition", opt_int, LOFF(size_addition) },
124 { "socks_proxy", opt_stringptr, LOFF(socks_proxy) },
127 { "tls_alpn", opt_stringptr, LOFF(tls_alpn) },
128 { "tls_certificate", opt_stringptr, LOFF(tls_certificate) },
129 { "tls_crl", opt_stringptr, LOFF(tls_crl) },
130 { "tls_dh_min_bits", opt_int, LOFF(tls_dh_min_bits) },
131 { "tls_privatekey", opt_stringptr, LOFF(tls_privatekey) },
132 { "tls_require_ciphers", opt_stringptr, LOFF(tls_require_ciphers) },
133 # ifndef DISABLE_TLS_RESUME
134 { "tls_resumption_hosts", opt_stringptr, LOFF(tls_resumption_hosts) },
136 { "tls_sni", opt_stringptr, LOFF(tls_sni) },
137 { "tls_tempfail_tryclear", opt_bool, LOFF(tls_tempfail_tryclear) },
138 { "tls_try_verify_hosts", opt_stringptr, LOFF(tls_try_verify_hosts) },
139 { "tls_verify_cert_hostnames", opt_stringptr, LOFF(tls_verify_cert_hostnames)},
140 { "tls_verify_certificates", opt_stringptr, LOFF(tls_verify_certificates) },
141 { "tls_verify_hosts", opt_stringptr, LOFF(tls_verify_hosts) },
144 { "utf8_downconvert", opt_stringptr, LOFF(utf8_downconvert) },
148 /* Size of the options list. An extern variable has to be used so that its
149 address can appear in the tables drtables.c. */
151 int smtp_transport_options_count = nelem(smtp_transport_options);
157 smtp_transport_options_block smtp_transport_option_defaults = {0};
158 void smtp_transport_init(transport_instance *tblock) {}
159 BOOL smtp_transport_entry(transport_instance *tblock, address_item *addr) {return FALSE;}
160 void smtp_transport_closedown(transport_instance *tblock) {}
162 #else /*!MACRO_PREDEF*/
165 /* Default private options block for the smtp transport. */
167 smtp_transport_options_block smtp_transport_option_defaults = {
168 /* All non-mentioned elements 0/NULL/FALSE */
169 .helo_data = US"$primary_hostname",
170 .protocol = US"smtp",
171 .hosts_try_chunking = US"*",
173 .hosts_try_dane = US"*",
175 .hosts_try_fastopen = US"*",
177 .hosts_try_prdr = US"*",
180 .hosts_request_ocsp = US"*", /* hosts_request_ocsp (except under DANE; tls_client_start()) */
182 .command_timeout = 5*60,
183 .connect_timeout = 5*60,
184 .data_timeout = 5*60,
185 .final_timeout = 10*60,
186 .size_addition = 1024,
188 .hosts_max_try_hardlimit = 50,
189 .message_linelength_limit = 998,
190 .address_retry_include_sender = TRUE,
191 .dns_qualify_single = TRUE,
192 .dnssec = { .request= US"*", .require=NULL },
193 .delay_after_cutoff = TRUE,
195 .retry_include_ip_address = TRUE,
197 .tls_verify_certificates = US"system",
198 .tls_dh_min_bits = EXIM_CLIENT_DH_DEFAULT_MIN_BITS,
199 .tls_tempfail_tryclear = TRUE,
200 .tls_try_verify_hosts = US"*",
201 .tls_verify_cert_hostnames = US"*",
204 .utf8_downconvert = US"-1",
208 { .dkim_hash = US"sha256", },
212 /* some DSN flags for use later */
214 static int rf_list[] = {rf_notify_never, rf_notify_success,
215 rf_notify_failure, rf_notify_delay };
217 static uschar *rf_names[] = { US"NEVER", US"SUCCESS", US"FAILURE", US"DELAY" };
223 static uschar *smtp_command; /* Points to last cmd for error messages */
224 static uschar *mail_command; /* Points to MAIL cmd for error messages */
225 static uschar *data_command = US""; /* Points to DATA cmd for error messages */
226 static BOOL update_waiting; /* TRUE to update the "wait" database */
228 /*XXX move to smtp_context */
229 static BOOL pipelining_active; /* current transaction is in pipe mode */
232 static unsigned ehlo_response(uschar * buf, unsigned checks);
235 /******************************************************************************/
238 smtp_deliver_init(void)
242 const pcre2_code ** re;
243 const uschar * string;
246 { ®ex_AUTH, AUTHS_REGEX },
247 { ®ex_CHUNKING, US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)" },
248 { ®ex_DSN, US"\\n250[\\s\\-]DSN(\\s|\\n|$)" },
249 { ®ex_IGNOREQUOTA, US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)" },
250 { ®ex_PIPELINING, US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)" },
251 { ®ex_SIZE, US"\\n250[\\s\\-]SIZE(\\s|\\n|$)" },
254 { ®ex_STARTTLS, US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)" },
257 { ®ex_PRDR, US"\\n250[\\s\\-]PRDR(\\s|\\n|$)" },
260 { ®ex_UTF8, US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)" },
262 #ifndef DISABLE_PIPE_CONNECT
263 { ®ex_EARLY_PIPE, US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)" },
265 #ifdef EXPERIMENTAL_ESMTP_LIMITS
266 { ®ex_LIMITS, US"\\n250[\\s\\-]LIMITS\\s" },
270 for (struct list * l = list; l < list + nelem(list); l++)
272 *l->re = regex_must_compile(l->string, FALSE, TRUE);
276 /*************************************************
277 * Setup entry point *
278 *************************************************/
280 /* This function is called when the transport is about to be used,
281 but before running it in a sub-process. It is used for two things:
283 (1) To set the fallback host list in addresses, when delivering.
284 (2) To pass back the interface, port, protocol, and other options, for use
285 during callout verification.
288 tblock pointer to the transport instance block
289 addrlist list of addresses about to be transported
290 tf if not NULL, pointer to block in which to return options
291 uid the uid that will be set (not used)
292 gid the gid that will be set (not used)
293 errmsg place for error message (not used)
295 Returns: OK always (FAIL, DEFER not used)
299 smtp_transport_setup(transport_instance *tblock, address_item *addrlist,
300 transport_feedback *tf, uid_t uid, gid_t gid, uschar **errmsg)
302 smtp_transport_options_block *ob = SOB tblock->options_block;
304 /* Pass back options if required. This interface is getting very messy. */
308 tf->interface = ob->interface;
310 tf->protocol = ob->protocol;
311 tf->hosts = ob->hosts;
312 tf->hosts_override = ob->hosts_override;
313 tf->hosts_randomize = ob->hosts_randomize;
314 tf->gethostbyname = ob->gethostbyname;
315 tf->qualify_single = ob->dns_qualify_single;
316 tf->search_parents = ob->dns_search_parents;
317 tf->helo_data = ob->helo_data;
320 /* Set the fallback host list for all the addresses that don't have fallback
321 host lists, provided that the local host wasn't present in the original host
324 if (!testflag(addrlist, af_local_host_removed))
325 for (; addrlist; addrlist = addrlist->next)
326 if (!addrlist->fallback_hosts) addrlist->fallback_hosts = ob->fallback_hostlist;
333 /*************************************************
334 * Initialization entry point *
335 *************************************************/
337 /* Called for each instance, after its options have been read, to
338 enable consistency checks to be done, or anything else that needs
341 Argument: pointer to the transport instance block
346 smtp_transport_init(transport_instance *tblock)
348 smtp_transport_options_block *ob = SOB tblock->options_block;
349 int old_pool = store_pool;
351 /* Retry_use_local_part defaults FALSE if unset */
353 if (tblock->retry_use_local_part == TRUE_UNSET)
354 tblock->retry_use_local_part = FALSE;
356 /* Set the default port according to the protocol */
359 ob->port = strcmpic(ob->protocol, US"lmtp") == 0
361 : strcmpic(ob->protocol, US"smtps") == 0
362 ? US"smtps" : US"smtp";
364 /* Set up the setup entry point, to be called before subprocesses for this
367 tblock->setup = smtp_transport_setup;
369 /* Complain if any of the timeouts are zero. */
371 if (ob->command_timeout <= 0 || ob->data_timeout <= 0 ||
372 ob->final_timeout <= 0)
373 log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
374 "command, data, or final timeout value is zero for %s transport",
377 /* If hosts_override is set and there are local hosts, set the global
378 flag that stops verify from showing router hosts. */
380 if (ob->hosts_override && ob->hosts) tblock->overrides_hosts = TRUE;
382 /* If there are any fallback hosts listed, build a chain of host items
383 for them, but do not do any lookups at this time. */
385 store_pool = POOL_PERM;
386 host_build_hostlist(&ob->fallback_hostlist, ob->fallback_hosts, FALSE);
387 store_pool = old_pool;
394 /*************************************************
395 * Set delivery info into all active addresses *
396 *************************************************/
398 /* Only addresses whose status is >= PENDING are relevant. A lesser
399 status means that an address is not currently being processed.
402 addrlist points to a chain of addresses
403 errno_value to put in each address's errno field
404 msg to put in each address's message field
405 rc to put in each address's transport_return field
406 pass_message if TRUE, set the "pass message" flag in the address
407 host if set, mark addrs as having used this host
408 smtp_greeting from peer
409 helo_response from peer
410 start points to timestamp of delivery start
412 If errno_value has the special value ERRNO_CONNECTTIMEOUT, ETIMEDOUT is put in
413 the errno field, and RTEF_CTOUT is ORed into the more_errno field, to indicate
414 this particular type of timeout.
420 set_errno(address_item *addrlist, int errno_value, uschar *msg, int rc,
421 BOOL pass_message, host_item * host,
422 #ifdef EXPERIMENTAL_DSN_INFO
423 const uschar * smtp_greeting, const uschar * helo_response,
425 struct timeval * start
429 struct timeval deliver_time;
431 if (errno_value == ERRNO_CONNECTTIMEOUT)
433 errno_value = ETIMEDOUT;
434 orvalue = RTEF_CTOUT;
436 timesince(&deliver_time, start);
438 for (address_item * addr = addrlist; addr; addr = addr->next)
439 if (addr->transport_return >= PENDING)
441 addr->basic_errno = errno_value;
442 addr->more_errno |= orvalue;
443 addr->delivery_time = deliver_time;
447 if (pass_message) setflag(addr, af_pass_message);
449 addr->transport_return = rc;
452 addr->host_used = host;
453 #ifdef EXPERIMENTAL_DSN_INFO
455 {uschar * s = Ustrchr(smtp_greeting, '\n'); if (s) *s = '\0';}
456 addr->smtp_greeting = smtp_greeting;
459 {uschar * s = Ustrchr(helo_response, '\n'); if (s) *s = '\0';}
460 addr->helo_response = helo_response;
467 set_errno_nohost(address_item * addrlist, int errno_value, uschar * msg, int rc,
468 BOOL pass_message, struct timeval * start)
470 set_errno(addrlist, errno_value, msg, rc, pass_message, NULL,
471 #ifdef EXPERIMENTAL_DSN_INFO
478 /*************************************************
479 * Check an SMTP response *
480 *************************************************/
482 /* This function is given an errno code and the SMTP response buffer
483 to analyse, together with the host identification for generating messages. It
484 sets an appropriate message and puts the first digit of the response code into
485 the yield variable. If no response was actually read, a suitable digit is
489 host the current host, to get its name for messages
490 errno_value pointer to the errno value
491 more_errno from the top address for use with ERRNO_FILTER_FAIL
492 buffer the SMTP response buffer
493 yield where to put a one-digit SMTP response code
494 message where to put an error message
495 pass_message set TRUE if message is an SMTP response
497 Returns: TRUE if an SMTP "QUIT" command should be sent, else FALSE
501 check_response(host_item *host, int *errno_value, int more_errno,
502 uschar *buffer, int *yield, uschar **message, BOOL *pass_message)
504 uschar * pl = pipelining_active ? US"pipelined " : US"";
507 *yield = '4'; /* Default setting is to give a temporary error */
511 case ETIMEDOUT: /* Handle response timeout */
512 *message = US string_sprintf("SMTP timeout after %s%s",
514 if (transport_count > 0)
515 *message = US string_sprintf("%s (%d bytes written)", *message,
519 case ERRNO_SMTPFORMAT: /* Handle malformed SMTP response */
520 s = string_printing(buffer);
521 while (isspace(*s)) s++;
523 ? string_sprintf("Malformed SMTP reply (an empty line) "
524 "in response to %s%s", pl, smtp_command)
525 : string_sprintf("Malformed SMTP reply in response to %s%s: %s",
526 pl, smtp_command, s);
529 case ERRNO_TLSFAILURE: /* Handle bad first read; can happen with
531 *message = US"bad first read from TLS conn";
534 case ERRNO_FILTER_FAIL: /* Handle a failed filter process error;
535 can't send QUIT as we mustn't end the DATA. */
536 *message = string_sprintf("transport filter process failed (%d)%s",
538 more_errno == EX_EXECFAILED ? ": unable to execute command" : "");
541 case ERRNO_CHHEADER_FAIL: /* Handle a failed add_headers expansion;
542 can't send QUIT as we mustn't end the DATA. */
544 string_sprintf("failed to expand headers_add or headers_remove: %s",
545 expand_string_message);
548 case ERRNO_WRITEINCOMPLETE: /* failure to write a complete data block */
549 *message = US"failed to write a data block";
553 case ERRNO_UTF8_FWD: /* no advertised SMTPUTF8, for international message */
554 *message = US"utf8 support required but not offered for forwarding";
555 DEBUG(D_deliver|D_transport) debug_printf("%s\n", *message);
560 /* Handle error responses from the remote mailer. */
564 *message = string_sprintf("SMTP error from remote mail server after %s%s: "
565 "%s", pl, smtp_command, s = string_printing(buffer));
566 *pass_message = TRUE;
571 /* No data was read. If there is no errno, this must be the EOF (i.e.
572 connection closed) case, which causes deferral. An explicit connection reset
573 error has the same effect. Otherwise, put the host's identity in the message,
574 leaving the errno value to be interpreted as well. In all cases, we have to
575 assume the connection is now dead. */
577 if (*errno_value == 0 || *errno_value == ECONNRESET)
579 *errno_value = ERRNO_SMTPCLOSED;
580 *message = US string_sprintf("Remote host closed connection "
581 "in response to %s%s", pl, smtp_command);
584 *message = US string_sprintf("%s [%s]", host->name, host->address);
591 /*************************************************
592 * Write error message to logs *
593 *************************************************/
595 /* This writes to the main log and to the message log.
598 host the current host
599 detail the current message (addr_item->message)
600 basic_errno the errno (addr_item->basic_errno)
606 write_logs(const host_item *host, const uschar *suffix, int basic_errno)
608 gstring * message = LOGGING(outgoing_port)
609 ? string_fmt_append(NULL, "H=%s [%s]:%d", host->name, host->address,
610 host->port == PORT_NONE ? 25 : host->port)
611 : string_fmt_append(NULL, "H=%s [%s]", host->name, host->address);
615 message = string_fmt_append(message, ": %s", suffix);
617 message = string_fmt_append(message, ": %s", strerror(basic_errno));
620 message = string_fmt_append(message, " %s", exim_errstr(basic_errno));
622 log_write(0, LOG_MAIN, "%s", string_from_gstring(message));
623 deliver_msglog("%s %s\n", tod_stamp(tod_log), message->s);
627 msglog_line(host_item * host, uschar * message)
629 deliver_msglog("%s H=%s [%s] %s\n", tod_stamp(tod_log),
630 host->name, host->address, message);
635 #ifndef DISABLE_EVENT
636 /*************************************************
637 * Post-defer action *
638 *************************************************/
640 /* This expands an arbitrary per-transport string.
641 It might, for example, be used to write to the database log.
644 addr the address item containing error information
645 host the current host
652 deferred_event_raise(address_item * addr, host_item * host, uschar * evstr)
654 uschar * action = addr->transport->event_action;
655 const uschar * save_domain;
661 save_domain = deliver_domain;
662 save_local = deliver_localpart;
664 /*XXX would ip & port already be set up? */
665 deliver_host_address = string_copy(host->address);
666 deliver_host_port = host->port == PORT_NONE ? 25 : host->port;
667 event_defer_errno = addr->basic_errno;
669 router_name = addr->router->name;
670 transport_name = addr->transport->name;
671 deliver_domain = addr->domain;
672 deliver_localpart = addr->local_part;
674 (void) event_raise(action, evstr,
676 ? addr->basic_errno > 0
677 ? string_sprintf("%s: %s", addr->message, strerror(addr->basic_errno))
678 : string_copy(addr->message)
679 : addr->basic_errno > 0
680 ? string_copy(US strerror(addr->basic_errno))
684 deliver_localpart = save_local;
685 deliver_domain = save_domain;
686 router_name = transport_name = NULL;
690 /*************************************************
691 * Reap SMTP specific responses *
692 *************************************************/
694 smtp_discard_responses(smtp_context * sx, smtp_transport_options_block * ob,
697 uschar flushbuffer[4096];
701 if (!smtp_read_response(sx, flushbuffer, sizeof(flushbuffer),
702 '2', ob->command_timeout)
703 && (errno != 0 || flushbuffer[0] == 0))
710 /* Return boolean success */
713 smtp_reap_banner(smtp_context * sx)
716 #if defined(__linux__) && defined(TCP_QUICKACK)
717 { /* Hack to get QUICKACK disabled; has to be right after 3whs, and has to on->off */
718 int sock = sx->cctx.sock;
719 struct pollfd p = {.fd = sock, .events = POLLOUT};
720 if (poll(&p, 1, 1000) >= 0) /* retval test solely for compiler quitening */
722 (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &on, sizeof(on));
723 (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off));
727 good_response = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer),
728 '2', (SOB sx->conn_args.ob)->command_timeout);
729 #ifdef EXPERIMENTAL_DSN_INFO
730 sx->smtp_greeting = string_copy(sx->buffer);
732 return good_response;
736 smtp_reap_ehlo(smtp_context * sx)
738 if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
739 (SOB sx->conn_args.ob)->command_timeout))
741 if (errno != 0 || sx->buffer[0] == 0 || sx->lmtp)
743 #ifdef EXPERIMENTAL_DSN_INFO
744 sx->helo_response = string_copy(sx->buffer);
750 #ifdef EXPERIMENTAL_DSN_INFO
751 sx->helo_response = string_copy(sx->buffer);
753 #ifndef DISABLE_EVENT
754 (void) event_raise(sx->conn_args.tblock->event_action,
755 US"smtp:ehlo", sx->buffer, NULL);
761 /******************************************************************************/
763 #ifdef EXPERIMENTAL_ESMTP_LIMITS
764 /* If TLS, or TLS not offered, called with the EHLO response in the buffer.
765 Check it for a LIMITS keyword and parse values into the smtp context structure.
767 We don't bother with peers that we won't talk TLS to, even though they can,
768 just ignore their LIMITS advice (if any) and treat them as if they do not.
769 This saves us dealing with a duplicate set of values. */
772 ehlo_response_limits_read(smtp_context * sx)
776 /* matches up to just after the first space after the keyword */
778 if (regex_match(regex_LIMITS, sx->buffer, -1, &match))
779 for (const uschar * s = sx->buffer + Ustrlen(match); *s; )
781 while (isspace(*s)) s++;
782 if (*s == '\n') break;
784 if (strncmpic(s, US"MAILMAX=", 8) == 0)
786 sx->peer_limit_mail = atoi(CS (s += 8));
787 while (isdigit(*s)) s++;
789 else if (strncmpic(s, US"RCPTMAX=", 8) == 0)
791 sx->peer_limit_rcpt = atoi(CS (s += 8));
792 while (isdigit(*s)) s++;
794 else if (strncmpic(s, US"RCPTDOMAINMAX=", 14) == 0)
796 sx->peer_limit_rcptdom = atoi(CS (s += 14));
797 while (isdigit(*s)) s++;
800 while (*s && !isspace(*s)) s++;
804 /* Apply given values to the current connection */
806 ehlo_limits_apply(smtp_context * sx,
807 unsigned limit_mail, unsigned limit_rcpt, unsigned limit_rcptdom)
809 if (limit_mail && limit_mail < sx->max_mail) sx->max_mail = limit_mail;
810 if (limit_rcpt && limit_rcpt < sx->max_rcpt) sx->max_rcpt = limit_rcpt;
813 DEBUG(D_transport) debug_printf("will treat as !multi_domain\n");
814 sx->single_rcpt_domain = TRUE;
818 /* Apply values from EHLO-resp to the current connection */
820 ehlo_response_limits_apply(smtp_context * sx)
822 ehlo_limits_apply(sx, sx->peer_limit_mail, sx->peer_limit_rcpt,
823 sx->peer_limit_rcptdom);
826 /* Apply values read from cache to the current connection */
828 ehlo_cache_limits_apply(smtp_context * sx)
830 ehlo_limits_apply(sx, sx->ehlo_resp.limit_mail, sx->ehlo_resp.limit_rcpt,
831 sx->ehlo_resp.limit_rcptdom);
835 /******************************************************************************/
837 #ifndef DISABLE_PIPE_CONNECT
839 ehlo_cache_key(const smtp_context * sx)
841 host_item * host = sx->conn_args.host;
842 return Ustrchr(host->address, ':')
843 ? string_sprintf("[%s]:%d.EHLO", host->address,
844 host->port == PORT_NONE ? sx->port : host->port)
845 : string_sprintf("%s:%d.EHLO", host->address,
846 host->port == PORT_NONE ? sx->port : host->port);
849 /* Cache EHLO-response info for use by early-pipe.
851 - During a normal flow on EHLO response (either cleartext or under TLS),
852 when we are willing to do PIPECONNECT and it is offered
853 - During an early-pipe flow on receiving the actual EHLO response and noting
854 disparity versus the cached info used, when PIPECONNECT is still being offered
856 We assume that suitable values have been set in the sx.ehlo_resp structure for
857 features and auths; we handle the copy of limits. */
860 write_ehlo_cache_entry(smtp_context * sx)
862 open_db dbblock, * dbm_file;
864 #ifdef EXPERIMENTAL_ESMTP_LIMITS
865 sx->ehlo_resp.limit_mail = sx->peer_limit_mail;
866 sx->ehlo_resp.limit_rcpt = sx->peer_limit_rcpt;
867 sx->ehlo_resp.limit_rcptdom = sx->peer_limit_rcptdom;
870 if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE)))
872 uschar * ehlo_resp_key = ehlo_cache_key(sx);
873 dbdata_ehlo_resp er = { .data = sx->ehlo_resp };
876 #ifdef EXPERIMENTAL_ESMTP_LIMITS
877 if (sx->ehlo_resp.limit_mail || sx->ehlo_resp.limit_rcpt || sx->ehlo_resp.limit_rcptdom)
878 debug_printf("writing clr %04x/%04x cry %04x/%04x lim %05d/%05d/%05d\n",
879 sx->ehlo_resp.cleartext_features, sx->ehlo_resp.cleartext_auths,
880 sx->ehlo_resp.crypted_features, sx->ehlo_resp.crypted_auths,
881 sx->ehlo_resp.limit_mail, sx->ehlo_resp.limit_rcpt,
882 sx->ehlo_resp.limit_rcptdom);
885 debug_printf("writing clr %04x/%04x cry %04x/%04x\n",
886 sx->ehlo_resp.cleartext_features, sx->ehlo_resp.cleartext_auths,
887 sx->ehlo_resp.crypted_features, sx->ehlo_resp.crypted_auths);
889 dbfn_write(dbm_file, ehlo_resp_key, &er, (int)sizeof(er));
890 dbfn_close(dbm_file);
895 invalidate_ehlo_cache_entry(smtp_context * sx)
897 open_db dbblock, * dbm_file;
899 if ( sx->early_pipe_active
900 && (dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE)))
902 uschar * ehlo_resp_key = ehlo_cache_key(sx);
903 dbfn_delete(dbm_file, ehlo_resp_key);
904 dbfn_close(dbm_file);
909 read_ehlo_cache_entry(smtp_context * sx)
914 if (!(dbm_file = dbfn_open(US"misc", O_RDONLY, &dbblock, FALSE, TRUE)))
915 { DEBUG(D_transport) debug_printf("ehlo-cache: no misc DB\n"); }
918 uschar * ehlo_resp_key = ehlo_cache_key(sx);
919 dbdata_ehlo_resp * er;
921 if (!(er = dbfn_read_enforce_length(dbm_file, ehlo_resp_key, sizeof(dbdata_ehlo_resp))))
922 { DEBUG(D_transport) debug_printf("no ehlo-resp record\n"); }
923 else if (time(NULL) - er->time_stamp > retry_data_expire)
925 DEBUG(D_transport) debug_printf("ehlo-resp record too old\n");
926 dbfn_close(dbm_file);
927 if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE)))
928 dbfn_delete(dbm_file, ehlo_resp_key);
933 #ifdef EXPERIMENTAL_ESMTP_LIMITS
934 if (er->data.limit_mail || er->data.limit_rcpt || er->data.limit_rcptdom)
935 debug_printf("EHLO response bits from cache:"
936 " cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x lim %05d/%05d/%05d\n",
937 er->data.cleartext_features, er->data.cleartext_auths,
938 er->data.crypted_features, er->data.crypted_auths,
939 er->data.limit_mail, er->data.limit_rcpt, er->data.limit_rcptdom);
942 debug_printf("EHLO response bits from cache:"
943 " cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x\n",
944 er->data.cleartext_features, er->data.cleartext_auths,
945 er->data.crypted_features, er->data.crypted_auths);
947 sx->ehlo_resp = er->data;
948 #ifdef EXPERIMENTAL_ESMTP_LIMITS
949 ehlo_cache_limits_apply(sx);
951 dbfn_close(dbm_file);
954 dbfn_close(dbm_file);
961 /* Return an auths bitmap for the set of AUTH methods offered by the server
962 which match our authenticators. */
964 static unsigned short
965 study_ehlo_auths(smtp_context * sx)
970 unsigned short authbits = 0;
972 if (!sx->esmtp) return 0;
973 if (!regex_AUTH) regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE);
974 if (!regex_match_and_setup(regex_AUTH, sx->buffer, 0, -1)) return 0;
975 expand_nmax = -1; /* reset */
976 names = string_copyn(expand_nstring[1], expand_nlength[1]);
978 for (au = auths, authnum = 0; au; au = au->next, authnum++) if (au->client)
980 const uschar * list = names;
982 for (int sep = ' '; s = string_nextinlist(&list, &sep, NULL, 0); )
983 if (strcmpic(au->public_name, s) == 0)
984 { authbits |= BIT(authnum); break; }
988 debug_printf("server offers %s AUTH, methods '%s', bitmap 0x%04x\n",
989 tls_out.active.sock >= 0 ? "crypted" : "plaintext", names, authbits);
991 if (tls_out.active.sock >= 0)
992 sx->ehlo_resp.crypted_auths = authbits;
994 sx->ehlo_resp.cleartext_auths = authbits;
1001 /* Wait for and check responses for early-pipelining.
1003 Called from the lower-level smtp_read_response() function
1004 used for general code that assume synchronisation, if context
1005 flags indicate outstanding early-pipelining commands. Also
1006 called fom sync_responses() which handles pipelined commands.
1009 sx smtp connection context
1010 countp number of outstanding responses, adjusted on return
1014 DEFER error on first read of TLS'd conn
1015 FAIL SMTP error in response
1018 smtp_reap_early_pipe(smtp_context * sx, int * countp)
1020 BOOL pending_BANNER = sx->pending_BANNER;
1021 BOOL pending_EHLO = sx->pending_EHLO;
1024 sx->pending_BANNER = FALSE; /* clear early to avoid recursion */
1025 sx->pending_EHLO = FALSE;
1029 DEBUG(D_transport) debug_printf("%s expect banner\n", __FUNCTION__);
1031 if (!smtp_reap_banner(sx))
1033 DEBUG(D_transport) debug_printf("bad banner\n");
1034 if (tls_out.active.sock >= 0) rc = DEFER;
1041 unsigned peer_offered;
1042 unsigned short authbits = 0, * ap;
1044 DEBUG(D_transport) debug_printf("%s expect ehlo\n", __FUNCTION__);
1046 if (!smtp_reap_ehlo(sx))
1048 DEBUG(D_transport) debug_printf("bad response for EHLO\n");
1049 if (tls_out.active.sock >= 0) rc = DEFER;
1053 /* Compare the actual EHLO response extensions and AUTH methods to the cached
1054 value we assumed; on difference, dump or rewrite the cache and arrange for a
1057 ap = tls_out.active.sock < 0
1058 ? &sx->ehlo_resp.cleartext_auths : &sx->ehlo_resp.crypted_auths;
1060 peer_offered = ehlo_response(sx->buffer,
1061 (tls_out.active.sock < 0 ? OPTION_TLS : 0)
1062 | OPTION_CHUNKING | OPTION_PRDR | OPTION_DSN | OPTION_PIPE | OPTION_SIZE
1063 | OPTION_UTF8 | OPTION_EARLY_PIPE
1065 #ifdef EXPERIMENTAL_ESMTP_LIMITS
1066 if (tls_out.active.sock >= 0 || !(peer_offered & OPTION_TLS))
1067 ehlo_response_limits_read(sx);
1069 if ( peer_offered != sx->peer_offered
1070 || (authbits = study_ehlo_auths(sx)) != *ap)
1073 debug_printf("EHLO %s extensions changed, 0x%04x/0x%04x -> 0x%04x/0x%04x\n",
1074 tls_out.active.sock < 0 ? "cleartext" : "crypted",
1075 sx->peer_offered, *ap, peer_offered, authbits);
1076 if (peer_offered & OPTION_EARLY_PIPE)
1078 *(tls_out.active.sock < 0
1079 ? &sx->ehlo_resp.cleartext_features : &sx->ehlo_resp.crypted_features) =
1082 write_ehlo_cache_entry(sx);
1085 invalidate_ehlo_cache_entry(sx);
1087 return OK; /* just carry on */
1089 #ifdef EXPERIMENTAL_ESMTP_LIMITS
1090 /* If we are handling LIMITS, compare the actual EHLO LIMITS values with the
1091 cached values and invalidate cache if different. OK to carry on with
1092 connect since values are advisory. */
1094 if ( (tls_out.active.sock >= 0 || !(peer_offered & OPTION_TLS))
1095 && ( sx->peer_limit_mail != sx->ehlo_resp.limit_mail
1096 || sx->peer_limit_rcpt != sx->ehlo_resp.limit_rcpt
1097 || sx->peer_limit_rcptdom != sx->ehlo_resp.limit_rcptdom
1102 debug_printf("EHLO LIMITS changed:");
1103 if (sx->peer_limit_mail != sx->ehlo_resp.limit_mail)
1104 debug_printf(" MAILMAX %u -> %u\n", sx->ehlo_resp.limit_mail, sx->peer_limit_mail);
1105 else if (sx->peer_limit_rcpt != sx->ehlo_resp.limit_rcpt)
1106 debug_printf(" RCPTMAX %u -> %u\n", sx->ehlo_resp.limit_rcpt, sx->peer_limit_rcpt);
1108 debug_printf(" RCPTDOMAINMAX %u -> %u\n", sx->ehlo_resp.limit_rcptdom, sx->peer_limit_rcptdom);
1110 invalidate_ehlo_cache_entry(sx);
1118 invalidate_ehlo_cache_entry(sx);
1119 (void) smtp_discard_responses(sx, sx->conn_args.ob, *countp);
1122 #endif /*!DISABLE_PIPE_CONNECT*/
1125 /*************************************************
1126 * Synchronize SMTP responses *
1127 *************************************************/
1129 /* This function is called from smtp_deliver() to receive SMTP responses from
1130 the server, and match them up with the commands to which they relate. When
1131 PIPELINING is not in use, this function is called after every command, and is
1132 therefore somewhat over-engineered, but it is simpler to use a single scheme
1133 that works both with and without PIPELINING instead of having two separate sets
1136 The set of commands that are buffered up with pipelining may start with MAIL
1137 and may end with DATA; in between are RCPT commands that correspond to the
1138 addresses whose status is PENDING_DEFER. All other commands (STARTTLS, AUTH,
1139 etc.) are never buffered.
1141 Errors after MAIL or DATA abort the whole process leaving the response in the
1142 buffer. After MAIL, pending responses are flushed, and the original command is
1143 re-instated in big_buffer for error messages. For RCPT commands, the remote is
1144 permitted to reject some recipient addresses while accepting others. However
1145 certain errors clearly abort the whole process. Set the value in
1146 transport_return to PENDING_OK if the address is accepted. If there is a
1147 subsequent general error, it will get reset accordingly. If not, it will get
1148 converted to OK at the end.
1151 sx smtp connection context
1152 count the number of responses to read
1153 pending_DATA 0 if last command sent was not DATA
1154 +1 if previously had a good recipient
1155 -1 if not previously had a good recipient
1157 Returns: 3 if at least one address had 2xx and one had 5xx
1158 2 if at least one address had 5xx but none had 2xx
1159 1 if at least one host had a 2xx response, but none had 5xx
1160 0 no address had 2xx or 5xx but no errors (all 4xx, or just DATA)
1161 -1 timeout while reading RCPT response
1162 -2 I/O or other non-response error for RCPT
1163 -3 DATA or MAIL failed - errno and buffer set
1164 -4 banner or EHLO failed (early-pipelining)
1165 -5 banner or EHLO failed (early-pipelining, TLS)
1169 sync_responses(smtp_context * sx, int count, int pending_DATA)
1171 address_item * addr = sx->sync_addr;
1172 smtp_transport_options_block * ob = sx->conn_args.ob;
1175 #ifndef DISABLE_PIPE_CONNECT
1177 if ((rc = smtp_reap_early_pipe(sx, &count)) != OK)
1178 return rc == FAIL ? -4 : -5;
1181 /* Handle the response for a MAIL command. On error, reinstate the original
1182 command in big_buffer for error message use, and flush any further pending
1183 responses before returning, except after I/O errors and timeouts. */
1185 if (sx->pending_MAIL)
1187 DEBUG(D_transport) debug_printf("%s expect mail\n", __FUNCTION__);
1189 sx->pending_MAIL = sx->RCPT_452 = FALSE;
1190 if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer),
1191 '2', ob->command_timeout))
1193 DEBUG(D_transport) debug_printf("bad response for MAIL\n");
1194 Ustrcpy(big_buffer, mail_command); /* Fits, because it came from there! */
1195 if (errno == ERRNO_TLSFAILURE)
1197 if (errno == 0 && sx->buffer[0] != 0)
1200 if (sx->buffer[0] == '4')
1202 save_errno = ERRNO_MAIL4XX;
1203 addr->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
1205 count = smtp_discard_responses(sx, ob, count);
1209 if (pending_DATA) count--; /* Number of RCPT responses to come */
1210 while (count-- > 0) /* Mark any pending addrs with the host used */
1212 while (addr->transport_return != PENDING_DEFER) addr = addr->next;
1213 addr->host_used = sx->conn_args.host;
1220 if (pending_DATA) count--; /* Number of RCPT responses to come */
1222 /* Read and handle the required number of RCPT responses, matching each one up
1223 with an address by scanning for the next address whose status is PENDING_DEFER.
1228 while (addr->transport_return != PENDING_DEFER)
1229 if (!(addr = addr->next))
1232 /* The address was accepted */
1233 addr->host_used = sx->conn_args.host;
1235 DEBUG(D_transport) debug_printf("%s expect rcpt for %s\n", __FUNCTION__, addr->address);
1236 if (smtp_read_response(sx, sx->buffer, sizeof(sx->buffer),
1237 '2', ob->command_timeout))
1240 addr->transport_return = PENDING_OK;
1242 /* If af_dr_retry_exists is set, there was a routing delay on this address;
1243 ensure that any address-specific retry record is expunged. We do this both
1244 for the basic key and for the version that also includes the sender. */
1246 if (testflag(addr, af_dr_retry_exists))
1248 uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key,
1250 retry_add_item(addr, altkey, rf_delete);
1251 retry_add_item(addr, addr->address_retry_key, rf_delete);
1255 /* Error on first TLS read */
1257 else if (errno == ERRNO_TLSFAILURE)
1260 /* Timeout while reading the response */
1262 else if (errno == ETIMEDOUT)
1264 uschar *message = string_sprintf("SMTP timeout after RCPT TO:<%s>",
1265 transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes));
1266 set_errno_nohost(sx->first_addr, ETIMEDOUT, message, DEFER, FALSE, &sx->delivery_start);
1267 retry_add_item(addr, addr->address_retry_key, 0);
1268 update_waiting = FALSE;
1272 /* Handle other errors in obtaining an SMTP response by returning -1. This
1273 will cause all the addresses to be deferred. Restore the SMTP command in
1274 big_buffer for which we are checking the response, so the error message
1277 else if (errno != 0 || sx->buffer[0] == 0)
1279 gstring gs = { .size = big_buffer_size, .ptr = 0, .s = big_buffer }, * g = &gs;
1281 /* Use taint-unchecked routines for writing into big_buffer, trusting
1282 that we'll never expand it. */
1284 g = string_fmt_append_f(g, SVFMT_TAINT_NOCHK, "RCPT TO:<%s>",
1285 transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes));
1286 string_from_gstring(g);
1290 /* Handle SMTP permanent and temporary response codes. */
1295 string_sprintf("SMTP error from remote mail server after RCPT TO:<%s>: "
1296 "%s", transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes),
1297 string_printing(sx->buffer));
1298 setflag(addr, af_pass_message);
1300 msglog_line(sx->conn_args.host, addr->message);
1302 /* The response was 5xx */
1304 if (sx->buffer[0] == '5')
1306 addr->transport_return = FAIL;
1310 /* The response was 4xx */
1314 addr->transport_return = DEFER;
1315 addr->basic_errno = ERRNO_RCPT4XX;
1316 addr->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
1320 #ifndef DISABLE_EVENT
1321 event_defer_errno = addr->more_errno;
1322 msg_event_raise(US"msg:rcpt:host:defer", addr);
1324 /* If a 452 and we've had at least one 2xx or 5xx, set next_addr to the
1325 start point for another MAIL command. */
1327 if (addr->more_errno >> 8 == 52 && yield & 3)
1329 if (!sx->RCPT_452) /* initialised at MAIL-ack above */
1332 debug_printf("%s: seen first 452 too-many-rcpts\n", __FUNCTION__);
1333 sx->RCPT_452 = TRUE;
1334 sx->next_addr = addr;
1336 addr->transport_return = PENDING_DEFER;
1337 addr->basic_errno = 0;
1341 /* Log temporary errors if there are more hosts to be tried.
1342 If not, log this last one in the == line. */
1344 if (sx->conn_args.host->next)
1345 if (LOGGING(outgoing_port))
1346 log_write(0, LOG_MAIN, "H=%s [%s]:%d %s", sx->conn_args.host->name,
1347 sx->conn_args.host->address,
1348 sx->port == PORT_NONE ? 25 : sx->port, addr->message);
1350 log_write(0, LOG_MAIN, "H=%s [%s]: %s", sx->conn_args.host->name,
1351 sx->conn_args.host->address, addr->message);
1353 #ifndef DISABLE_EVENT
1355 msg_event_raise(US"msg:rcpt:defer", addr);
1358 /* Do not put this message on the list of those waiting for specific
1359 hosts, as otherwise it is likely to be tried too often. */
1361 update_waiting = FALSE;
1363 /* Add a retry item for the address so that it doesn't get tried again
1364 too soon. If address_retry_include_sender is true, add the sender address
1365 to the retry key. */
1367 retry_add_item(addr,
1368 ob->address_retry_include_sender
1369 ? string_sprintf("%s:<%s>", addr->address_retry_key, sender_address)
1370 : addr->address_retry_key,
1376 if (count && !(addr = addr->next))
1378 } /* Loop for next RCPT response */
1380 /* Update where to start at for the next block of responses, unless we
1381 have already handled all the addresses. */
1383 if (addr) sx->sync_addr = addr->next;
1385 /* Handle a response to DATA. If we have not had any good recipients, either
1386 previously or in this block, the response is ignored. */
1388 if (pending_DATA != 0)
1390 DEBUG(D_transport) debug_printf("%s expect data\n", __FUNCTION__);
1391 if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer),
1392 '3', ob->command_timeout))
1398 if (errno == ERRNO_TLSFAILURE) /* Error on first TLS read */
1401 if (pending_DATA > 0 || (yield & 1) != 0)
1403 if (errno == 0 && sx->buffer[0] == '4')
1405 errno = ERRNO_DATA4XX;
1406 sx->first_addr->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
1410 (void)check_response(sx->conn_args.host, &errno, 0, sx->buffer, &code, &msg, &pass_message);
1411 DEBUG(D_transport) debug_printf("%s\nerror for DATA ignored: pipelining "
1412 "is in use and there were no good recipients\n", msg);
1416 /* All responses read and handled; MAIL (if present) received 2xx and DATA (if
1417 present) received 3xx. If any RCPTs were handled and yielded anything other
1418 than 4xx, yield will be set non-zero. */
1427 /* Try an authenticator's client entry */
1430 try_authenticator(smtp_context * sx, auth_instance * au)
1432 smtp_transport_options_block * ob = sx->conn_args.ob; /* transport options */
1433 host_item * host = sx->conn_args.host; /* host to deliver to */
1436 sx->outblock.authenticating = TRUE;
1437 rc = (au->info->clientcode)(au, sx, ob->command_timeout,
1438 sx->buffer, sizeof(sx->buffer));
1439 sx->outblock.authenticating = FALSE;
1440 DEBUG(D_transport) debug_printf("%s authenticator yielded %d\n", au->name, rc);
1442 /* A temporary authentication failure must hold up delivery to
1443 this host. After a permanent authentication failure, we carry on
1444 to try other authentication methods. If all fail hard, try to
1445 deliver the message unauthenticated unless require_auth was set. */
1450 f.smtp_authenticated = TRUE; /* stops the outer loop */
1451 client_authenticator = au->name;
1452 if (au->set_client_id)
1453 client_authenticated_id = expand_string(au->set_client_id);
1456 /* Failure after writing a command */
1461 /* Failure after reading a response */
1464 if (errno != 0 || sx->buffer[0] != '5') return FAIL;
1465 log_write(0, LOG_MAIN, "%s authenticator failed H=%s [%s] %s",
1466 au->name, host->name, host->address, sx->buffer);
1469 /* Failure by some other means. In effect, the authenticator
1470 decided it wasn't prepared to handle this case. Typically this
1471 is the result of "fail" in an expansion string. Do we need to
1472 log anything here? Feb 2006: a message is now put in the buffer
1473 if logging is required. */
1476 if (*sx->buffer != 0)
1477 log_write(0, LOG_MAIN, "%s authenticator cancelled "
1478 "authentication H=%s [%s] %s", au->name, host->name,
1479 host->address, sx->buffer);
1482 /* Internal problem, message in buffer. */
1485 set_errno_nohost(sx->addrlist, ERRNO_AUTHPROB, string_copy(sx->buffer),
1486 DEFER, FALSE, &sx->delivery_start);
1495 /* Do the client side of smtp-level authentication.
1498 sx smtp connection context
1500 sx->buffer should have the EHLO response from server (gets overwritten)
1503 OK Success, or failed (but not required): global "smtp_authenticated" set
1504 DEFER Failed authentication (and was required)
1505 ERROR Internal problem
1507 FAIL_SEND Failed communications - transmit
1512 smtp_auth(smtp_context * sx)
1514 host_item * host = sx->conn_args.host; /* host to deliver to */
1515 smtp_transport_options_block * ob = sx->conn_args.ob; /* transport options */
1516 int require_auth = verify_check_given_host(CUSS &ob->hosts_require_auth, host);
1517 #ifndef DISABLE_PIPE_CONNECT
1518 unsigned short authbits = tls_out.active.sock >= 0
1519 ? sx->ehlo_resp.crypted_auths : sx->ehlo_resp.cleartext_auths;
1521 uschar * fail_reason = US"server did not advertise AUTH support";
1523 f.smtp_authenticated = FALSE;
1524 client_authenticator = client_authenticated_id = client_authenticated_sender = NULL;
1527 regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE);
1529 /* Is the server offering AUTH? */
1533 #ifndef DISABLE_PIPE_CONNECT
1534 sx->early_pipe_active ? authbits
1537 regex_match_and_setup(regex_AUTH, sx->buffer, 0, -1)
1540 uschar * names = NULL;
1541 expand_nmax = -1; /* reset */
1543 #ifndef DISABLE_PIPE_CONNECT
1544 if (!sx->early_pipe_active)
1546 names = string_copyn(expand_nstring[1], expand_nlength[1]);
1548 /* Must not do this check until after we have saved the result of the
1549 regex match above as the check could be another RE. */
1551 if ( require_auth == OK
1552 || verify_check_given_host(CUSS &ob->hosts_try_auth, host) == OK)
1554 DEBUG(D_transport) debug_printf("scanning authentication mechanisms\n");
1555 fail_reason = US"no common mechanisms were found";
1557 #ifndef DISABLE_PIPE_CONNECT
1558 if (sx->early_pipe_active)
1560 /* Scan our authenticators (which support use by a client and were offered
1561 by the server (checked at cache-write time)), not suppressed by
1562 client_condition. If one is found, attempt to authenticate by calling its
1563 client function. We are limited to supporting up to 16 authenticator
1564 public-names by the number of bits in a short. */
1570 for (bitnum = 0, au = auths;
1571 !f.smtp_authenticated && au && bitnum < 16;
1572 bitnum++, au = au->next) if (authbits & BIT(bitnum))
1574 if ( au->client_condition
1575 && !expand_check_condition(au->client_condition, au->name,
1576 US"client authenticator"))
1578 DEBUG(D_transport) debug_printf("skipping %s authenticator: %s\n",
1579 au->name, "client_condition is false");
1583 /* Found data for a listed mechanism. Call its client entry. Set
1584 a flag in the outblock so that data is overwritten after sending so
1585 that reflections don't show it. */
1587 fail_reason = US"authentication attempt(s) failed";
1589 if ((rc = try_authenticator(sx, au)) != OK)
1596 /* Scan the configured authenticators looking for one which is configured
1597 for use as a client, which is not suppressed by client_condition, and
1598 whose name matches an authentication mechanism supported by the server.
1599 If one is found, attempt to authenticate by calling its client function.
1602 for (auth_instance * au = auths; !f.smtp_authenticated && au; au = au->next)
1607 || ( au->client_condition
1608 && !expand_check_condition(au->client_condition, au->name,
1609 US"client authenticator")))
1611 DEBUG(D_transport) debug_printf("skipping %s authenticator: %s\n",
1613 (au->client)? "client_condition is false" :
1614 "not configured as a client");
1618 /* Loop to scan supported server mechanisms */
1622 int len = Ustrlen(au->public_name);
1625 while (isspace(*p)) p++;
1627 if (strncmpic(au->public_name, p, len) != 0 ||
1628 (p[len] != 0 && !isspace(p[len])))
1630 while (*p != 0 && !isspace(*p)) p++;
1634 /* Found data for a listed mechanism. Call its client entry. Set
1635 a flag in the outblock so that data is overwritten after sending so
1636 that reflections don't show it. */
1638 fail_reason = US"authentication attempt(s) failed";
1640 if ((rc = try_authenticator(sx, au)) != OK)
1643 break; /* If not authenticated, try next authenticator */
1644 } /* Loop for scanning supported server mechanisms */
1645 } /* Loop for further authenticators */
1649 /* If we haven't authenticated, but are required to, give up. */
1651 if (require_auth == OK && !f.smtp_authenticated)
1653 #ifndef DISABLE_PIPE_CONNECT
1654 invalidate_ehlo_cache_entry(sx);
1656 set_errno_nohost(sx->addrlist, ERRNO_AUTHFAIL,
1657 string_sprintf("authentication required but %s", fail_reason), DEFER,
1658 FALSE, &sx->delivery_start);
1666 /* Construct AUTH appendix string for MAIL TO */
1669 sx context for smtp connection
1670 p point in sx->buffer to build string
1671 addrlist chain of potential addresses to deliver
1673 Globals f.smtp_authenticated
1674 client_authenticated_sender
1675 Return True on error, otherwise buffer has (possibly empty) terminated string
1679 smtp_mail_auth_str(smtp_context * sx, uschar * p, address_item * addrlist)
1681 smtp_transport_options_block * ob = sx->conn_args.ob;
1682 uschar * local_authenticated_sender = authenticated_sender;
1685 debug_printf("smtp_mail_auth_str: as<%s> os<%s> SA<%s>\n",
1686 authenticated_sender, ob->authenticated_sender, f.smtp_authenticated?"Y":"N");
1689 if (ob->authenticated_sender)
1691 uschar * new = expand_string(ob->authenticated_sender);
1694 if (!f.expand_string_forcedfail)
1696 uschar *message = string_sprintf("failed to expand "
1697 "authenticated_sender: %s", expand_string_message);
1698 set_errno_nohost(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
1703 local_authenticated_sender = new;
1706 /* Add the authenticated sender address if present */
1708 if ( (f.smtp_authenticated || ob->authenticated_sender_force)
1709 && local_authenticated_sender)
1711 string_format_nt(p, sizeof(sx->buffer) - (p-sx->buffer), " AUTH=%s",
1712 auth_xtextencode(local_authenticated_sender,
1713 Ustrlen(local_authenticated_sender)));
1714 client_authenticated_sender = string_copy(local_authenticated_sender);
1724 typedef struct smtp_compare_s
1726 uschar * current_sender_address;
1727 struct transport_instance * tblock;
1731 /* Create a unique string that identifies this message, it is based on
1732 sender_address, helo_data and tls_certificate if enabled.
1736 smtp_local_identity(uschar * sender, struct transport_instance * tblock)
1738 address_item * addr1;
1739 uschar * if1 = US"";
1740 uschar * helo1 = US"";
1742 uschar * tlsc1 = US"";
1744 uschar * save_sender_address = sender_address;
1745 uschar * local_identity = NULL;
1746 smtp_transport_options_block * ob = SOB tblock->options_block;
1748 sender_address = sender;
1750 addr1 = deliver_make_addr (sender, TRUE);
1751 deliver_set_expansions(addr1);
1754 if1 = expand_string(ob->interface);
1757 helo1 = expand_string(ob->helo_data);
1760 if (ob->tls_certificate)
1761 tlsc1 = expand_string(ob->tls_certificate);
1762 local_identity = string_sprintf ("%s^%s^%s", if1, helo1, tlsc1);
1764 local_identity = string_sprintf ("%s^%s", if1, helo1);
1767 deliver_set_expansions(NULL);
1768 sender_address = save_sender_address;
1770 return local_identity;
1775 /* This routine is a callback that is called from transport_check_waiting.
1776 This function will evaluate the incoming message versus the previous
1777 message. If the incoming message is using a different local identity then
1778 we will veto this new message. */
1781 smtp_are_same_identities(uschar * message_id, smtp_compare_t * s_compare)
1783 uschar * message_local_identity,
1784 * current_local_identity,
1785 * new_sender_address;
1787 current_local_identity =
1788 smtp_local_identity(s_compare->current_sender_address, s_compare->tblock);
1790 if (!(new_sender_address = spool_sender_from_msgid(message_id)))
1794 message_local_identity =
1795 smtp_local_identity(new_sender_address, s_compare->tblock);
1797 return Ustrcmp(current_local_identity, message_local_identity) == 0;
1803 ehlo_response(uschar * buf, unsigned checks)
1805 PCRE2_SIZE bsize = Ustrlen(buf);
1806 pcre2_match_data * md = pcre2_match_data_create(1, pcre_gen_ctx);
1808 /* debug_printf("%s: check for 0x%04x\n", __FUNCTION__, checks); */
1811 if ( checks & OPTION_TLS
1812 && pcre2_match(regex_STARTTLS,
1813 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1815 checks &= ~OPTION_TLS;
1817 if ( checks & OPTION_IGNQ
1818 && pcre2_match(regex_IGNOREQUOTA,
1819 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1820 checks &= ~OPTION_IGNQ;
1822 if ( checks & OPTION_CHUNKING
1823 && pcre2_match(regex_CHUNKING,
1824 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1825 checks &= ~OPTION_CHUNKING;
1827 #ifndef DISABLE_PRDR
1828 if ( checks & OPTION_PRDR
1829 && pcre2_match(regex_PRDR,
1830 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1832 checks &= ~OPTION_PRDR;
1835 if ( checks & OPTION_UTF8
1836 && pcre2_match(regex_UTF8,
1837 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1839 checks &= ~OPTION_UTF8;
1841 if ( checks & OPTION_DSN
1842 && pcre2_match(regex_DSN,
1843 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1844 checks &= ~OPTION_DSN;
1846 if ( checks & OPTION_PIPE
1847 && pcre2_match(regex_PIPELINING,
1848 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1849 checks &= ~OPTION_PIPE;
1851 if ( checks & OPTION_SIZE
1852 && pcre2_match(regex_SIZE,
1853 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1854 checks &= ~OPTION_SIZE;
1856 #ifndef DISABLE_PIPE_CONNECT
1857 if ( checks & OPTION_EARLY_PIPE
1858 && pcre2_match(regex_EARLY_PIPE,
1859 (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_mtc_ctx) < 0)
1861 checks &= ~OPTION_EARLY_PIPE;
1863 pcre2_match_data_free(md);
1864 /* debug_printf("%s: found 0x%04x\n", __FUNCTION__, checks); */
1870 /* Callback for emitting a BDAT data chunk header.
1872 If given a nonzero size, first flush any buffered SMTP commands
1873 then emit the command.
1875 Reap previous SMTP command responses if requested, and always reap
1876 the response from a previous BDAT command.
1879 tctx transport context
1880 chunk_size value for SMTP BDAT command
1882 tc_chunk_last add LAST option to SMTP BDAT command
1883 tc_reap_prev reap response to previous SMTP commands
1887 DEFER TLS error on first read (EHLO-resp); errno set
1891 smtp_chunk_cmd_callback(transport_ctx * tctx, unsigned chunk_size,
1894 smtp_transport_options_block * ob = SOB tctx->tblock->options_block;
1895 smtp_context * sx = tctx->smtp_context;
1899 /* Write SMTP chunk header command. If not reaping responses, note that
1900 there may be more writes (like, the chunk data) done soon. */
1904 #ifndef DISABLE_PIPE_CONNECT
1905 BOOL new_conn = !!(sx->outblock.conn_args);
1907 if((cmd_count = smtp_write_command(sx,
1908 flags & tc_reap_prev ? SCMD_FLUSH : SCMD_MORE,
1909 "BDAT %u%s\r\n", chunk_size, flags & tc_chunk_last ? " LAST" : "")
1910 ) < 0) return ERROR;
1911 if (flags & tc_chunk_last)
1912 data_command = string_copy(big_buffer); /* Save for later error message */
1913 #ifndef DISABLE_PIPE_CONNECT
1914 /* That command write could have been the one that made the connection.
1915 Copy the fd from the client conn ctx (smtp transport specific) to the
1916 generic transport ctx. */
1919 tctx->u.fd = sx->outblock.cctx->sock;
1923 prev_cmd_count = cmd_count += sx->cmd_count;
1925 /* Reap responses for any previous, but not one we just emitted */
1929 if (sx->pending_BDAT)
1932 if (flags & tc_reap_prev && prev_cmd_count > 0)
1934 DEBUG(D_transport) debug_printf("look for %d responses"
1935 " for previous pipelined cmds\n", prev_cmd_count);
1937 switch(sync_responses(sx, prev_cmd_count, 0))
1939 case 1: /* 2xx (only) => OK */
1940 case 3: sx->good_RCPT = TRUE; /* 2xx & 5xx => OK & progress made */
1941 case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */
1942 case 0: break; /* No 2xx or 5xx, but no probs */
1944 case -5: errno = ERRNO_TLSFAILURE;
1946 #ifndef DISABLE_PIPE_CONNECT
1947 case -4: /* non-2xx for pipelined banner or EHLO */
1949 case -1: /* Timeout on RCPT */
1950 default: return ERROR; /* I/O error, or any MAIL/DATA error */
1953 if (!sx->pending_BDAT)
1954 pipelining_active = FALSE;
1957 /* Reap response for an outstanding BDAT */
1959 if (sx->pending_BDAT)
1961 DEBUG(D_transport) debug_printf("look for one response for BDAT\n");
1963 if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
1964 ob->command_timeout))
1966 if (errno == 0 && sx->buffer[0] == '4')
1968 errno = ERRNO_DATA4XX; /*XXX does this actually get used? */
1969 sx->addrlist->more_errno |=
1970 ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
1975 sx->pending_BDAT = FALSE;
1976 pipelining_active = FALSE;
1978 else if (chunk_size > 0)
1979 sx->pending_BDAT = TRUE;
1982 sx->cmd_count = cmd_count;
1988 /*************************************************
1989 * Make connection for given message *
1990 *************************************************/
1994 sx connection context
1995 suppress_tls if TRUE, don't attempt a TLS connection - this is set for
1996 a second attempt after TLS initialization fails
1998 Returns: OK - the connection was made and the delivery attempted;
1999 fd is set in the conn context, tls_out set up.
2000 DEFER - the connection could not be made, or something failed
2001 while setting up the SMTP session, or there was a
2002 non-message-specific error, such as a timeout.
2003 ERROR - helo_data or add_headers or authenticated_sender is
2004 specified for this transport, and the string failed
2008 smtp_setup_conn(smtp_context * sx, BOOL suppress_tls)
2010 smtp_transport_options_block * ob = sx->conn_args.tblock->options_block;
2011 BOOL pass_message = FALSE;
2012 uschar * message = NULL;
2015 uschar * tls_errstr;
2018 sx->conn_args.ob = ob;
2020 sx->lmtp = strcmpic(ob->protocol, US"lmtp") == 0;
2021 sx->smtps = strcmpic(ob->protocol, US"smtps") == 0;
2022 /* sx->ok = FALSE; */
2023 sx->send_rset = TRUE;
2024 sx->send_quit = TRUE;
2025 sx->setting_up = TRUE;
2027 /* sx->esmtp_sent = FALSE; */
2029 /* sx->utf8_needed = FALSE; */
2031 sx->dsn_all_lasthop = TRUE;
2033 /* sx->conn_args.dane = FALSE; */
2035 verify_check_given_host(CUSS &ob->hosts_require_dane, sx->conn_args.host) == OK;
2037 #ifndef DISABLE_PIPE_CONNECT
2038 /* sx->early_pipe_active = sx->early_pipe_ok = FALSE; */
2039 /* sx->ehlo_resp.cleartext_features = sx->ehlo_resp.crypted_features = 0; */
2040 /* sx->pending_BANNER = sx->pending_EHLO = sx->pending_MAIL = FALSE; */
2043 if ((sx->max_mail = sx->conn_args.tblock->connection_max_messages) == 0) sx->max_mail = 999999;
2044 if ((sx->max_rcpt = sx->conn_args.tblock->max_addresses) == 0) sx->max_rcpt = 999999;
2045 /* sx->peer_offered = 0; */
2046 /* sx->avoid_option = 0; */
2047 sx->igquotstr = US"";
2048 if (!sx->helo_data) sx->helo_data = ob->helo_data;
2049 #ifdef EXPERIMENTAL_DSN_INFO
2050 /* sx->smtp_greeting = NULL; */
2051 /* sx->helo_response = NULL; */
2054 smtp_command = US"initial connection";
2055 /* sx->buffer[0] = '\0'; */
2057 /* Set up the buffer for reading SMTP response packets. */
2059 sx->inblock.buffer = sx->inbuffer;
2060 sx->inblock.buffersize = sizeof(sx->inbuffer);
2061 sx->inblock.ptr = sx->inbuffer;
2062 sx->inblock.ptrend = sx->inbuffer;
2064 /* Set up the buffer for holding SMTP commands while pipelining */
2066 sx->outblock.buffer = sx->outbuffer;
2067 sx->outblock.buffersize = sizeof(sx->outbuffer);
2068 sx->outblock.ptr = sx->outbuffer;
2069 /* sx->outblock.cmd_count = 0; */
2070 /* sx->outblock.authenticating = FALSE; */
2071 /* sx->outblock.conn_args = NULL; */
2073 /* Reset the parameters of a TLS session. */
2076 tls_out.cipher = NULL; /* the one we may use for this transport */
2077 tls_out.ourcert = NULL;
2078 tls_out.peercert = NULL;
2079 tls_out.peerdn = NULL;
2083 tls_out.ocsp = OCSP_NOT_REQ;
2084 #ifndef DISABLE_TLS_RESUME
2085 tls_out.resumption = 0;
2089 /* Flip the legacy TLS-related variables over to the outbound set in case
2090 they're used in the context of the transport. Don't bother resetting
2091 afterward (when being used by a transport) as we're in a subprocess.
2092 For verify, unflipped once the callout is dealt with */
2094 tls_modify_variables(&tls_out);
2099 set_errno_nohost(sx->addrlist, ERRNO_TLSFAILURE, US"TLS support not available",
2100 DEFER, FALSE, &sx->delivery_start);
2105 /* If we have a proxied TLS connection, check usability for this message */
2107 if (continue_hostname && continue_proxy_cipher)
2110 const uschar * sni = US"";
2112 # ifdef SUPPORT_DANE
2113 /* Check if the message will be DANE-verified; if so force its SNI */
2115 tls_out.dane_verified = FALSE;
2116 smtp_port_for_connect(sx->conn_args.host, sx->port);
2117 if ( sx->conn_args.host->dnssec == DS_YES
2118 && ( sx->dane_required
2119 || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK
2121 switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required))
2123 case OK: sx->conn_args.dane = TRUE;
2124 ob->tls_tempfail_tryclear = FALSE; /* force TLS */
2125 ob->tls_sni = sx->conn_args.host->name; /* force SNI */
2127 case FAIL_FORCED: break;
2128 default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER,
2129 string_sprintf("DANE error: tlsa lookup %s",
2131 rc, FALSE, &sx->delivery_start);
2132 # ifndef DISABLE_EVENT
2133 (void) event_raise(sx->conn_args.tblock->event_action,
2134 US"dane:fail", sx->dane_required
2135 ? US"dane-required" : US"dnssec-invalid",
2142 /* If the SNI or the DANE status required for the new message differs from the
2143 existing conn drop the connection to force a new one. */
2145 if (ob->tls_sni && !(sni = expand_cstring(ob->tls_sni)))
2146 log_write(0, LOG_MAIN|LOG_PANIC,
2147 "<%s>: failed to expand transport's tls_sni value: %s",
2148 sx->addrlist->address, expand_string_message);
2150 # ifdef SUPPORT_DANE
2151 if ( (continue_proxy_sni ? (Ustrcmp(continue_proxy_sni, sni) == 0) : !*sni)
2152 && continue_proxy_dane == sx->conn_args.dane)
2154 tls_out.sni = US sni;
2155 if ((tls_out.dane_verified = continue_proxy_dane))
2156 sx->conn_args.host->dnssec = DS_YES;
2159 if ((continue_proxy_sni ? (Ustrcmp(continue_proxy_sni, sni) == 0) : !*sni))
2160 tls_out.sni = US sni;
2165 debug_printf("Closing proxied-TLS connection due to SNI mismatch\n");
2167 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> QUIT\n");
2168 write(0, "QUIT\r\n", 6);
2170 continue_hostname = continue_proxy_cipher = NULL;
2171 f.continue_more = FALSE;
2172 continue_sequence = 1; /* Unfortunately, this process cannot affect success log
2173 which is done by delivery proc. Would have to pass this
2174 back through reporting pipe. */
2177 #endif /*!DISABLE_TLS*/
2179 /* Make a connection to the host if this isn't a continued delivery, and handle
2180 the initial interaction and HELO/EHLO/LHLO. Connect timeout errors are handled
2181 specially so they can be identified for retries. */
2183 if (!continue_hostname)
2186 HDEBUG(D_verify) debug_printf("interface=%s port=%d\n", sx->conn_args.interface, sx->port);
2188 /* Arrange to report to calling process this is a new connection */
2190 clearflag(sx->first_addr, af_cont_conn);
2191 setflag(sx->first_addr, af_new_conn);
2193 /* Get the actual port the connection will use, into sx->conn_args.host */
2195 smtp_port_for_connect(sx->conn_args.host, sx->port);
2198 /* Do TLSA lookup for DANE */
2200 tls_out.dane_verified = FALSE;
2201 tls_out.tlsa_usage = 0;
2203 if (sx->conn_args.host->dnssec == DS_YES)
2206 if( sx->dane_required
2207 || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK
2209 switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required))
2211 case OK: sx->conn_args.dane = TRUE;
2212 ob->tls_tempfail_tryclear = FALSE; /* force TLS */
2213 ob->tls_sni = sx->conn_args.host->name; /* force SNI */
2215 case FAIL_FORCED: break;
2216 default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER,
2217 string_sprintf("DANE error: tlsa lookup %s",
2219 rc, FALSE, &sx->delivery_start);
2220 # ifndef DISABLE_EVENT
2221 (void) event_raise(sx->conn_args.tblock->event_action,
2222 US"dane:fail", sx->dane_required
2223 ? US"dane-required" : US"dnssec-invalid",
2229 else if (sx->dane_required)
2231 set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER,
2232 string_sprintf("DANE error: %s lookup not DNSSEC", sx->conn_args.host->name),
2233 FAIL, FALSE, &sx->delivery_start);
2234 # ifndef DISABLE_EVENT
2235 (void) event_raise(sx->conn_args.tblock->event_action,
2236 US"dane:fail", US"dane-required", NULL);
2243 /* Make the TCP connection */
2245 sx->cctx.tls_ctx = NULL;
2246 sx->inblock.cctx = sx->outblock.cctx = &sx->cctx;
2247 #ifdef EXPERIMENTAL_ESMTP_LIMITS
2248 sx->peer_limit_mail = sx->peer_limit_rcpt = sx->peer_limit_rcptdom =
2250 sx->avoid_option = sx->peer_offered = smtp_peer_options = 0;
2252 #ifndef DISABLE_PIPE_CONNECT
2253 if ( verify_check_given_host(CUSS &ob->hosts_pipe_connect,
2254 sx->conn_args.host) == OK)
2256 /* We don't find out the local ip address until the connect, so if
2257 the helo string might use it avoid doing early-pipelining. */
2260 || !Ustrstr(sx->helo_data, "$sending_ip_address")
2261 || Ustrstr(sx->helo_data, "def:sending_ip_address")
2264 sx->early_pipe_ok = TRUE;
2265 if ( read_ehlo_cache_entry(sx)
2266 && sx->ehlo_resp.cleartext_features & OPTION_EARLY_PIPE)
2269 debug_printf("Using cached cleartext PIPECONNECT\n");
2270 sx->early_pipe_active = TRUE;
2271 sx->peer_offered = sx->ehlo_resp.cleartext_features;
2274 else DEBUG(D_transport)
2275 debug_printf("helo needs $sending_ip_address\n");
2278 if (sx->early_pipe_active)
2279 sx->outblock.conn_args = &sx->conn_args;
2283 blob lazy_conn = {.data = NULL};
2284 /* For TLS-connect, a TFO lazy-connect is useful since the Client Hello
2285 can go on the TCP SYN. */
2287 if ((sx->cctx.sock = smtp_connect(&sx->conn_args,
2288 sx->smtps ? &lazy_conn : NULL)) < 0)
2290 set_errno_nohost(sx->addrlist,
2291 errno == ETIMEDOUT ? ERRNO_CONNECTTIMEOUT : errno,
2292 sx->verify ? US strerror(errno) : NULL,
2293 DEFER, FALSE, &sx->delivery_start);
2294 sx->send_quit = FALSE;
2298 (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, TCP_QUICKACK, US &off,
2302 /* Expand the greeting message while waiting for the initial response. (Makes
2303 sense if helo_data contains ${lookup dnsdb ...} stuff). The expansion is
2304 delayed till here so that $sending_interface and $sending_port are set. */
2305 /*XXX early-pipe: they still will not be. Is there any way to find out what they
2306 will be? Somehow I doubt it. */
2309 if (!(sx->helo_data = expand_string(sx->helo_data)))
2311 log_write(0, LOG_MAIN|LOG_PANIC,
2312 "<%s>: failed to expand transport's helo_data value for callout: %s",
2313 sx->addrlist->address, expand_string_message);
2318 expand_string_message = NULL;
2319 if ((sx->helo_data = string_domain_utf8_to_alabel(sx->helo_data,
2320 &expand_string_message)),
2321 expand_string_message)
2323 log_write(0, LOG_MAIN|LOG_PANIC,
2324 "<%s>: failed to expand transport's helo_data value for callout: %s",
2325 sx->addrlist->address, expand_string_message);
2327 sx->helo_data = NULL;
2331 /* The first thing is to wait for an initial OK response. The dreaded "goto"
2332 is nevertheless a reasonably clean way of programming this kind of logic,
2333 where you want to escape on any error. */
2337 #ifndef DISABLE_PIPE_CONNECT
2338 if (sx->early_pipe_active)
2340 sx->pending_BANNER = TRUE; /* sync_responses() must eventually handle */
2341 sx->outblock.cmd_count = 1;
2346 if (!smtp_reap_banner(sx))
2347 goto RESPONSE_FAILED;
2350 #ifndef DISABLE_EVENT
2353 lookup_dnssec_authenticated = sx->conn_args.host->dnssec==DS_YES ? US"yes"
2354 : sx->conn_args.host->dnssec==DS_NO ? US"no" : NULL;
2355 s = event_raise(sx->conn_args.tblock->event_action, US"smtp:connect", sx->buffer, NULL);
2358 set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL,
2359 string_sprintf("deferred by smtp:connect event expansion: %s", s),
2360 DEFER, FALSE, &sx->delivery_start);
2367 /* Now check if the helo_data expansion went well, and sign off cleanly if
2372 message = string_sprintf("failed to expand helo_data: %s",
2373 expand_string_message);
2374 set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
2380 /** Debugging without sending a message
2381 sx->addrlist->transport_return = DEFER;
2385 /* Errors that occur after this point follow an SMTP command, which is
2386 left in big_buffer by smtp_write_command() for use in error messages. */
2388 smtp_command = big_buffer;
2390 /* Tell the remote who we are...
2392 February 1998: A convention has evolved that ESMTP-speaking MTAs include the
2393 string "ESMTP" in their greeting lines, so make Exim send EHLO if the
2394 greeting is of this form. The assumption was that the far end supports it
2395 properly... but experience shows that there are some that give 5xx responses,
2396 even though the banner includes "ESMTP" (there's a bloody-minded one that
2397 says "ESMTP not spoken here"). Cope with that case.
2399 September 2000: Time has passed, and it seems reasonable now to always send
2400 EHLO at the start. It is also convenient to make the change while installing
2403 July 2003: Joachim Wieland met a broken server that advertises "PIPELINING"
2404 but times out after sending MAIL FROM, RCPT TO and DATA all together. There
2405 would be no way to send out the mails, so there is now a host list
2406 "hosts_avoid_esmtp" that disables ESMTP for special hosts and solves the
2407 PIPELINING problem as well. Maybe it can also be useful to cure other
2408 problems with broken servers.
2410 Exim originally sent "Helo" at this point and ran for nearly a year that way.
2411 Then somebody tried it with a Microsoft mailer... It seems that all other
2412 mailers use upper case for some reason (the RFC is quite clear about case
2413 independence) so, for peace of mind, I gave in. */
2415 sx->esmtp = verify_check_given_host(CUSS &ob->hosts_avoid_esmtp, sx->conn_args.host) != OK;
2417 /* Alas; be careful, since this goto is not an error-out, so conceivably
2418 we might set data between here and the target which we assume to exist
2419 and be usable. I can see this coming back to bite us. */
2423 smtp_peer_options |= OPTION_TLS;
2424 suppress_tls = FALSE;
2425 ob->tls_tempfail_tryclear = FALSE;
2426 smtp_command = US"SSL-on-connect";
2433 if (smtp_write_command(sx,
2434 #ifndef DISABLE_PIPE_CONNECT
2435 sx->early_pipe_active ? SCMD_BUFFER :
2438 "%s %s\r\n", sx->lmtp ? "LHLO" : "EHLO", sx->helo_data) < 0)
2440 sx->esmtp_sent = TRUE;
2442 #ifndef DISABLE_PIPE_CONNECT
2443 if (sx->early_pipe_active)
2445 sx->pending_EHLO = TRUE;
2447 /* If we have too many authenticators to handle and might need to AUTH
2448 for this transport, pipeline no further as we will need the
2449 list of auth methods offered. Reap the banner and EHLO. */
2451 if ( (ob->hosts_require_auth || ob->hosts_try_auth)
2452 && f.smtp_in_early_pipe_no_auth)
2454 DEBUG(D_transport) debug_printf("may need to auth, so pipeline no further\n");
2455 if (smtp_write_command(sx, SCMD_FLUSH, NULL) < 0)
2457 if (sync_responses(sx, 2, 0) != 0)
2460 debug_printf("failed reaping pipelined cmd responses\n");
2461 goto RESPONSE_FAILED;
2463 sx->early_pipe_active = FALSE;
2468 if (!smtp_reap_ehlo(sx))
2469 goto RESPONSE_FAILED;
2473 debug_printf("not sending EHLO (host matches hosts_avoid_esmtp)\n");
2475 #ifndef DISABLE_PIPE_CONNECT
2476 if (!sx->early_pipe_active)
2481 int n = sizeof(sx->buffer);
2482 uschar * rsp = sx->buffer;
2484 if (sx->esmtp_sent && (n = Ustrlen(sx->buffer) + 1) < sizeof(sx->buffer)/2)
2485 { rsp = sx->buffer + n; n = sizeof(sx->buffer) - n; }
2487 if (smtp_write_command(sx, SCMD_FLUSH, "HELO %s\r\n", sx->helo_data) < 0)
2489 good_response = smtp_read_response(sx, rsp, n, '2', ob->command_timeout);
2490 #ifdef EXPERIMENTAL_DSN_INFO
2491 sx->helo_response = string_copy(rsp);
2495 /* Handle special logging for a closed connection after HELO
2496 when had previously sent EHLO */
2498 if (rsp != sx->buffer && rsp[0] == 0 && (errno == 0 || errno == ECONNRESET))
2500 errno = ERRNO_SMTPCLOSED;
2501 goto EHLOHELO_FAILED;
2503 memmove(sx->buffer, rsp, Ustrlen(rsp));
2504 goto RESPONSE_FAILED;
2508 if (sx->esmtp || sx->lmtp)
2510 #ifndef DISABLE_PIPE_CONNECT
2511 if (!sx->early_pipe_active)
2514 sx->peer_offered = ehlo_response(sx->buffer,
2515 OPTION_TLS /* others checked later */
2516 #ifndef DISABLE_PIPE_CONNECT
2517 | (sx->early_pipe_ok
2519 | OPTION_CHUNKING | OPTION_PRDR | OPTION_DSN | OPTION_PIPE | OPTION_SIZE
2528 #ifdef EXPERIMENTAL_ESMTP_LIMITS
2529 if (tls_out.active.sock >= 0 || !(sx->peer_offered & OPTION_TLS))
2531 ehlo_response_limits_read(sx);
2532 ehlo_response_limits_apply(sx);
2535 #ifndef DISABLE_PIPE_CONNECT
2536 if (sx->early_pipe_ok)
2538 sx->ehlo_resp.cleartext_features = sx->peer_offered;
2540 if ( (sx->peer_offered & (OPTION_PIPE | OPTION_EARLY_PIPE))
2541 == (OPTION_PIPE | OPTION_EARLY_PIPE))
2543 DEBUG(D_transport) debug_printf("PIPE_CONNECT usable in future for this IP\n");
2544 sx->ehlo_resp.cleartext_auths = study_ehlo_auths(sx);
2545 write_ehlo_cache_entry(sx);
2551 /* Set tls_offered if the response to EHLO specifies support for STARTTLS. */
2554 smtp_peer_options |= sx->peer_offered & OPTION_TLS;
2559 /* For continuing deliveries down the same channel, having re-exec'd the socket
2560 is the standard input; for a socket held open from verify it is recorded
2561 in the cutthrough context block. Either way we don't need to redo EHLO here
2562 (but may need to do so for TLS - see below).
2563 Set up the pointer to where subsequent commands will be left, for
2564 error messages. Note that smtp_peer_options will have been
2565 set from the command line if they were set in the process that passed the
2568 /*XXX continue case needs to propagate DSN_INFO, prob. in deliver.c
2569 as the continue goes via transport_pass_socket() and doublefork and exec.
2570 It does not wait. Unclear how we keep separate host's responses
2571 separate - we could match up by host ip+port as a bodge. */
2575 if (cutthrough.cctx.sock >= 0 && cutthrough.callout_hold_only)
2577 sx->cctx = cutthrough.cctx;
2578 sx->conn_args.host->port = sx->port = cutthrough.host.port;
2582 sx->cctx.sock = 0; /* stdin */
2583 sx->cctx.tls_ctx = NULL;
2584 smtp_port_for_connect(sx->conn_args.host, sx->port); /* Record the port that was used */
2586 sx->inblock.cctx = sx->outblock.cctx = &sx->cctx;
2587 smtp_command = big_buffer;
2588 sx->peer_offered = smtp_peer_options;
2589 #ifdef EXPERIMENTAL_ESMTP_LIMITS
2590 /* Limits passed by cmdline over exec. */
2591 ehlo_limits_apply(sx,
2592 sx->peer_limit_mail = continue_limit_mail,
2593 sx->peer_limit_rcpt = continue_limit_rcpt,
2594 sx->peer_limit_rcptdom = continue_limit_rcptdom);
2596 sx->helo_data = NULL; /* ensure we re-expand ob->helo_data */
2598 /* For a continued connection with TLS being proxied for us, or a
2599 held-open verify connection with TLS, nothing more to do. */
2601 if ( continue_proxy_cipher
2602 || (cutthrough.cctx.sock >= 0 && cutthrough.callout_hold_only
2603 && cutthrough.is_tls)
2606 sx->pipelining_used = pipelining_active = !!(smtp_peer_options & OPTION_PIPE);
2607 HDEBUG(D_transport) debug_printf("continued connection, %s TLS\n",
2608 continue_proxy_cipher ? "proxied" : "verify conn with");
2611 HDEBUG(D_transport) debug_printf("continued connection, no TLS\n");
2614 /* If TLS is available on this connection, whether continued or not, attempt to
2615 start up a TLS session, unless the host is in hosts_avoid_tls. If successful,
2616 send another EHLO - the server may give a different answer in secure mode. We
2617 use a separate buffer for reading the response to STARTTLS so that if it is
2618 negative, the original EHLO data is available for subsequent analysis, should
2619 the client not be required to use TLS. If the response is bad, copy the buffer
2620 for error analysis. */
2623 if ( smtp_peer_options & OPTION_TLS
2625 && verify_check_given_host(CUSS &ob->hosts_avoid_tls, sx->conn_args.host) != OK
2627 || verify_check_given_host(CUSS &ob->hosts_verify_avoid_tls, sx->conn_args.host) != OK
2630 uschar buffer2[4096];
2632 if (smtp_write_command(sx, SCMD_FLUSH, "STARTTLS\r\n") < 0)
2635 #ifndef DISABLE_PIPE_CONNECT
2636 /* If doing early-pipelining reap the banner and EHLO-response but leave
2637 the response for the STARTTLS we just sent alone. On fail, assume wrong
2638 cached capability and retry with the pipelining disabled. */
2640 if (sx->early_pipe_active && sync_responses(sx, 2, 0) != 0)
2643 debug_printf("failed reaping pipelined cmd responses\n");
2644 close(sx->cctx.sock);
2646 sx->early_pipe_active = FALSE;
2647 goto PIPE_CONNECT_RETRY;
2651 /* If there is an I/O error, transmission of this message is deferred. If
2652 there is a temporary rejection of STARRTLS and tls_tempfail_tryclear is
2653 false, we also defer. However, if there is a temporary rejection of STARTTLS
2654 and tls_tempfail_tryclear is true, or if there is an outright rejection of
2655 STARTTLS, we carry on. This means we will try to send the message in clear,
2656 unless the host is in hosts_require_tls (tested below). */
2658 if (!smtp_read_response(sx, buffer2, sizeof(buffer2), '2', ob->command_timeout))
2662 || (buffer2[0] == '4' && !ob->tls_tempfail_tryclear)
2665 Ustrncpy(sx->buffer, buffer2, sizeof(sx->buffer));
2666 sx->buffer[sizeof(sx->buffer)-1] = '\0';
2667 goto RESPONSE_FAILED;
2671 /* STARTTLS accepted: try to negotiate a TLS session. */
2676 if (!tls_client_start(&sx->cctx, &sx->conn_args, sx->addrlist, &tls_out, &tls_errstr))
2678 /* TLS negotiation failed; give an error. From outside, this function may
2679 be called again to try in clear on a new connection, if the options permit
2680 it for this host. */
2682 DEBUG(D_tls) debug_printf("TLS session fail: %s\n", tls_errstr);
2684 # ifdef SUPPORT_DANE
2685 if (sx->conn_args.dane)
2687 log_write(0, LOG_MAIN,
2688 "DANE attempt failed; TLS connection to %s [%s]: %s",
2689 sx->conn_args.host->name, sx->conn_args.host->address, tls_errstr);
2690 # ifndef DISABLE_EVENT
2691 (void) event_raise(sx->conn_args.tblock->event_action,
2692 US"dane:fail", US"validation-failure", NULL); /* could do with better detail */
2697 errno = ERRNO_TLSFAILURE;
2698 message = string_sprintf("TLS session: %s", tls_errstr);
2699 sx->send_quit = FALSE;
2702 sx->send_tlsclose = TRUE;
2704 /* TLS session is set up. Check the inblock fill level. If there is
2705 content then as we have not yet done a tls read it must have arrived before
2706 the TLS handshake, in-clear. That violates the sync requirement of the
2707 STARTTLS RFC, so fail. */
2709 if (sx->inblock.ptr != sx->inblock.ptrend)
2713 int i = sx->inblock.ptrend - sx->inblock.ptr;
2714 debug_printf("unused data in input buffer after ack for STARTTLS:\n"
2716 i > 100 ? 100 : i, sx->inblock.ptr, i > 100 ? "..." : "");
2718 tls_errstr = US"synch error before connect";
2719 goto TLS_CONN_FAILED;
2722 smtp_peer_options_wrap = smtp_peer_options;
2723 for (address_item * addr = sx->addrlist; addr; addr = addr->next)
2724 if (addr->transport_return == PENDING_DEFER)
2726 addr->cipher = tls_out.cipher;
2727 addr->ourcert = tls_out.ourcert;
2728 addr->peercert = tls_out.peercert;
2729 addr->peerdn = tls_out.peerdn;
2730 addr->ocsp = tls_out.ocsp;
2731 addr->tlsver = tls_out.ver;
2736 /* if smtps, we'll have smtp_command set to something else; always safe to
2738 smtp_command = big_buffer;
2740 /* If we started TLS, redo the EHLO/LHLO exchange over the secure channel. If
2741 helo_data is null, we are dealing with a connection that was passed from
2742 another process, and so we won't have expanded helo_data above. We have to
2743 expand it here. $sending_ip_address and $sending_port are set up right at the
2744 start of the Exim process (in exim.c). */
2746 if (tls_out.active.sock >= 0)
2748 uschar * greeting_cmd;
2750 if (!sx->helo_data && !(sx->helo_data = expand_string(ob->helo_data)))
2752 uschar *message = string_sprintf("failed to expand helo_data: %s",
2753 expand_string_message);
2754 set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
2759 #ifndef DISABLE_PIPE_CONNECT
2760 /* For SMTPS there is no cleartext early-pipe; use the crypted permission bit.
2761 We're unlikely to get the group sent and delivered before the server sends its
2762 banner, but it's still worth sending as a group.
2763 For STARTTLS allow for cleartext early-pipe but no crypted early-pipe, but not
2766 if (sx->smtps ? sx->early_pipe_ok : sx->early_pipe_active)
2768 sx->peer_offered = sx->ehlo_resp.crypted_features;
2769 if ((sx->early_pipe_active =
2770 !!(sx->ehlo_resp.crypted_features & OPTION_EARLY_PIPE)))
2771 DEBUG(D_transport) debug_printf("Using cached crypted PIPECONNECT\n");
2774 #ifdef EXPERIMMENTAL_ESMTP_LIMITS
2775 /* As we are about to send another EHLO, forget any LIMITS received so far. */
2776 sx->peer_limit_mail = sx->peer_limit_rcpt = sx->peer_limit_rcptdom = 0;
2777 if ((sx->max_mail = sx->conn_args.tblock->connection_max_message) == 0) sx->max_mail = 999999;
2778 if ((sx->max_rcpt = sx->conn_args.tblock->max_addresses) == 0) sx->max_rcpt = 999999;
2779 sx->single_rcpt_domain = FALSE;
2782 /* For SMTPS we need to wait for the initial OK response. */
2784 #ifndef DISABLE_PIPE_CONNECT
2785 if (sx->early_pipe_active)
2787 sx->pending_BANNER = TRUE;
2788 sx->outblock.cmd_count = 1;
2792 if (!smtp_reap_banner(sx))
2793 goto RESPONSE_FAILED;
2796 greeting_cmd = US"LHLO";
2798 greeting_cmd = US"EHLO";
2801 greeting_cmd = US"HELO";
2803 debug_printf("not sending EHLO (host matches hosts_avoid_esmtp)\n");
2806 if (smtp_write_command(sx,
2807 #ifndef DISABLE_PIPE_CONNECT
2808 sx->early_pipe_active ? SCMD_BUFFER :
2811 "%s %s\r\n", greeting_cmd, sx->helo_data) < 0)
2814 #ifndef DISABLE_PIPE_CONNECT
2815 if (sx->early_pipe_active)
2816 sx->pending_EHLO = TRUE;
2820 if (!smtp_reap_ehlo(sx))
2823 /* The GnuTLS layer in Exim only spots a server-rejection of a client
2824 cert late, under TLS1.3 - which means here; the first time we try to
2825 receive crypted data. Treat it as if it was a connect-time failure.
2826 See also the early-pipe equivalent... which will be hard; every call
2827 to sync_responses will need to check the result.
2828 It would be nicer to have GnuTLS check the cert during the handshake.
2829 Can it do that, with all the flexibility we need? */
2831 tls_errstr = US"error on first read";
2832 goto TLS_CONN_FAILED;
2835 goto RESPONSE_FAILED;
2837 smtp_peer_options = 0;
2841 /* If the host is required to use a secure channel, ensure that we
2845 # ifdef SUPPORT_DANE
2846 || sx->conn_args.dane
2848 || verify_check_given_host(CUSS &ob->hosts_require_tls, sx->conn_args.host) == OK
2851 errno = ERRNO_TLSREQUIRED;
2852 message = string_sprintf("a TLS session is required, but %s",
2853 smtp_peer_options & OPTION_TLS
2854 ? "an attempt to start TLS failed" : "the server did not offer TLS support");
2855 # if defined(SUPPORT_DANE) && !defined(DISABLE_EVENT)
2856 if (sx->conn_args.dane)
2857 (void) event_raise(sx->conn_args.tblock->event_action, US"dane:fail",
2858 smtp_peer_options & OPTION_TLS
2859 ? US"validation-failure" /* could do with better detail */
2860 : US"starttls-not-supported",
2865 #endif /*DISABLE_TLS*/
2867 /* If TLS is active, we have just started it up and re-done the EHLO command,
2868 so its response needs to be analyzed. If TLS is not active and this is a
2869 continued session down a previously-used socket, we haven't just done EHLO, so
2872 if ( !continue_hostname
2874 || tls_out.active.sock >= 0
2878 if (sx->esmtp || sx->lmtp)
2880 #ifndef DISABLE_PIPE_CONNECT
2881 if (!sx->early_pipe_active)
2884 sx->peer_offered = ehlo_response(sx->buffer,
2886 #ifndef DISABLE_PIPE_CONNECT
2887 | (sx->lmtp && ob->lmtp_ignore_quota ? OPTION_IGNQ : 0)
2888 | OPTION_DSN | OPTION_PIPE | OPTION_SIZE
2889 | OPTION_CHUNKING | OPTION_PRDR | OPTION_UTF8
2890 | (tls_out.active.sock >= 0 ? OPTION_EARLY_PIPE : 0) /* not for lmtp */
2894 | (sx->lmtp && ob->lmtp_ignore_quota ? OPTION_IGNQ : 0)
2897 # ifdef SUPPORT_I18N
2898 | (sx->addrlist->prop.utf8_msg ? OPTION_UTF8 : 0)
2899 /*XXX if we hand peercaps on to continued-conn processes,
2900 must not depend on this addr */
2904 | (ob->size_addition >= 0 ? OPTION_SIZE : 0)
2907 #ifndef DISABLE_PIPE_CONNECT
2908 if (tls_out.active.sock >= 0)
2909 sx->ehlo_resp.crypted_features = sx->peer_offered;
2912 #ifdef EXPERIMENTAL_ESMTP_LIMITS
2913 if (tls_out.active.sock >= 0 || !(sx->peer_offered & OPTION_TLS))
2915 ehlo_response_limits_read(sx);
2916 ehlo_response_limits_apply(sx);
2921 /* Set for IGNOREQUOTA if the response to LHLO specifies support and the
2922 lmtp_ignore_quota option was set. */
2924 sx->igquotstr = sx->peer_offered & OPTION_IGNQ ? US" IGNOREQUOTA" : US"";
2926 /* If the response to EHLO specified support for the SIZE parameter, note
2927 this, provided size_addition is non-negative. */
2929 smtp_peer_options |= sx->peer_offered & OPTION_SIZE;
2931 /* Note whether the server supports PIPELINING. If hosts_avoid_esmtp matched
2932 the current host, esmtp will be false, so PIPELINING can never be used. If
2933 the current host matches hosts_avoid_pipelining, don't do it. */
2935 if ( sx->peer_offered & OPTION_PIPE
2936 && verify_check_given_host(CUSS &ob->hosts_avoid_pipelining, sx->conn_args.host) != OK)
2937 smtp_peer_options |= OPTION_PIPE;
2939 DEBUG(D_transport) debug_printf("%susing PIPELINING\n",
2940 smtp_peer_options & OPTION_PIPE ? "" : "not ");
2942 if ( sx->peer_offered & OPTION_CHUNKING
2943 && verify_check_given_host(CUSS &ob->hosts_try_chunking, sx->conn_args.host) != OK)
2944 sx->peer_offered &= ~OPTION_CHUNKING;
2946 if (sx->peer_offered & OPTION_CHUNKING)
2947 DEBUG(D_transport) debug_printf("CHUNKING usable\n");
2949 #ifndef DISABLE_PRDR
2950 if ( sx->peer_offered & OPTION_PRDR
2951 && verify_check_given_host(CUSS &ob->hosts_try_prdr, sx->conn_args.host) != OK)
2952 sx->peer_offered &= ~OPTION_PRDR;
2954 if (sx->peer_offered & OPTION_PRDR)
2955 DEBUG(D_transport) debug_printf("PRDR usable\n");
2958 /* Note if the server supports DSN */
2959 smtp_peer_options |= sx->peer_offered & OPTION_DSN;
2960 DEBUG(D_transport) debug_printf("%susing DSN\n",
2961 sx->peer_offered & OPTION_DSN ? "" : "not ");
2963 #ifndef DISABLE_PIPE_CONNECT
2964 if ( sx->early_pipe_ok
2965 && !sx->early_pipe_active
2966 && tls_out.active.sock >= 0
2967 && smtp_peer_options & OPTION_PIPE
2968 && ( sx->ehlo_resp.cleartext_features | sx->ehlo_resp.crypted_features)
2969 & OPTION_EARLY_PIPE)
2971 DEBUG(D_transport) debug_printf("PIPECONNECT usable in future for this IP\n");
2972 sx->ehlo_resp.crypted_auths = study_ehlo_auths(sx);
2973 write_ehlo_cache_entry(sx);
2977 /* Note if the response to EHLO specifies support for the AUTH extension.
2978 If it has, check that this host is one we want to authenticate to, and do
2979 the business. The host name and address must be available when the
2980 authenticator's client driver is running. */
2982 switch (yield = smtp_auth(sx))
2984 default: goto SEND_QUIT;
2986 case FAIL_SEND: goto SEND_FAILED;
2987 case FAIL: goto RESPONSE_FAILED;
2991 sx->pipelining_used = pipelining_active = !!(smtp_peer_options & OPTION_PIPE);
2993 /* The setting up of the SMTP call is now complete. Any subsequent errors are
2994 message-specific. */
2996 sx->setting_up = FALSE;
2999 if (sx->addrlist->prop.utf8_msg)
3003 /* If the transport sets a downconversion mode it overrides any set by ACL
3006 if ((s = ob->utf8_downconvert))
3008 if (!(s = expand_string(s)))
3010 message = string_sprintf("failed to expand utf8_downconvert: %s",
3011 expand_string_message);
3012 set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
3018 case '1': sx->addrlist->prop.utf8_downcvt = TRUE;
3019 sx->addrlist->prop.utf8_downcvt_maybe = FALSE;
3021 case '0': sx->addrlist->prop.utf8_downcvt = FALSE;
3022 sx->addrlist->prop.utf8_downcvt_maybe = FALSE;
3024 case '-': if (s[1] == '1')
3026 sx->addrlist->prop.utf8_downcvt = FALSE;
3027 sx->addrlist->prop.utf8_downcvt_maybe = TRUE;
3033 sx->utf8_needed = !sx->addrlist->prop.utf8_downcvt
3034 && !sx->addrlist->prop.utf8_downcvt_maybe;
3035 DEBUG(D_transport) if (!sx->utf8_needed)
3036 debug_printf("utf8: %s downconvert\n",
3037 sx->addrlist->prop.utf8_downcvt ? "mandatory" : "optional");
3040 /* If this is an international message we need the host to speak SMTPUTF8 */
3041 if (sx->utf8_needed && !(sx->peer_offered & OPTION_UTF8))
3043 errno = ERRNO_UTF8_FWD;
3044 goto RESPONSE_FAILED;
3046 #endif /*SUPPORT_I18N*/
3055 if (errno == ECONNREFUSED) /* first-read error on a TFO conn */
3057 /* There is a testing facility for simulating a connection timeout, as I
3058 can't think of any other way of doing this. It converts a connection
3059 refused into a timeout if the timeout is set to 999999. This is done for
3060 a 3whs connection in ip_connect(), but a TFO connection does not error
3061 there - instead it gets ECONNREFUSED on the first data read. Tracking
3062 that a TFO really was done is too hard, or we would set a
3063 sx->pending_conn_done bit and test that in smtp_reap_banner() and
3064 smtp_reap_ehlo(). That would let us also add the conn-timeout to the
3067 if (f.running_in_test_harness && ob->connect_timeout == 999999)
3069 set_errno_nohost(sx->addrlist,
3070 errno == ETIMEDOUT ? ERRNO_CONNECTTIMEOUT : errno,
3071 sx->verify ? US strerror(errno) : NULL,
3072 DEFER, FALSE, &sx->delivery_start);
3073 sx->send_quit = FALSE;
3077 /* really an error on an SMTP read */
3079 sx->send_quit = check_response(sx->conn_args.host, &errno, sx->addrlist->more_errno,
3080 sx->buffer, &code, &message, &pass_message);
3086 message = US string_sprintf("smtp send to %s [%s] failed: %s",
3087 sx->conn_args.host->name, sx->conn_args.host->address, strerror(errno));
3088 sx->send_quit = FALSE;
3094 message = string_sprintf("Remote host closed connection in response to %s"
3095 " (EHLO response was: %s)", smtp_command, sx->buffer);
3096 sx->send_quit = FALSE;
3100 /* This label is jumped to directly when a TLS negotiation has failed,
3101 or was not done for a host for which it is required. Values will be set
3102 in message and errno, and setting_up will always be true. Treat as
3103 a temporary error. */
3107 code = '4', yield = DEFER;
3111 /* The failure happened while setting up the call; see if the failure was
3112 a 5xx response (this will either be on connection, or following HELO - a 5xx
3113 after EHLO causes it to try HELO). If so, and there are no more hosts to try,
3114 fail all addresses, as this host is never going to accept them. For other
3115 errors during setting up (timeouts or whatever), defer all addresses, and
3116 yield DEFER, so that the host is not tried again for a while.
3118 XXX This peeking for another host feels like a layering violation. We want
3119 to note the host as unusable, but down here we shouldn't know if this was
3120 the last host to try for the addr(list). Perhaps the upper layer should be
3121 the one to do set_errno() ? The problem is that currently the addr is where
3122 errno etc. are stashed, but until we run out of hosts to try the errors are
3123 host-specific. Maybe we should enhance the host_item definition? */
3126 sx->ok = FALSE; /* For when reached by GOTO */
3127 set_errno(sx->addrlist, errno, message,
3128 sx->conn_args.host->next
3132 || errno == ERRNO_UTF8_FWD
3136 errno == ECONNREFUSED ? NULL : sx->conn_args.host,
3137 #ifdef EXPERIMENTAL_DSN_INFO
3138 sx->smtp_greeting, sx->helo_response,
3140 &sx->delivery_start);
3147 (void)smtp_write_command(sx, SCMD_FLUSH, "QUIT\r\n");
3150 if (sx->cctx.tls_ctx)
3152 if (sx->send_tlsclose)
3154 tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
3155 sx->send_tlsclose = FALSE;
3157 sx->cctx.tls_ctx = NULL;
3161 /* Close the socket, and return the appropriate value, first setting
3162 works because the NULL setting is passed back to the calling process, and
3163 remote_max_parallel is forced to 1 when delivering over an existing connection,
3166 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
3169 shutdown(sx->cctx.sock, SHUT_WR);
3170 if (fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0)
3171 for (int i = 16; read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer)) > 0 && i > 0;)
3172 i--; /* drain socket */
3173 sx->send_quit = FALSE;
3175 (void)close(sx->cctx.sock);
3178 #ifndef DISABLE_EVENT
3179 (void) event_raise(sx->conn_args.tblock->event_action, US"tcp:close", NULL, NULL);
3182 continue_transport = NULL;
3183 continue_hostname = NULL;
3190 /* Create the string of options that will be appended to the MAIL FROM:
3191 in the connection context buffer */
3194 build_mailcmd_options(smtp_context * sx, address_item * addrlist)
3196 uschar * p = sx->buffer;
3197 address_item * addr;
3202 /* If we know the receiving MTA supports the SIZE qualification, and we know it,
3203 send it, adding something to the message size to allow for imprecision
3204 and things that get added en route. Exim keeps the number of lines
3205 in a message, so we can give an accurate value for the original message, but we
3206 need some additional to handle added headers. (Double "." characters don't get
3207 included in the count.) */
3209 if ( message_size > 0
3210 && sx->peer_offered & OPTION_SIZE && !(sx->avoid_option & OPTION_SIZE))
3212 /*XXX problem here under spool_files_wireformat?
3213 Or just forget about lines? Or inflate by a fixed proportion? */
3215 sprintf(CS p, " SIZE=%d", message_size+message_linecount+(SOB sx->conn_args.ob)->size_addition);
3219 #ifndef DISABLE_PRDR
3220 /* If it supports Per-Recipient Data Responses, and we have more than one recipient,
3223 sx->prdr_active = FALSE;
3224 if (sx->peer_offered & OPTION_PRDR)
3225 for (address_item * addr = addrlist; addr; addr = addr->next)
3226 if (addr->transport_return == PENDING_DEFER)
3228 for (addr = addr->next; addr; addr = addr->next)
3229 if (addr->transport_return == PENDING_DEFER)
3230 { /* at least two recipients to send */
3231 sx->prdr_active = TRUE;
3232 sprintf(CS p, " PRDR"); p += 5;
3240 /* If it supports internationalised messages, and this meesage need that,
3243 if ( sx->peer_offered & OPTION_UTF8
3244 && addrlist->prop.utf8_msg
3245 && !addrlist->prop.utf8_downcvt
3247 Ustrcpy(p, US" SMTPUTF8"), p += 9;
3250 /* check if all addresses have DSN-lasthop flag; do not send RET and ENVID if so */
3251 for (sx->dsn_all_lasthop = TRUE, addr = addrlist, address_count = 0;
3252 addr && address_count < sx->max_rcpt; /*XXX maybe also || sx->single_rcpt_domain ? */
3253 addr = addr->next) if (addr->transport_return == PENDING_DEFER)
3256 if (!(addr->dsn_flags & rf_dsnlasthop))
3258 sx->dsn_all_lasthop = FALSE;
3263 /* Add any DSN flags to the mail command */
3265 if (sx->peer_offered & OPTION_DSN && !sx->dsn_all_lasthop)
3267 if (dsn_ret == dsn_ret_hdrs)
3268 { Ustrcpy(p, US" RET=HDRS"); p += 9; }
3269 else if (dsn_ret == dsn_ret_full)
3270 { Ustrcpy(p, US" RET=FULL"); p += 9; }
3274 string_format(p, sizeof(sx->buffer) - (p-sx->buffer), " ENVID=%s", dsn_envid);
3279 /* If an authenticated_sender override has been specified for this transport
3280 instance, expand it. If the expansion is forced to fail, and there was already
3281 an authenticated_sender for this message, the original value will be used.
3282 Other expansion failures are serious. An empty result is ignored, but there is
3283 otherwise no check - this feature is expected to be used with LMTP and other
3284 cases where non-standard addresses (e.g. without domains) might be required. */
3286 return smtp_mail_auth_str(sx, p, addrlist) ? ERROR : OK;
3291 build_rcptcmd_options(smtp_context * sx, const address_item * addr)
3293 uschar * p = sx->buffer;
3296 /* Add any DSN flags to the rcpt command */
3298 if (sx->peer_offered & OPTION_DSN && !(addr->dsn_flags & rf_dsnlasthop))
3300 if (addr->dsn_flags & rf_dsnflags)
3304 Ustrcpy(p, US" NOTIFY=");
3306 for (int i = 0; i < nelem(rf_list); i++) if (addr->dsn_flags & rf_list[i])
3308 if (!first) *p++ = ',';
3310 Ustrcpy(p, rf_names[i]);
3315 if (addr->dsn_orcpt)
3317 string_format(p, sizeof(sx->buffer) - (p-sx->buffer), " ORCPT=%s",
3328 0 good, rcpt results in addr->transport_return (PENDING_OK, DEFER, FAIL)
3329 -1 MAIL response error
3330 -2 any non-MAIL read i/o error
3331 -3 non-MAIL response timeout
3332 -4 internal error; channel still usable
3337 smtp_write_mail_and_rcpt_cmds(smtp_context * sx, int * yield)
3339 address_item * addr;
3340 #ifdef EXPERIMENTAL_ESMTP_LIMITS
3341 address_item * restart_addr = NULL;
3343 int address_count, pipe_limit;
3346 if (build_mailcmd_options(sx, sx->first_addr) != OK)
3352 /* From here until we send the DATA command, we can make use of PIPELINING
3353 if the server host supports it. The code has to be able to check the responses
3354 at any point, for when the buffer fills up, so we write it totally generally.
3355 When PIPELINING is off, each command written reports that it has flushed the
3358 sx->pending_MAIL = TRUE; /* The block starts with MAIL */
3361 uschar * s = sx->from_addr;
3363 uschar * errstr = NULL;
3365 /* If we must downconvert, do the from-address here. Remember we had to
3366 for the to-addresses (done below), and also (ugly) for re-doing when building
3367 the delivery log line. */
3369 if ( sx->addrlist->prop.utf8_msg
3370 && (sx->addrlist->prop.utf8_downcvt || !(sx->peer_offered & OPTION_UTF8))
3373 if (s = string_address_utf8_to_alabel(s, &errstr), errstr)
3375 set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, &sx->delivery_start);
3379 setflag(sx->addrlist, af_utf8_downcvt);
3383 rc = smtp_write_command(sx, pipelining_active ? SCMD_BUFFER : SCMD_FLUSH,
3384 "MAIL FROM:<%s>%s\r\n", s, sx->buffer);
3387 mail_command = string_copy(big_buffer); /* Save for later error message */
3391 case -1: /* Transmission error */
3394 case +1: /* Cmd was sent */
3395 if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
3396 (SOB sx->conn_args.ob)->command_timeout))
3398 if (errno == 0 && sx->buffer[0] == '4')
3400 errno = ERRNO_MAIL4XX;
3401 sx->addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
3405 sx->pending_MAIL = FALSE;
3408 /* otherwise zero: command queued for pipeline */
3411 /* Pass over all the relevant recipient addresses for this host, which are the
3412 ones that have status PENDING_DEFER. If we are using PIPELINING, we can send
3413 several before we have to read the responses for those seen so far. This
3414 checking is done by a subroutine because it also needs to be done at the end.
3415 Send only up to max_rcpt addresses at a time, leaving next_addr pointing to
3416 the next one if not all are sent.
3418 In the MUA wrapper situation, we want to flush the PIPELINING buffer for the
3419 last address because we want to abort if any recipients have any kind of
3420 problem, temporary or permanent. We know that all recipient addresses will have
3421 the PENDING_DEFER status, because only one attempt is ever made, and we know
3422 that max_rcpt will be large, so all addresses will be done at once.
3424 For verify we flush the pipeline after any (the only) rcpt address. */
3426 for (addr = sx->first_addr, address_count = 0, pipe_limit = 100;
3427 addr && address_count < sx->max_rcpt;
3428 addr = addr->next) if (addr->transport_return == PENDING_DEFER)
3434 #ifdef EXPERIMENTAL_ESMTP_LIMITS
3435 if ( sx->single_rcpt_domain /* restriction on domains */
3436 && address_count > 0 /* not first being sent */
3437 && Ustrcmp(addr->domain, sx->first_addr->domain) != 0 /* dom diff from first */
3440 DEBUG(D_transport) debug_printf("skipping different domain %s\n", addr->domain);
3442 /* Ensure the smtp-response reaper does not think the address had a RCPT
3443 command sent for it. Reset to PENDING_DEFER in smtp_deliver(), where we
3444 goto SEND_MESSAGE. */
3446 addr->transport_return = SKIP;
3447 if (!restart_addr) restart_addr = addr; /* note restart point */
3448 continue; /* skip this one */
3452 addr->dsn_aware = sx->peer_offered & OPTION_DSN
3453 ? dsn_support_yes : dsn_support_no;
3456 if (pipe_limit-- <= 0)
3457 { no_flush = FALSE; pipe_limit = 100; }
3459 no_flush = pipelining_active && !sx->verify
3460 && (!mua_wrapper || addr->next && address_count < sx->max_rcpt);
3462 build_rcptcmd_options(sx, addr);
3464 /* Now send the RCPT command, and process outstanding responses when
3465 necessary. After a timeout on RCPT, we just end the function, leaving the
3466 yield as OK, because this error can often mean that there is a problem with
3467 just one address, so we don't want to delay the host. */
3469 rcpt_addr = transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes);
3472 if ( testflag(sx->addrlist, af_utf8_downcvt)
3473 && !(rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, NULL))
3476 /*XXX could we use a per-address errstr here? Not fail the whole send? */
3477 errno = ERRNO_EXPANDFAIL;
3478 return -5; /*XXX too harsh? */
3482 cmds_sent = smtp_write_command(sx, no_flush ? SCMD_BUFFER : SCMD_FLUSH,
3483 "RCPT TO:<%s>%s%s\r\n", rcpt_addr, sx->igquotstr, sx->buffer);
3485 if (cmds_sent < 0) return -5;
3488 switch(sync_responses(sx, cmds_sent, 0))
3490 case 3: sx->ok = TRUE; /* 2xx & 5xx => OK & progress made */
3491 case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */
3494 case 1: sx->ok = TRUE; /* 2xx (only) => OK, but if LMTP, */
3495 if (!sx->lmtp) /* can't tell about progress yet */
3496 sx->completed_addr = TRUE;
3497 case 0: /* No 2xx or 5xx, but no probs */
3498 /* If any RCPT got a 452 response then next_addr has been updated
3499 for restarting with a new MAIL on the same connection. Send no more
3500 RCPTs for this MAIL. */
3504 DEBUG(D_transport) debug_printf("seen 452 too-many-rcpts\n");
3505 sx->RCPT_452 = FALSE;
3506 /* sx->next_addr has been reset for fast_retry */
3511 case -1: return -3; /* Timeout on RCPT */
3512 case -2: return -2; /* non-MAIL read i/o error */
3513 default: return -1; /* any MAIL error */
3515 #ifndef DISABLE_PIPE_CONNECT
3516 case -4: return -1; /* non-2xx for pipelined banner or EHLO */
3517 case -5: return -1; /* TLS first-read error */
3521 } /* Loop for next address */
3523 #ifdef EXPERIMENTAL_ESMTP_LIMITS
3524 sx->next_addr = restart_addr ? restart_addr : addr;
3526 sx->next_addr = addr;
3533 /*****************************************************
3534 * Proxy TLS connection for another transport process *
3535 ******************************************************/
3537 Close the unused end of the pipe, fork once more, then use the given buffer
3538 as a staging area, and select on both the given fd and the TLS'd client-fd for
3539 data to read (per the coding in ip_recv() and fd_ready() this is legitimate).
3540 Do blocking full-size writes, and reads under a timeout. Once both input
3541 channels are closed, exit the process.
3545 buf space to use for buffering
3546 bufsiz size of buffer
3547 pfd pipe filedescriptor array; [0] is comms to proxied process
3548 timeout per-read timeout, seconds
3554 smtp_proxy_tls(void * ct_ctx, uschar * buf, size_t bsize, int * pfd,
3557 struct pollfd p[2] = {{.fd = tls_out.active.sock, .events = POLLIN},
3558 {.fd = pfd[0], .events = POLLIN}};
3560 BOOL send_tls_shutdown = TRUE;
3563 if ((rc = exim_fork(US"tls-proxy")))
3564 _exit(rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS);
3566 set_process_info("proxying TLS connection for continued transport");
3570 time_t time_left = timeout;
3571 time_t time_start = time(NULL);
3576 rc = poll(p, 2, time_left * 1000);
3578 if (rc < 0 && errno == EINTR)
3579 if ((time_left -= time(NULL) - time_start) > 0) continue;
3583 DEBUG(D_transport) if (rc == 0) debug_printf("%s: timed out\n", __FUNCTION__);
3587 /* For errors where not readable, bomb out */
3589 if (p[0].revents & POLLERR || p[1].revents & POLLERR)
3591 DEBUG(D_transport) debug_printf("select: exceptional cond on %s fd\n",
3592 p[0].revents & POLLERR ? "tls" : "proxy");
3593 if (!(p[0].revents & POLLIN || p[1].events & POLLIN))
3595 DEBUG(D_transport) debug_printf("- but also readable; no exit yet\n");
3598 while (rc < 0 || !(p[0].revents & POLLIN || p[1].revents & POLLIN));
3600 /* handle inbound data */
3601 if (p[0].revents & POLLIN)
3602 if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0) /* Expect -1 for EOF; */
3603 { /* that reaps the TLS Close Notify record */
3605 shutdown(pfd[0], SHUT_WR);
3609 for (int nbytes = 0; rc - nbytes > 0; nbytes += i)
3610 if ((i = write(pfd[0], buf + nbytes, rc - nbytes)) < 0) goto done;
3612 /* Handle outbound data. We cannot combine payload and the TLS-close
3613 due to the limitations of the (pipe) channel feeding us. Maybe use a unix-domain
3615 if (p[1].revents & POLLIN)
3616 if ((rc = read(pfd[0], buf, bsize)) <= 0)
3620 # ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */
3621 (void) setsockopt(tls_out.active.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
3623 tls_shutdown_wr(ct_ctx);
3624 send_tls_shutdown = FALSE;
3625 shutdown(tls_out.active.sock, SHUT_WR);
3628 for (int nbytes = 0; rc - nbytes > 0; nbytes += i)
3629 if ((i = tls_write(ct_ctx, buf + nbytes, rc - nbytes, FALSE)) < 0)
3632 while (p[0].fd >= 0 || p[1].fd >= 0);
3635 if (send_tls_shutdown) tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT);
3637 testharness_pause_ms(100); /* let logging complete */
3638 exim_exit(EXIT_SUCCESS);
3643 /*************************************************
3644 * Deliver address list to given host *
3645 *************************************************/
3647 /* If continue_hostname is not null, we get here only when continuing to
3648 deliver down an existing channel. The channel was passed as the standard
3649 input. TLS is never active on a passed channel; the previous process either
3650 closes it down before passing the connection on, or inserts a TLS-proxy
3651 process and passes on a cleartext conection.
3653 Otherwise, we have to make a connection to the remote host, and do the
3654 initial protocol exchange.
3656 When running as an MUA wrapper, if the sender or any recipient is rejected,
3657 temporarily or permanently, we force failure for all recipients.
3660 addrlist chain of potential addresses to deliver; only those whose
3661 transport_return field is set to PENDING_DEFER are currently
3662 being processed; others should be skipped - they have either
3663 been delivered to an earlier host or IP address, or been
3664 failed by one of them.
3665 host host to deliver to
3666 host_af AF_INET or AF_INET6
3667 defport default TCP/IP port to use if host does not specify, in host
3669 interface interface to bind to, or NULL
3670 tblock transport instance block
3671 message_defer set TRUE if yield is OK, but all addresses were deferred
3672 because of a non-recipient, non-host failure, that is, a
3673 4xx response to MAIL FROM, DATA, or ".". This is a defer
3674 that is specific to the message.
3675 suppress_tls if TRUE, don't attempt a TLS connection - this is set for
3676 a second attempt after TLS initialization fails
3678 Returns: OK - the connection was made and the delivery attempted;
3679 the result for each address is in its data block.
3680 DEFER - the connection could not be made, or something failed
3681 while setting up the SMTP session, or there was a
3682 non-message-specific error, such as a timeout.
3683 ERROR - a filter command is specified for this transport,
3684 and there was a problem setting it up; OR helo_data
3685 or add_headers or authenticated_sender is specified
3686 for this transport, and the string failed to expand
3688 For all non-OK returns the first addr of the list carries the
3689 time taken for the attempt.
3693 smtp_deliver(address_item *addrlist, host_item *host, int host_af, int defport,
3694 uschar *interface, transport_instance *tblock,
3695 BOOL *message_defer, BOOL suppress_tls)
3697 smtp_transport_options_block * ob = SOB tblock->options_block;
3702 uschar *message = NULL;
3703 uschar new_message_id[MESSAGE_ID_LENGTH + 1];
3704 smtp_context * sx = store_get(sizeof(*sx), TRUE); /* tainted, for the data buffers */
3705 BOOL pass_message = FALSE;
3706 #ifdef EXPERIMENTAL_ESMTP_LIMITS
3707 BOOL mail_limit = FALSE;
3712 BOOL tcw_done = FALSE, tcw = FALSE;
3714 *message_defer = FALSE;
3716 memset(sx, 0, sizeof(*sx));
3717 sx->addrlist = addrlist;
3718 sx->conn_args.host = host;
3719 sx->conn_args.host_af = host_af,
3721 sx->conn_args.interface = interface;
3722 sx->helo_data = NULL;
3723 sx->conn_args.tblock = tblock;
3724 /* sx->verify = FALSE; */
3725 gettimeofday(&sx->delivery_start, NULL);
3726 sx->sync_addr = sx->first_addr = addrlist;
3733 /* Get the channel set up ready for a message, MAIL FROM being the next
3734 SMTP command to send. */
3736 if ((rc = smtp_setup_conn(sx, suppress_tls)) != OK)
3738 timesince(&addrlist->delivery_time, &sx->delivery_start);
3744 /* If the connection used DANE, ignore for now any addresses with incompatible
3745 domains. The SNI has to be the domain. Arrange a whole new TCP conn later,
3746 just in case only TLS isn't enough. */
3748 if (sx->conn_args.dane)
3750 const uschar * dane_domain = sx->first_addr->domain;
3752 for (address_item * a = sx->first_addr->next; a; a = a->next)
3753 if ( a->transport_return == PENDING_DEFER
3754 && Ustrcmp(dane_domain, a->domain) != 0)
3756 DEBUG(D_transport) debug_printf("DANE: holding %s for later\n", a->domain);
3758 a->transport_return = DANE;
3763 /* If there is a filter command specified for this transport, we can now
3764 set it up. This cannot be done until the identity of the host is known. */
3766 if (tblock->filter_command)
3768 transport_filter_timeout = tblock->filter_timeout;
3770 /* On failure, copy the error to all addresses, abandon the SMTP call, and
3773 if (!transport_set_up_command(&transport_filter_argv,
3774 tblock->filter_command, TRUE, DEFER, addrlist,
3775 string_sprintf("%.50s transport", tblock->name), NULL))
3777 set_errno_nohost(addrlist->next, addrlist->basic_errno, addrlist->message, DEFER,
3778 FALSE, &sx->delivery_start);
3783 if ( transport_filter_argv
3784 && *transport_filter_argv
3785 && **transport_filter_argv
3786 && sx->peer_offered & OPTION_CHUNKING
3787 #ifndef DISABLE_DKIM
3788 /* When dkim signing, chunking is handled even with a transport-filter */
3789 && !(ob->dkim.dkim_private_key && ob->dkim.dkim_domain && ob->dkim.dkim_selector)
3790 && !ob->dkim.force_bodyhash
3794 sx->peer_offered &= ~OPTION_CHUNKING;
3795 DEBUG(D_transport) debug_printf("CHUNKING not usable due to transport filter\n");
3799 /* For messages that have more than the maximum number of envelope recipients,
3800 we want to send several transactions down the same SMTP connection. (See
3801 comments in deliver.c as to how this reconciles, heuristically, with
3802 remote_max_parallel.) This optimization was added to Exim after the following
3803 code was already working. The simplest way to put it in without disturbing the
3804 code was to use a goto to jump back to this point when there is another
3805 transaction to handle. */
3808 sx->from_addr = return_path;
3809 sx->sync_addr = sx->first_addr;
3811 sx->send_rset = TRUE;
3812 sx->completed_addr = FALSE;
3815 /* If we are a continued-connection-after-verify the MAIL and RCPT
3816 commands were already sent; do not re-send but do mark the addrs as
3817 having been accepted up to RCPT stage. A traditional cont-conn
3818 always has a sequence number greater than one. */
3820 if (continue_hostname && continue_sequence == 1)
3822 /* sx->pending_MAIL = FALSE; */
3824 /* sx->next_addr = NULL; */
3826 for (address_item * addr = addrlist; addr; addr = addr->next)
3827 addr->transport_return = PENDING_OK;
3831 /* Initiate a message transfer. */
3833 switch(smtp_write_mail_and_rcpt_cmds(sx, &yield))
3836 case -1: case -2: goto RESPONSE_FAILED;
3837 case -3: goto END_OFF;
3838 case -4: goto SEND_QUIT;
3839 default: goto SEND_FAILED;
3842 /* If we are an MUA wrapper, abort if any RCPTs were rejected, either
3843 permanently or temporarily. We should have flushed and synced after the last
3851 for (a = sx->first_addr, cnt = 0; a && cnt < sx->max_rcpt; a = a->next, cnt++)
3852 if (a->transport_return != PENDING_OK)
3854 /*XXX could we find a better errno than 0 here? */
3855 set_errno_nohost(addrlist, 0, a->message, FAIL,
3856 testflag(a, af_pass_message), &sx->delivery_start);
3863 /* If ok is TRUE, we know we have got at least one good recipient, and must now
3864 send DATA, but if it is FALSE (in the normal, non-wrapper case), we may still
3865 have a good recipient buffered up if we are pipelining. We don't want to waste
3866 time sending DATA needlessly, so we only send it if either ok is TRUE or if we
3867 are pipelining. The responses are all handled by sync_responses().
3868 If using CHUNKING, do not send a BDAT until we know how big a chunk we want
3871 if ( !(sx->peer_offered & OPTION_CHUNKING)
3872 && (sx->ok || (pipelining_active && !mua_wrapper)))
3874 int count = smtp_write_command(sx, SCMD_FLUSH, "DATA\r\n");
3876 if (count < 0) goto SEND_FAILED;
3877 switch(sync_responses(sx, count, sx->ok ? +1 : -1))
3879 case 3: sx->ok = TRUE; /* 2xx & 5xx => OK & progress made */
3880 case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */
3883 case 1: sx->ok = TRUE; /* 2xx (only) => OK, but if LMTP, */
3884 if (!sx->lmtp) sx->completed_addr = TRUE; /* can't tell about progress yet */
3885 case 0: break; /* No 2xx or 5xx, but no probs */
3887 case -1: goto END_OFF; /* Timeout on RCPT */
3889 #ifndef DISABLE_PIPE_CONNECT
3890 case -5: /* TLS first-read error */
3891 case -4: HDEBUG(D_transport)
3892 debug_printf("failed reaping pipelined cmd responses\n");
3894 default: goto RESPONSE_FAILED; /* I/O error, or any MAIL/DATA error */
3896 pipelining_active = FALSE;
3897 data_command = string_copy(big_buffer); /* Save for later error message */
3900 /* If there were no good recipients (but otherwise there have been no
3901 problems), just set ok TRUE, since we have handled address-specific errors
3902 already. Otherwise, it's OK to send the message. Use the check/escape mechanism
3903 for handling the SMTP dot-handling protocol, flagging to apply to headers as
3904 well as body. Set the appropriate timeout value to be used for each chunk.
3905 (Haven't been able to make it work using select() for writing yet.) */
3908 && (!(sx->peer_offered & OPTION_CHUNKING) || !pipelining_active))
3910 /* Save the first address of the next batch. */
3911 sx->first_addr = sx->next_addr;
3917 transport_ctx tctx = {
3918 .u = {.fd = sx->cctx.sock}, /*XXX will this need TLS info? */
3921 .check_string = US".",
3922 .escape_string = US"..", /* Escaping strings */
3924 topt_use_crlf | topt_escape_headers
3925 | (tblock->body_only ? topt_no_headers : 0)
3926 | (tblock->headers_only ? topt_no_body : 0)
3927 | (tblock->return_path_add ? topt_add_return_path : 0)
3928 | (tblock->delivery_date_add ? topt_add_delivery_date : 0)
3929 | (tblock->envelope_to_add ? topt_add_envelope_to : 0)
3932 /* If using CHUNKING we need a callback from the generic transport
3933 support to us, for the sending of BDAT smtp commands and the reaping
3934 of responses. The callback needs a whole bunch of state so set up
3935 a transport-context structure to be passed around. */
3937 if (sx->peer_offered & OPTION_CHUNKING)
3939 tctx.check_string = tctx.escape_string = NULL;
3940 tctx.options |= topt_use_bdat;
3941 tctx.chunk_cb = smtp_chunk_cmd_callback;
3942 sx->pending_BDAT = FALSE;
3943 sx->good_RCPT = sx->ok;
3945 tctx.smtp_context = sx;
3948 tctx.options |= topt_end_dot;
3950 /* Save the first address of the next batch. */
3951 sx->first_addr = sx->next_addr;
3953 /* Responses from CHUNKING commands go in buffer. Otherwise,
3954 there has not been a response. */
3958 sigalrm_seen = FALSE;
3959 transport_write_timeout = ob->data_timeout;
3960 smtp_command = US"sending data block"; /* For error messages */
3961 DEBUG(D_transport|D_v)
3962 if (sx->peer_offered & OPTION_CHUNKING)
3963 debug_printf(" will write message using CHUNKING\n");
3965 debug_printf(" SMTP>> writing message and terminating \".\"\n");
3966 transport_count = 0;
3968 #ifndef DISABLE_DKIM
3970 # ifdef MEASURE_TIMING
3972 gettimeofday(&t0, NULL);
3974 dkim_exim_sign_init();
3975 # ifdef EXPERIMENTAL_ARC
3977 uschar * s = ob->arc_sign;
3980 if (!(ob->dkim.arc_signspec = s = expand_string(s)))
3982 if (!f.expand_string_forcedfail)
3984 message = US"failed to expand arc_sign";
3991 /* Ask dkim code to hash the body for ARC */
3992 (void) arc_ams_setup_sign_bodyhash();
3993 ob->dkim.force_bodyhash = TRUE;
3998 # ifdef MEASURE_TIMING
3999 report_time_since(&t0, US"dkim_exim_sign_init (delta)");
4004 /* See if we can pipeline QUIT. Reasons not to are
4005 - pipelining not active
4006 - not ok to send quit
4007 - errors in amtp transation responses
4008 - more addrs to send for this message or this host
4009 - this message was being retried
4010 - more messages for this host
4011 If we can, we want the message-write to not flush (the tail end of) its data out. */
4013 if ( sx->pipelining_used
4014 && (sx->ok && sx->completed_addr || sx->peer_offered & OPTION_CHUNKING)
4016 && !(sx->first_addr || f.continue_more)
4017 && f.deliver_firsttime
4020 smtp_compare_t t_compare =
4021 {.tblock = tblock, .current_sender_address = sender_address};
4026 ( tls_out.active.sock < 0 && !continue_proxy_cipher
4027 || verify_check_given_host(CUSS &ob->hosts_nopass_tls, host) != OK
4031 transport_check_waiting(tblock->name, host->name,
4032 tblock->connection_max_messages, new_message_id,
4033 (oicf)smtp_are_same_identities, (void*)&t_compare);
4036 HDEBUG(D_transport) debug_printf("will pipeline QUIT\n");
4037 tctx.options |= topt_no_flush;
4041 #ifndef DISABLE_DKIM
4042 sx->ok = dkim_transport_write_message(&tctx, &ob->dkim, CUSS &message);
4044 sx->ok = transport_write_message(&tctx, 0);
4047 /* transport_write_message() uses write() because it is called from other
4048 places to write to non-sockets. This means that under some OS (e.g. Solaris)
4049 it can exit with "Broken pipe" as its error. This really means that the
4050 socket got closed at the far end. */
4052 transport_write_timeout = 0; /* for subsequent transports */
4054 /* Failure can either be some kind of I/O disaster (including timeout),
4055 or the failure of a transport filter or the expansion of added headers.
4056 Or, when CHUNKING, it can be a protocol-detected failure. */
4059 if (message) goto SEND_FAILED;
4060 else goto RESPONSE_FAILED;
4062 /* We used to send the terminating "." explicitly here, but because of
4063 buffering effects at both ends of TCP/IP connections, you don't gain
4064 anything by keeping it separate, so it might as well go in the final
4065 data buffer for efficiency. This is now done by setting the topt_end_dot
4068 smtp_command = US"end of data";
4070 /* If we can pipeline a QUIT with the data them send it now. If a new message
4071 for this host appeared in the queue while data was being sent, we will not see
4072 it and it will have to wait for a queue run. If there was one but another
4073 thread took it, we might attempt to send it - but locking of spoolfiles will
4074 detect that. Use _MORE to get QUIT in FIN segment. */
4076 if (tcw_done && !tcw)
4078 /*XXX jgh 2021/03/10 google et. al screwup. G, at least, sends TCP FIN in response to TLS
4079 close-notify. Under TLS 1.3, violating RFC.
4080 However, TLS 1.2 does not have half-close semantics. */
4082 if ( sx->cctx.tls_ctx
4083 #if 0 && !defined(DISABLE_TLS)
4084 && Ustrcmp(tls_out.ver, "TLS1.3") != 0
4086 || !f.deliver_firsttime
4088 { /* Send QUIT now and not later */
4089 (void)smtp_write_command(sx, SCMD_FLUSH, "QUIT\r\n");
4090 sx->send_quit = FALSE;
4093 { /* add QUIT to the output buffer */
4094 (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n");
4095 sx->send_quit = FALSE; /* avoid sending it later */
4098 if (sx->cctx.tls_ctx) /* need to send TLS Close Notify */
4100 # ifdef EXIM_TCP_CORK /* Use _CORK to get Close Notify in FIN segment */
4101 (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
4103 tls_shutdown_wr(sx->cctx.tls_ctx);
4104 sx->send_tlsclose = FALSE; /* avoid later repeat */
4107 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(shutdown)>>\n");
4108 shutdown(sx->cctx.sock, SHUT_WR); /* flush output buffer, with TCP FIN */
4112 if (sx->peer_offered & OPTION_CHUNKING && sx->cmd_count > 1)
4114 /* Reap any outstanding MAIL & RCPT commands, but not a DATA-go-ahead */
4115 switch(sync_responses(sx, sx->cmd_count-1, 0))
4117 case 3: sx->ok = TRUE; /* 2xx & 5xx => OK & progress made */
4118 case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */
4121 case 1: sx->ok = TRUE; /* 2xx (only) => OK, but if LMTP, */
4122 if (!sx->lmtp) sx->completed_addr = TRUE; /* can't tell about progress yet */
4123 case 0: break; /* No 2xx or 5xx, but no probs */
4125 case -1: goto END_OFF; /* Timeout on RCPT */
4127 #ifndef DISABLE_PIPE_CONNECT
4128 case -5: /* TLS first-read error */
4129 case -4: HDEBUG(D_transport)
4130 debug_printf("failed reaping pipelined cmd responses\n");
4132 default: goto RESPONSE_FAILED; /* I/O error, or any MAIL/DATA error */
4136 #ifndef DISABLE_PRDR
4137 /* For PRDR we optionally get a partial-responses warning followed by the
4138 individual responses, before going on with the overall response. If we don't
4139 get the warning then deal with per non-PRDR. */
4143 sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '3', ob->final_timeout);
4144 if (!sx->ok && errno == 0) switch(sx->buffer[0])
4146 case '2': sx->prdr_active = FALSE;
4149 case '4': errno = ERRNO_DATA4XX;
4150 addrlist->more_errno |=
4151 ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
4158 /* For non-PRDR SMTP, we now read a single response that applies to the
4159 whole message. If it is OK, then all the addresses have been delivered. */
4163 sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
4165 if (!sx->ok && errno == 0 && sx->buffer[0] == '4')
4167 errno = ERRNO_DATA4XX;
4168 addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
4172 /* For LMTP, we get back a response for every RCPT command that we sent;
4173 some may be accepted and some rejected. For those that get a response, their
4174 status is fixed; any that are accepted have been handed over, even if later
4175 responses crash - at least, that's how I read RFC 2033.
4177 If all went well, mark the recipient addresses as completed, record which
4178 host/IPaddress they were delivered to, and cut out RSET when sending another
4179 message down the same channel. Write the completed addresses to the journal
4180 now so that they are recorded in case there is a crash of hardware or
4181 software before the spool gets updated. Also record the final SMTP
4182 confirmation if needed (for SMTP only). */
4187 struct timeval delivery_time;
4189 uschar * conf = NULL;
4191 timesince(&delivery_time, &sx->delivery_start);
4192 sx->send_rset = FALSE;
4193 pipelining_active = FALSE;
4195 /* Set up confirmation if needed - applies only to SMTP */
4198 #ifdef DISABLE_EVENT
4199 LOGGING(smtp_confirmation) &&
4204 const uschar * s = string_printing(sx->buffer);
4205 /* deconst cast ok here as string_printing was checked to have alloc'n'copied */
4206 conf = s == sx->buffer ? US string_copy(s) : US s;
4209 /* Process all transported addresses - for LMTP or PRDR, read a status for
4210 each one. We used to drop out at first_addr, until someone returned a 452
4211 followed by a 250... and we screwed up the accepted addresses. */
4213 for (address_item * addr = addrlist; addr; addr = addr->next)
4215 if (addr->transport_return != PENDING_OK) continue;
4217 /* LMTP - if the response fails badly (e.g. timeout), use it for all the
4218 remaining addresses. Otherwise, it's a return code for just the one
4219 address. For temporary errors, add a retry item for the address so that
4220 it doesn't get tried again too soon. */
4222 #ifndef DISABLE_PRDR
4223 if (sx->lmtp || sx->prdr_active)
4228 if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
4231 if (errno != 0 || sx->buffer[0] == 0) goto RESPONSE_FAILED;
4232 addr->message = string_sprintf(
4233 #ifndef DISABLE_PRDR
4234 "%s error after %s: %s", sx->prdr_active ? "PRDR":"LMTP",
4236 "LMTP error after %s: %s",
4238 data_command, string_printing(sx->buffer));
4239 setflag(addr, af_pass_message); /* Allow message to go to user */
4240 if (sx->buffer[0] == '5')
4241 addr->transport_return = FAIL;
4244 errno = ERRNO_DATA4XX;
4245 addr->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
4246 addr->transport_return = DEFER;
4247 #ifndef DISABLE_PRDR
4248 if (!sx->prdr_active)
4250 retry_add_item(addr, addr->address_retry_key, 0);
4254 sx->completed_addr = TRUE; /* NOW we can set this flag */
4255 if (LOGGING(smtp_confirmation))
4257 const uschar *s = string_printing(sx->buffer);
4258 /* deconst cast ok here as string_printing was checked to have alloc'n'copied */
4259 conf = (s == sx->buffer) ? US string_copy(s) : US s;
4263 /* SMTP, or success return from LMTP for this address. Pass back the
4264 actual host that was used. */
4266 addr->transport_return = OK;
4267 addr->host_used = host;
4268 addr->delivery_time = delivery_time;
4269 addr->special_action = flag;
4270 addr->message = conf;
4272 if (tcp_out_fastopen)
4274 setflag(addr, af_tcp_fastopen_conn);
4275 if (tcp_out_fastopen >= TFO_USED_NODATA) setflag(addr, af_tcp_fastopen);
4276 if (tcp_out_fastopen >= TFO_USED_DATA) setflag(addr, af_tcp_fastopen_data);
4278 if (sx->pipelining_used) setflag(addr, af_pipelining);
4279 #ifndef DISABLE_PIPE_CONNECT
4280 if (sx->early_pipe_active) setflag(addr, af_early_pipe);
4282 #ifndef DISABLE_PRDR
4283 if (sx->prdr_active) setflag(addr, af_prdr_used);
4285 if (sx->peer_offered & OPTION_CHUNKING) setflag(addr, af_chunking_used);
4288 #ifndef DISABLE_PRDR
4289 if (!sx->prdr_active)
4292 /* Update the journal. For homonymic addresses, use the base address plus
4293 the transport name. See lots of comments in deliver.c about the reasons
4294 for the complications when homonyms are involved. Just carry on after
4295 write error, as it may prove possible to update the spool file later. */
4297 if (testflag(addr, af_homonym))
4298 sprintf(CS sx->buffer, "%.500s/%s\n", addr->unique + 3, tblock->name);
4300 sprintf(CS sx->buffer, "%.500s\n", addr->unique);
4302 DEBUG(D_deliver) debug_printf("S:journalling %s", sx->buffer);
4303 len = Ustrlen(CS sx->buffer);
4304 if (write(journal_fd, sx->buffer, len) != len)
4305 log_write(0, LOG_MAIN|LOG_PANIC, "failed to write journal for "
4306 "%s: %s", sx->buffer, strerror(errno));
4310 #ifndef DISABLE_PRDR
4311 if (sx->prdr_active)
4313 const uschar * overall_message;
4315 /* PRDR - get the final, overall response. For any non-success
4316 upgrade all the address statuses. */
4318 sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
4322 if(errno == 0 && sx->buffer[0] == '4')
4324 errno = ERRNO_DATA4XX;
4325 addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
4327 for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
4328 if (sx->buffer[0] == '5' || addr->transport_return == OK)
4329 addr->transport_return = PENDING_OK; /* allow set_errno action */
4330 goto RESPONSE_FAILED;
4333 /* Append the overall response to the individual PRDR response for logging
4334 and update the journal, or setup retry. */
4336 overall_message = string_printing(sx->buffer);
4337 for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
4338 if (addr->transport_return == OK)
4339 addr->message = string_sprintf("%s\\n%s", addr->message, overall_message);
4341 for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
4342 if (addr->transport_return == OK)
4344 if (testflag(addr, af_homonym))
4345 sprintf(CS sx->buffer, "%.500s/%s\n", addr->unique + 3, tblock->name);
4347 sprintf(CS sx->buffer, "%.500s\n", addr->unique);
4349 DEBUG(D_deliver) debug_printf("journalling(PRDR) %s\n", sx->buffer);
4350 len = Ustrlen(CS sx->buffer);
4351 if (write(journal_fd, sx->buffer, len) != len)
4352 log_write(0, LOG_MAIN|LOG_PANIC, "failed to write journal for "
4353 "%s: %s", sx->buffer, strerror(errno));
4355 else if (addr->transport_return == DEFER)
4356 /*XXX magic value -2 ? maybe host+message ? */
4357 retry_add_item(addr, addr->address_retry_key, -2);
4361 /* Ensure the journal file is pushed out to disk. */
4363 if (EXIMfsync(journal_fd) < 0)
4364 log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
4370 /* Handle general (not specific to one address) failures here. The value of ok
4371 is used to skip over this code on the falling through case. A timeout causes a
4372 deferral. Other errors may defer or fail according to the response code, and
4373 may set up a special errno value, e.g. after connection chopped, which is
4374 assumed if errno == 0 and there is no text in the buffer. If control reaches
4375 here during the setting up phase (i.e. before MAIL FROM) then always defer, as
4376 the problem is not related to this specific message. */
4381 uschar * set_message;
4387 /* Clear send_quit flag if needed. Do not set. */
4388 sx->send_quit &= check_response(host, &save_errno, addrlist->more_errno,
4389 sx->buffer, &code, &message, &pass_message);
4397 message = string_sprintf("smtp send to %s [%s] failed: %s",
4398 host->name, host->address, message ? message : US strerror(save_errno));
4399 sx->send_quit = FALSE;
4407 sx->ok = FALSE; /* For when reached by GOTO */
4408 set_message = message;
4410 /* We want to handle timeouts after MAIL or "." and loss of connection after
4411 "." specially. They can indicate a problem with the sender address or with
4412 the contents of the message rather than a real error on the connection. These
4413 cases are treated in the same way as a 4xx response. This next bit of code
4414 does the classification. */
4421 message_error = TRUE;
4425 message_error = Ustrncmp(smtp_command,"MAIL",4) == 0 ||
4426 Ustrncmp(smtp_command,"end ",4) == 0;
4429 case ERRNO_SMTPCLOSED:
4430 message_error = Ustrncmp(smtp_command,"end ",4) == 0;
4433 #ifndef DISABLE_DKIM
4435 /* DKIM signing failure: avoid thinking we pipelined quit,
4436 just abandon the message and close the socket. */
4438 message_error = FALSE;
4439 # ifndef DISABLE_TLS
4440 if (sx->cctx.tls_ctx)
4442 tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT);
4443 sx->cctx.tls_ctx = NULL;
4449 message_error = FALSE;
4453 /* Handle the cases that are treated as message errors. These are:
4455 (a) negative response or timeout after MAIL
4456 (b) negative response after DATA
4457 (c) negative response or timeout or dropped connection after "."
4458 (d) utf8 support required and not offered
4460 It won't be a negative response or timeout after RCPT, as that is dealt
4461 with separately above. The action in all cases is to set an appropriate
4462 error code for all the addresses, but to leave yield set to OK because the
4463 host itself has not failed. Of course, it might in practice have failed
4464 when we've had a timeout, but if so, we'll discover that at the next
4465 delivery attempt. For a temporary error, set the message_defer flag, and
4466 write to the logs for information if this is not the last host. The error
4467 for the last host will be logged as part of the address's log line. */
4471 if (mua_wrapper) code = '5'; /* Force hard failure in wrapper mode */
4473 /* If there's an errno, the message contains just the identity of
4478 else /* Anything other than 5 is treated as temporary */
4482 message = US string_sprintf("%s: %s", message, strerror(save_errno));
4484 write_logs(host, message, sx->first_addr ? sx->first_addr->basic_errno : 0);
4486 *message_defer = TRUE;
4489 DEBUG(D_transport) if (sx->cctx.sock >= 0)
4492 if (ioctl(sx->cctx.sock, TIOCOUTQ, &n) == 0)
4493 debug_printf("%d bytes remain in socket output buffer\n", n);
4497 /* Otherwise, we have an I/O error or a timeout other than after MAIL or
4498 ".", or some other transportation error. We defer all addresses and yield
4499 DEFER, except for the case of failed add_headers expansion, or a transport
4500 filter failure, when the yield should be ERROR, to stop it trying other
4505 #ifndef DISABLE_PIPE_CONNECT
4506 /* If we were early-pipelinng and the actual EHLO response did not match
4507 the cached value we assumed, we could have detected it and passed a
4508 custom errno through to here. It would be nice to RSET and retry right
4509 away, but to reliably do that we eould need an extra synch point before
4510 we committed to data and that would discard half the gained roundrips.
4511 Or we could summarily drop the TCP connection. but that is also ugly.
4512 Instead, we ignore the possibility (having freshened the cache) and rely
4513 on the server telling us with a nonmessage error if we have tried to
4514 do something it no longer supports. */
4517 yield = (save_errno == ERRNO_CHHEADER_FAIL ||
4518 save_errno == ERRNO_FILTER_FAIL) ? ERROR : DEFER;
4522 set_errno(addrlist, save_errno, set_message, set_rc, pass_message, host,
4523 #ifdef EXPERIMENTAL_DSN_INFO
4524 sx->smtp_greeting, sx->helo_response,
4526 &sx->delivery_start);
4529 /* If all has gone well, send_quit will be set TRUE, implying we can end the
4530 SMTP session tidily. However, if there were too many addresses to send in one
4531 message (indicated by first_addr being non-NULL) we want to carry on with the
4532 rest of them. Also, it is desirable to send more than one message down the SMTP
4533 connection if there are several waiting, provided we haven't already sent so
4534 many as to hit the configured limit. The function transport_check_waiting looks
4535 for a waiting message and returns its id. Then transport_pass_socket tries to
4536 set up a continued delivery by passing the socket on to another process. The
4537 variable send_rset is FALSE if a message has just been successfully transferred.
4539 If we are already sending down a continued channel, there may be further
4540 addresses not yet delivered that are aimed at the same host, but which have not
4541 been passed in this run of the transport. In this case, continue_more will be
4542 true, and all we should do is send RSET if necessary, and return, leaving the
4545 However, if no address was disposed of, i.e. all addresses got 4xx errors, we
4546 do not want to continue with other messages down the same channel, because that
4547 can lead to looping between two or more messages, all with the same,
4548 temporarily failing address(es). [The retry information isn't updated yet, so
4549 new processes keep on trying.] We probably also don't want to try more of this
4550 message's addresses either.
4552 If we have started a TLS session, we have to end it before passing the
4553 connection to a new process. However, not all servers can handle this (Exim
4554 can), so we do not pass such a connection on if the host matches
4555 hosts_nopass_tls. */
4558 debug_printf("ok=%d send_quit=%d send_rset=%d continue_more=%d "
4559 "yield=%d first_address is %sNULL\n", sx->ok, sx->send_quit,
4560 sx->send_rset, f.continue_more, yield, sx->first_addr ? "not " : "");
4562 if (sx->completed_addr && sx->ok && sx->send_quit)
4563 #ifdef EXPERIMENTAL_ESMTP_LIMITS
4564 if (mail_limit = continue_sequence >= sx->max_mail)
4567 debug_printf("reached limit %u for MAILs per conn\n", sx->max_mail);
4572 smtp_compare_t t_compare =
4573 {.tblock = tblock, .current_sender_address = sender_address};
4575 if ( sx->first_addr /* more addrs for this message */
4576 || f.continue_more /* more addrs for continued-host */
4577 || tcw_done && tcw /* more messages for host */
4580 ( tls_out.active.sock < 0 && !continue_proxy_cipher
4581 || verify_check_given_host(CUSS &ob->hosts_nopass_tls, host) != OK
4585 transport_check_waiting(tblock->name, host->name,
4586 sx->max_mail, new_message_id,
4587 (oicf)smtp_are_same_identities, (void*)&t_compare)
4594 if (! (sx->ok = smtp_write_command(sx, SCMD_FLUSH, "RSET\r\n") >= 0))
4596 msg = US string_sprintf("smtp send to %s [%s] failed: %s", host->name,
4597 host->address, strerror(errno));
4598 sx->send_quit = FALSE;
4600 else if (! (sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer),
4601 '2', ob->command_timeout)))
4604 sx->send_quit = check_response(host, &errno, 0, sx->buffer, &code, &msg,
4608 DEBUG(D_transport) debug_printf("H=%s [%s] %s\n",
4609 host->name, host->address, msg);
4613 /* Either RSET was not needed, or it succeeded */
4620 int socket_fd = sx->cctx.sock;
4622 if (sx->first_addr) /* More addresses still to be sent */
4623 { /* for this message */
4624 #ifdef EXPERIMENTAL_ESMTP_LIMITS
4625 /* Any that we marked as skipped, reset to do now */
4626 for (address_item * a = sx->first_addr; a; a = a->next)
4627 if (a->transport_return == SKIP)
4628 a->transport_return = PENDING_DEFER;
4630 continue_sequence++; /* for consistency */
4631 clearflag(sx->first_addr, af_new_conn);
4632 setflag(sx->first_addr, af_cont_conn); /* Causes * in logging */
4633 pipelining_active = sx->pipelining_used; /* was cleared at DATA */
4637 /* Unless caller said it already has more messages listed for this host,
4638 pass the connection on to a new Exim process (below, the call to
4639 transport_pass_socket). If the caller has more ready, just return with
4640 the connection still open. */
4643 if (tls_out.active.sock >= 0)
4644 if ( f.continue_more
4645 || verify_check_given_host(CUSS &ob->hosts_noproxy_tls, host) == OK)
4647 /* Before passing the socket on, or returning to caller with it still
4648 open, we must shut down TLS. Not all MTAs allow for the continuation
4649 of the SMTP session when TLS is shut down. We test for this by sending
4650 a new EHLO. If we don't get a good response, we don't attempt to pass
4653 tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT);
4654 sx->send_tlsclose = FALSE;
4655 sx->cctx.tls_ctx = NULL;
4656 tls_out.active.sock = -1;
4657 smtp_peer_options = smtp_peer_options_wrap;
4659 && smtp_write_command(sx, SCMD_FLUSH, "EHLO %s\r\n", sx->helo_data)
4661 && smtp_read_response(sx, sx->buffer, sizeof(sx->buffer),
4662 '2', ob->command_timeout);
4664 if (sx->ok && f.continue_more)
4665 goto TIDYUP; /* More addresses for another run */
4669 /* Set up a pipe for proxying TLS for the new transport process */
4671 smtp_peer_options |= OPTION_TLS;
4672 if ((sx->ok = socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) == 0))
4675 set_errno(sx->first_addr, errno, US"internal allocation problem",
4677 # ifdef EXPERIMENTAL_DSN_INFO
4678 sx->smtp_greeting, sx->helo_response,
4680 &sx->delivery_start);
4684 if (f.continue_more)
4685 goto TIDYUP; /* More addresses for another run */
4687 /* If the socket is successfully passed, we mustn't send QUIT (or
4688 indeed anything!) from here. */
4690 /*XXX DSN_INFO: assume likely to do new HELO; but for greet we'll want to
4691 propagate it from the initial
4693 if (sx->ok && transport_pass_socket(tblock->name, host->name,
4694 host->address, new_message_id, socket_fd
4695 #ifdef EXPERIMENTAL_ESMTP_LIMITS
4696 , sx->peer_limit_mail, sx->peer_limit_rcpt, sx->peer_limit_rcptdom
4700 sx->send_quit = FALSE;
4702 /* We have passed the client socket to a fresh transport process.
4703 If TLS is still active, we need to proxy it for the transport we
4704 just passed the baton to. Fork a child to to do it, and return to
4705 get logging done asap. Which way to place the work makes assumptions
4706 about post-fork prioritisation which may not hold on all platforms. */
4708 if (tls_out.active.sock >= 0)
4710 int pid = exim_fork(US"tls-proxy-interproc");
4711 if (pid == 0) /* child; fork again to disconnect totally */
4713 /* does not return */
4714 smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), pfd,
4715 ob->command_timeout);
4718 if (pid > 0) /* parent */
4721 /* tidy the inter-proc to disconn the proxy proc */
4722 waitpid(pid, NULL, 0);
4723 tls_close(sx->cctx.tls_ctx, TLS_NO_SHUTDOWN);
4724 sx->cctx.tls_ctx = NULL;
4725 (void)close(sx->cctx.sock);
4727 continue_transport = NULL;
4728 continue_hostname = NULL;
4731 log_write(0, LOG_PANIC_DIE, "fork failed");
4737 /* If RSET failed and there are addresses left, they get deferred. */
4739 set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host,
4740 #ifdef EXPERIMENTAL_DSN_INFO
4741 sx->smtp_greeting, sx->helo_response,
4743 &sx->delivery_start);
4747 /* End off tidily with QUIT unless the connection has died or the socket has
4748 been passed to another process. */
4752 { /* Use _MORE to get QUIT in FIN segment */
4753 (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n");
4755 if (sx->cctx.tls_ctx)
4757 # ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */
4758 (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
4760 tls_shutdown_wr(sx->cctx.tls_ctx);
4761 sx->send_tlsclose = FALSE;
4768 /* Close the socket, and return the appropriate value, first setting
4769 works because the NULL setting is passed back to the calling process, and
4770 remote_max_parallel is forced to 1 when delivering over an existing connection,
4772 If all went well and continue_more is set, we shouldn't actually get here if
4773 there are further addresses, as the return above will be taken. However,
4774 writing RSET might have failed, or there may be other addresses whose hosts are
4775 specified in the transports, and therefore not visible at top level, in which
4776 case continue_more won't get set. */
4780 /* This flushes data queued in the socket, being the QUIT and any TLS Close,
4781 sending them along with the client FIN flag. Us (we hope) sending FIN first
4782 means we (client) take the TIME_WAIT state, so the server (which likely has a
4783 higher connection rate) does not have to. */
4785 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(shutdown)>>\n");
4786 shutdown(sx->cctx.sock, SHUT_WR);
4789 if (sx->send_quit || tcw_done && !tcw)
4791 /* Wait for (we hope) ack of our QUIT, and a server FIN. Discard any data
4792 received, then discard the socket. Any packet received after then, or receive
4793 data still in the socket, will get a RST - hence the pause/drain. */
4795 /* Reap the response to QUIT, timing out after one second */
4796 (void) smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', 1);
4798 if (sx->cctx.tls_ctx)
4802 /* Reap the TLS Close Notify from the server, timing out after one second */
4803 sigalrm_seen = FALSE;
4806 n = tls_read(sx->cctx.tls_ctx, sx->inbuffer, sizeof(sx->inbuffer));
4807 while (!sigalrm_seen && n > 0);
4810 # ifdef EXIM_TCP_CORK
4811 (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
4813 tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT);
4814 sx->cctx.tls_ctx = NULL;
4818 /* Drain any trailing data from the socket before close, to avoid sending a RST */
4820 if ( poll_one_fd(sx->cctx.sock, POLLIN, 20) != 0 /* 20ms */
4821 && fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0)
4822 for (int i = 16, n; /* drain socket */
4823 (n = read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer))) > 0 && i > 0;
4824 i--) HDEBUG(D_transport|D_acl|D_v)
4827 debug_printf_indent(" SMTP(drain %d bytes)<< %.*s\n", n, m, sx->inbuffer);
4828 for (m = 0; m < n; m++)
4829 debug_printf("0x%02x\n", sx->inbuffer[m]);
4832 HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n");
4833 (void)close(sx->cctx.sock);
4835 continue_transport = NULL;
4836 continue_hostname = NULL;
4838 #ifndef DISABLE_EVENT
4839 (void) event_raise(tblock->event_action, US"tcp:close", NULL, NULL);
4845 sx->first_addr = NULL;
4846 for (address_item * a = sx->addrlist->next; a; a = a->next)
4847 if (a->transport_return == DANE)
4849 a->transport_return = PENDING_DEFER;
4850 if (!sx->first_addr)
4852 /* Remember the new start-point in the addrlist, for smtp_setup_conn()
4853 to get the domain string for SNI */
4856 clearflag(a, af_cont_conn);
4857 setflag(a, af_new_conn); /* clear * from logging */
4858 DEBUG(D_transport) debug_printf("DANE: go-around for %s\n", a->domain);
4861 continue_sequence = 1; /* for consistency */
4866 #ifdef EXPERIMENTAL_ESMTP_LIMITS
4867 if (mail_limit && sx->first_addr)
4869 /* Reset the sequence count since we closed the connection. This is flagged
4870 on the pipe back to the delivery process so that a non-continued-conn delivery
4873 continue_sequence = 1; /* for consistency */
4874 clearflag(sx->first_addr, af_cont_conn);
4875 setflag(sx->first_addr, af_new_conn); /* clear * from logging */
4884 if (dane_held) for (address_item * a = sx->addrlist->next; a; a = a->next)
4885 if (a->transport_return == DANE)
4886 a->transport_return = PENDING_DEFER;
4894 /*************************************************
4895 * Closedown entry point *
4896 *************************************************/
4898 /* This function is called when exim is passed an open smtp channel
4899 from another incarnation, but the message which it has been asked
4900 to deliver no longer exists. The channel is on stdin.
4902 We might do fancy things like looking for another message to send down
4903 the channel, but if the one we sought has gone, it has probably been
4904 delivered by some other process that itself will seek further messages,
4905 so just close down our connection.
4907 Argument: pointer to the transport instance block
4912 smtp_transport_closedown(transport_instance *tblock)
4914 smtp_transport_options_block * ob = SOB tblock->options_block;
4915 client_conn_ctx cctx;
4918 uschar inbuffer[4096];
4919 uschar outbuffer[16];
4921 /*XXX really we need an active-smtp-client ctx, rather than assuming stdout */
4922 cctx.sock = fileno(stdin);
4923 cctx.tls_ctx = cctx.sock == tls_out.active.sock ? tls_out.active.tls_ctx : NULL;
4925 sx.inblock.cctx = &cctx;
4926 sx.inblock.buffer = inbuffer;
4927 sx.inblock.buffersize = sizeof(inbuffer);
4928 sx.inblock.ptr = inbuffer;
4929 sx.inblock.ptrend = inbuffer;
4931 sx.outblock.cctx = &cctx;
4932 sx.outblock.buffersize = sizeof(outbuffer);
4933 sx.outblock.buffer = outbuffer;
4934 sx.outblock.ptr = outbuffer;
4935 sx.outblock.cmd_count = 0;
4936 sx.outblock.authenticating = FALSE;
4938 (void)smtp_write_command(&sx, SCMD_FLUSH, "QUIT\r\n");
4939 (void)smtp_read_response(&sx, buffer, sizeof(buffer), '2', ob->command_timeout);
4940 (void)close(cctx.sock);
4945 /*************************************************
4946 * Prepare addresses for delivery *
4947 *************************************************/
4949 /* This function is called to flush out error settings from previous delivery
4950 attempts to other hosts. It also records whether we got here via an MX record
4951 or not in the more_errno field of the address. We are interested only in
4952 addresses that are still marked DEFER - others may have got delivered to a
4953 previously considered IP address. Set their status to PENDING_DEFER to indicate
4954 which ones are relevant this time.
4957 addrlist the list of addresses
4958 host the host we are delivering to
4960 Returns: the first address for this delivery
4963 static address_item *
4964 prepare_addresses(address_item *addrlist, host_item *host)
4966 address_item *first_addr = NULL;
4967 for (address_item * addr = addrlist; addr; addr = addr->next)
4968 if (addr->transport_return == DEFER)
4970 if (!first_addr) first_addr = addr;
4971 addr->transport_return = PENDING_DEFER;
4972 addr->basic_errno = 0;
4973 addr->more_errno = (host->mx >= 0)? 'M' : 'A';
4974 addr->message = NULL;
4976 addr->cipher = NULL;
4977 addr->ourcert = NULL;
4978 addr->peercert = NULL;
4979 addr->peerdn = NULL;
4980 addr->ocsp = OCSP_NOT_REQ;
4981 addr->tlsver = NULL;
4983 #ifdef EXPERIMENTAL_DSN_INFO
4984 addr->smtp_greeting = NULL;
4985 addr->helo_response = NULL;
4993 /*************************************************
4994 * Main entry point *
4995 *************************************************/
4997 /* See local README for interface details. As this is a remote transport, it is
4998 given a chain of addresses to be delivered in one connection, if possible. It
4999 always returns TRUE, indicating that each address has its own independent
5000 status set, except if there is a setting up problem, in which case it returns
5004 smtp_transport_entry(
5005 transport_instance *tblock, /* data for this instantiation */
5006 address_item *addrlist) /* addresses we are working on */
5009 int hosts_defer = 0;
5011 int hosts_looked_up = 0;
5012 int hosts_retry = 0;
5013 int hosts_serial = 0;
5014 int hosts_total = 0;
5015 int total_hosts_tried = 0;
5016 BOOL expired = TRUE;
5017 uschar *expanded_hosts = NULL;
5019 uschar *tid = string_sprintf("%s transport", tblock->name);
5020 smtp_transport_options_block *ob = SOB tblock->options_block;
5021 host_item *hostlist = addrlist->host_list;
5022 host_item *host = NULL;
5026 debug_printf("%s transport entered\n", tblock->name);
5027 for (address_item * addr = addrlist; addr; addr = addr->next)
5028 debug_printf(" %s\n", addr->address);
5031 debug_printf("hostlist:\n");
5032 for (host_item * host = hostlist; host; host = host->next)
5033 debug_printf(" '%s' IP %s port %d\n", host->name, host->address, host->port);
5035 if (continue_hostname)
5036 debug_printf("already connected to %s [%s] (on fd %d)\n",
5037 continue_hostname, continue_host_address,
5038 cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0);
5041 /* Check the restrictions on line length */
5043 if (max_received_linelength > ob->message_linelength_limit)
5046 gettimeofday(&now, NULL);
5048 for (address_item * addr = addrlist; addr; addr = addr->next)
5049 if (addr->transport_return == DEFER)
5050 addr->transport_return = PENDING_DEFER;
5052 set_errno_nohost(addrlist, ERRNO_SMTPFORMAT,
5053 US"message has lines too long for transport", FAIL, TRUE, &now);
5057 /* Set the flag requesting that these hosts be added to the waiting
5058 database if the delivery fails temporarily or if we are running with
5059 queue_smtp or a 2-stage queue run. This gets unset for certain
5060 kinds of error, typically those that are specific to the message. */
5062 update_waiting = TRUE;
5064 /* If a host list is not defined for the addresses - they must all have the
5065 same one in order to be passed to a single transport - or if the transport has
5066 a host list with hosts_override set, use the host list supplied with the
5067 transport. It is an error for this not to exist. */
5069 if (!hostlist || (ob->hosts_override && ob->hosts))
5073 addrlist->message = string_sprintf("%s transport called with no hosts set",
5075 addrlist->transport_return = PANIC;
5076 return FALSE; /* Only top address has status */
5079 DEBUG(D_transport) debug_printf("using the transport's hosts: %s\n",
5082 /* If the transport's host list contains no '$' characters, and we are not
5083 randomizing, it is fixed and therefore a chain of hosts can be built once
5084 and for all, and remembered for subsequent use by other calls to this
5085 transport. If, on the other hand, the host list does contain '$', or we are
5086 randomizing its order, we have to rebuild it each time. In the fixed case,
5087 as the hosts string will never be used again, it doesn't matter that we
5088 replace all the : characters with zeros. */
5092 uschar *s = ob->hosts;
5094 if (Ustrchr(s, '$'))
5096 if (!(expanded_hosts = expand_string(s)))
5098 addrlist->message = string_sprintf("failed to expand list of hosts "
5099 "\"%s\" in %s transport: %s", s, tblock->name, expand_string_message);
5100 addrlist->transport_return = f.search_find_defer ? DEFER : PANIC;
5101 return FALSE; /* Only top address has status */
5103 DEBUG(D_transport) debug_printf("expanded list of hosts \"%s\" to "
5104 "\"%s\"\n", s, expanded_hosts);
5108 if (ob->hosts_randomize) s = expanded_hosts = string_copy(s);
5110 if (is_tainted2(s, LOG_MAIN|LOG_PANIC, "Tainted host list '%s' from '%s' in transport %s", s, ob->hosts, tblock->name))
5112 /* Avoid leaking info to an attacker */
5113 addrlist->message = US"internal configuration error";
5114 addrlist->transport_return = PANIC;
5118 host_build_hostlist(&hostlist, s, ob->hosts_randomize);
5120 /* Check that the expansion yielded something useful. */
5124 string_sprintf("%s transport has empty hosts setting", tblock->name);
5125 addrlist->transport_return = PANIC;
5126 return FALSE; /* Only top address has status */
5129 /* If there was no expansion of hosts, save the host list for
5132 if (!expanded_hosts) ob->hostlist = hostlist;
5135 /* This is not the first time this transport has been run in this delivery;
5136 the host list was built previously. */
5139 hostlist = ob->hostlist;
5142 /* The host list was supplied with the address. If hosts_randomize is set, we
5143 must sort it into a random order if it did not come from MX records and has not
5144 already been randomized (but don't bother if continuing down an existing
5147 else if (ob->hosts_randomize && hostlist->mx == MX_NONE && !continue_hostname)
5149 host_item *newlist = NULL;
5152 host_item *h = hostlist;
5153 hostlist = hostlist->next;
5155 h->sort_key = random_number(100);
5162 else if (h->sort_key < newlist->sort_key)
5169 host_item *hh = newlist;
5172 if (h->sort_key < hh->next->sort_key) break;
5180 hostlist = addrlist->host_list = newlist;
5183 /* Sort out the default port. */
5185 if (!smtp_get_port(ob->port, addrlist, &defport, tid)) return FALSE;
5187 /* For each host-plus-IP-address on the list:
5189 . If this is a continued delivery and the host isn't the one with the
5190 current connection, skip.
5192 . If the status is unusable (i.e. previously failed or retry checked), skip.
5194 . If no IP address set, get the address, either by turning the name into
5195 an address, calling gethostbyname if gethostbyname is on, or by calling
5196 the DNS. The DNS may yield multiple addresses, in which case insert the
5197 extra ones into the list.
5199 . Get the retry data if not previously obtained for this address and set the
5200 field which remembers the state of this address. Skip if the retry time is
5201 not reached. If not, remember whether retry data was found. The retry string
5202 contains both the name and the IP address.
5204 . Scan the list of addresses and mark those whose status is DEFER as
5205 PENDING_DEFER. These are the only ones that will be processed in this cycle
5208 . Make a delivery attempt - addresses marked PENDING_DEFER will be tried.
5209 Some addresses may be successfully delivered, others may fail, and yet
5210 others may get temporary errors and so get marked DEFER.
5212 . The return from the delivery attempt is OK if a connection was made and a
5213 valid SMTP dialogue was completed. Otherwise it is DEFER.
5215 . If OK, add a "remove" retry item for this host/IPaddress, if any.
5217 . If fail to connect, or other defer state, add a retry item.
5219 . If there are any addresses whose status is still DEFER, carry on to the
5220 next host/IPaddress, unless we have tried the number of hosts given
5221 by hosts_max_try or hosts_max_try_hardlimit; otherwise return. Note that
5222 there is some fancy logic for hosts_max_try that means its limit can be
5223 overstepped in some circumstances.
5225 If we get to the end of the list, all hosts have deferred at least one address,
5226 or not reached their retry times. If delay_after_cutoff is unset, it requests a
5227 delivery attempt to those hosts whose last try was before the arrival time of
5228 the current message. To cope with this, we have to go round the loop a second
5229 time. After that, set the status and error data for any addresses that haven't
5230 had it set already. */
5232 for (int cutoff_retry = 0;
5233 expired && cutoff_retry < (ob->delay_after_cutoff ? 1 : 2);
5236 host_item *nexthost = NULL;
5237 int unexpired_hosts_tried = 0;
5238 BOOL continue_host_tried = FALSE;
5240 retry_non_continued:
5241 for (host = hostlist;
5243 && unexpired_hosts_tried < ob->hosts_max_try
5244 && total_hosts_tried < ob->hosts_max_try_hardlimit;
5249 BOOL host_is_expired = FALSE;
5250 BOOL message_defer = FALSE;
5251 BOOL some_deferred = FALSE;
5252 address_item *first_addr = NULL;
5253 uschar *interface = NULL;
5254 uschar *retry_host_key = NULL;
5255 uschar *retry_message_key = NULL;
5256 uschar *serialize_key = NULL;
5258 /* Default next host is next host. :-) But this can vary if the
5259 hosts_max_try limit is hit (see below). It may also be reset if a host
5260 address is looked up here (in case the host was multihomed). */
5262 nexthost = host->next;
5264 /* If the address hasn't yet been obtained from the host name, look it up
5265 now, unless the host is already marked as unusable. If it is marked as
5266 unusable, it means that the router was unable to find its IP address (in
5267 the DNS or wherever) OR we are in the 2nd time round the cutoff loop, and
5268 the lookup failed last time. We don't get this far if *all* MX records
5269 point to non-existent hosts; that is treated as a hard error.
5271 We can just skip this host entirely. When the hosts came from the router,
5272 the address will timeout based on the other host(s); when the address is
5273 looked up below, there is an explicit retry record added.
5275 Note that we mustn't skip unusable hosts if the address is not unset; they
5276 may be needed as expired hosts on the 2nd time round the cutoff loop. */
5280 int new_port, flags;
5282 if (host->status >= hstatus_unusable)
5284 DEBUG(D_transport) debug_printf("%s has no address and is unusable - skipping\n",
5289 DEBUG(D_transport) debug_printf("getting address for %s\n", host->name);
5291 /* The host name is permitted to have an attached port. Find it, and
5292 strip it from the name. Just remember it for now. */
5294 new_port = host_item_get_port(host);
5296 /* Count hosts looked up */
5300 /* Find by name if so configured, or if it's an IP address. We don't
5301 just copy the IP address, because we need the test-for-local to happen. */
5303 flags = HOST_FIND_BY_A | HOST_FIND_BY_AAAA;
5304 if (ob->dns_qualify_single) flags |= HOST_FIND_QUALIFY_SINGLE;
5305 if (ob->dns_search_parents) flags |= HOST_FIND_SEARCH_PARENTS;
5307 if (ob->gethostbyname || string_is_ip_address(host->name, NULL) != 0)
5308 rc = host_find_byname(host, NULL, flags, NULL, TRUE);
5310 rc = host_find_bydns(host, NULL, flags, NULL, NULL, NULL,
5311 &ob->dnssec, /* domains for request/require */
5314 /* Update the host (and any additional blocks, resulting from
5315 multihoming) with a host-specific port, if any. */
5317 for (host_item * hh = host; hh != nexthost; hh = hh->next) hh->port = new_port;
5319 /* Failure to find the host at this time (usually DNS temporary failure)
5320 is really a kind of routing failure rather than a transport failure.
5321 Therefore we add a retry item of the routing kind, not to stop us trying
5322 to look this name up here again, but to ensure the address gets timed
5323 out if the failures go on long enough. A complete failure at this point
5324 commonly points to a configuration error, but the best action is still
5325 to carry on for the next host. */
5327 if (rc == HOST_FIND_AGAIN || rc == HOST_FIND_SECURITY || rc == HOST_FIND_FAILED)
5329 retry_add_item(addrlist, string_sprintf("R:%s", host->name), 0);
5331 if (rc == HOST_FIND_AGAIN) hosts_defer++; else hosts_fail++;
5332 DEBUG(D_transport) debug_printf("rc = %s for %s\n", (rc == HOST_FIND_AGAIN)?
5333 "HOST_FIND_AGAIN" : "HOST_FIND_FAILED", host->name);
5334 host->status = hstatus_unusable;
5336 for (address_item * addr = addrlist; addr; addr = addr->next)
5338 if (addr->transport_return != DEFER) continue;
5339 addr->basic_errno = ERRNO_UNKNOWNHOST;
5340 addr->message = string_sprintf(
5341 rc == HOST_FIND_SECURITY
5342 ? "lookup of IP address for %s was insecure"
5343 : "failed to lookup IP address for %s",
5349 /* If the host is actually the local host, we may have a problem, or
5350 there may be some cunning configuration going on. In the problem case,
5351 log things and give up. The default transport status is already DEFER. */
5353 if (rc == HOST_FOUND_LOCAL && !ob->allow_localhost)
5355 for (address_item * addr = addrlist; addr; addr = addr->next)
5357 addr->basic_errno = ERRNO_HOST_IS_LOCAL;
5358 addr->message = string_sprintf("%s transport found host %s to be "
5359 "local", tblock->name, host->name);
5363 } /* End of block for IP address lookup */
5365 /* If this is a continued delivery, we are interested only in the host
5366 which matches the name of the existing open channel. The check is put
5367 here after the local host lookup, in case the name gets expanded as a
5368 result of the lookup. Set expired FALSE, to save the outer loop executing
5371 if (continue_hostname)
5372 if ( Ustrcmp(continue_hostname, host->name) != 0
5373 || Ustrcmp(continue_host_address, host->address) != 0
5377 continue; /* With next host */
5380 continue_host_tried = TRUE;
5382 /* Reset the default next host in case a multihomed host whose addresses
5383 are not looked up till just above added to the host list. */
5385 nexthost = host->next;
5387 /* If queue_smtp is set (-odqs or the first part of a 2-stage run), or the
5388 domain is in queue_smtp_domains, we don't actually want to attempt any
5389 deliveries. When doing a queue run, queue_smtp_domains is always unset. If
5390 there is a lookup defer in queue_smtp_domains, proceed as if the domain
5391 were not in it. We don't want to hold up all SMTP deliveries! Except when
5392 doing a two-stage queue run, don't do this if forcing. */
5394 if ( (!f.deliver_force || f.queue_2stage)
5396 || match_isinlist(addrlist->domain,
5397 CUSS &queue_smtp_domains, 0,
5398 &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
5401 DEBUG(D_transport) debug_printf("first-pass routing only\n");
5403 for (address_item * addr = addrlist; addr; addr = addr->next)
5404 if (addr->transport_return == DEFER)
5405 addr->message = US"first-pass only routing due to -odqs, "
5406 "queue_smtp_domains or control=queue";
5407 continue; /* With next host */
5410 /* Count hosts being considered - purely for an intelligent comment
5411 if none are usable. */
5415 /* Set $host and $host address now in case they are needed for the
5416 interface expansion or the serialize_hosts check; they remain set if an
5417 actual delivery happens. */
5419 deliver_host = host->name;
5420 deliver_host_address = host->address;
5421 lookup_dnssec_authenticated = host->dnssec == DS_YES ? US"yes"
5422 : host->dnssec == DS_NO ? US"no"
5425 /* Set up a string for adding to the retry key if the port number is not
5426 the standard SMTP port. A host may have its own port setting that overrides
5429 pistring = string_sprintf(":%d", host->port == PORT_NONE
5430 ? defport : host->port);
5431 if (Ustrcmp(pistring, ":25") == 0) pistring = US"";
5433 /* Select IPv4 or IPv6, and choose an outgoing interface. If the interface
5434 string is set, even if constant (as different transports can have different
5435 constant settings), we must add it to the key that is used for retries,
5436 because connections to the same host from a different interface should be
5437 treated separately. */
5439 host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET;
5441 uschar * s = ob->interface;
5444 if (!smtp_get_interface(s, host_af, addrlist, &interface, tid))
5446 pistring = string_sprintf("%s/%s", pistring, interface);
5450 /* The first time round the outer loop, check the status of the host by
5451 inspecting the retry data. The second time round, we are interested only
5452 in expired hosts that haven't been tried since this message arrived. */
5454 if (cutoff_retry == 0)
5457 /* Ensure the status of the address is set by checking retry data if
5458 necessary. There may be host-specific retry data (applicable to all
5459 messages) and also data for retries of a specific message at this host.
5460 If either of these retry records are actually read, the keys used are
5461 returned to save recomputing them later. */
5463 if (exp_bool(addrlist, US"transport", tblock->name, D_transport,
5464 US"retry_include_ip_address", ob->retry_include_ip_address,
5465 ob->expand_retry_include_ip_address, &incl_ip) != OK)
5466 continue; /* with next host */
5468 host_is_expired = retry_check_address(addrlist->domain, host, pistring,
5469 incl_ip, &retry_host_key, &retry_message_key);
5471 DEBUG(D_transport) debug_printf("%s [%s]%s retry-status = %s\n", host->name,
5472 host->address ? host->address : US"", pistring,
5473 host->status == hstatus_usable ? "usable"
5474 : host->status == hstatus_unusable ? "unusable"
5475 : host->status == hstatus_unusable_expired ? "unusable (expired)" : "?");
5477 /* Skip this address if not usable at this time, noting if it wasn't
5478 actually expired, both locally and in the address. */
5480 switch (host->status)
5482 case hstatus_unusable:
5484 setflag(addrlist, af_retry_skipped);
5487 case hstatus_unusable_expired:
5490 case hwhy_retry: hosts_retry++; break;
5491 case hwhy_failed: hosts_fail++; break;
5493 case hwhy_deferred: hosts_defer++; break;
5496 /* If there was a retry message key, implying that previously there
5497 was a message-specific defer, we don't want to update the list of
5498 messages waiting for these hosts. */
5500 if (retry_message_key) update_waiting = FALSE;
5501 continue; /* With the next host or IP address */
5505 /* Second time round the loop: if the address is set but expired, and
5506 the message is newer than the last try, let it through. */
5511 || host->status != hstatus_unusable_expired
5512 || host->last_try > received_time.tv_sec)
5514 DEBUG(D_transport) debug_printf("trying expired host %s [%s]%s\n",
5515 host->name, host->address, pistring);
5516 host_is_expired = TRUE;
5519 /* Setting "expired=FALSE" doesn't actually mean not all hosts are expired;
5520 it remains TRUE only if all hosts are expired and none are actually tried.
5525 /* If this host is listed as one to which access must be serialized,
5526 see if another Exim process has a connection to it, and if so, skip
5527 this host. If not, update the database to record our connection to it
5528 and remember this for later deletion. Do not do any of this if we are
5529 sending the message down a pre-existing connection. */
5531 if ( !continue_hostname
5532 && verify_check_given_host(CUSS &ob->serialize_hosts, host) == OK)
5534 serialize_key = string_sprintf("host-serialize-%s", host->name);
5535 if (!enq_start(serialize_key, 1))
5538 debug_printf("skipping host %s because another Exim process "
5539 "is connected to it\n", host->name);
5545 /* OK, we have an IP address that is not waiting for its retry time to
5546 arrive (it might be expired) OR (second time round the loop) we have an
5547 expired host that hasn't been tried since the message arrived. Have a go
5548 at delivering the message to it. First prepare the addresses by flushing
5549 out the result of previous attempts, and finding the first address that
5550 is still to be delivered. */
5552 first_addr = prepare_addresses(addrlist, host);
5554 DEBUG(D_transport) debug_printf("delivering %s to %s [%s] (%s%s)\n",
5555 message_id, host->name, host->address, addrlist->address,
5556 addrlist->next ? ", ..." : "");
5558 set_process_info("delivering %s to %s [%s]%s (%s%s)",
5559 message_id, host->name, host->address, pistring, addrlist->address,
5560 addrlist->next ? ", ..." : "");
5562 /* This is not for real; don't do the delivery. If there are
5563 any remaining hosts, list them. */
5568 gettimeofday(&now, NULL);
5569 set_errno_nohost(addrlist, 0, NULL, OK, FALSE, &now);
5570 for (address_item * addr = addrlist; addr; addr = addr->next)
5572 addr->host_used = host;
5573 addr->special_action = '*';
5574 addr->message = US"delivery bypassed by -N option";
5578 debug_printf("*** delivery by %s transport bypassed by -N option\n"
5579 "*** host and remaining hosts:\n", tblock->name);
5580 for (host_item * host2 = host; host2; host2 = host2->next)
5581 debug_printf(" %s [%s]\n", host2->name,
5582 host2->address ? host2->address : US"unset");
5587 /* This is for real. If the host is expired, we don't count it for
5588 hosts_max_retry. This ensures that all hosts must expire before an address
5589 is timed out, unless hosts_max_try_hardlimit (which protects against
5590 lunatic DNS configurations) is reached.
5592 If the host is not expired and we are about to hit the hosts_max_retry
5593 limit, check to see if there is a subsequent hosts with a different MX
5594 value. If so, make that the next host, and don't count this one. This is a
5595 heuristic to make sure that different MXs do get tried. With a normal kind
5596 of retry rule, they would get tried anyway when the earlier hosts were
5597 delayed, but if the domain has a "retry every time" type of rule - as is
5598 often used for the the very large ISPs, that won't happen. */
5603 /* Make a copy of the host if it is local to this invocation
5604 of the transport. */
5608 thost = store_get(sizeof(host_item), FALSE);
5610 thost->name = string_copy(host->name);
5611 thost->address = string_copy(host->address);
5616 if (!host_is_expired && ++unexpired_hosts_tried >= ob->hosts_max_try)
5619 debug_printf("hosts_max_try limit reached with this host\n");
5620 for (host_item * h = host; h; h = h->next) if (h->mx != host->mx)
5623 unexpired_hosts_tried--;
5624 DEBUG(D_transport) debug_printf("however, a higher MX host exists "
5625 "and will be tried\n");
5630 /* Attempt the delivery. */
5632 total_hosts_tried++;
5633 rc = smtp_deliver(addrlist, thost, host_af, defport, interface, tblock,
5634 &message_defer, FALSE);
5637 OK => connection made, each address contains its result;
5638 message_defer is set for message-specific defers (when all
5639 recipients are marked defer)
5640 DEFER => there was a non-message-specific delivery problem;
5641 ERROR => there was a problem setting up the arguments for a filter,
5642 or there was a problem with expanding added headers
5645 /* If the result is not OK, there was a non-message-specific problem.
5646 If the result is DEFER, we need to write to the logs saying what happened
5647 for this particular host, except in the case of authentication and TLS
5648 failures, where the log has already been written. If all hosts defer a
5649 general message is written at the end. */
5651 if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL
5652 && first_addr->basic_errno != ERRNO_TLSFAILURE)
5653 write_logs(host, first_addr->message, first_addr->basic_errno);
5655 #ifndef DISABLE_EVENT
5657 deferred_event_raise(first_addr, host, US"msg:host:defer");
5660 /* If STARTTLS was accepted, but there was a failure in setting up the
5661 TLS session (usually a certificate screwup), and the host is not in
5662 hosts_require_tls, and tls_tempfail_tryclear is true, try again, with
5663 TLS forcibly turned off. We have to start from scratch with a new SMTP
5664 connection. That's why the retry is done from here, not from within
5665 smtp_deliver(). [Rejections of STARTTLS itself don't screw up the
5666 session, so the in-clear transmission after those errors, if permitted,
5667 happens inside smtp_deliver().] */
5671 && first_addr->basic_errno == ERRNO_TLSFAILURE
5672 && ob->tls_tempfail_tryclear
5673 && verify_check_given_host(CUSS &ob->hosts_require_tls, host) != OK
5676 log_write(0, LOG_MAIN,
5677 "%s: delivering unencrypted to H=%s [%s] (not in hosts_require_tls)",
5678 first_addr->message, host->name, host->address);
5679 first_addr = prepare_addresses(addrlist, host);
5680 rc = smtp_deliver(addrlist, thost, host_af, defport, interface, tblock,
5681 &message_defer, TRUE);
5682 if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL)
5683 write_logs(host, first_addr->message, first_addr->basic_errno);
5684 # ifndef DISABLE_EVENT
5686 deferred_event_raise(first_addr, host, US"msg:host:defer");
5689 #endif /*DISABLE_TLS*/
5691 #ifndef DISABLE_EVENT
5692 /* If the last host gave a defer raise a per-message event */
5695 && unexpired_hosts_tried < ob->hosts_max_try
5696 && total_hosts_tried < ob->hosts_max_try_hardlimit
5698 && (message_defer || rc == DEFER)
5700 deferred_event_raise(first_addr, host, US"msg:defer");
5704 /* Delivery attempt finished */
5706 set_process_info("delivering %s: just tried %s [%s]%s for %s%s: result %s",
5707 message_id, host->name, host->address, pistring, addrlist->address,
5708 addrlist->next ? " (& others)" : "", rc_to_string(rc));
5710 /* Release serialization if set up */
5712 if (serialize_key) enq_end(serialize_key);
5714 /* If the result is DEFER, or if a host retry record is known to exist, we
5715 need to add an item to the retry chain for updating the retry database
5716 at the end of delivery. We only need to add the item to the top address,
5717 of course. Also, if DEFER, we mark the IP address unusable so as to skip it
5718 for any other delivery attempts using the same address. (It is copied into
5719 the unusable tree at the outer level, so even if different address blocks
5720 contain the same address, it still won't get tried again.) */
5722 if (rc == DEFER || retry_host_key)
5724 int delete_flag = rc != DEFER ? rf_delete : 0;
5725 if (!retry_host_key)
5728 if (exp_bool(addrlist, US"transport", tblock->name, D_transport,
5729 US"retry_include_ip_address", ob->retry_include_ip_address,
5730 ob->expand_retry_include_ip_address, &incl_ip) != OK)
5731 incl_ip = TRUE; /* error; use most-specific retry record */
5733 retry_host_key = incl_ip
5734 ? string_sprintf("T:%S:%s%s", host->name, host->address, pistring)
5735 : string_sprintf("T:%S%s", host->name, pistring);
5738 /* If a delivery of another message over an existing SMTP connection
5739 yields DEFER, we do NOT set up retry data for the host. This covers the
5740 case when there are delays in routing the addresses in the second message
5741 that are so long that the server times out. This is alleviated by not
5742 routing addresses that previously had routing defers when handling an
5743 existing connection, but even so, this case may occur (e.g. if a
5744 previously happily routed address starts giving routing defers). If the
5745 host is genuinely down, another non-continued message delivery will
5746 notice it soon enough. */
5748 if (delete_flag != 0 || !continue_hostname)
5749 retry_add_item(first_addr, retry_host_key, rf_host | delete_flag);
5751 /* We may have tried an expired host, if its retry time has come; ensure
5752 the status reflects the expiry for the benefit of any other addresses. */
5756 host->status = host_is_expired
5757 ? hstatus_unusable_expired : hstatus_unusable;
5758 host->why = hwhy_deferred;
5762 /* If message_defer is set (host was OK, but every recipient got deferred
5763 because of some message-specific problem), or if that had happened
5764 previously so that a message retry key exists, add an appropriate item
5765 to the retry chain. Note that if there was a message defer but now there is
5766 a host defer, the message defer record gets deleted. That seems perfectly
5767 reasonable. Also, stop the message from being remembered as waiting
5768 for specific hosts. */
5770 if (message_defer || retry_message_key)
5772 int delete_flag = message_defer ? 0 : rf_delete;
5773 if (!retry_message_key)
5776 if (exp_bool(addrlist, US"transport", tblock->name, D_transport,
5777 US"retry_include_ip_address", ob->retry_include_ip_address,
5778 ob->expand_retry_include_ip_address, &incl_ip) != OK)
5779 incl_ip = TRUE; /* error; use most-specific retry record */
5781 retry_message_key = incl_ip
5782 ? string_sprintf("T:%S:%s%s:%s", host->name, host->address, pistring,
5784 : string_sprintf("T:%S%s:%s", host->name, pistring, message_id);
5786 retry_add_item(addrlist, retry_message_key,
5787 rf_message | rf_host | delete_flag);
5788 update_waiting = FALSE;
5791 /* Any return other than DEFER (that is, OK or ERROR) means that the
5792 addresses have got their final statuses filled in for this host. In the OK
5793 case, see if any of them are deferred. */
5796 for (address_item * addr = addrlist; addr; addr = addr->next)
5797 if (addr->transport_return == DEFER)
5799 some_deferred = TRUE;
5803 /* If no addresses deferred or the result was ERROR, return. We do this for
5804 ERROR because a failing filter set-up or add_headers expansion is likely to
5805 fail for any host we try. */
5807 if (rc == ERROR || (rc == OK && !some_deferred))
5809 DEBUG(D_transport) debug_printf("Leaving %s transport\n", tblock->name);
5810 return TRUE; /* Each address has its status */
5813 /* If the result was DEFER or some individual addresses deferred, let
5814 the loop run to try other hosts with the deferred addresses, except for the
5815 case when we were trying to deliver down an existing channel and failed.
5816 Don't try any other hosts in this case. */
5818 if (continue_hostname) break;
5820 /* If the whole delivery, or some individual addresses, were deferred and
5821 there are more hosts that could be tried, do not count this host towards
5822 the hosts_max_try limit if the age of the message is greater than the
5823 maximum retry time for this host. This means we may try try all hosts,
5824 ignoring the limit, when messages have been around for some time. This is
5825 important because if we don't try all hosts, the address will never time
5826 out. NOTE: this does not apply to hosts_max_try_hardlimit. */
5828 if ((rc == DEFER || some_deferred) && nexthost)
5831 retry_config *retry = retry_find_config(host->name, NULL, 0, 0);
5833 if (retry && retry->rules)
5835 retry_rule *last_rule;
5836 for (last_rule = retry->rules;
5838 last_rule = last_rule->next);
5839 timedout = time(NULL) - received_time.tv_sec > last_rule->timeout;
5841 else timedout = TRUE; /* No rule => timed out */
5845 unexpired_hosts_tried--;
5846 DEBUG(D_transport) debug_printf("temporary delivery error(s) override "
5847 "hosts_max_try (message older than host's retry time)\n");
5853 if (unexpired_hosts_tried >= ob->hosts_max_try)
5854 debug_printf("reached transport hosts_max_try limit %d\n",
5856 if (total_hosts_tried >= ob->hosts_max_try_hardlimit)
5857 debug_printf("reached transport hosts_max_try_hardlimit limit %d\n",
5858 ob->hosts_max_try_hardlimit);
5861 testharness_pause_ms(500); /* let server debug out */
5862 } /* End of loop for trying multiple hosts. */
5864 /* If we failed to find a matching host in the list, for an already-open
5865 connection, just close it and start over with the list. This can happen
5866 for routing that changes from run to run, or big multi-IP sites with
5869 if (continue_hostname && !continue_host_tried)
5871 int fd = cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0;
5873 DEBUG(D_transport) debug_printf("no hosts match already-open connection\n");
5875 /* A TLS conn could be open for a cutthrough, but not for a plain continued-
5877 /*XXX doublecheck that! */
5879 if (cutthrough.cctx.sock >= 0 && cutthrough.is_tls)
5881 (void) tls_write(cutthrough.cctx.tls_ctx, US"QUIT\r\n", 6, FALSE);
5882 tls_close(cutthrough.cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
5883 cutthrough.cctx.tls_ctx = NULL;
5884 cutthrough.is_tls = FALSE;
5888 (void) write(fd, US"QUIT\r\n", 6);
5891 cutthrough.cctx.sock = -1;
5892 continue_hostname = NULL;
5893 goto retry_non_continued;
5896 /* This is the end of the loop that repeats iff expired is TRUE and
5897 ob->delay_after_cutoff is FALSE. The second time round we will
5898 try those hosts that haven't been tried since the message arrived. */
5902 debug_printf("all IP addresses skipped or deferred at least one address\n");
5903 if (expired && !ob->delay_after_cutoff && cutoff_retry == 0)
5904 debug_printf("retrying IP addresses not tried since message arrived\n");
5909 /* Get here if all IP addresses are skipped or defer at least one address. In
5910 MUA wrapper mode, this will happen only for connection or other non-message-
5911 specific failures. Force the delivery status for all addresses to FAIL. */
5915 for (address_item * addr = addrlist; addr; addr = addr->next)
5916 addr->transport_return = FAIL;
5920 /* In the normal, non-wrapper case, add a standard message to each deferred
5921 address if there hasn't been an error, that is, if it hasn't actually been
5922 tried this time. The variable "expired" will be FALSE if any deliveries were
5923 actually tried, or if there was at least one host that was not expired. That
5924 is, it is TRUE only if no deliveries were tried and all hosts were expired. If
5925 a delivery has been tried, an error code will be set, and the failing of the
5926 message is handled by the retry code later.
5928 If queue_smtp is set, or this transport was called to send a subsequent message
5929 down an existing TCP/IP connection, and something caused the host not to be
5930 found, we end up here, but can detect these cases and handle them specially. */
5932 for (address_item * addr = addrlist; addr; addr = addr->next)
5934 /* If host is not NULL, it means that we stopped processing the host list
5935 because of hosts_max_try or hosts_max_try_hardlimit. In the former case, this
5936 means we need to behave as if some hosts were skipped because their retry
5937 time had not come. Specifically, this prevents the address from timing out.
5938 However, if we have hit hosts_max_try_hardlimit, we want to behave as if all
5939 hosts were tried. */
5942 if (total_hosts_tried >= ob->hosts_max_try_hardlimit)
5945 debug_printf("hosts_max_try_hardlimit reached: behave as if all "
5946 "hosts were tried\n");
5951 debug_printf("hosts_max_try limit caused some hosts to be skipped\n");
5952 setflag(addr, af_retry_skipped);
5955 if (f.queue_smtp) /* no deliveries attempted */
5957 addr->transport_return = DEFER;
5958 addr->basic_errno = 0;
5959 addr->message = US"SMTP delivery explicitly queued";
5962 else if ( addr->transport_return == DEFER
5963 && (addr->basic_errno == ERRNO_UNKNOWNERROR || addr->basic_errno == 0)
5967 addr->basic_errno = ERRNO_HRETRY;
5968 if (continue_hostname)
5969 addr->message = US"no host found for existing SMTP connection";
5972 setflag(addr, af_pass_message); /* This is not a security risk */
5973 addr->message = string_sprintf(
5974 "all hosts%s have been failing for a long time %s",
5975 addr->domain ? string_sprintf(" for '%s'", addr->domain) : US"",
5976 ob->delay_after_cutoff
5977 ? US"(and retry time not reached)"
5978 : US"and were last tried after this message arrived");
5980 /* If we are already using fallback hosts, or there are no fallback hosts
5981 defined, convert the result to FAIL to cause a bounce. */
5983 if (addr->host_list == addr->fallback_hosts || !addr->fallback_hosts)
5984 addr->transport_return = FAIL;
5989 if (hosts_retry == hosts_total)
5990 s = "retry time not reached for any host%s";
5991 else if (hosts_fail == hosts_total)
5992 s = "all host address lookups%s failed permanently";
5993 else if (hosts_defer == hosts_total)
5994 s = "all host address lookups%s failed temporarily";
5995 else if (hosts_serial == hosts_total)
5996 s = "connection limit reached for all hosts%s";
5997 else if (hosts_fail+hosts_defer == hosts_total)
5998 s = "all host address lookups%s failed";
6000 s = "some host address lookups failed and retry time "
6001 "not reached for other hosts or connection limit reached%s";
6003 addr->message = string_sprintf(s,
6004 addr->domain ? string_sprintf(" for '%s'", addr->domain) : US"");
6009 /* Update the database which keeps information about which messages are waiting
6010 for which hosts to become available. For some message-specific errors, the
6011 update_waiting flag is turned off because we don't want follow-on deliveries in
6012 those cases. If this transport instance is explicitly limited to one message
6013 per connection then follow-on deliveries are not possible and there's no need
6014 to create/update the per-transport wait-<transport_name> database. */
6016 if (update_waiting && tblock->connection_max_messages != 1)
6017 transport_update_waiting(hostlist, tblock->name);
6021 DEBUG(D_transport) debug_printf("Leaving %s transport\n", tblock->name);
6023 return TRUE; /* Each address has its status */
6026 #endif /*!MACRO_PREDEF*/
6029 /* End of transport/smtp.c */