Redis: move from Experimental to mainlineexim-4_87_RC2

17 files changed, 83 insertions, 139 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index 13a426c00..33f35bf89 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -6625,6 +6625,14 @@ password value. For example:
 PostgreSQL database. See section &<<SECTsql>>&.
 
 .next
+.new
+.cindex "Redis lookup type"
+.cindex lookup Redis
+&(redis)&: The format of the query is an SQL statement that is passed to a
+Redis database. See section &<<SECTsql>>&.
+.wen
+
+.next
 .cindex "sqlite lookup type"
 .cindex "lookup" "sqlite"
 &(sqlite)&: The format of the query is a file name followed by an SQL statement
@@ -7489,7 +7497,10 @@ operator is to double any quote characters within the text.
 .cindex "lookup" "Oracle"
 .cindex "InterBase lookup type"
 .cindex "lookup" "InterBase"
-Exim can support lookups in InterBase, MySQL, Oracle, PostgreSQL, and SQLite
+.cindex "Redis lookup type"
+.cindex lookup Redis
+Exim can support lookups in InterBase, MySQL, Oracle, PostgreSQL, Redis,
+and SQLite
 databases. Queries for these databases contain SQL statements, so an example
 might be
 .code
@@ -7516,7 +7527,7 @@ If the result of the query yields more than one row, it is all concatenated,
 with a newline between the data for each row.
 
 
-.section "More about MySQL, PostgreSQL, Oracle, and InterBase" "SECID72"
+.section "More about MySQL, PostgreSQL, Oracle, InterBase, and Redis" "SECID72"
 .cindex "MySQL" "lookup type"
 .cindex "PostgreSQL lookup type"
 .cindex "lookup" "MySQL"
@@ -7525,13 +7536,18 @@ with a newline between the data for each row.
 .cindex "lookup" "Oracle"
 .cindex "InterBase lookup type"
 .cindex "lookup" "InterBase"
-If any MySQL, PostgreSQL, Oracle, or InterBase lookups are used, the
-&%mysql_servers%&, &%pgsql_servers%&, &%oracle_servers%&, or &%ibase_servers%&
+.cindex "Redis lookup type"
+.cindex lookup Redis
+If any MySQL, PostgreSQL, Oracle, InterBase or Redis lookups are used, the
+&%mysql_servers%&, &%pgsql_servers%&, &%oracle_servers%&, &%ibase_servers%&,
+or &%redis_servers%&
 option (as appropriate) must be set to a colon-separated list of server
 information.
-(For MySQL and PostgreSQL only, the global option need not be set if all
+(For MySQL and PostgreSQL, the global option need not be set if all
 queries contain their own server information &-- see section
-&<<SECTspeserque>>&.) Each item in the list is a slash-separated list of four
+&<<SECTspeserque>>&.)
+For all but Redis
+each item in the list is a slash-separated list of four
 items: host name, database name, user name, and password. In the case of
 Oracle, the host name field is used for the &"service name"&, and the database
 name field is not used and should be empty. For example:
@@ -7553,14 +7569,35 @@ found, but that is still a successful query. In other words, the list of
 servers provides a backup facility, not a list of different places to look.
 
 .new
+For Redis the global option need not be specified if all queries contain their
+own server information &-- see section &<<SECTspeserque>>&.
+If specified, the option must be set to a colon-separated list of server
+information.
+Each item in the list is a slash-separated list of three items:
+host, database number, and password.
+.olist
+The host is required and may be either an IPv4 address and optional
+port number (separated by a colon, which needs doubling due to the
+higher-level list), or a Unix socket pathname enclosed in parentheses
+.next
+The database number is optional; if present that number is selected in the backend
+.next
+The password is optional; if present it is used to authenticate to the backend
+.endlist
+.wen
+
+.new
 The &%quote_mysql%&, &%quote_pgsql%&, and &%quote_oracle%& expansion operators
 convert newline, tab, carriage return, and backspace to \n, \t, \r, and \b
 respectively, and the characters single-quote, double-quote, and backslash
 itself are escaped with backslashes.
+
+The &%quote_redis%& expansion operator
+escapes whitespace and backslash characters with a backslash.
 .wen
 
 .section "Specifying the server in the query" "SECTspeserque"
-For MySQL and PostgreSQL lookups (but not currently for Oracle and InterBase),
+For MySQL, PostgreSQL and Redis lookups (but not currently for Oracle and InterBase),
 it is possible to specify a list of servers with an individual query. This is
 done by starting the query with
 .display
diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 4622fbd36..6976d4bfe 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -134,6 +134,9 @@ JH/30 Updated DANE implementation code to current from Viktor Dukhovni.
 JH/31 Fix bug with hosts_connection_nolog and named-lists which were wrongly
       cached by the daemon.
 
+JH/32 Move Redis support from Experimental to mainline, enabled for a build
+      by defining LOOKUP_REDIS. The libhiredis library is required.
+
 
 Exim version 4.86
 -----------------
diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index 81acfda5e..4836a7d51 100644
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -771,89 +771,6 @@ b. Configure, somewhere before the DATA ACL, the control option to
 
 
 
-Redis Lookup
---------------------------------------------------------------
-
-Redis is open source advanced key-value data store. This document
-does not explain the fundamentals, you should read and understand how
-it works by visiting the website at http://www.redis.io/.
-
-Redis lookup support is added via the hiredis library.  Visit:
-
-  https://github.com/redis/hiredis
-
-to obtain a copy, or find it in your operating systems package repository.
-If building from source, this description assumes that headers will be in
-/usr/local/include, and that the libraries are in /usr/local/lib.
-
-1. In order to build exim with Redis lookup support add
-
-EXPERIMENTAL_REDIS=yes
-
-to your Local/Makefile. (Re-)build/install exim. exim -d should show
-Experimental_Redis in the line "Support for:".
-
-EXPERIMENTAL_REDIS=yes
-LDFLAGS += -lhiredis
-# CFLAGS += -I/usr/local/include
-# LDFLAGS += -L/usr/local/lib
-
-The first line sets the feature to include the correct code, and
-the second line says to link the hiredis libraries into the
-exim binary.  The commented out lines should be uncommented if you
-built hiredis from source and installed in the default location.
-Adjust the paths if you installed them elsewhere, but you do not
-need to uncomment them if an rpm (or you) installed them in the
-package controlled locations (/usr/include and /usr/lib).
-
-
-2. Use the following global settings to configure Redis lookup support:
-
-Required:
-redis_servers       This option provides a list of Redis servers
-                    and associated connection data, to be used in
-                    conjunction with redis lookups. The option is
-                    only available if Exim is configured with Redis
-                    support.
-
-For example:
-
-redis_servers = 127.0.0.1/10/ - using database 10 with no password
-redis_servers = 127.0.0.1//password - to make use of the default database of 0 with a password
-redis_servers = 127.0.0.1// - for default database of 0 with no password
-
-3. Once you have the Redis servers defined you can then make use of the
-experimental Redis lookup by specifying ${lookup redis{}} in a lookup query.
-
-4. Example usage:
-
-(Host List)
-hostlist relay_from_ips = <\n ${lookup redis{SMEMBERS relay_from_ips}}
-
-Where relay_from_ips is a Redis set which contains entries such as "192.168.0.0/24" "10.0.0.0/8" and so on.
-The result set is returned as
-192.168.0.0/24
-10.0.0.0/8
-..
-.
-
-(Domain list)
-domainlist virtual_domains = ${lookup redis {HGET $domain domain}}
-
-Where $domain is a hash which includes the key 'domain' and the value '$domain'.
-
-(Adding or updating an existing key)
-set acl_c_spammer = ${if eq{${lookup redis{SPAMMER_SET}}}{OK}}
-
-Where SPAMMER_SET is a macro and it is defined as
-
-"SET SPAMMER <some_value>"
-
-(Getting a value from Redis)
-
-set acl_c_spam_host = ${lookup redis{GET...}}
-
-
 DANE
 ------------------------------------------------------------
 DNS-based Authentication of Named Entities, as applied
diff --git a/src/scripts/lookups-Makefile b/src/scripts/lookups-Makefile
index 61493c632..4e69a9a77 100755
--- a/src/scripts/lookups-Makefile
+++ b/src/scripts/lookups-Makefile
@@ -161,7 +161,7 @@ sed -n "1,/$tag_marker/p" < "$input"
 
 for name_mod in \
     CDB DBM:dbmdb DNSDB DSEARCH IBASE LSEARCH MYSQL NIS NISPLUS ORACLE \
-    PASSWD PGSQL SQLITE TESTDB WHOSON
+    PASSWD PGSQL REDIS SQLITE TESTDB WHOSON
 do
   emit_module_rule $name_mod
 done
@@ -177,14 +177,6 @@ fi
 
 OBJ="${OBJ} spf.o"
 
-# Because the variable is EXPERIMENTAL_REDIS and not LOOKUP_REDIS we
-# use a different function to check for EXPERIMENTAL_* features
-# requested. Don't use the SPF method with dummy functions above.
-if want_experimental REDIS
-then
-  OBJ="${OBJ} redis.o"
-fi
-
 echo "MODS = $MODS"
 echo "OBJ = $OBJ"
 
diff --git a/src/src/EDITME b/src/src/EDITME
index 7cdcfc93a..ab7f18588 100644
--- a/src/src/EDITME
+++ b/src/src/EDITME
@@ -288,6 +288,11 @@ TRANSPORT_SMTP=yes
 # library.
 # NOTE: LDAP cannot be built as a module!
 #
+# For Redis you need to have hiredis installed on your system
+# (https://github.com/redis/hiredis).
+# Depending on where it is installed you may have to edit the CFLAGS
+# (often += -I/usr/local/include) and LDFLAGS (-lhiredis) lines.
+
 # If your system has pkg-config then the _INCLUDE/_LIBS setting can be
 # handled for you automatically by also defining the _PC variable to reference
 # the name of the pkg-config package, if such is available.
@@ -306,6 +311,7 @@ LOOKUP_DNSDB=yes
 # LOOKUP_ORACLE=yes
 # LOOKUP_PASSWD=yes
 # LOOKUP_PGSQL=yes
+# LOOKUP_REDIS=yes
 # LOOKUP_SQLITE=yes
 # LOOKUP_SQLITE_PC=sqlite3
 # LOOKUP_WHOSON=yes
@@ -357,7 +363,8 @@ PCRE_CONFIG=yes
 # the command for linking Exim itself, not on any auxiliary programs. You
 # don't need to set LOOKUP_INCLUDE if the relevant directories are already
 # specified in INCLUDE. The settings below are just examples; -lpq is for
-# PostgreSQL, -lgds is for Interbase, -lsqlite3 is for SQLite.
+# PostgreSQL, -lgds is for Interbase, -lsqlite3 is for SQLite, -lhiredis
+# is for Redis.
 #
 # You do not need to use this for any lookup information added via pkg-config.
 
@@ -480,14 +487,6 @@ EXIM_MONITOR=eximon.bin
 # CFLAGS += -I/usr/local/include
 # LDFLAGS += -lopendmarc
 
-
-# Uncomment the following line to add Redis lookup support
-# You need to have hiredis installed on your system (https://github.com/redis/hiredis).
-# Depending on where it is installed you may have to edit the CFLAGS and LDFLAGS lines.
-# EXPERIMENTAL_REDIS=yes
-# CFLAGS += -I/usr/local/include
-# LDFLAGS += -lhiredis
-
 # Uncomment the following line to enable support for checking certificate
 # ownership
 # EXPERIMENTAL_CERTNAMES=yes
diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults
index 35c5559fb..14de083fe 100644
--- a/src/src/config.h.defaults
+++ b/src/src/config.h.defaults
@@ -95,6 +95,7 @@ it's a default value. */
 #define LOOKUP_ORACLE
 #define LOOKUP_PASSWD
 #define LOOKUP_PGSQL
+#define LOOKUP_REDIS
 #define LOOKUP_SQLITE
 #define LOOKUP_TESTDB
 #define LOOKUP_WHOSON
@@ -177,7 +178,6 @@ it's a default value. */
 #define EXPERIMENTAL_DCC
 #define EXPERIMENTAL_DSN_INFO
 #define EXPERIMENTAL_DMARC
-#define EXPERIMENTAL_REDIS
 #define EXPERIMENTAL_SPF
 #define EXPERIMENTAL_SRS
 
diff --git a/src/src/deliver.c b/src/src/deliver.c
index d85f010e6..e67c06183 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -1085,9 +1085,7 @@ if (addr->message)
 	)
      && (  Ustrstr(s, "mysql")   != NULL
         || Ustrstr(s, "pgsql")   != NULL
-#ifdef EXPERIMENTAL_REDIS
 	|| Ustrstr(s, "redis")   != NULL
-#endif
 	|| Ustrstr(s, "sqlite")  != NULL
 	|| Ustrstr(s, "ldap:")   != NULL
 	|| Ustrstr(s, "ldapdn:") != NULL
diff --git a/src/src/drtables.c b/src/src/drtables.c
index 3e978243d..ec4e72683 100644
--- a/src/src/drtables.c
+++ b/src/src/drtables.c
@@ -465,15 +465,15 @@ extern lookup_module_info sqlite_lookup_module_info;
 #ifdef EXPERIMENTAL_SPF
 extern lookup_module_info spf_lookup_module_info;
 #endif
-#ifdef EXPERIMENTAL_REDIS
-extern lookup_module_info redis_lookup_module_info;
-#endif
 #if defined(LOOKUP_PGSQL) && LOOKUP_PGSQL!=2
 extern lookup_module_info pgsql_lookup_module_info;
 #endif
 #if defined(LOOKUP_PASSWD) && LOOKUP_PASSWD!=2
 extern lookup_module_info passwd_lookup_module_info;
 #endif
+#if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2
+extern lookup_module_info redis_lookup_module_info;
+#endif
 #if defined(LOOKUP_ORACLE) && LOOKUP_ORACLE!=2
 extern lookup_module_info oracle_lookup_module_info;
 #endif
@@ -576,7 +576,7 @@ void init_lookup_list(void)
   addlookupmodule(NULL, &pgsql_lookup_module_info);
 #endif
 
-#ifdef EXPERIMENTAL_REDIS
+#if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2
   addlookupmodule(NULL, &redis_lookup_module_info);
 #endif
 
diff --git a/src/src/exim.c b/src/src/exim.c
index f8cfda8a8..28617a510 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -862,9 +862,6 @@ fprintf(f, "Support for:");
 #ifdef EXPERIMENTAL_DSN_INFO
   fprintf(f, " Experimental_DSN_info");
 #endif
-#ifdef EXPERIMENTAL_REDIS
-  fprintf(f, " Experimental_Redis");
-#endif
 fprintf(f, "\n");
 
 fprintf(f, "Lookups (built-in):");
@@ -907,6 +904,9 @@ fprintf(f, "Lookups (built-in):");
 #if defined(LOOKUP_PGSQL) && LOOKUP_PGSQL!=2
   fprintf(f, " pgsql");
 #endif
+#if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2
+  fprintf(f, " redis");
+#endif
 #if defined(LOOKUP_SQLITE) && LOOKUP_SQLITE!=2
   fprintf(f, " sqlite");
 #endif
diff --git a/src/src/expand.c b/src/src/expand.c
index f09271782..a51dac570 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -7482,22 +7482,22 @@ for (i = 1; i < argc; i++)
     if (Ustrspn(argv[i], "abcdefghijklmnopqrtsuvwxyz0123456789-.:/") ==
         Ustrlen(argv[i]))
       {
-      #ifdef LOOKUP_LDAP
+#ifdef LOOKUP_LDAP
       eldap_default_servers = argv[i];
-      #endif
-      #ifdef LOOKUP_MYSQL
+#endif
+#ifdef LOOKUP_MYSQL
       mysql_servers = argv[i];
-      #endif
-      #ifdef LOOKUP_PGSQL
+#endif
+#ifdef LOOKUP_PGSQL
       pgsql_servers = argv[i];
-      #endif
-      #ifdef EXPERIMENTAL_REDIS
+#endif
+#ifdef LOOKUP_REDIS
       redis_servers = argv[i];
-      #endif
+#endif
       }
-  #ifdef EXIM_PERL
+#ifdef EXIM_PERL
   else opt_perl_startup = argv[i];
-  #endif
+#endif
   }
 
 printf("Testing string expansion: debug_level = %d\n\n", debug_level);
diff --git a/src/src/globals.c b/src/src/globals.c
index 0edb16962..14a821a19 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -83,7 +83,7 @@ uschar *oracle_servers         = NULL;
 uschar *pgsql_servers          = NULL;
 #endif
 
-#ifdef EXPERIMENTAL_REDIS
+#ifdef LOOKUP_REDIS
 uschar *redis_servers          = NULL;
 #endif
 
diff --git a/src/src/globals.h b/src/src/globals.h
index eb85eefa9..9ae78a920 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -62,7 +62,7 @@ extern uschar *oracle_servers;         /* List of servers and connect info */
 extern uschar *pgsql_servers;          /* List of servers and connect info */
 #endif
 
-#ifdef EXPERIMENTAL_REDIS
+#ifdef LOOKUP_REDIS
 extern uschar *redis_servers;          /* List of servers and connect info */
 #endif
 
diff --git a/src/src/lookups/redis.c b/src/src/lookups/redis.c
index 5f677ff06..7bab34971 100644
--- a/src/src/lookups/redis.c
+++ b/src/src/lookups/redis.c
@@ -7,7 +7,7 @@
 
 #include "../exim.h"
 
-#ifdef EXPERIMENTAL_REDIS
+#ifdef LOOKUP_REDIS
 
 #include "lf_functions.h"
 
@@ -118,7 +118,7 @@ if (sdata[2][0] == 0) sdata[2] = NULL;
 
 /* See if we have a cached connection to the server */
 
-for (cn = redis_connections; cn != NULL; cn = cn->next)
+for (cn = redis_connections; cn; cn = cn->next)
   if (Ustrcmp(cn->server, server_copy) == 0)
     {
     redis_handle = cn->handle;
@@ -458,11 +458,11 @@ static lookup_info redis_lookup_info = {
 };
 
 #ifdef DYNLOOKUP
-#define redis_lookup_module_info _lookup_module_info
+# define redis_lookup_module_info _lookup_module_info
 #endif /* DYNLOOKUP */
 
 static lookup_info *_lookup_list[] = { &redis_lookup_info };
 lookup_module_info redis_lookup_module_info = { LOOKUP_MODULE_INFO_MAGIC, _lookup_list, 1 };
 
-#endif /* EXPERIMENTAL_REDIS */
+#endif /* LOOKUP_REDIS */
 /* End of lookups/redis.c */
diff --git a/src/src/readconf.c b/src/src/readconf.c
index f2bf23bbb..6f97dad69 100644
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -375,7 +375,7 @@ static optionlist optionlist_config[] = {
   { "recipient_unqualified_hosts", opt_stringptr, &recipient_unqualified_hosts },
   { "recipients_max",           opt_int,         &recipients_max },
   { "recipients_max_reject",    opt_bool,        &recipients_max_reject },
-#ifdef EXPERIMENTAL_REDIS
+#ifdef LOOKUP_REDIS
   { "redis_servers",            opt_stringptr,   &redis_servers },
 #endif
   { "remote_max_parallel",      opt_int,         &remote_max_parallel },
diff --git a/src/src/route.c b/src/src/route.c
index 0d96e2365..b289b0f34 100644
--- a/src/src/route.c
+++ b/src/src/route.c
@@ -1904,9 +1904,7 @@ if (  yield == DEFER
       )
    && (  Ustrstr(addr->message, "mysql") != NULL
       || Ustrstr(addr->message, "pgsql") != NULL
-#ifdef EXPERIMENTAL_REDIS
       || Ustrstr(addr->message, "redis") != NULL
-#endif
       || Ustrstr(addr->message, "sqlite") != NULL
       || Ustrstr(addr->message, "ldap:") != NULL
       || Ustrstr(addr->message, "ldapdn:") != NULL
diff --git a/test/runtest b/test/runtest
index 724ccd9ed..88c26e273 100755
--- a/test/runtest
+++ b/test/runtest
@@ -2857,7 +2857,7 @@ if (defined $parm_support{'Content_Scanning'})
 ##################################################
 #       Check for redis                          #
 ##################################################
-if (defined $parm_support{'Experimental_Redis'})
+if (defined $parm_lookups{'redis'})
   {
   if (system("redis-server -v 2>/dev/null >/dev/null") == 0)
     {
diff --git a/test/scripts/2700-redis/REQUIRES b/test/scripts/2700-redis/REQUIRES
index 9c48ef88e..55fbcfdcc 100644
--- a/test/scripts/2700-redis/REQUIRES
+++ b/test/scripts/2700-redis/REQUIRES
@@ -1,2 +1,2 @@
-support Experimental_Redis
+lookup redis
 running redis