Patch by Phoenix

* adds line 8 for additional UTF-8 ranges
* adds cjk-utf8 locale
* fixes check 3.2 and adds check 3.3

git-svn-id: http://svn.inspircd.org/repository/trunk/inspircd@11082 e03df62e-2008-0410-955e-edbf42e46eb7

diff --git a/locales/readme.txt b/locales/readme.txt
index 1304adbb5b2978a072a5d3c9c60bbdf01767b08c..069e664b590dfa9fbb891ded7c0a4d9cff2b8e07 100755 (executable)
--- a/locales/readme.txt
+++ b/locales/readme.txt
@@ -23,7 +23,9 @@ Can be usefull for example for comparing nicknames that contains similar-looking
 
 6: List of additional UTF-8 allowed characters
 
-7: List of additional UTF-8 ranges (character followed by "range"). Strongly experimental. May be replaced in future versions.
+7: List of additional UTF-8 ranges (character followed by 1-byte "range").
+
+8: List of additional UTF-8 ranges (i.e. start1, end1, start2, end2,... UTF8-characters between each start-end pair assumed valid).
 
 *** Line format ***
 
@@ -44,5 +46,4 @@ In this case every character of line except first dot specifies one character-co
 
 *** TODO ***
 
-- Maybe replace line 7 with <interval start> <interval end> form?
 - UTF-8 collation rules (Inapplieable to InspIRCd atm).

diff --git a/src/modules/m_nationalchars.cpp b/src/modules/m_nationalchars.cpp
index 17be414a603e79852cce379e9f97a1213ddca34b..5d2be96d75abd0cbaec0a11327f154288c0fb32f 100755 (executable)
--- a/src/modules/m_nationalchars.cpp
+++ b/src/modules/m_nationalchars.cpp
@@ -12,7 +12,8 @@
  */
 
 /* Contains a code of Unreal IRCd + Bynets patch ( http://www.unrealircd.com/ and http://www.bynets.org/ )
-   Changed at 2008-06-15 - 2008-12-15
+   Original patch is made by Dmitry "Killer{R}" Kononko. ( http://killprog.com/ )
+   Changed at 2008-06-15 - 2009-02-11
    by Chernov-Phoenix Alexey (Phoenix@RusNet) mailto:phoenix /email address separator/ pravmail.ru */
 
 #include "inspircd.h"
@@ -31,13 +32,14 @@ class lwbNickHandler : public HandlerBase2<bool, const char*, size_t>
 };
 
                                                                 /*,m_reverse_additionalUp[256];*/
-static unsigned char m_reverse_additional[256],m_additionalMB[256],m_additionalUtf8[256],m_additionalUtf8range[256];
+static unsigned char m_reverse_additional[256],m_additionalMB[256],m_additionalUtf8[256],m_additionalUtf8range[256],m_additionalUtf8interval[256];
 
 char utf8checkrest(unsigned char * mb, unsigned char cnt)
 {
        for (unsigned char * tmp=mb; tmp<mb+cnt; tmp++)
        {
-               if ((*tmp < 128) || (*tmp > 191))
+               /* & is faster! -- Phoenix (char & b11000000 == b10000000) */
+               if ((*tmp & 192) != 128)
                        return -1;
        }
        return cnt + 1;
@@ -107,7 +109,7 @@ bool lwbNickHandler::Call(const char* n, size_t max)
                        continue;
 
                /* 3.1. Check against a simple UTF-8 characters enumeration */
-               int cursize, ncursize = utf8size((unsigned char *)i);
+               int cursize, cursize2, ncursize = utf8size((unsigned char *)i);
                /* do check only if current multibyte character is valid UTF-8 only */
                if (ncursize != -1)
                {
@@ -130,25 +132,39 @@ bool lwbNickHandler::Call(const char* n, size_t max)
                        if (found)
                                continue;
 
-                       /* 3.2. Check against an UTF-8 ranges: <start character> and <lenght of the range>.
-                       Also char. is to be checked if it is a valid UTF-8 one */
+                       /* 3.2. Check against an UTF-8 ranges: <start character> and <length of the range>. */
                        found = false;
                        for (unsigned char * mb = m_additionalUtf8range; (utf8size(mb) != -1) && (mb < m_additionalUtf8range + sizeof(m_additionalUtf8range)); mb += cursize + 1)
                        {
                                cursize = utf8size(mb);
-                               /* Size differs? Pick the next! */
+                               /* Size differs (or lengthbyte is zero)? Pick the next! */
                                if ((cursize != ncursize) || (!mb[cursize]))
                                        continue;
 
-                               unsigned char uright[5] = {0,0,0,0,0};
-
+                               unsigned char uright[5] = {0,0,0,0,0}, range = mb[cursize] - 1;
                                strncpy((char* ) uright, (char *) mb, cursize);
 
-                               if ((uright[cursize-1] + mb[cursize]-1>0xff) && (cursize != 1))
+                               for (int temp = cursize - 1; (temp >= 0) && range; --temp)
                                {
-                                       uright[cursize - 2]+=1;
+                                       /* all but the first char are 64-based */
+                                       if (temp)
+                                       {
+                                               char part64 = range & 63; /* i.e. % 64 */
+                                               /* handle carrying over */
+                                               if (uright[temp] + part64 - 1 > 191)
+                                               {
+                                                       uright[temp] -= 64;
+                                                       range += 64;
+                                               }
+                                               uright[temp] += part64;
+                                               range >>= 6; /* divide it on a 64 */
+                                       }
+                                       /* the first char of UTF-8 doesn't follow the rule */
+                                       else
+                                       {
+                                               uright[temp] += range;
+                                       }
                                }
-                               uright[cursize - 1] = (uright[cursize - 1]+mb[cursize] - 1) % 0x100;
 
                                if ((strncmp(i, (char *) mb, cursize) >= 0) && (strncmp(i, (char *) uright, cursize) <= 0))
                                {
@@ -160,6 +176,30 @@ bool lwbNickHandler::Call(const char* n, size_t max)
                        }
                        if (found)
                                continue;
+
+                       /* 3.3. Check against an UTF-8 intervals: <start character> and <end character>. */
+                       found = false;
+                       for (unsigned char * mb = m_additionalUtf8interval; (utf8size(mb) != -1) && (utf8size(mb+utf8size(mb)) != -1)
+                               && (mb < m_additionalUtf8interval + sizeof(m_additionalUtf8interval)); mb += (cursize+cursize2) )
+                       {
+                               cursize = utf8size(mb);
+                               cursize2= utf8size(mb+cursize);
+
+                               int minlen  = cursize  > ncursize ? ncursize : cursize;
+                               int minlen2 = cursize2 > ncursize ? ncursize : cursize2;
+
+                               unsigned char* uright = mb + cursize;
+
+                               if ((strncmp(i, (char *) mb, minlen) >= 0) && (strncmp(i, (char *) uright, minlen2) <= 0))
+                               {
+                                       i += cursize - 1;
+                                       p += cursize - 1;
+                                       found = true;
+                                       break;
+                               }
+                       }
+                       if (found)
+                               continue;
                }
 
                /* invalid character! abort */
@@ -216,8 +256,8 @@ class ModuleNationalChars : public Module
                charset = conf->ReadValue("nationalchars", "file", 0);
                casemapping = conf->ReadValue("nationalchars", "casemapping", charset, 0, false);
                charset.insert(0, "../locales/");
-               unsigned char * tables[7] = { m_additional, m_additionalMB, m_additionalUp, m_lower, m_upper, m_additionalUtf8, m_additionalUtf8range };
-               loadtables(charset, tables, 7, 5);
+               unsigned char * tables[8] = { m_additional, m_additionalMB, m_additionalUp, m_lower, m_upper, m_additionalUtf8, m_additionalUtf8range, m_additionalUtf8interval };
+               loadtables(charset, tables, 8, 5);
                forcequit = conf->ReadFlag("nationalchars", "forcequit", 0);
                CheckForceQuit("National character set changed");
                delete conf;
@@ -319,7 +359,7 @@ class ModuleNationalChars : public Module
                if (buf[0] == '.')      /* simple plain-text string after dot */
                {
                        i = buf.size() - 1;
-       
+
                        if (i > (maxindex + 1))
                                i = maxindex + 1;