* <many>: fix source code checks; move mu_msg_str_normalize to separate file

6 files changed, 685 insertions, 646 deletions

diff --git a/Makefile.am b/Makefile.am
index d1ec008..5f841dd 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -32,13 +32,18 @@ tags:
 # which is a sign that it needs some refactoring. requires the pmccabe
 # tool. If all is fine, it outputs nothing
 cc10:
-	@$(PMCCABE) `find -name '*.c' -o -name '*.cc'` | sort -nr | awk '($$1 > 10)'
+	@$(PMCCABE) `find -name '*.c' -o -name '*.cc'` \
+	| grep -v mu-msg-str-normalize.c \
+	| sort -nr | awk '($$1 > 10)'
 
 # this warns about functions that are over 33 non-comment lines long, which is a
 # sign that they need some refactoring. requires the pmccabe tool. if
 # all is fine, it outputs nothing
+# note, mu_msg_str_normalize is excempted from this rule.
 line33:
-	@$(PMCCABE) -c `find -name '*.c' -o -name '*.cc'` | awk '($$5 > 33)'
+	@$(PMCCABE) -c `find -name '*.c' -o -name '*.cc'` \
+	 | grep -v mu-msg-str-normalize.c \
+         | awk '($$5 > 33)'
 
 # get all todo/fixme messages
 fixme:
diff --git a/src/Makefile.am b/src/Makefile.am
index a7582ea..2e9a835 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -80,6 +80,7 @@ libmu_la_SOURCES=			\
 	mu-msg-part.c			\
 	mu-msg-part.h			\
 	mu-msg-str.c			\
+	mu-msg-str-normalize.c		\
 	mu-msg-str.h			\
 	mu-msg.h			\
 	mu-output-link.c		\
diff --git a/src/mu-cmd-find.c b/src/mu-cmd-find.c
index 48d5dda..fa42845 100644
--- a/src/mu-cmd-find.c
+++ b/src/mu-cmd-find.c
@@ -243,6 +243,23 @@ get_query (MuConfigOptions *opts)
 	return query;
 }
 
+static gboolean
+db_is_ready (const char *xpath)
+{	
+	if (mu_util_db_is_empty (xpath)) {
+		g_warning ("Database is empty; use 'mu index' to "
+			   "add messages");
+		return FALSE;
+	}
+		
+	if (!mu_util_db_version_up_to_date (xpath)) {
+		update_warning ();
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
 
 gboolean
 mu_cmd_find (MuConfigOptions *opts)
@@ -259,16 +276,8 @@ mu_cmd_find (MuConfigOptions *opts)
 		return FALSE;
 
 	xpath = mu_runtime_xapian_dir ();
-	
-	if (mu_util_db_is_empty (xpath)) {
-		g_warning ("Database is empty; use 'mu index' to add messages");
-		return FALSE;
-	}
-		
-	if (!mu_util_db_version_up_to_date (xpath)) {
-		update_warning ();
+	if (!db_is_ready(xpath))
 		return FALSE;
-	}
 	
 	/* first param is 'query', search params are after that */
 	query = get_query (opts);
diff --git a/src/mu-msg-flags.c b/src/mu-msg-flags.c
index 11af61d..a677ee6 100644
--- a/src/mu-msg-flags.c
+++ b/src/mu-msg-flags.c
@@ -181,8 +181,10 @@ check_msg_type (const char* path, char **info)
 	 * documentation at http://docs.python.org/lib/mailbox-maildir.html
 	 * mentions the '!' as well as a 'popular choice'
 	 */
-	dir =  g_path_get_dirname (path);
-	file = g_path_get_basename (path);
+	
+	*info = NULL;
+	dir   = g_path_get_dirname (path);
+	file  = g_path_get_basename (path);
 	
 	if (!(*info = strrchr(file, ':')))
 		*info = strrchr (file, '!'); /* Tinymail */
@@ -221,36 +223,26 @@ mu_msg_flags_from_file (const char* path)
 			      MU_MSG_FLAG_NONE);
 	
 	mtype = check_msg_type (path, &info);
-	
-	/* we ignore any flags for a new message */
-	if (mtype == MSG_TYPE_NEW) {
+	if (mtype == MSG_TYPE_NEW) { /* we ignore any new-msg flags */
 		g_free (info);
 		return MU_MSG_FLAG_NEW;
 	}
 	
-	flags = 0;
+	flags = MU_MSG_FLAG_NONE;
 	if (mtype == MSG_TYPE_CUR || mtype == MSG_TYPE_OTHER) {
 		char *cursor = info;
 		/* only support the "2," format */
 		if (cursor && cursor[0]=='2' && cursor[1]==',') {
 			cursor += 2; /* jump past 2, */
-			while (*cursor) {
+			for (; *cursor; ++cursor)
 				switch (*cursor) {
-				case 'P': flags |= MU_MSG_FLAG_PASSED;
-					break;
-				case 'T': flags |= MU_MSG_FLAG_TRASHED;
-					break;
-				case 'R': flags |= MU_MSG_FLAG_REPLIED;
-					break;
-				case 'S': flags |= MU_MSG_FLAG_SEEN;
-					break;
-				case 'D': flags |= MU_MSG_FLAG_DRAFT;
-					break;
-				case 'F': flags |= MU_MSG_FLAG_FLAGGED;
-					break;
+				case 'P': flags|= MU_MSG_FLAG_PASSED; break;
+				case 'T': flags|= MU_MSG_FLAG_TRASHED; break;
+				case 'R': flags|= MU_MSG_FLAG_REPLIED; break;
+				case 'S': flags|= MU_MSG_FLAG_SEEN; break;
+				case 'D': flags|= MU_MSG_FLAG_DRAFT;break;
+				case 'F': flags|= MU_MSG_FLAG_FLAGGED; break;
 				}
-				++cursor;
-			}
 		}
 	} 
 	g_free (info);
diff --git a/src/mu-msg-str-normalize.c b/src/mu-msg-str-normalize.c
new file mode 100644
index 0000000..fd05847
--- /dev/null
+++ b/src/mu-msg-str-normalize.c
@@ -0,0 +1,644 @@
+/* 
+** Copyright (C) 2010 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License as published by
+** the Free Software Foundation; either version 3 of the License, or
+** (at your option) any later version.
+**  
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**  
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  
+**  
+*/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif /*HAVE_CONFIG_H*/
+
+
+#include <glib.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "mu-msg-str.h"
+
+static char* normalize_in_place_downcase (char *str);
+
+char*
+mu_msg_str_normalize (const char *str, gboolean downcase)
+{
+	g_return_val_if_fail (str, NULL);
+
+	return mu_msg_str_normalize_in_place (g_strdup(str), downcase);
+}
+
+/* we can normalize in-place, as the normalized string will never be
+ * longer than the original.  even for replacements that are 2 chars
+ * wide (e.g. German ß => ss), the replacement is 2 bytes, like the
+ * original 0xc3 0x9f
+ */
+char*
+mu_msg_str_normalize_in_place (char *str, gboolean downcase)
+{
+	const guchar *cur;
+	int i;
+	
+	g_return_val_if_fail (str, NULL);
+	
+	if (*str == '\0')
+		return str;
+
+#ifdef MU_DISABLE_NORMALIZATION
+	{
+		if (downcase) {
+			gchar *c;
+			for (c = str; *c; ++c)
+				*c = tolower (*c);
+		}
+		return str;	
+	}
+#endif /*MU_DISABLE_NORMALIZATION*/
+
+	
+	if (downcase)
+		return normalize_in_place_downcase (str);
+	
+	for (i = 0, cur = (const guchar*)str; *cur; ++cur) {
+		if (*cur < 0xc3 || *cur > 0xc5) { 
+			str[i++] = *cur;
+			continue; 
+		}
+
+		if (*cur == 0xc3) { /* latin-1 supplement */
+			++cur;
+			switch (*cur) {
+				
+			case 0x80:
+			case 0x81:
+			case 0x82:
+			case 0x83:
+			case 0x84:
+			case 0x85: str[i++] = 'A'; break;
+
+			case 0x86: str[i++] = 'A'; str[i++] = 'e'; break;
+			case 0x87: str[i++] = 'C'; break;
+				
+			case 0x88:
+			case 0x89:
+			case 0x8a:
+			case 0x8b: str[i++] = 'E'; break;
+				
+			case 0x8c:
+			case 0x8d:
+			case 0x8e:
+			case 0x8f: str[i++] = 'I'; break;
+				
+			case 0x90: str[i++] = 'D'; break;
+			case 0x91: str[i++] = 'N'; break;
+				
+			case 0x92:
+			case 0x93:
+			case 0x94:
+			case 0x95:
+			case 0x96: str[i++] = 'O'; break;
+				
+			case 0x99:
+			case 0x9a:
+			case 0x9b:
+			case 0x9c: str[i++] = 'U'; break;
+			
+			case 0x9d: str[i++] = 'Y'; break;
+			case 0x9e: str[i++] = 'T'; str[i++] = 'h'; break;
+			case 0x9f: str[i++] = 's'; str[i++] = 's'; break;
+				
+			case 0xa0:
+			case 0xa1:
+			case 0xa2:
+			case 0xa3:
+			case 0xa4:
+			case 0xa5: str[i++] = 'a'; break;
+				
+			case 0xa6: str[i++] = 'a'; str[i++] = 'e'; break;
+			case 0xa7: str[i++] = 'c'; break;
+				
+			case 0xa8:
+			case 0xa9:
+			case 0xaa:
+			case 0xab: str[i++] = 'e'; break;
+				
+			case 0xac:
+			case 0xad:
+			case 0xae:
+			case 0xaf: str[i++] = 'i'; break;
+				
+			case 0xb0: str[i++] = 'd'; break;
+			case 0xb1: str[i++] = 'n'; break;
+				
+			case 0xb2:
+			case 0xb3:
+			case 0xb4:
+			case 0xb5:
+			case 0xb6: str[i++] = 'o'; break;
+				
+			case 0xb9:
+			case 0xba:
+			case 0xbb:
+			case 0xbc: str[i++] = 'u'; break;
+				
+			case 0xbd: str[i++] = 'y'; break;
+			case 0xbe: str[i++] = 't'; str[i++] = 'h'; break;
+			case 0xbf: str[i++] = 'y'; break;
+				
+			default:
+				str[i++] = *cur;
+			}
+		} else if (*cur == 0xc4) {  /* Latin Extended-A (0x04) */
+			++cur;
+			switch (*cur) {	
+			case 0x80:
+			case 0x82:
+			case 0x84: str[i++] = 'A'; break;
+
+			case 0x86:
+			case 0x88:
+			case 0x8a:
+			case 0x8c: str[i++] = 'C'; break;
+
+			case 0x8e:
+			case 0x90: str[i++] = 'D'; break;
+
+			case 0x92:
+			case 0x94:
+			case 0x96:
+			case 0x98:
+			case 0x9a: str[i++] = 'E'; break;
+
+			case 0x9c:
+			case 0x9e:
+			case 0xa0:
+			case 0xa2: str[i++] = 'G'; break;
+
+			case 0xa4:
+			case 0xa6: str[i++] = 'H'; break;
+
+			case 0xa8:
+			case 0xaa:
+			case 0xac:
+			case 0xae:
+			case 0xb0: str[i++] = 'I'; break;
+	
+			case 0xb2: str[i++] = 'I'; str[i++] = 'J'; break;
+
+			case 0xb4: str[i++] = 'J'; break;
+
+			case 0xb6: str[i++] = 'K'; break;
+
+			case 0xb9:
+			case 0xbb:
+			case 0xbd:
+			case 0xbf: str[i++] = 'L'; break;
+				
+			case 0x81:
+			case 0x83:
+			case 0x85: str[i++] = 'a'; break;
+
+			case 0x87:
+			case 0x89:
+			case 0x8b:
+			case 0x8d: str[i++] = 'c'; break;
+
+			case 0x8f:
+			case 0x91: str[i++] = 'd'; break;
+
+			case 0x93:
+			case 0x95:
+			case 0x97:
+			case 0x99:
+			case 0x9b: str[i++] = 'e'; break;
+
+			case 0x9d:
+			case 0x9f:
+			case 0xa1:
+			case 0xa: str[i++] = 'g'; break;
+
+			case 0xa5:
+			case 0xa7: str[i++] = 'h'; break;
+
+			case 0xa9:
+			case 0xab:
+			case 0xad:
+			case 0xaf:
+			case 0xb1: str[i++] = 'i'; break;
+
+			case 0xb3: str[i++] = 'i'; str[i++] = 'j'; break;
+
+			case 0xb5: str[i++] = 'j'; break;
+
+			case 0xb7:
+			case 0xb8: str[i++] = 'k'; break;
+
+			case 0xba:
+			case 0xbc:
+			case 0xbe: str[i++] = 'l'; break;
+
+			default:
+				str[i++] = *cur;
+			}
+
+		} else { /* Latin Extended-A (0xc5) */
+			++cur;
+			switch (*cur) {
+			case 0x81: str[i++] = 'L'; break;
+				
+			case 0x83:
+			case 0x85:
+			case 0x87: str[i++] = 'N'; break;
+
+			case 0x8c:
+			case 0x8e:
+			case 0x90: str[i++] = 'O'; break;
+				
+			case 0x92: str[i++] = 'O'; str[i++] = 'e'; break;
+
+			case 0x94:
+			case 0x96:
+			case 0x98: str[i++] = 'R'; break;
+
+			case 0x9a:
+			case 0x9c:
+			case 0x9e:
+			case 0xa0: str[i++] = 'S'; break;
+
+			case 0xa2:
+			case 0xa4:
+			case 0xa6: str[i++] = 'T'; break;
+
+			case 0xa8:
+			case 0xaa:
+			case 0xac:
+			case 0xae:
+			case 0xb0:	
+			case 0xb2: str[i++] = 'U'; break;
+			case 0xb4: str[i++] = 'W'; break;
+
+			case 0xb6:
+			case 0xb8:
+				str[i++] = 'Y'; break;
+
+			case 0xb9:
+			case 0xbb: 
+			case 0xbd: str[i++] = 'Z'; break;
+				
+			case 0x80:
+			case 0x82: str[i++] = 'l'; break;
+
+			case 0x84:
+			case 0x86:
+			case 0x88:
+			case 0x89:
+			case 0x8a:
+			case 0x8b: str[i++] = 'n'; break;
+
+			case 0x8d:
+			case 0x8f:
+			case 0x91: str[i++] = 'o'; break;
+
+			case 0x93: str[i++] = 'o'; str[i++] = 'e'; break;
+
+			case 0x95:
+			case 0x97:
+			case 0x99: str[i++] = 'r'; break;
+
+			case 0x9b:
+			case 0x9d:
+			case 0x9f:
+			case 0xa1: str[i++] = 's'; break;
+
+			case 0xa3:
+			case 0xa5:
+			case 0xa7: str[i++] = 't'; break;
+
+			case 0xa9:
+			case 0xab:
+			case 0xad:
+			case 0xaf:
+			case 0xb1:	
+			case 0xb3: str[i++] = 'u'; break;
+				
+			case 0xb5: str[i++] = 'w'; break;
+
+			case 0xb7: str[i++] = 'y'; break;
+
+			case 0xba:
+			case 0xbc: 
+			case 0xbe: str[i++] = 'z'; break;
+
+			case 0xbf: str[i++] = 's'; break;
+				
+			}
+		}
+	}
+
+	str [i] = '\0';
+	
+	return str;
+}
+
+
+static char*
+normalize_in_place_downcase (char *str)
+{
+	const guchar *cur;
+	int i;
+
+	if (*str == '\0')
+		return str;
+	
+	for (i = 0, cur = (const guchar*)str; *cur; ++cur) {
+
+		if (G_UNLIKELY (*cur == 0xc3)) { /* latin-1 supplement */
+			++cur;
+			switch (*cur) {
+				
+			case 0x80:
+			case 0x81:
+			case 0x82:
+			case 0x83:
+			case 0x84:
+			case 0x85: str[i++] = 'a'; break;
+
+			case 0x86: str[i++] = 'a'; str[i++] = 'e'; break;
+			case 0x87: str[i++] = 'c'; break;
+				
+			case 0x88:
+			case 0x89:
+			case 0x8a:
+			case 0x8b: str[i++] = 'e'; break;
+				
+			case 0x8c:
+			case 0x8d:
+			case 0x8e:
+			case 0x8f: str[i++] = 'i'; break;
+				
+			case 0x90: str[i++] = 'd'; break;
+			case 0x91: str[i++] = 'n'; break;
+				
+			case 0x92:
+			case 0x93:
+			case 0x94:
+			case 0x95:
+			case 0x96: str[i++] = 'o'; break;
+				
+			case 0x99:
+			case 0x9a:
+			case 0x9b:
+			case 0x9c: str[i++] = 'u'; break;
+			
+			case 0x9d: str[i++] = 'y'; break;
+			case 0x9e: str[i++] = 't'; str[i++] = 'h'; break;
+			case 0x9f: str[i++] = 't'; str[i++] = 's'; break;
+				
+			case 0xa0:
+			case 0xa1:
+			case 0xa2:
+			case 0xa3:
+			case 0xa4:
+			case 0xa5: str[i++] = 'a'; break;
+				
+			case 0xa6: str[i++] = 'a'; str[i++] = 'e'; break;
+			case 0xa7: str[i++] = 'c'; break;
+				
+			case 0xa8:
+			case 0xa9:
+			case 0xaa:
+			case 0xab: str[i++] = 'e'; break;
+				
+			case 0xac:
+			case 0xad:
+			case 0xae:
+			case 0xaf: str[i++] = 'i'; break;
+				
+			case 0xb0: str[i++] = 'd'; break;
+			case 0xb1: str[i++] = 'n'; break;
+				
+			case 0xb2:
+			case 0xb3:
+			case 0xb4:
+			case 0xb5:
+			case 0xb6: str[i++] = 'o'; break;
+				
+			case 0xb9:
+			case 0xba:
+			case 0xbb:
+			case 0xbc: str[i++] = 'u'; break;
+				
+			case 0xbd: str[i++] = 'y'; break;
+			case 0xbe: str[i++] = 't'; str[i++] = 'h'; break;
+			case 0xbf: str[i++] = 'y'; break;
+				
+			default:
+				str[i++] = tolower(*cur);
+			}
+
+			/* Latin Extended-A (0x04) */
+		} else if (G_UNLIKELY ((*cur == 0xc4))) {  
+			++cur;
+			switch (*cur) {	
+			case 0x80:
+			case 0x82:
+			case 0x84: str[i++] = 'a'; break;
+
+			case 0x86:
+			case 0x88:
+			case 0x8a:
+			case 0x8c: str[i++] = 'c'; break;
+
+			case 0x8e:
+			case 0x90: str[i++] = 'd'; break;
+
+			case 0x92:
+			case 0x94:
+			case 0x96:
+			case 0x98:
+			case 0x9a: str[i++] = 'e'; break;
+
+			case 0x9c:
+			case 0x9e:
+			case 0xa0:
+			case 0xa2: str[i++] = 'g'; break;
+
+			case 0xa4:
+			case 0xa6: str[i++] = 'h'; break;
+
+			case 0xa8:
+			case 0xaa:
+			case 0xac:
+			case 0xae:
+			case 0xb0: str[i++] = 'i'; break;
+	
+			case 0xb2: str[i++] = 'i'; str[i++] = 'j'; break;
+
+			case 0xb4: str[i++] = 'j'; break;
+
+			case 0xb6: str[i++] = 'k'; break;
+
+			case 0xb9:
+			case 0xbb:
+			case 0xbd:
+			case 0xbf: str[i++] = 'l'; break;
+				
+			case 0x81:
+			case 0x83:
+			case 0x85: str[i++] = 'a'; break;
+
+			case 0x87:
+			case 0x89:
+			case 0x8b:
+			case 0x8d: str[i++] = 'c'; break;
+
+			case 0x8f:
+			case 0x91: str[i++] = 'd'; break;
+
+			case 0x93:
+			case 0x95:
+			case 0x97:
+			case 0x99:
+			case 0x9b: str[i++] = 'e'; break;
+
+			case 0x9d:
+			case 0x9f:
+			case 0xa1:
+			case 0xa: str[i++] = 'g'; break;
+
+			case 0xa5:
+			case 0xa7: str[i++] = 'h'; break;
+
+			case 0xa9:
+			case 0xab:
+			case 0xad:
+			case 0xaf:
+			case 0xb1: str[i++] = 'i'; break;
+
+			case 0xb3: str[i++] = 'i'; str[i++] = 'j'; break;
+
+			case 0xb5: str[i++] = 'j'; break;
+
+			case 0xb7:
+			case 0xb8: str[i++] = 'k'; break;
+
+			case 0xba:
+			case 0xbc:
+			case 0xbe: str[i++] = 'l'; break;
+
+			default:
+				str[i++] = tolower(*cur);
+			}
+
+			/* Latin Extended-A (0xc5) */
+		} else if (G_UNLIKELY ((*cur == 0xc5))) { 
+			++cur;
+			switch (*cur) {
+			case 0x81: str[i++] = 'l'; break;
+				
+			case 0x83:
+			case 0x85:
+			case 0x87: str[i++] = 'n'; break;
+
+			case 0x8c:
+			case 0x8e:
+			case 0x90: str[i++] = 'o'; break;
+				
+			case 0x92: str[i++] = 'o'; str[i++] = 'e'; break;
+
+			case 0x94:
+			case 0x96:
+			case 0x98: str[i++] = 'r'; break;
+
+			case 0x9a:
+			case 0x9c:
+			case 0x9e:
+			case 0xa0: str[i++] = 's'; break;
+
+			case 0xa2:
+			case 0xa4:
+			case 0xa6: str[i++] = 't'; break;
+
+			case 0xa8:
+			case 0xaa:
+			case 0xac:
+			case 0xae:
+			case 0xb0:	
+			case 0xb2: str[i++] = 'u'; break;
+			case 0xb4: str[i++] = 'w'; break;
+			case 0xb6:
+			case 0xb8:
+				str[i++] = 'y'; break;
+
+			case 0xb9:
+			case 0xbb: 
+			case 0xbd: str[i++] = 'z'; break;
+				
+			case 0x80:
+			case 0x82: str[i++] = 'l'; break;
+
+			case 0x84:
+			case 0x86:
+			case 0x88:
+			case 0x89:
+			case 0x8a:
+			case 0x8b: str[i++] = 'n'; break;
+
+			case 0x8d:
+			case 0x8f:
+			case 0x91: str[i++] = 'o'; break;
+
+			case 0x93: str[i++] = 'o'; str[i++] = 'e'; break;
+
+			case 0x95:
+			case 0x97:
+			case 0x99: str[i++] = 'r'; break;
+
+			case 0x9b:
+			case 0x9d:
+			case 0x9f:
+			case 0xa1: str[i++] = 's'; break;
+
+			case 0xa3:
+			case 0xa5:
+			case 0xa7: str[i++] = 't'; break;
+
+			case 0xa9:
+			case 0xab:
+			case 0xad:
+			case 0xaf:
+			case 0xb1:	
+			case 0xb3: str[i++] = 'u'; break;
+				
+			case 0xb5: str[i++] = 'w'; break;
+
+			case 0xb7: str[i++] = 'y'; break;
+
+			case 0xba:
+			case 0xbc: 
+			case 0xbe: str[i++] = 'z'; break;
+
+			case 0xbf: str[i++] = 's'; break;
+
+			default:
+				str[i++] = tolower(*cur);
+			}				
+		} else			
+			str[i++] = tolower(*cur);
+	}
+
+	str [i] = '\0';
+	
+	return str;
+}
+
diff --git a/src/mu-msg-str.c b/src/mu-msg-str.c
index 84be447..0011d62 100644
--- a/src/mu-msg-str.c
+++ b/src/mu-msg-str.c
@@ -16,6 +16,8 @@
 ** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  
 **  
 */
+
+
 #if HAVE_CONFIG_H
 #include "config.h"
 #endif /*HAVE_CONFIG_H*/
@@ -28,620 +30,6 @@
 #include "mu-msg-str.h"
 #include "mu-msg-flags.h"
 
-static char* normalize_in_place_downcase (char *str);
-
-char*
-mu_msg_str_normalize (const char *str, gboolean downcase)
-{
-	g_return_val_if_fail (str, NULL);
-
-	return mu_msg_str_normalize_in_place (g_strdup(str), downcase);
-}
-
-/* we can normalize in-place, as the normalized string will never be
- * longer than the original.  even for replacements that are 2 chars
- * wide (e.g. German ß => ss), the replacement is 2 bytes, like the
- * original 0xc3 0x9f
- */
-char*
-mu_msg_str_normalize_in_place (char *str, gboolean downcase)
-{
-	const guchar *cur;
-	int i;
-	
-	g_return_val_if_fail (str, NULL);
-	
-	if (*str == '\0')
-		return str;
-
-#ifdef MU_DISABLE_NORMALIZATION
-	{
-		if (downcase) {
-			gchar *c;
-			for (c = str; *c; ++c)
-				*c = tolower (*c);
-		}
-		return str;	
-	}
-#endif /*MU_DISABLE_NORMALIZATION*/
-
-	
-	if (downcase)
-		return normalize_in_place_downcase (str);
-	
-	for (i = 0, cur = (const guchar*)str; *cur; ++cur) {
-		if (*cur < 0xc3 || *cur > 0xc5) { 
-			str[i++] = *cur;
-			continue; 
-		}
-
-		if (*cur == 0xc3) { /* latin-1 supplement */
-			++cur;
-			switch (*cur) {
-				
-			case 0x80:
-			case 0x81:
-			case 0x82:
-			case 0x83:
-			case 0x84:
-			case 0x85: str[i++] = 'A'; break;
-
-			case 0x86: str[i++] = 'A'; str[i++] = 'e'; break;
-			case 0x87: str[i++] = 'C'; break;
-				
-			case 0x88:
-			case 0x89:
-			case 0x8a:
-			case 0x8b: str[i++] = 'E'; break;
-				
-			case 0x8c:
-			case 0x8d:
-			case 0x8e:
-			case 0x8f: str[i++] = 'I'; break;
-				
-			case 0x90: str[i++] = 'D'; break;
-			case 0x91: str[i++] = 'N'; break;
-				
-			case 0x92:
-			case 0x93:
-			case 0x94:
-			case 0x95:
-			case 0x96: str[i++] = 'O'; break;
-				
-			case 0x99:
-			case 0x9a:
-			case 0x9b:
-			case 0x9c: str[i++] = 'U'; break;
-			
-			case 0x9d: str[i++] = 'Y'; break;
-			case 0x9e: str[i++] = 'T'; str[i++] = 'h'; break;
-			case 0x9f: str[i++] = 's'; str[i++] = 's'; break;
-				
-			case 0xa0:
-			case 0xa1:
-			case 0xa2:
-			case 0xa3:
-			case 0xa4:
-			case 0xa5: str[i++] = 'a'; break;
-				
-			case 0xa6: str[i++] = 'a'; str[i++] = 'e'; break;
-			case 0xa7: str[i++] = 'c'; break;
-				
-			case 0xa8:
-			case 0xa9:
-			case 0xaa:
-			case 0xab: str[i++] = 'e'; break;
-				
-			case 0xac:
-			case 0xad:
-			case 0xae:
-			case 0xaf: str[i++] = 'i'; break;
-				
-			case 0xb0: str[i++] = 'd'; break;
-			case 0xb1: str[i++] = 'n'; break;
-				
-			case 0xb2:
-			case 0xb3:
-			case 0xb4:
-			case 0xb5:
-			case 0xb6: str[i++] = 'o'; break;
-				
-			case 0xb9:
-			case 0xba:
-			case 0xbb:
-			case 0xbc: str[i++] = 'u'; break;
-				
-			case 0xbd: str[i++] = 'y'; break;
-			case 0xbe: str[i++] = 't'; str[i++] = 'h'; break;
-			case 0xbf: str[i++] = 'y'; break;
-				
-			default:
-				str[i++] = *cur;
-			}
-		} else if (*cur == 0xc4) {  /* Latin Extended-A (0x04) */
-			++cur;
-			switch (*cur) {	
-			case 0x80:
-			case 0x82:
-			case 0x84: str[i++] = 'A'; break;
-
-			case 0x86:
-			case 0x88:
-			case 0x8a:
-			case 0x8c: str[i++] = 'C'; break;
-
-			case 0x8e:
-			case 0x90: str[i++] = 'D'; break;
-
-			case 0x92:
-			case 0x94:
-			case 0x96:
-			case 0x98:
-			case 0x9a: str[i++] = 'E'; break;
-
-			case 0x9c:
-			case 0x9e:
-			case 0xa0:
-			case 0xa2: str[i++] = 'G'; break;
-
-			case 0xa4:
-			case 0xa6: str[i++] = 'H'; break;
-
-			case 0xa8:
-			case 0xaa:
-			case 0xac:
-			case 0xae:
-			case 0xb0: str[i++] = 'I'; break;
-	
-			case 0xb2: str[i++] = 'I'; str[i++] = 'J'; break;		
-
-			case 0xb4: str[i++] = 'J'; break;
-
-			case 0xb6: str[i++] = 'K'; break;
-
-			case 0xb9:
-			case 0xbb:
-			case 0xbd:
-			case 0xbf: str[i++] = 'L'; break;
-				
-			case 0x81:
-			case 0x83:
-			case 0x85: str[i++] = 'a'; break;
-
-			case 0x87:
-			case 0x89:
-			case 0x8b:
-			case 0x8d: str[i++] = 'c'; break;
-
-			case 0x8f:
-			case 0x91: str[i++] = 'd'; break;
-
-			case 0x93:
-			case 0x95:
-			case 0x97:
-			case 0x99:
-			case 0x9b: str[i++] = 'e'; break;
-
-			case 0x9d:
-			case 0x9f:
-			case 0xa1:
-			case 0xa: str[i++] = 'g'; break;
-
-			case 0xa5:
-			case 0xa7: str[i++] = 'h'; break;
-
-			case 0xa9:
-			case 0xab:
-			case 0xad:
-			case 0xaf:
-			case 0xb1: str[i++] = 'i'; break;
-
-			case 0xb3: str[i++] = 'i'; str[i++] = 'j'; break;		
-
-			case 0xb5: str[i++] = 'j'; break;
-
-			case 0xb7:
-			case 0xb8: str[i++] = 'k'; break;
-
-			case 0xba:
-			case 0xbc:
-			case 0xbe: str[i++] = 'l'; break;
-
-			default:
-				str[i++] = *cur;
-			}
-
-		} else { /* Latin Extended-A (0xc5) */
-			++cur;
-			switch (*cur) {
-			case 0x81: str[i++] = 'L'; break;
-				
-			case 0x83:
-			case 0x85:
-			case 0x87: str[i++] = 'N'; break;
-
-			case 0x8c:
-			case 0x8e:
-			case 0x90: str[i++] = 'O'; break;
-				
-			case 0x92: str[i++] = 'O'; str[i++] = 'e'; break;		
-
-			case 0x94:
-			case 0x96:
-			case 0x98: str[i++] = 'R'; break;
-
-			case 0x9a:
-			case 0x9c:
-			case 0x9e:
-			case 0xa0: str[i++] = 'S'; break;
-
-			case 0xa2:
-			case 0xa4:
-			case 0xa6: str[i++] = 'T'; break;
-
-			case 0xa8:
-			case 0xaa:
-			case 0xac:
-			case 0xae:
-			case 0xb0:	
-			case 0xb2: str[i++] = 'U'; break;				
-
-			case 0xb4: str[i++] = 'W'; break;				
-
-			case 0xb6:
-			case 0xb8:
-				str[i++] = 'Y'; break;				
-
-			case 0xb9:
-			case 0xbb: 
-			case 0xbd: str[i++] = 'Z'; break;
-				
-			case 0x80:
-			case 0x82: str[i++] = 'l'; break;
-
-			case 0x84:
-			case 0x86:
-			case 0x88:
-			case 0x89:
-			case 0x8a:
-			case 0x8b: str[i++] = 'n'; break;
-
-			case 0x8d:
-			case 0x8f:
-			case 0x91: str[i++] = 'o'; break;
-
-			case 0x93: str[i++] = 'o'; str[i++] = 'e'; break;		
-
-			case 0x95:
-			case 0x97:
-			case 0x99: str[i++] = 'r'; break;
-
-			case 0x9b:
-			case 0x9d:
-			case 0x9f:
-			case 0xa1: str[i++] = 's'; break;
-
-			case 0xa3:
-			case 0xa5:
-			case 0xa7: str[i++] = 't'; break;
-
-			case 0xa9:
-			case 0xab:
-			case 0xad:
-			case 0xaf:
-			case 0xb1:	
-			case 0xb3: str[i++] = 'u'; break;
-				
-			case 0xb5: str[i++] = 'w'; break;
-
-			case 0xb7: str[i++] = 'y'; break;
-
-			case 0xba:
-			case 0xbc: 
-			case 0xbe: str[i++] = 'z'; break;
-
-			case 0xbf: str[i++] = 's'; break;
-				
-			}
-		}
-	}
-
-	str [i] = '\0';
-	
-	return str;
-}
-
-
-static char*
-normalize_in_place_downcase (char *str)
-{
-	const guchar *cur;
-	int i;
-
-	if (*str == '\0')
-		return str;
-	
-	for (i = 0, cur = (const guchar*)str; *cur; ++cur) {
-
-		if (G_UNLIKELY (*cur == 0xc3)) { /* latin-1 supplement */
-			++cur;
-			switch (*cur) {
-				
-			case 0x80:
-			case 0x81:
-			case 0x82:
-			case 0x83:
-			case 0x84:
-			case 0x85: str[i++] = 'a'; break;
-
-			case 0x86: str[i++] = 'a'; str[i++] = 'e'; break;
-			case 0x87: str[i++] = 'c'; break;
-				
-			case 0x88:
-			case 0x89:
-			case 0x8a:
-			case 0x8b: str[i++] = 'e'; break;
-				
-			case 0x8c:
-			case 0x8d:
-			case 0x8e:
-			case 0x8f: str[i++] = 'i'; break;
-				
-			case 0x90: str[i++] = 'd'; break;
-			case 0x91: str[i++] = 'n'; break;
-				
-			case 0x92:
-			case 0x93:
-			case 0x94:
-			case 0x95:
-			case 0x96: str[i++] = 'o'; break;
-				
-			case 0x99:
-			case 0x9a:
-			case 0x9b:
-			case 0x9c: str[i++] = 'u'; break;
-			
-			case 0x9d: str[i++] = 'y'; break;
-			case 0x9e: str[i++] = 't'; str[i++] = 'h'; break;
-			case 0x9f: str[i++] = 't'; str[i++] = 's'; break;
-				
-			case 0xa0:
-			case 0xa1:
-			case 0xa2:
-			case 0xa3:
-			case 0xa4:
-			case 0xa5: str[i++] = 'a'; break;
-				
-			case 0xa6: str[i++] = 'a'; str[i++] = 'e'; break;
-			case 0xa7: str[i++] = 'c'; break;
-				
-			case 0xa8:
-			case 0xa9:
-			case 0xaa:
-			case 0xab: str[i++] = 'e'; break;
-				
-			case 0xac:
-			case 0xad:
-			case 0xae:
-			case 0xaf: str[i++] = 'i'; break;
-				
-			case 0xb0: str[i++] = 'd'; break;
-			case 0xb1: str[i++] = 'n'; break;
-				
-			case 0xb2:
-			case 0xb3:
-			case 0xb4:
-			case 0xb5:
-			case 0xb6: str[i++] = 'o'; break;
-				
-			case 0xb9:
-			case 0xba:
-			case 0xbb:
-			case 0xbc: str[i++] = 'u'; break;
-				
-			case 0xbd: str[i++] = 'y'; break;
-			case 0xbe: str[i++] = 't'; str[i++] = 'h'; break;
-			case 0xbf: str[i++] = 'y'; break;
-				
-			default:
-				str[i++] = tolower(*cur);
-			}
-		} else if (G_UNLIKELY ((*cur == 0xc4))) {  /* Latin Extended-A (0x04) */
-			++cur;
-			switch (*cur) {	
-			case 0x80:
-			case 0x82:
-			case 0x84: str[i++] = 'a'; break;
-
-			case 0x86:
-			case 0x88:
-			case 0x8a:
-			case 0x8c: str[i++] = 'c'; break;
-
-			case 0x8e:
-			case 0x90: str[i++] = 'd'; break;
-
-			case 0x92:
-			case 0x94:
-			case 0x96:
-			case 0x98:
-			case 0x9a: str[i++] = 'e'; break;
-
-			case 0x9c:
-			case 0x9e:
-			case 0xa0:
-			case 0xa2: str[i++] = 'g'; break;
-
-			case 0xa4:
-			case 0xa6: str[i++] = 'h'; break;
-
-			case 0xa8:
-			case 0xaa:
-			case 0xac:
-			case 0xae:
-			case 0xb0: str[i++] = 'i'; break;
-	
-			case 0xb2: str[i++] = 'i'; str[i++] = 'j'; break;		
-
-			case 0xb4: str[i++] = 'j'; break;
-
-			case 0xb6: str[i++] = 'k'; break;
-
-			case 0xb9:
-			case 0xbb:
-			case 0xbd:
-			case 0xbf: str[i++] = 'l'; break;
-				
-			case 0x81:
-			case 0x83:
-			case 0x85: str[i++] = 'a'; break;
-
-			case 0x87:
-			case 0x89:
-			case 0x8b:
-			case 0x8d: str[i++] = 'c'; break;
-
-			case 0x8f:
-			case 0x91: str[i++] = 'd'; break;
-
-			case 0x93:
-			case 0x95:
-			case 0x97:
-			case 0x99:
-			case 0x9b: str[i++] = 'e'; break;
-
-			case 0x9d:
-			case 0x9f:
-			case 0xa1:
-			case 0xa: str[i++] = 'g'; break;
-
-			case 0xa5:
-			case 0xa7: str[i++] = 'h'; break;
-
-			case 0xa9:
-			case 0xab:
-			case 0xad:
-			case 0xaf:
-			case 0xb1: str[i++] = 'i'; break;
-
-			case 0xb3: str[i++] = 'i'; str[i++] = 'j'; break;		
-
-			case 0xb5: str[i++] = 'j'; break;
-
-			case 0xb7:
-			case 0xb8: str[i++] = 'k'; break;
-
-			case 0xba:
-			case 0xbc:
-			case 0xbe: str[i++] = 'l'; break;
-
-			default:
-				str[i++] = tolower(*cur);
-			}
-			
-		} else if (G_UNLIKELY ((*cur == 0xc5))) { /* Latin Extended-A (0xc5) */
-			++cur;
-			switch (*cur) {
-			case 0x81: str[i++] = 'l'; break;
-				
-			case 0x83:
-			case 0x85:
-			case 0x87: str[i++] = 'n'; break;
-
-			case 0x8c:
-			case 0x8e:
-			case 0x90: str[i++] = 'o'; break;
-				
-			case 0x92: str[i++] = 'o'; str[i++] = 'e'; break;		
-
-			case 0x94:
-			case 0x96:
-			case 0x98: str[i++] = 'r'; break;
-
-			case 0x9a:
-			case 0x9c:
-			case 0x9e:
-			case 0xa0: str[i++] = 's'; break;
-
-			case 0xa2:
-			case 0xa4:
-			case 0xa6: str[i++] = 't'; break;
-
-			case 0xa8:
-			case 0xaa:
-			case 0xac:
-			case 0xae:
-			case 0xb0:	
-			case 0xb2: str[i++] = 'u'; break;				
-
-			case 0xb4: str[i++] = 'w'; break;				
-
-			case 0xb6:
-			case 0xb8:
-				str[i++] = 'y'; break;				
-
-			case 0xb9:
-			case 0xbb: 
-			case 0xbd: str[i++] = 'z'; break;
-				
-			case 0x80:
-			case 0x82: str[i++] = 'l'; break;
-
-			case 0x84:
-			case 0x86:
-			case 0x88:
-			case 0x89:
-			case 0x8a:
-			case 0x8b: str[i++] = 'n'; break;
-
-			case 0x8d:
-			case 0x8f:
-			case 0x91: str[i++] = 'o'; break;
-
-			case 0x93: str[i++] = 'o'; str[i++] = 'e'; break;		
-
-			case 0x95:
-			case 0x97:
-			case 0x99: str[i++] = 'r'; break;
-
-			case 0x9b:
-			case 0x9d:
-			case 0x9f:
-			case 0xa1: str[i++] = 's'; break;
-
-			case 0xa3:
-			case 0xa5:
-			case 0xa7: str[i++] = 't'; break;
-
-			case 0xa9:
-			case 0xab:
-			case 0xad:
-			case 0xaf:
-			case 0xb1:	
-			case 0xb3: str[i++] = 'u'; break;
-				
-			case 0xb5: str[i++] = 'w'; break;
-
-			case 0xb7: str[i++] = 'y'; break;
-
-			case 0xba:
-			case 0xbc: 
-			case 0xbe: str[i++] = 'z'; break;
-
-			case 0xbf: str[i++] = 's'; break;
-
-			default:
-				str[i++] = tolower(*cur);
-			}				
-		} else			
-			str[i++] = tolower(*cur);
-	}
-
-	str [i] = '\0';
-	
-	return str;
-}
-
 
 const char* 
 mu_msg_str_date_s (const char* frm, time_t t)