/* search_sphinx.c -- glue code for searching with Sphinx
 *
 * Copyright (c) 1994-2012 Carnegie Mellon University.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. The name "Carnegie Mellon University" must not be used to
 *    endorse or promote products derived from this software without
 *    prior written permission. For permission or any legal
 *    details, please contact
 *      Carnegie Mellon University
 *      Center for Technology Transfer and Enterprise Creation
 *      4615 Forbes Avenue
 *      Suite 302
 *      Pittsburgh, PA  15213
 *      (412) 268-7393, fax: (412) 268-7395
 *      innovation@andrew.cmu.edu
 *
 * 4. Redistributions of any form whatsoever must retain the following
 *    acknowledgment:
 *    "This product includes software developed by Computing Services
 *     at Carnegie Mellon University (http://www.cmu.edu/computing/)."
 *
 * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
 * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
 * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
 * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <config.h>

#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <syslog.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#include "index.h"
#include "global.h"
#include "retry.h"
#include "command.h"
#include "xmalloc.h"
#include "xstrlcpy.h"
#include "xstrlcat.h"
#include "ptrarray.h"
#include "bitvector.h"
#include "mboxlist.h"
#include "xstats.h"
#include "search_engines.h"
#include "cyr_lock.h"

/* generated headers are not necessarily in current directory */
#include "imap/imap_err.h"
#include "imap/imap_err.h"

#include <mysql/mysql.h>

struct connection
{
    MYSQL *mysql;
};
#define CONNECTION_INITIALIZER  { 0 }

struct latestdb
{
    struct db *db;
    char *path;
};
#define LATESTDB_INITIALIZER { 0, 0 }
#define LATESTDB_VERSION        1
#define LATESTDB_FNAME          "/latest.db"
#define LATESTDB_LASTID_KEY     "LASTID"

#define SPHINX_CONFIG       "/sphinx.conf"
#define SEARCHD             "/usr/bin/searchd"

#define INDEXING_LOCK_SUFFIX    ".indexing.lock"

static int open_latest(struct mailbox *, struct latestdb *);
static void close_latest(struct latestdb *);
static int read_latest(struct latestdb *, struct mailbox *, uint32_t *, int);
static int write_latest(struct latestdb *, struct mailbox *, uint32_t, int);
static int doquery(struct connection *, int, const struct buf *);
static int sphinx_basedir(const struct mailbox *mailbox, char **basedirp, char **indexnamep);
static int sphinx_setup(const struct mailbox *mailbox, int verbose, int create);

/* Name of columns */
#define COL_CYRUSID     "cyrusid"
static const char * const column_by_part[SEARCH_NUM_PARTS] = {
    NULL,
    "header_from",
    "header_to",
    "header_cc",
    "header_bcc",
    "header_subject",
    "header_listid",
    "header_type",
    "headers",
    "body",
    "location",
    "attachmentname"
};

static void close_connection(struct connection *conn)
{
    if (conn->mysql) {
        xstats_inc(SPHINX_CLOSE);
        mysql_close(conn->mysql);
        conn->mysql = NULL;
        mysql_library_end();
    }
}

static int get_connection(struct connection *conn)
{
    MYSQL *c = NULL;
    const char *socket_path = config_getstring(IMAPOPT_SPHINX_SOCKET);

    if (conn->mysql) return 0;      /* already open */

    close_connection(conn);

    xstats_inc(SPHINX_CONNECT);
    c = mysql_init(NULL);

    if (!mysql_real_connect(c,
                           /*host*/NULL,
                           /*user*/"", /*password*/"",
                           /*database*/NULL,
                           /*port*/0, socket_path,
                           /*client_flag*/0)) {
        syslog(LOG_ERR, "IOERROR: failed to connect to Sphinx: %s",
               mysql_error(c));
        mysql_close(c);
        mysql_library_end();
        return IMAP_IOERROR;
    }

    conn->mysql = c;
    return 0;
}

static int parse_cyrusid(const char *cyrusid,
                         const char **mboxnamep,
                         unsigned int *uidvalidityp,
                         unsigned int *uidp)
{
    // user.cassandane.1320711192.196715
    static struct buf buf = BUF_INITIALIZER;
    char *p;

    buf_reset(&buf);
    buf_appendcstr(&buf, cyrusid);

    p = strrchr(buf_cstring(&buf), '.');
    if (!p)
        return 0;
    *p++ = '\0';
    *uidp = strtoul(p, NULL, 10);

    p = strrchr(buf.s, '.');
    if (!p)
        return 0;
    *p++ = '\0';
    *uidvalidityp = strtoul(p, NULL, 10);

    *mboxnamep = buf.s;

    return 1;
}

static const struct buf *make_cyrusid(struct mailbox *mailbox, uint32_t uid)
{
    static struct buf buf = BUF_INITIALIZER;
    // user.cassandane.1320711192.196715
    buf_reset(&buf);
    buf_printf(&buf, "%s.%u.%u",
                     mailbox->name,
                     mailbox->i.uidvalidity,
                     uid);
    return &buf;
}

/*
 * Escape a string for MySQL.  Note that mysql_real_escape_string
 * requires a live connection, and we now want to be able to build a
 * query string before we have a connection.  From the MySQL
 * documentation:
 *
 *      Strictly speaking, MySQL requires only that backslash and
 *      the quote character used to quote the string in the query
 *      be escaped. mysql_real_escape_string() quotes the other
 *      characters to make them easier to read in log files.
 *
 * Note that we need to escape a number of SphinxQL extended query
 * syntax metacharacters like ^ and $.
 */
static void append_escaped_map(struct buf *buf,
                               const char *base, unsigned int len,
                               int quote)
{
    static const char metacharacters[] = "!\"$'-/<=@[\\]^|~";
    buf_ensure(buf, len+1);

    buf_putc(buf, quote);
    for ( ; len ; len--, base++) {
        int c = *(unsigned char *)base;
        if (strchr(metacharacters, c))
            buf_putc(buf, '\\');
        buf_putc(buf, c);
    }
    buf_putc(buf, quote);
    buf_cstring(buf);
}

static void append_escaped(struct buf *to, const struct buf *from, int quote)
{
    append_escaped_map(to, from->s, from->len, quote);
}

static void append_escaped_cstr(struct buf *to, const char *str, int quote)
{
    if (str)
        append_escaped_map(to, str, strlen(str), quote);
}

struct opnode {
    int op;     /* SEARCH_OP_* or SEARCH_PART_* constant */
    char *arg;
    struct opnode *next;
    struct opnode *children;
};

typedef struct sphinx_builder sphinx_builder_t;
struct sphinx_builder {
    search_builder_t super;
    struct mailbox *mailbox;
    char *indexname;
    int opts;
    struct opnode *root;
    ptrarray_t stack;       /* points to opnode* */
};

static struct opnode *opnode_new(int op, const char *arg)
{
    struct opnode *on = xzmalloc(sizeof(struct opnode));
    on->op = op;
    on->arg = xstrdupnull(arg);
    return on;
}

static void opnode_delete(struct opnode *on)
{
    struct opnode *child;
    struct opnode *next;

    for (child = on->children ; child ; child = next) {
        next = child->next;
        opnode_delete(child);
    }
    free(on->arg);
    free(on);
}

static void opnode_detach_child(struct opnode *parent, struct opnode *child)
{
    struct opnode **prevp;

    for (prevp = &parent->children ; *prevp ; prevp = &((*prevp)->next)) {
        if (*prevp == child) {
            *prevp = child->next;
            child->next = NULL;
            return;
        }
    }
}

static void opnode_append_child(struct opnode *parent, struct opnode *child)
{
    struct opnode **tailp;

    for (tailp = &parent->children ; *tailp ; tailp = &((*tailp)->next))
        ;
    *tailp = child;
    child->next = NULL;
}

static void opnode_insert_child(struct opnode *parent __attribute__((unused)),
                                struct opnode *after,
                                struct opnode *child)
{
    child->next = after->next;
    after->next = child;
}

static void begin_boolean(search_builder_t *bx, int op)
{
    sphinx_builder_t *bb = (sphinx_builder_t *)bx;
    struct opnode *top = ptrarray_tail(&bb->stack);
    struct opnode *on = opnode_new(op, NULL);
    if (top)
        opnode_append_child(top, on);
    else
        bb->root = on;
    ptrarray_push(&bb->stack, on);
    if (SEARCH_VERBOSE(bb->opts))
        syslog(LOG_INFO, "begin_boolean(op=%s)", search_op_as_string(op));
}

static void end_boolean(search_builder_t *bx, int op __attribute__((unused)))
{
    sphinx_builder_t *bb = (sphinx_builder_t *)bx;
    if (SEARCH_VERBOSE(bb->opts))
        syslog(LOG_INFO, "end_boolean");
    ptrarray_pop(&bb->stack);
}

static void match(search_builder_t *bx, int part, const char *str)
{
    sphinx_builder_t *bb = (sphinx_builder_t *)bx;
    struct opnode *top = ptrarray_tail(&bb->stack);
    struct opnode *on;

    if (!str) return;
    if (SEARCH_VERBOSE(bb->opts))
        syslog(LOG_INFO, "match(part=%s, str=\"%s\")",
               search_part_as_string(part), str);

    xstats_inc(SPHINX_MATCH);

    on = opnode_new(part, str);
    if (top)
        opnode_append_child(top, on);
    else
        bb->root = on;
}

static void optimise_nodes(struct opnode *parent, struct opnode *on)
{
    struct opnode *child;
    struct opnode *next;

    switch (on->op) {
    case SEARCH_OP_NOT:
    case SEARCH_OP_OR:
    case SEARCH_OP_AND:
        for (child = on->children ; child ; child = next) {
            next = child->next;
            optimise_nodes(on, child);
        }
        if (parent) {
            if (!on->children) {
                /* empty node - remove it */
                opnode_detach_child(parent, on);
                opnode_delete(on);
            }
            else if (on->op != SEARCH_OP_NOT && !on->children->next) {
                /* logical AND or OR with only one child - replace
                 * the node with its child */
                struct opnode *child = on->children;
                opnode_detach_child(on, child);
                opnode_insert_child(parent, on, child);
                opnode_detach_child(parent, on);
                opnode_delete(on);
            }
        }
        break;
    }
}

/* Sphinx extended query syntax, not SphinxQL */
static void generate_query(struct buf *query,
                           struct opnode *on,
                           struct opnode *parent)
{
    struct opnode *child;
    struct buf arg = BUF_INITIALIZER;
    int need_paren;
    int sep;

    buf_init_ro_cstr(&arg, on->arg);

    switch (on->op) {

    case SEARCH_OP_NOT:
        if (on->children) buf_appendcstr(query, "!");
        /* fall through - note we treat multiple children to a NOT
         * node as if they were ANDed together because that's the
         * closest match to IMAP semantics. */
    case SEARCH_OP_OR:
    case SEARCH_OP_AND:

        need_paren = 0;
        if (parent && parent->op > on->op)
            need_paren = 1;
        if (on->op == SEARCH_OP_NOT)
            need_paren = 1;
        /* Note that we might have 0, 1, or >1 children, the caller will
         * not have optimised the silly cases out. */
        if (!on->children || !on->children->next)
            need_paren = 0;

        sep = (on->op == SEARCH_OP_OR ? '|' : ' ');

        if (need_paren) buf_putc(query, '(');
        for (child = on->children ; child ; child = child->next) {
            generate_query(query, child, on);
            if (child->next) buf_putc(query, sep);
        }
        if (need_paren) buf_putc(query, ')');
        break;

    case SEARCH_PART_ANY:
        assert(on->children == NULL);
        if (config_getswitch(IMAPOPT_SPHINX_TEXT_EXCLUDES_ODD_HEADERS)) {
            /* This horrible hack makes TEXT searches match FROM, TO, CC, BCC
             * and SUBJECT but not any other random headers, which is more
             * like what users expect. */
            int i;
            const char *sep = "(";
            buf_appendcstr(query, "@");
            for (i = 0 ; i < SEARCH_NUM_PARTS ; i++) {
                if (column_by_part[i] && i != SEARCH_PART_HEADERS) {
                    buf_appendcstr(query, sep);
                    buf_appendcstr(query, column_by_part[i]);
                    sep = ",";
                }
            }
            buf_appendcstr(query, ") ");
        }
        append_escaped(query, &arg, '"');
        break;

    default:
        /* other SEARCH_PART_* constants */
        assert(on->op >= 0 && on->op < SEARCH_NUM_PARTS);
        assert(on->children == NULL);
        buf_appendcstr(query, "@");
        buf_appendcstr(query, column_by_part[on->op]);
        buf_appendcstr(query, " ");
        append_escaped(query, &arg, '"');
        break;
    }

    buf_free(&arg);
}

static void *get_internalised(search_builder_t *bx)
{
    sphinx_builder_t *bb = (sphinx_builder_t *)bx;
    struct opnode *on = bb->root;
    bb->root = NULL;
    optimise_nodes(NULL, on);
    return on;
}

static char *describe_internalised(void *internalised)
{
    struct opnode *on = (struct opnode *)internalised;
    struct buf buf = BUF_INITIALIZER;

    generate_query(&buf, on, NULL);
    return buf_release(&buf);
}

static void free_internalised(void *internalised)
{
    struct opnode *on = (struct opnode *)internalised;
    if (on) opnode_delete(on);
}

static int run(search_builder_t *bx, search_hit_cb_t proc, void *rock);

static search_builder_t *begin_search(struct mailbox *mailbox, int opts)
{
    sphinx_builder_t *bb;
    int r;

    bb = xzmalloc(sizeof(sphinx_builder_t));
    bb->super.begin_boolean = begin_boolean;
    bb->super.end_boolean = end_boolean;
    bb->super.match = match;
    bb->super.get_internalised = get_internalised;
    bb->super.run = run;

    bb->mailbox = mailbox;
    bb->opts = opts;

    r = sphinx_basedir(mailbox, NULL, &bb->indexname);
    if (r) {
        free(bb);
        return NULL;
    }

    if ((opts & SEARCH_MULTIPLE))
        xstats_inc(SPHINX_MULTIPLE);
    else
        xstats_inc(SPHINX_SINGLE);

    return &bb->super;
}

static int run(search_builder_t *bx, search_hit_cb_t proc, void *rock)
{
    sphinx_builder_t *bb = (sphinx_builder_t *)bx;
    struct connection conn = CONNECTION_INITIALIZER;
    struct buf query = BUF_INITIALIZER; /* SphinxQL query */
    struct buf inner_query = BUF_INITIALIZER;   /* Sphinx extended match syntax  */
    MYSQL_RES *res = NULL;
    MYSQL_ROW row;
    uint32_t uid;
    uint32_t latest = 0;
    int r = 0;

    r = sphinx_setup(bb->mailbox, SEARCH_VERBOSE(bb->opts), /*create*/0);
    if (r == IMAP_NOTFOUND) {
        /* there's no index for this user */
        r = 0;
        goto out;
    }
    if (r) goto out;

    r = get_connection(&conn);
    if (r) goto out;

    if ((bb->opts & SEARCH_UNINDEXED)) {
        /* To avoid races, we want the 'latest' uid we use to be
         * an underestimate, because the caller can handle false
         * positives but not false negatives.  So we fetch it
         * first before the main query. */
        struct latestdb ldb = LATESTDB_INITIALIZER;
        r = open_latest(bb->mailbox, &ldb);
        if (!r) goto out;
        r = read_latest(&ldb, bb->mailbox, &latest,
                        SEARCH_VERBOSE(bb->opts));
        close_latest(&ldb);
        if (r) goto out;
    }

    optimise_nodes(NULL, bb->root);
    generate_query(&inner_query, bb->root, NULL);

    buf_printf(&query, "SELECT "COL_CYRUSID" FROM %s WHERE MATCH(", bb->indexname);
    append_escaped(&query, &inner_query, '\'');
    buf_appendcstr(&query, ")");
    // get sphinx to sort by most recent date first
    buf_appendcstr(&query, " ORDER BY "COL_CYRUSID" DESC "
                               " LIMIT " SPHINX_MAX_MATCHES
                               " OPTION max_matches=" SPHINX_MAX_MATCHES);
    buf_cstring(&query);

    if (SEARCH_VERBOSE(bb->opts))
        syslog(LOG_NOTICE, "Sphinx query %s", query.s);
    xstats_inc(SPHINX_QUERY);

    r = mysql_real_query(conn.mysql, query.s, query.len);
    if (r) {
        syslog(LOG_ERR, "IOERROR: Sphinx query %s failed: %s",
               query.s, mysql_error(conn.mysql));
        r = IMAP_IOERROR;
        goto out;
    }

    res = mysql_use_result(conn.mysql);
    while ((row = mysql_fetch_row(res))) {
        const char *mboxname;
        unsigned int uidvalidity;
        unsigned int uid;
        if (SEARCH_VERBOSE(bb->opts) > 1)
            syslog(LOG_NOTICE, "Sphinx row cyrusid=%s", row[0]);
        xstats_inc(SPHINX_ROW);
        if (!parse_cyrusid(row[0], &mboxname, &uidvalidity, &uid))
            // TODO: whine
            continue;
        if (!(bb->opts & SEARCH_MULTIPLE)) {
            if (strcmp(mboxname, bb->mailbox->name))
                continue;
            if (uidvalidity != bb->mailbox->i.uidvalidity)
                continue;
        }
        xstats_inc(SPHINX_RESULT);
        r = proc(mboxname, uidvalidity, uid, rock);
        if (r) goto out;
    }
    r = 0;

    if ((bb->opts & SEARCH_UNINDEXED)) {
        /* add in the unindexed uids as false positives */
        for (uid = latest+1 ; uid <= bb->mailbox->i.last_uid ; uid++) {
            xstats_inc(SPHINX_UNINDEXED);
            r = proc(bb->mailbox->name, bb->mailbox->i.uidvalidity, uid, rock);
            if (r) goto out;
        }
    }

out:
    if (res) mysql_free_result(res);
    close_connection(&conn);
    buf_free(&query);
    buf_free(&inner_query);
    return r;
}

static void end_search(search_builder_t *bx)
{
    sphinx_builder_t *bb = (sphinx_builder_t *)bx;

    ptrarray_fini(&bb->stack);
    if (bb->root) opnode_delete(bb->root);
    free(bb->indexname);
    free(bx);
}

/* base class for both update and snippet receivers */
typedef struct sphinx_receiver sphinx_receiver_t;
struct sphinx_receiver
{
    search_text_receiver_t super;
    int verbose;
    struct mailbox *mailbox;
    char *indexname;
    uint32_t uid;
    int part;
    unsigned int parts_total;
    int truncate_warning;
    struct buf parts[SEARCH_NUM_PARTS];
    struct buf tmp;             /* SphinxQL query */
};

/* receiver used for updating the index */
typedef struct sphinx_update_receiver sphinx_update_receiver_t;
struct sphinx_update_receiver
{
    sphinx_receiver_t super;
    struct connection conn;
    int indexing_lock_fd;
    unsigned int uncommitted;
    uint32_t latest;
    struct latestdb latestdb;
    uint32_t lastid;        /* largest document ID in the 'tr' table,
                             * used to assign new document IDs when
                             * INSERTing into the table */
};

/* receiver used for extracting snippets after a search */
typedef struct sphinx_snippet_receiver sphinx_snippet_receiver_t;
struct sphinx_snippet_receiver
{
    sphinx_receiver_t super;
    struct connection conn;
    struct opnode *root;
    search_snippet_cb_t proc;
    void *rock;
};

/* Maximum size of a query, determined empirically, is a little bit
 * under 8MB.  That seems like more than enough, so let's limit the
 * total amount of parts text to 4 MB. */
#define MAX_PARTS_SIZE      (4*1024*1024)

static const char *describe_query(struct buf *desc,
                                  const struct buf *query,
                                  unsigned maxlen)
{
    buf_reset(desc);
    buf_appendcstr(desc, "Sphinx query ");
    if (maxlen && query->len > maxlen) {
        buf_appendmap(desc, query->s, maxlen);
        buf_appendcstr(desc, "...");
    }
    else {
        buf_append(desc, query);
    }
    return buf_cstring(desc);
}

static int doquery(struct connection *conn, int verbose, const struct buf *query)
{
    int r;
    struct buf desc = BUF_INITIALIZER;
    unsigned int maxlen = verbose > 2 ? /*unlimited*/0 : 128;

    if (verbose > 1)
        syslog(LOG_NOTICE, "%s", describe_query(&desc, query, maxlen));

    r = mysql_real_query(conn->mysql, query->s, query->len);
    if (r) {
        syslog(LOG_ERR, "IOERROR: %s failed: %s",
                        describe_query(&desc, query, maxlen),
                        mysql_error(conn->mysql));
        r = IMAP_IOERROR;
    }

    buf_free(&desc);
    return r;
}

#if 0
/* Dump a result which has had mysql_store_result() called on it */
static void dump_result(MYSQL_RES *res)
{
    uint64_t nrows = mysql_num_rows(res);
    unsigned int nfields = mysql_num_fields(res);
    unsigned int i;
    unsigned int j;
    MYSQL_FIELD *fields = mysql_fetch_fields(res);
    MYSQL_ROW *row;
    struct buf buf = BUF_INITIALIZER;

    syslog(LOG_NOTICE, "Sphinx result: %u rows", (unsigned int)nrows);
    i = 0;
    while ((row = mysql_fetch_row(res))) {
        buf_reset(&buf);
        for (j = 0 ; j < nfields ; j++)
            buf_printf(&buf, " %s=\"%s\"", fields[j].name, row[j]);
        syslog(LOG_NOTICE, "    [%u]%s", ++i, buf_cstring(&buf));
    }

    buf_free(&buf);
    mysql_data_seek(res, 0);    /* rewind */
}
#endif

static const char *sphinx_config_file(void)
{
    static const char *config_file = NULL;

    if (!config_file)
        config_file = strconcat(config_dir, SPHINX_CONFIG, (char *)NULL);
    return config_file;
}

static const char *sphinx_rootdir(const char *partition)
{
    char *confkey;
    const char *root;
    if (!partition)
        partition = config_getstring(IMAPOPT_DEFAULTPARTITION);
    confkey = strconcat("searchpartition-", partition, NULL);
    root = config_getoverflowstring(confkey, NULL);
    free(confkey);
    return root;
}

/* When building an index name we need to escape any @ or . characters
 * in the username.  We're slightly more general than that.  */
static void z_escape(struct buf *buf, const char *s)
{
    for ( ; *s ; s++) {
        if (*s == 'Z' || !Uisalnum(*s))
            buf_printf(buf, "Z%02X", *(const unsigned char *)s);
        else
            buf_putc(buf, *s);
    }
}

/* Returns in *basedir and *indexname new strings which must be free()d */
static int sphinx_basedir(const struct mailbox *mailbox,
                          char **basedirp,
                          char **indexnamep)
{
    const char *root;
    char *basedir = NULL;
    struct buf indexname = BUF_INITIALIZER;
    struct mboxname_parts parts;
    char c[2], d[2];
    int r;

    mboxname_init_parts(&parts);

    root = sphinx_rootdir(mailbox->part);
    if (!root) {
        r = IMAP_PARTITION_UNKNOWN;
        goto out;
    }

    r = mboxname_to_parts(mailbox->name, &parts);
    if (r) goto out;
    if (!parts.userid) {
        r = IMAP_PARTITION_UNKNOWN;
        goto out;
    }

    if (parts.domain)
        basedir = strconcat(root,
                            FNAME_DOMAINDIR,
                            dir_hash_b(parts.domain, config_fulldirhash, d),
                            "/", parts.domain,
                            FNAME_USERDIR,
                            dir_hash_b(parts.userid, config_fulldirhash, c),
                            "/", parts.userid,
                            (char *)NULL);
    else
        basedir = strconcat(root,
                            FNAME_USERDIR,
                            dir_hash_b(parts.userid, config_fulldirhash, c),
                            "/", parts.userid,
                            (char *)NULL);

    buf_appendcstr(&indexname, "X");
    z_escape(&indexname, parts.userid);
    if (parts.domain) {
        z_escape(&indexname, "@");
        z_escape(&indexname, parts.domain);
    }

    r = 0;

out:
    if (!r && indexnamep)
        *indexnamep = buf_release(&indexname);
    buf_free(&indexname);
    if (!r && basedirp)
        *basedirp = basedir;
    else
        free(basedir);
    mboxname_free_parts(&parts);
    return r;
}

static const char *sphinx_syslog_prefix(void)
{
    static const char *prefix = NULL;

    if (!prefix) {
        prefix = config_getstring(IMAPOPT_SYSLOG_PREFIX);
        if (!prefix)
            prefix = "cyrus";
    }
    return prefix;
}

static int check_directory(const char *dir, int verbose, int create)
{
    int r;
    char *dummyfile = NULL;
    struct stat sb;

    r = stat(dir, &sb);
    if (r < 0) {
        if (r != ENOENT) {
            /* something went wrong - permissions problem most likely */
            syslog(LOG_ERR, "IOERROR: unable to stat %s: %m", dir);
            r = IMAP_IOERROR;
            goto out;
        }
        /* the directory is just missing */
        if (!create) {
            /* caller doesn't care that much */
            r = IMAP_NOTFOUND;
            goto out;
        }
        if (verbose)
            syslog(LOG_INFO, "Building directory %s", dir);
        dummyfile = strconcat(dir, "/dummy", (char *)NULL);
        cyrus_mkdir(dummyfile, 0700);
        r = stat(dir, &sb);
        if (r < 0) {
            /* something went wrong - permissions problem most likely */
            syslog(LOG_ERR, "IOERROR: unable to stat %s: %m", dir);
            r = IMAP_IOERROR;
            goto out;
        }
    }

out:
    free(dummyfile);
    return r;
}

static int sphinx_signal(int sig, int verbose)
{
    const char *pidfile = config_getstring(IMAPOPT_SPHINX_PIDFILE);
    int fd = -1;
    pid_t pid;
    int r;
    char buf[33];

    fd = open(pidfile, O_RDONLY, 0);
    if (fd < 0) {
        if (fd == ENOENT) {
            r = IMAP_NOTFOUND;
        }
        else {
            syslog(LOG_ERR, "IOERROR: unable to open %s for reading: %m", pidfile);
            r = IMAP_IOERROR;
        }
        goto out;
    }

    memset(buf, 0, sizeof(buf));
    r = read(fd, buf, sizeof(buf)-1);
    if (r < 0) {
        syslog(LOG_ERR, "IOERROR: unable to read %s: %m", pidfile);
        r = IMAP_IOERROR;
        goto out;
    }
    if (r == 0) {
        syslog(LOG_ERR, "IOERROR: short file %s", pidfile);
        r = IMAP_IOERROR;
        goto out;
    }

    pid = strtoul(buf, NULL, 10);
    if (pid <= 0) {
        syslog(LOG_ERR, "IOERROR: invalid contents of %s", pidfile);
        r = IMAP_IOERROR;
        goto out;
    }

    if (verbose)
        syslog(LOG_INFO, "Sending signal %d to pid %d from %s",
                sig, (int)pid, pidfile);

    r = kill(pid, sig);
    if (r < 0) {
        if (r == ESRCH) {
            r = IMAP_NOTFOUND;
        }
        else {
            syslog(LOG_ERR, "IOERROR: failed to send signal %d "
                            "to searchd pid %d: %m",
                            sig, (int)pid);
            r = IMAP_SYS_ERROR;
        }
        goto out;
    }

    r = 0;

out:
    if (fd >= 0) close(fd);
    return r;
}


static int sphinx_setup(const struct mailbox *mailbox, int verbose, int create)
{
    static const char user_config[] =
        "index $indexname\n"
        "{\n"
        "    type = rt\n"
        "    path = $basedir/rt\n"
        "    morphology = stem_en\n"
        "    charset_type = utf-8\n"
        "    charset_table = 0..9, A..Z->a..z, _, a..z, \\\n"
        /* Support for Cyrillic from
         * http://sphinxsearch.com/wiki/doku.php?id=charset_tables#cyrillic */
        "       U+0400->U+0435, U+0401->U+0435, U+0402->U+0452, U+0452, \\\n"
        "       U+0403->U+0433, U+0404->U+0454, U+0454, U+0405->U+0455, \\\n"
        "       U+0455, U+0406->U+0456, U+0407->U+0456, U+0457->U+0456, \\\n"
        "       U+0456, U+0408..U+040B->U+0458..U+045B, U+0458..U+045B, \\\n"
        "       U+040C->U+043A, U+040D->U+0438, U+040E->U+0443, U+040F->U+045F, \\\n"
        "       U+045F, U+0450->U+0435, U+0451->U+0435, U+0453->U+0433, \\\n"
        "       U+045C->U+043A, U+045D->U+0438, U+045E->U+0443, U+0460->U+0461, \\\n"
        "       U+0461, U+0462->U+0463, U+0463, U+0464->U+0465, U+0465, \\\n"
        "       U+0466->U+0467, U+0467, U+0468->U+0469, U+0469, U+046A->U+046B, \\\n"
        "       U+046B, U+046C->U+046D, U+046D, U+046E->U+046F, U+046F, \\\n"
        "       U+0470->U+0471, U+0471, U+0472->U+0473, U+0473, U+0474->U+0475, \\\n"
        "       U+0476->U+0475, U+0477->U+0475, U+0475, U+0478->U+0479, U+0479, \\\n"
        "       U+047A->U+047B, U+047B, U+047C->U+047D, U+047D, U+047E->U+047F, \\\n"
        "       U+047F, U+0480->U+0481, U+0481, U+048A->U+0438, U+048B->U+0438, \\\n"
        "       U+048C->U+044C, U+048D->U+044C, U+048E->U+0440, U+048F->U+0440, \\\n"
        "       U+0490->U+0433, U+0491->U+0433, U+0490->U+0433, U+0491->U+0433, \\\n"
        "       U+0492->U+0433, U+0493->U+0433, U+0494->U+0433, U+0495->U+0433, \\\n"
        "       U+0496->U+0436, U+0497->U+0436, U+0498->U+0437, U+0499->U+0437, \\\n"
        "       U+049A->U+043A, U+049B->U+043A, U+049C->U+043A, U+049D->U+043A, \\\n"
        "       U+049E->U+043A, U+049F->U+043A, U+04A0->U+043A, U+04A1->U+043A, \\\n"
        "       U+04A2->U+043D, U+04A3->U+043D, U+04A4->U+043D, U+04A5->U+043D, \\\n"
        "       U+04A6->U+043F, U+04A7->U+043F, U+04A8->U+04A9, U+04A9, \\\n"
        "       U+04AA->U+0441, U+04AB->U+0441, U+04AC->U+0442, U+04AD->U+0442, \\\n"
        "       U+04AE->U+0443, U+04AF->U+0443, U+04B0->U+0443, U+04B1->U+0443, \\\n"
        "       U+04B2->U+0445, U+04B3->U+0445, U+04B4->U+04B5, U+04B5, \\\n"
        "       U+04B6->U+0447, U+04B7->U+0447, U+04B8->U+0447, U+04B9->U+0447, \\\n"
        "       U+04BA->U+04BB, U+04BB, U+04BC->U+04BD, U+04BE->U+04BD, \\\n"
        "       U+04BF->U+04BD, U+04BD, U+04C0->U+04CF, U+04CF, U+04C1->U+0436, \\\n"
        "       U+04C2->U+0436, U+04C3->U+043A, U+04C4->U+043A, U+04C5->U+043B, \\\n"
        "       U+04C6->U+043B, U+04C7->U+043D, U+04C8->U+043D, U+04C9->U+043D, \\\n"
        "       U+04CA->U+043D, U+04CB->U+0447, U+04CC->U+0447, U+04CD->U+043C, \\\n"
        "       U+04CE->U+043C, U+04D0->U+0430, U+04D1->U+0430, U+04D2->U+0430, \\\n"
        "       U+04D3->U+0430, U+04D4->U+00E6, U+04D5->U+00E6, U+04D6->U+0435, \\\n"
        "       U+04D7->U+0435, U+04D8->U+04D9, U+04DA->U+04D9, U+04DB->U+04D9, \\\n"
        "       U+04D9, U+04DC->U+0436, U+04DD->U+0436, U+04DE->U+0437, \\\n"
        "       U+04DF->U+0437, U+04E0->U+04E1, U+04E1, U+04E2->U+0438, \\\n"
        "       U+04E3->U+0438, U+04E4->U+0438, U+04E5->U+0438, U+04E6->U+043E, \\\n"
        "       U+04E7->U+043E, U+04E8->U+043E, U+04E9->U+043E, U+04EA->U+043E, \\\n"
        "       U+04EB->U+043E, U+04EC->U+044D, U+04ED->U+044D, U+04EE->U+0443, \\\n"
        "       U+04EF->U+0443, U+04F0->U+0443, U+04F1->U+0443, U+04F2->U+0443, \\\n"
        "       U+04F3->U+0443, U+04F4->U+0447, U+04F5->U+0447, U+04F6->U+0433, \\\n"
        "       U+04F7->U+0433, U+04F8->U+044B, U+04F9->U+044B, U+04FA->U+0433, \\\n"
        "       U+04FB->U+0433, U+04FC->U+0445, U+04FD->U+0445, U+04FE->U+0445, \\\n"
        "       U+04FF->U+0445, U+0410..U+0418->U+0430..U+0438, U+0419->U+0438, \\\n"
        "       U+0430..U+0438, U+041A..U+042F->U+043A..U+044F, U+043A..U+044F,\\\n"
        /* and this one was missing dammit */
        "       U+0418->U+0438, U+0439->U+0438, \\\n"
        /* Sumerian cuneiform which is fun for testing but seems
         * to be broken in Spinx. */
        /*
        "       U+12000..U+1237F, \\\n"
        */
        /* Support for Chinese/Japanese/Korean, from
         * http://sphinxsearch.com/wiki/doku.php?id=charset_tables#cjk */
        "       U+F900->U+8C48, U+F901->U+66F4, U+F902->U+8ECA, U+F903->U+8CC8, \\\n"
        "       U+F904->U+6ED1, U+F905->U+4E32, U+F906->U+53E5, U+F907->U+9F9C, \\\n"
        "       U+F908->U+9F9C, U+F909->U+5951, U+F90A->U+91D1, U+F90B->U+5587, \\\n"
        "       U+F90C->U+5948, U+F90D->U+61F6, U+F90E->U+7669, U+F90F->U+7F85, \\\n"
        "       U+F910->U+863F, U+F911->U+87BA, U+F912->U+88F8, U+F913->U+908F, \\\n"
        "       U+F914->U+6A02, U+F915->U+6D1B, U+F916->U+70D9, U+F917->U+73DE, \\\n"
        "       U+F918->U+843D, U+F919->U+916A, U+F91A->U+99F1, U+F91B->U+4E82, \\\n"
        "       U+F91C->U+5375, U+F91D->U+6B04, U+F91E->U+721B, U+F91F->U+862D, \\\n"
        "       U+F920->U+9E1E, U+F921->U+5D50, U+F922->U+6FEB, U+F923->U+85CD, \\\n"
        "       U+F924->U+8964, U+F925->U+62C9, U+F926->U+81D8, U+F927->U+881F, \\\n"
        "       U+F928->U+5ECA, U+F929->U+6717, U+F92A->U+6D6A, U+F92B->U+72FC, \\\n"
        "       U+F92C->U+90CE, U+F92D->U+4F86, U+F92E->U+51B7, U+F92F->U+52DE, \\\n"
        "       U+F930->U+64C4, U+F931->U+6AD3, U+F932->U+7210, U+F933->U+76E7, \\\n"
        "       U+F934->U+8001, U+F935->U+8606, U+F936->U+865C, U+F937->U+8DEF, \\\n"
        "       U+F938->U+9732, U+F939->U+9B6F, U+F93A->U+9DFA, U+F93B->U+788C, \\\n"
        "       U+F93C->U+797F, U+F93D->U+7DA0, U+F93E->U+83C9, U+F93F->U+9304, \\\n"
        "       U+F940->U+9E7F, U+F941->U+8AD6, U+F942->U+58DF, U+F943->U+5F04, \\\n"
        "       U+F944->U+7C60, U+F945->U+807E, U+F946->U+7262, U+F947->U+78CA, \\\n"
        "       U+F948->U+8CC2, U+F949->U+96F7, U+F94A->U+58D8, U+F94B->U+5C62, \\\n"
        "       U+F94C->U+6A13, U+F94D->U+6DDA, U+F94E->U+6F0F, U+F94F->U+7D2F, \\\n"
        "       U+F950->U+7E37, U+F951->U+964B, U+F952->U+52D2, U+F953->U+808B, \\\n"
        "       U+F954->U+51DC, U+F955->U+51CC, U+F956->U+7A1C, U+F957->U+7DBE, \\\n"
        "       U+F958->U+83F1, U+F959->U+9675, U+F95A->U+8B80, U+F95B->U+62CF, \\\n"
        "       U+F95C->U+6A02, U+F95D->U+8AFE, U+F95E->U+4E39, U+F95F->U+5BE7, \\\n"
        "       U+F960->U+6012, U+F961->U+7387, U+F962->U+7570, U+F963->U+5317, \\\n"
        "       U+F964->U+78FB, U+F965->U+4FBF, U+F966->U+5FA9, U+F967->U+4E0D, \\\n"
        "       U+F968->U+6CCC, U+F969->U+6578, U+F96A->U+7D22, U+F96B->U+53C3, \\\n"
        "       U+F96C->U+585E, U+F96D->U+7701, U+F96E->U+8449, U+F96F->U+8AAA, \\\n"
        "       U+F970->U+6BBA, U+F971->U+8FB0, U+F972->U+6C88, U+F973->U+62FE, \\\n"
        "       U+F974->U+82E5, U+F975->U+63A0, U+F976->U+7565, U+F977->U+4EAE, \\\n"
        "       U+F978->U+5169, U+F979->U+51C9, U+F97A->U+6881, U+F97B->U+7CE7, \\\n"
        "       U+F97C->U+826F, U+F97D->U+8AD2, U+F97E->U+91CF, U+F97F->U+52F5, \\\n"
        "       U+F980->U+5442, U+F981->U+5973, U+F982->U+5EEC, U+F983->U+65C5, \\\n"
        "       U+F984->U+6FFE, U+F985->U+792A, U+F986->U+95AD, U+F987->U+9A6A, \\\n"
        "       U+F988->U+9E97, U+F989->U+9ECE, U+F98A->U+529B, U+F98B->U+66C6, \\\n"
        "       U+F98C->U+6B77, U+F98D->U+8F62, U+F98E->U+5E74, U+F98F->U+6190, \\\n"
        "       U+F990->U+6200, U+F991->U+649A, U+F992->U+6F23, U+F993->U+7149, \\\n"
        "       U+F994->U+7489, U+F995->U+79CA, U+F996->U+7DF4, U+F997->U+806F, \\\n"
        "       U+F998->U+8F26, U+F999->U+84EE, U+F99A->U+9023, U+F99B->U+934A, \\\n"
        "       U+F99C->U+5217, U+F99D->U+52A3, U+F99E->U+54BD, U+F99F->U+70C8, \\\n"
        "       U+F9A0->U+88C2, U+F9A1->U+8AAA, U+F9A2->U+5EC9, U+F9A3->U+5FF5, \\\n"
        "       U+F9A4->U+637B, U+F9A5->U+6BAE, U+F9A6->U+7C3E, U+F9A7->U+7375, \\\n"
        "       U+F9A8->U+4EE4, U+F9A9->U+56F9, U+F9AA->U+5BE7, U+F9AB->U+5DBA, \\\n"
        "       U+F9AC->U+601C, U+F9AD->U+73B2, U+F9AE->U+7469, U+F9AF->U+7F9A, \\\n"
        "       U+F9B0->U+8046, U+F9B1->U+9234, U+F9B2->U+96F6, U+F9B3->U+9748, \\\n"
        "       U+F9B4->U+9818, U+F9B5->U+4F8B, U+F9B6->U+79AE, U+F9B7->U+91B4, \\\n"
        "       U+F9B8->U+96B8, U+F9B9->U+60E1, U+F9BA->U+4E86, U+F9BB->U+50DA, \\\n"
        "       U+F9BC->U+5BEE, U+F9BD->U+5C3F, U+F9BE->U+6599, U+F9BF->U+6A02, \\\n"
        "       U+F9C0->U+71CE, U+F9C1->U+7642, U+F9C2->U+84FC, U+F9C3->U+907C, \\\n"
        "       U+F9C4->U+9F8D, U+F9C5->U+6688, U+F9C6->U+962E, U+F9C7->U+5289, \\\n"
        "       U+F9C8->U+677B, U+F9C9->U+67F3, U+F9CA->U+6D41, U+F9CB->U+6E9C, \\\n"
        "       U+F9CC->U+7409, U+F9CD->U+7559, U+F9CE->U+786B, U+F9CF->U+7D10, \\\n"
        "       U+F9D0->U+985E, U+F9D1->U+516D, U+F9D2->U+622E, U+F9D3->U+9678, \\\n"
        "       U+F9D4->U+502B, U+F9D5->U+5D19, U+F9D6->U+6DEA, U+F9D7->U+8F2A, \\\n"
        "       U+F9D8->U+5F8B, U+F9D9->U+6144, U+F9DA->U+6817, U+F9DB->U+7387, \\\n"
        "       U+F9DC->U+9686, U+F9DD->U+5229, U+F9DE->U+540F, U+F9DF->U+5C65, \\\n"
        "       U+F9E0->U+6613, U+F9E1->U+674E, U+F9E2->U+68A8, U+F9E3->U+6CE5, \\\n"
        "       U+F9E4->U+7406, U+F9E5->U+75E2, U+F9E6->U+7F79, U+F9E7->U+88CF, \\\n"
        "       U+F9E8->U+88E1, U+F9E9->U+91CC, U+F9EA->U+96E2, U+F9EB->U+533F, \\\n"
        "       U+F9EC->U+6EBA, U+F9ED->U+541D, U+F9EE->U+71D0, U+F9EF->U+7498, \\\n"
        "       U+F9F0->U+85FA, U+F9F1->U+96A3, U+F9F2->U+9C57, U+F9F3->U+9E9F, \\\n"
        "       U+F9F4->U+6797, U+F9F5->U+6DCB, U+F9F6->U+81E8, U+F9F7->U+7ACB, \\\n"
        "       U+F9F8->U+7B20, U+F9F9->U+7C92, U+F9FA->U+72C0, U+F9FB->U+7099, \\\n"
        "       U+F9FC->U+8B58, U+F9FD->U+4EC0, U+F9FE->U+8336, U+F9FF->U+523A, \\\n"
        "       U+FA00->U+5207, U+FA01->U+5EA6, U+FA02->U+62D3, U+FA03->U+7CD6, \\\n"
        "       U+FA04->U+5B85, U+FA05->U+6D1E, U+FA06->U+66B4, U+FA07->U+8F3B, \\\n"
        "       U+FA08->U+884C, U+FA09->U+964D, U+FA0A->U+898B, U+FA0B->U+5ED3, \\\n"
        "       U+FA0C->U+5140, U+FA0D->U+55C0, U+FA10->U+585A, U+FA12->U+6674, \\\n"
        "       U+FA15->U+51DE, U+FA16->U+732A, U+FA17->U+76CA, U+FA18->U+793C, \\\n"
        "       U+FA19->U+795E, U+FA1A->U+7965, U+FA1B->U+798F, U+FA1C->U+9756, \\\n"
        "       U+FA1D->U+7CBE, U+FA1E->U+7FBD, U+FA20->U+8612, U+FA22->U+8AF8, \\\n"
        "       U+FA25->U+9038, U+FA26->U+90FD, U+FA2A->U+98EF, U+FA2B->U+98FC, \\\n"
        "       U+FA2C->U+9928, U+FA2D->U+9DB4, U+FA30->U+4FAE, U+FA31->U+50E7, \\\n"
        "       U+FA32->U+514D, U+FA33->U+52C9, U+FA34->U+52E4, U+FA35->U+5351, \\\n"
        "       U+FA36->U+559D, U+FA37->U+5606, U+FA38->U+5668, U+FA39->U+5840, \\\n"
        "       U+FA3A->U+58A8, U+FA3B->U+5C64, U+FA3C->U+5C6E, U+FA3D->U+6094, \\\n"
        "       U+FA3E->U+6168, U+FA3F->U+618E, U+FA40->U+61F2, U+FA41->U+654F, \\\n"
        "       U+FA42->U+65E2, U+FA43->U+6691, U+FA44->U+6885, U+FA45->U+6D77, \\\n"
        "       U+FA46->U+6E1A, U+FA47->U+6F22, U+FA48->U+716E, U+FA49->U+722B, \\\n"
        "       U+FA4A->U+7422, U+FA4B->U+7891, U+FA4C->U+793E, U+FA4D->U+7949, \\\n"
        "       U+FA4E->U+7948, U+FA4F->U+7950, U+FA50->U+7956, U+FA51->U+795D, \\\n"
        "       U+FA52->U+798D, U+FA53->U+798E, U+FA54->U+7A40, U+FA55->U+7A81, \\\n"
        "       U+FA56->U+7BC0, U+FA57->U+7DF4, U+FA58->U+7E09, U+FA59->U+7E41, \\\n"
        "       U+FA5A->U+7F72, U+FA5B->U+8005, U+FA5C->U+81ED, U+FA5D->U+8279, \\\n"
        "       U+FA5E->U+8279, U+FA5F->U+8457, U+FA60->U+8910, U+FA61->U+8996, \\\n"
        "       U+FA62->U+8B01, U+FA63->U+8B39, U+FA64->U+8CD3, U+FA65->U+8D08, \\\n"
        "       U+FA66->U+8FB6, U+FA67->U+9038, U+FA68->U+96E3, U+FA69->U+97FF, \\\n"
        "       U+FA6A->U+983B, U+FA70->U+4E26, U+FA71->U+51B5, U+FA72->U+5168, \\\n"
        "       U+FA73->U+4F80, U+FA74->U+5145, U+FA75->U+5180, U+FA76->U+52C7, \\\n"
        "       U+FA77->U+52FA, U+FA78->U+559D, U+FA79->U+5555, U+FA7A->U+5599, \\\n"
        "       U+FA7B->U+55E2, U+FA7C->U+585A, U+FA7D->U+58B3, U+FA7E->U+5944, \\\n"
        "       U+FA7F->U+5954, U+FA80->U+5A62, U+FA81->U+5B28, U+FA82->U+5ED2, \\\n"
        "       U+FA83->U+5ED9, U+FA84->U+5F69, U+FA85->U+5FAD, U+FA86->U+60D8, \\\n"
        "       U+FA87->U+614E, U+FA88->U+6108, U+FA89->U+618E, U+FA8A->U+6160, \\\n"
        "       U+FA8B->U+61F2, U+FA8C->U+6234, U+FA8D->U+63C4, U+FA8E->U+641C, \\\n"
        "       U+FA8F->U+6452, U+FA90->U+6556, U+FA91->U+6674, U+FA92->U+6717, \\\n"
        "       U+FA93->U+671B, U+FA94->U+6756, U+FA95->U+6B79, U+FA96->U+6BBA, \\\n"
        "       U+FA97->U+6D41, U+FA98->U+6EDB, U+FA99->U+6ECB, U+FA9A->U+6F22, \\\n"
        "       U+FA9B->U+701E, U+FA9C->U+716E, U+FA9D->U+77A7, U+FA9E->U+7235, \\\n"
        "       U+FA9F->U+72AF, U+FAA0->U+732A, U+FAA1->U+7471, U+FAA2->U+7506, \\\n"
        "       U+FAA3->U+753B, U+FAA4->U+761D, U+FAA5->U+761F, U+FAA6->U+76CA, \\\n"
        "       U+FAA7->U+76DB, U+FAA8->U+76F4, U+FAA9->U+774A, U+FAAA->U+7740, \\\n"
        "       U+FAAB->U+78CC, U+FAAC->U+7AB1, U+FAAD->U+7BC0, U+FAAE->U+7C7B, \\\n"
        "       U+FAAF->U+7D5B, U+FAB0->U+7DF4, U+FAB1->U+7F3E, U+FAB2->U+8005, \\\n"
        "       U+FAB3->U+8352, U+FAB4->U+83EF, U+FAB5->U+8779, U+FAB6->U+8941, \\\n"
        "       U+FAB7->U+8986, U+FAB8->U+8996, U+FAB9->U+8ABF, U+FABA->U+8AF8, \\\n"
        "       U+FABB->U+8ACB, U+FABC->U+8B01, U+FABD->U+8AFE, U+FABE->U+8AED, \\\n"
        "       U+FABF->U+8B39, U+FAC0->U+8B8A, U+FAC1->U+8D08, U+FAC2->U+8F38, \\\n"
        "       U+FAC3->U+9072, U+FAC4->U+9199, U+FAC5->U+9276, U+FAC6->U+967C, \\\n"
        "       U+FAC7->U+96E3, U+FAC8->U+9756, U+FAC9->U+97DB, U+FACA->U+97FF, \\\n"
        "       U+FACB->U+980B, U+FACC->U+983B, U+FACD->U+9B12, U+FACE->U+9F9C, \\\n"
        "       U+FACF->U+2284A, U+FAD0->U+22844, U+FAD1->U+233D5, U+FAD2->U+3B9D, \\\n"
        "       U+FAD3->U+4018, U+FAD4->U+4039, U+FAD5->U+25249, U+FAD6->U+25CD0, \\\n"
        "       U+FAD7->U+27ED3, U+FAD8->U+9F43, U+FAD9->U+9F8E, U+2F800->U+4E3D, \\\n"
        "       U+2F801->U+4E38, U+2F802->U+4E41, U+2F803->U+20122, U+2F804->U+4F60, \\\n"
        "       U+2F805->U+4FAE, U+2F806->U+4FBB, U+2F807->U+5002, U+2F808->U+507A, \\\n"
        "       U+2F809->U+5099, U+2F80A->U+50E7, U+2F80B->U+50CF, U+2F80C->U+349E, \\\n"
        "       U+2F80D->U+2063A, U+2F80E->U+514D, U+2F80F->U+5154, U+2F810->U+5164, \\\n"
        "       U+2F811->U+5177, U+2F812->U+2051C, U+2F813->U+34B9, U+2F814->U+5167, \\\n"
        "       U+2F815->U+518D, U+2F816->U+2054B, U+2F817->U+5197, U+2F818->U+51A4, \\\n"
        "       U+2F819->U+4ECC, U+2F81A->U+51AC, U+2F81B->U+51B5, U+2F81C->U+291DF, \\\n"
        "       U+2F81D->U+51F5, U+2F81E->U+5203, U+2F81F->U+34DF, U+2F820->U+523B, \\\n"
        "       U+2F821->U+5246, U+2F822->U+5272, U+2F823->U+5277, U+2F824->U+3515, \\\n"
        "       U+2F825->U+52C7, U+2F826->U+52C9, U+2F827->U+52E4, U+2F828->U+52FA, \\\n"
        "       U+2F829->U+5305, U+2F82A->U+5306, U+2F82B->U+5317, U+2F82C->U+5349, \\\n"
        "       U+2F82D->U+5351, U+2F82E->U+535A, U+2F82F->U+5373, U+2F830->U+537D, \\\n"
        "       U+2F831->U+537F, U+2F832->U+537F, U+2F833->U+537F, U+2F834->U+20A2C, \\\n"
        "       U+2F835->U+7070, U+2F836->U+53CA, U+2F837->U+53DF, U+2F838->U+20B63, \\\n"
        "       U+2F839->U+53EB, U+2F83A->U+53F1, U+2F83B->U+5406, U+2F83C->U+549E, \\\n"
        "       U+2F83D->U+5438, U+2F83E->U+5448, U+2F83F->U+5468, U+2F840->U+54A2, \\\n"
        "       U+2F841->U+54F6, U+2F842->U+5510, U+2F843->U+5553, U+2F844->U+5563, \\\n"
        "       U+2F845->U+5584, U+2F846->U+5584, U+2F847->U+5599, U+2F848->U+55AB, \\\n"
        "       U+2F849->U+55B3, U+2F84A->U+55C2, U+2F84B->U+5716, U+2F84C->U+5606, \\\n"
        "       U+2F84D->U+5717, U+2F84E->U+5651, U+2F84F->U+5674, U+2F850->U+5207, \\\n"
        "       U+2F851->U+58EE, U+2F852->U+57CE, U+2F853->U+57F4, U+2F854->U+580D, \\\n"
        "       U+2F855->U+578B, U+2F856->U+5832, U+2F857->U+5831, U+2F858->U+58AC, \\\n"
        "       U+2F859->U+214E4, U+2F85A->U+58F2, U+2F85B->U+58F7, U+2F85C->U+5906, \\\n"
        "       U+2F85D->U+591A, U+2F85E->U+5922, U+2F85F->U+5962, U+2F860->U+216A8, \\\n"
        "       U+2F861->U+216EA, U+2F862->U+59EC, U+2F863->U+5A1B, U+2F864->U+5A27, \\\n"
        "       U+2F865->U+59D8, U+2F866->U+5A66, U+2F867->U+36EE, U+2F868->U+36FC, \\\n"
        "       U+2F869->U+5B08, U+2F86A->U+5B3E, U+2F86B->U+5B3E, U+2F86C->U+219C8, \\\n"
        "       U+2F86D->U+5BC3, U+2F86E->U+5BD8, U+2F86F->U+5BE7, U+2F870->U+5BF3, \\\n"
        "       U+2F871->U+21B18, U+2F872->U+5BFF, U+2F873->U+5C06, U+2F874->U+5F53, \\\n"
        "       U+2F875->U+5C22, U+2F876->U+3781, U+2F877->U+5C60, U+2F878->U+5C6E, \\\n"
        "       U+2F879->U+5CC0, U+2F87A->U+5C8D, U+2F87B->U+21DE4, U+2F87C->U+5D43, \\\n"
        "       U+2F87D->U+21DE6, U+2F87E->U+5D6E, U+2F87F->U+5D6B, U+2F880->U+5D7C, \\\n"
        "       U+2F881->U+5DE1, U+2F882->U+5DE2, U+2F883->U+382F, U+2F884->U+5DFD, \\\n"
        "       U+2F885->U+5E28, U+2F886->U+5E3D, U+2F887->U+5E69, U+2F888->U+3862, \\\n"
        "       U+2F889->U+22183, U+2F88A->U+387C, U+2F88B->U+5EB0, U+2F88C->U+5EB3, \\\n"
        "       U+2F88D->U+5EB6, U+2F88E->U+5ECA, U+2F88F->U+2A392, U+2F890->U+5EFE, \\\n"
        "       U+2F891->U+22331, U+2F892->U+22331, U+2F893->U+8201, U+2F894->U+5F22, \\\n"
        "       U+2F895->U+5F22, U+2F896->U+38C7, U+2F897->U+232B8, U+2F898->U+261DA, \\\n"
        "       U+2F899->U+5F62, U+2F89A->U+5F6B, U+2F89B->U+38E3, U+2F89C->U+5F9A, \\\n"
        "       U+2F89D->U+5FCD, U+2F89E->U+5FD7, U+2F89F->U+5FF9, U+2F8A0->U+6081, \\\n"
        "       U+2F8A1->U+393A, U+2F8A2->U+391C, U+2F8A3->U+6094, U+2F8A4->U+226D4, \\\n"
        "       U+2F8A5->U+60C7, U+2F8A6->U+6148, U+2F8A7->U+614C, U+2F8A8->U+614E, \\\n"
        "       U+2F8A9->U+614C, U+2F8AA->U+617A, U+2F8AB->U+618E, U+2F8AC->U+61B2, \\\n"
        "       U+2F8AD->U+61A4, U+2F8AE->U+61AF, U+2F8AF->U+61DE, U+2F8B0->U+61F2, \\\n"
        "       U+2F8B1->U+61F6, U+2F8B2->U+6210, U+2F8B3->U+621B, U+2F8B4->U+625D, \\\n"
        "       U+2F8B5->U+62B1, U+2F8B6->U+62D4, U+2F8B7->U+6350, U+2F8B8->U+22B0C, \\\n"
        "       U+2F8B9->U+633D, U+2F8BA->U+62FC, U+2F8BB->U+6368, U+2F8BC->U+6383, \\\n"
        "       U+2F8BD->U+63E4, U+2F8BE->U+22BF1, U+2F8BF->U+6422, U+2F8C0->U+63C5, \\\n"
        "       U+2F8C1->U+63A9, U+2F8C2->U+3A2E, U+2F8C3->U+6469, U+2F8C4->U+647E, \\\n"
        "       U+2F8C5->U+649D, U+2F8C6->U+6477, U+2F8C7->U+3A6C, U+2F8C8->U+654F, \\\n"
        "       U+2F8C9->U+656C, U+2F8CA->U+2300A, U+2F8CB->U+65E3, U+2F8CC->U+66F8, \\\n"
        "       U+2F8CD->U+6649, U+2F8CE->U+3B19, U+2F8CF->U+6691, U+2F8D0->U+3B08, \\\n"
        "       U+2F8D1->U+3AE4, U+2F8D2->U+5192, U+2F8D3->U+5195, U+2F8D4->U+6700, \\\n"
        "       U+2F8D5->U+669C, U+2F8D6->U+80AD, U+2F8D7->U+43D9, U+2F8D8->U+6717, \\\n"
        "       U+2F8D9->U+671B, U+2F8DA->U+6721, U+2F8DB->U+675E, U+2F8DC->U+6753, \\\n"
        "       U+2F8DD->U+233C3, U+2F8DE->U+3B49, U+2F8DF->U+67FA, U+2F8E0->U+6785, \\\n"
        "       U+2F8E1->U+6852, U+2F8E2->U+6885, U+2F8E3->U+2346D, U+2F8E4->U+688E, \\\n"
        "       U+2F8E5->U+681F, U+2F8E6->U+6914, U+2F8E7->U+3B9D, U+2F8E8->U+6942, \\\n"
        "       U+2F8E9->U+69A3, U+2F8EA->U+69EA, U+2F8EB->U+6AA8, U+2F8EC->U+236A3, \\\n"
        "       U+2F8ED->U+6ADB, U+2F8EE->U+3C18, U+2F8EF->U+6B21, U+2F8F0->U+238A7, \\\n"
        "       U+2F8F1->U+6B54, U+2F8F2->U+3C4E, U+2F8F3->U+6B72, U+2F8F4->U+6B9F, \\\n"
        "       U+2F8F5->U+6BBA, U+2F8F6->U+6BBB, U+2F8F7->U+23A8D, U+2F8F8->U+21D0B, \\\n"
        "       U+2F8F9->U+23AFA, U+2F8FA->U+6C4E, U+2F8FB->U+23CBC, U+2F8FC->U+6CBF, \\\n"
        "       U+2F8FD->U+6CCD, U+2F8FE->U+6C67, U+2F8FF->U+6D16, U+2F900->U+6D3E, \\\n"
        "       U+2F901->U+6D77, U+2F902->U+6D41, U+2F903->U+6D69, U+2F904->U+6D78, \\\n"
        "       U+2F905->U+6D85, U+2F906->U+23D1E, U+2F907->U+6D34, U+2F908->U+6E2F, \\\n"
        "       U+2F909->U+6E6E, U+2F90A->U+3D33, U+2F90B->U+6ECB, U+2F90C->U+6EC7, \\\n"
        "       U+2F90D->U+23ED1, U+2F90E->U+6DF9, U+2F90F->U+6F6E, U+2F910->U+23F5E, \\\n"
        "       U+2F911->U+23F8E, U+2F912->U+6FC6, U+2F913->U+7039, U+2F914->U+701E, \\\n"
        "       U+2F915->U+701B, U+2F916->U+3D96, U+2F917->U+704A, U+2F918->U+707D, \\\n"
        "       U+2F919->U+7077, U+2F91A->U+70AD, U+2F91B->U+20525, U+2F91C->U+7145, \\\n"
        "       U+2F91D->U+24263, U+2F91E->U+719C, U+2F91F->U+243AB, U+2F920->U+7228, \\\n"
        "       U+2F921->U+7235, U+2F922->U+7250, U+2F923->U+24608, U+2F924->U+7280, \\\n"
        "       U+2F925->U+7295, U+2F926->U+24735, U+2F927->U+24814, U+2F928->U+737A, \\\n"
        "       U+2F929->U+738B, U+2F92A->U+3EAC, U+2F92B->U+73A5, U+2F92C->U+3EB8, \\\n"
        "       U+2F92D->U+3EB8, U+2F92E->U+7447, U+2F92F->U+745C, U+2F930->U+7471, \\\n"
        "       U+2F931->U+7485, U+2F932->U+74CA, U+2F933->U+3F1B, U+2F934->U+7524, \\\n"
        "       U+2F935->U+24C36, U+2F936->U+753E, U+2F937->U+24C92, U+2F938->U+7570, \\\n"
        "       U+2F939->U+2219F, U+2F93A->U+7610, U+2F93B->U+24FA1, U+2F93C->U+24FB8, \\\n"
        "       U+2F93D->U+25044, U+2F93E->U+3FFC, U+2F93F->U+4008, U+2F940->U+76F4, \\\n"
        "       U+2F941->U+250F3, U+2F942->U+250F2, U+2F943->U+25119, U+2F944->U+25133, \\\n"
        "       U+2F945->U+771E, U+2F946->U+771F, U+2F947->U+771F, U+2F948->U+774A, \\\n"
        "       U+2F949->U+4039, U+2F94A->U+778B, U+2F94B->U+4046, U+2F94C->U+4096, \\\n"
        "       U+2F94D->U+2541D, U+2F94E->U+784E, U+2F94F->U+788C, U+2F950->U+78CC, \\\n"
        "       U+2F951->U+40E3, U+2F952->U+25626, U+2F953->U+7956, U+2F954->U+2569A, \\\n"
        "       U+2F955->U+256C5, U+2F956->U+798F, U+2F957->U+79EB, U+2F958->U+412F, \\\n"
        "       U+2F959->U+7A40, U+2F95A->U+7A4A, U+2F95B->U+7A4F, U+2F95C->U+2597C, \\\n"
        "       U+2F95D->U+25AA7, U+2F95E->U+25AA7, U+2F95F->U+7AEE, U+2F960->U+4202, \\\n"
        "       U+2F961->U+25BAB, U+2F962->U+7BC6, U+2F963->U+7BC9, U+2F964->U+4227, \\\n"
        "       U+2F965->U+25C80, U+2F966->U+7CD2, U+2F967->U+42A0, U+2F968->U+7CE8, \\\n"
        "       U+2F969->U+7CE3, U+2F96A->U+7D00, U+2F96B->U+25F86, U+2F96C->U+7D63, \\\n"
        "       U+2F96D->U+4301, U+2F96E->U+7DC7, U+2F96F->U+7E02, U+2F970->U+7E45, \\\n"
        "       U+2F971->U+4334, U+2F972->U+26228, U+2F973->U+26247, U+2F974->U+4359, \\\n"
        "       U+2F975->U+262D9, U+2F976->U+7F7A, U+2F977->U+2633E, U+2F978->U+7F95, \\\n"
        "       U+2F979->U+7FFA, U+2F97A->U+8005, U+2F97B->U+264DA, U+2F97C->U+26523, \\\n"
        "       U+2F97D->U+8060, U+2F97E->U+265A8, U+2F97F->U+8070, U+2F980->U+2335F, \\\n"
        "       U+2F981->U+43D5, U+2F982->U+80B2, U+2F983->U+8103, U+2F984->U+440B, \\\n"
        "       U+2F985->U+813E, U+2F986->U+5AB5, U+2F987->U+267A7, U+2F988->U+267B5, \\\n"
        "       U+2F989->U+23393, U+2F98A->U+2339C, U+2F98B->U+8201, U+2F98C->U+8204, \\\n"
        "       U+2F98D->U+8F9E, U+2F98E->U+446B, U+2F98F->U+8291, U+2F990->U+828B, \\\n"
        "       U+2F991->U+829D, U+2F992->U+52B3, U+2F993->U+82B1, U+2F994->U+82B3, \\\n"
        "       U+2F995->U+82BD, U+2F996->U+82E6, U+2F997->U+26B3C, U+2F998->U+82E5, \\\n"
        "       U+2F999->U+831D, U+2F99A->U+8363, U+2F99B->U+83AD, U+2F99C->U+8323, \\\n"
        "       U+2F99D->U+83BD, U+2F99E->U+83E7, U+2F99F->U+8457, U+2F9A0->U+8353, \\\n"
        "       U+2F9A1->U+83CA, U+2F9A2->U+83CC, U+2F9A3->U+83DC, U+2F9A4->U+26C36, \\\n"
        "       U+2F9A5->U+26D6B, U+2F9A6->U+26CD5, U+2F9A7->U+452B, U+2F9A8->U+84F1, \\\n"
        "       U+2F9A9->U+84F3, U+2F9AA->U+8516, U+2F9AB->U+273CA, U+2F9AC->U+8564, \\\n"
        "       U+2F9AD->U+26F2C, U+2F9AE->U+455D, U+2F9AF->U+4561, U+2F9B0->U+26FB1, \\\n"
        "       U+2F9B1->U+270D2, U+2F9B2->U+456B, U+2F9B3->U+8650, U+2F9B4->U+865C, \\\n"
        "       U+2F9B5->U+8667, U+2F9B6->U+8669, U+2F9B7->U+86A9, U+2F9B8->U+8688, \\\n"
        "       U+2F9B9->U+870E, U+2F9BA->U+86E2, U+2F9BB->U+8779, U+2F9BC->U+8728, \\\n"
        "       U+2F9BD->U+876B, U+2F9BE->U+8786, U+2F9BF->U+45D7, U+2F9C0->U+87E1, \\\n"
        "       U+2F9C1->U+8801, U+2F9C2->U+45F9, U+2F9C3->U+8860, U+2F9C4->U+8863, \\\n"
        "       U+2F9C5->U+27667, U+2F9C6->U+88D7, U+2F9C7->U+88DE, U+2F9C8->U+4635, \\\n"
        "       U+2F9C9->U+88FA, U+2F9CA->U+34BB, U+2F9CB->U+278AE, U+2F9CC->U+27966, \\\n"
        "       U+2F9CD->U+46BE, U+2F9CE->U+46C7, U+2F9CF->U+8AA0, U+2F9D0->U+8AED, \\\n"
        "       U+2F9D1->U+8B8A, U+2F9D2->U+8C55, U+2F9D3->U+27CA8, U+2F9D4->U+8CAB, \\\n"
        "       U+2F9D5->U+8CC1, U+2F9D6->U+8D1B, U+2F9D7->U+8D77, U+2F9D8->U+27F2F, \\\n"
        "       U+2F9D9->U+20804, U+2F9DA->U+8DCB, U+2F9DB->U+8DBC, U+2F9DC->U+8DF0, \\\n"
        "       U+2F9DD->U+208DE, U+2F9DE->U+8ED4, U+2F9DF->U+8F38, U+2F9E0->U+285D2, \\\n"
        "       U+2F9E1->U+285ED, U+2F9E2->U+9094, U+2F9E3->U+90F1, U+2F9E4->U+9111, \\\n"
        "       U+2F9E5->U+2872E, U+2F9E6->U+911B, U+2F9E7->U+9238, U+2F9E8->U+92D7, \\\n"
        "       U+2F9E9->U+92D8, U+2F9EA->U+927C, U+2F9EB->U+93F9, U+2F9EC->U+9415, \\\n"
        "       U+2F9ED->U+28BFA, U+2F9EE->U+958B, U+2F9EF->U+4995, U+2F9F0->U+95B7, \\\n"
        "       U+2F9F1->U+28D77, U+2F9F2->U+49E6, U+2F9F3->U+96C3, U+2F9F4->U+5DB2, \\\n"
        "       U+2F9F5->U+9723, U+2F9F6->U+29145, U+2F9F7->U+2921A, U+2F9F8->U+4A6E, \\\n"
        "       U+2F9F9->U+4A76, U+2F9FA->U+97E0, U+2F9FB->U+2940A, U+2F9FC->U+4AB2, \\\n"
        "       U+2F9FD->U+29496, U+2F9FE->U+980B, U+2F9FF->U+980B, U+2FA00->U+9829, \\\n"
        "       U+2FA01->U+295B6, U+2FA02->U+98E2, U+2FA03->U+4B33, U+2FA04->U+9929, \\\n"
        "       U+2FA05->U+99A7, U+2FA06->U+99C2, U+2FA07->U+99FE, U+2FA08->U+4BCE, \\\n"
        "       U+2FA09->U+29B30, U+2FA0A->U+9B12, U+2FA0B->U+9C40, U+2FA0C->U+9CFD, \\\n"
        "       U+2FA0D->U+4CCE, U+2FA0E->U+4CED, U+2FA0F->U+9D67, U+2FA10->U+2A0CE, \\\n"
        "       U+2FA11->U+4CF8, U+2FA12->U+2A105, U+2FA13->U+2A20E, U+2FA14->U+2A291, \\\n"
        "       U+2FA15->U+9EBB, U+2FA16->U+4D56, U+2FA17->U+9EF9, U+2FA18->U+9EFE, \\\n"
        "       U+2FA19->U+9F05, U+2FA1A->U+9F0F, U+2FA1B->U+9F16, U+2FA1C->U+9F3B, \\\n"
        "       U+2FA1D->U+2A600, U+2F00->U+4E00, U+2F01->U+4E28, U+2F02->U+4E36, \\\n"
        "       U+2F03->U+4E3F, U+2F04->U+4E59, U+2F05->U+4E85, U+2F06->U+4E8C, \\\n"
        "       U+2F07->U+4EA0, U+2F08->U+4EBA, U+2F09->U+513F, U+2F0A->U+5165, \\\n"
        "       U+2F0B->U+516B, U+2F0C->U+5182, U+2F0D->U+5196, U+2F0E->U+51AB, \\\n"
        "       U+2F0F->U+51E0, U+2F10->U+51F5, U+2F11->U+5200, U+2F12->U+529B, \\\n"
        "       U+2F13->U+52F9, U+2F14->U+5315, U+2F15->U+531A, U+2F16->U+5338, \\\n"
        "       U+2F17->U+5341, U+2F18->U+535C, U+2F19->U+5369, U+2F1A->U+5382, \\\n"
        "       U+2F1B->U+53B6, U+2F1C->U+53C8, U+2F1D->U+53E3, U+2F1E->U+56D7, \\\n"
        "       U+2F1F->U+571F, U+2F20->U+58EB, U+2F21->U+5902, U+2F22->U+590A, \\\n"
        "       U+2F23->U+5915, U+2F24->U+5927, U+2F25->U+5973, U+2F26->U+5B50, \\\n"
        "       U+2F27->U+5B80, U+2F28->U+5BF8, U+2F29->U+5C0F, U+2F2A->U+5C22, \\\n"
        "       U+2F2B->U+5C38, U+2F2C->U+5C6E, U+2F2D->U+5C71, U+2F2E->U+5DDB, \\\n"
        "       U+2F2F->U+5DE5, U+2F30->U+5DF1, U+2F31->U+5DFE, U+2F32->U+5E72, \\\n"
        "       U+2F33->U+5E7A, U+2F34->U+5E7F, U+2F35->U+5EF4, U+2F36->U+5EFE, \\\n"
        "       U+2F37->U+5F0B, U+2F38->U+5F13, U+2F39->U+5F50, U+2F3A->U+5F61, \\\n"
        "       U+2F3B->U+5F73, U+2F3C->U+5FC3, U+2F3D->U+6208, U+2F3E->U+6236, \\\n"
        "       U+2F3F->U+624B, U+2F40->U+652F, U+2F41->U+6534, U+2F42->U+6587, \\\n"
        "       U+2F43->U+6597, U+2F44->U+65A4, U+2F45->U+65B9, U+2F46->U+65E0, \\\n"
        "       U+2F47->U+65E5, U+2F48->U+66F0, U+2F49->U+6708, U+2F4A->U+6728, \\\n"
        "       U+2F4B->U+6B20, U+2F4C->U+6B62, U+2F4D->U+6B79, U+2F4E->U+6BB3, \\\n"
        "       U+2F4F->U+6BCB, U+2F50->U+6BD4, U+2F51->U+6BDB, U+2F52->U+6C0F, \\\n"
        "       U+2F53->U+6C14, U+2F54->U+6C34, U+2F55->U+706B, U+2F56->U+722A, \\\n"
        "       U+2F57->U+7236, U+2F58->U+723B, U+2F59->U+723F, U+2F5A->U+7247, \\\n"
        "       U+2F5B->U+7259, U+2F5C->U+725B, U+2F5D->U+72AC, U+2F5E->U+7384, \\\n"
        "       U+2F5F->U+7389, U+2F60->U+74DC, U+2F61->U+74E6, U+2F62->U+7518, \\\n"
        "       U+2F63->U+751F, U+2F64->U+7528, U+2F65->U+7530, U+2F66->U+758B, \\\n"
        "       U+2F67->U+7592, U+2F68->U+7676, U+2F69->U+767D, U+2F6A->U+76AE, \\\n"
        "       U+2F6B->U+76BF, U+2F6C->U+76EE, U+2F6D->U+77DB, U+2F6E->U+77E2, \\\n"
        "       U+2F6F->U+77F3, U+2F70->U+793A, U+2F71->U+79B8, U+2F72->U+79BE, \\\n"
        "       U+2F73->U+7A74, U+2F74->U+7ACB, U+2F75->U+7AF9, U+2F76->U+7C73, \\\n"
        "       U+2F77->U+7CF8, U+2F78->U+7F36, U+2F79->U+7F51, U+2F7A->U+7F8A, \\\n"
        "       U+2F7B->U+7FBD, U+2F7C->U+8001, U+2F7D->U+800C, U+2F7E->U+8012, \\\n"
        "       U+2F7F->U+8033, U+2F80->U+807F, U+2F81->U+8089, U+2F82->U+81E3, \\\n"
        "       U+2F83->U+81EA, U+2F84->U+81F3, U+2F85->U+81FC, U+2F86->U+820C, \\\n"
        "       U+2F87->U+821B, U+2F88->U+821F, U+2F89->U+826E, U+2F8A->U+8272, \\\n"
        "       U+2F8B->U+8278, U+2F8C->U+864D, U+2F8D->U+866B, U+2F8E->U+8840, \\\n"
        "       U+2F8F->U+884C, U+2F90->U+8863, U+2F91->U+897E, U+2F92->U+898B, \\\n"
        "       U+2F93->U+89D2, U+2F94->U+8A00, U+2F95->U+8C37, U+2F96->U+8C46, \\\n"
        "       U+2F97->U+8C55, U+2F98->U+8C78, U+2F99->U+8C9D, U+2F9A->U+8D64, \\\n"
        "       U+2F9B->U+8D70, U+2F9C->U+8DB3, U+2F9D->U+8EAB, U+2F9E->U+8ECA, \\\n"
        "       U+2F9F->U+8F9B, U+2FA0->U+8FB0, U+2FA1->U+8FB5, U+2FA2->U+9091, \\\n"
        "       U+2FA3->U+9149, U+2FA4->U+91C6, U+2FA5->U+91CC, U+2FA6->U+91D1, \\\n"
        "       U+2FA7->U+9577, U+2FA8->U+9580, U+2FA9->U+961C, U+2FAA->U+96B6, \\\n"
        "       U+2FAB->U+96B9, U+2FAC->U+96E8, U+2FAD->U+9751, U+2FAE->U+975E, \\\n"
        "       U+2FAF->U+9762, U+2FB0->U+9769, U+2FB1->U+97CB, U+2FB2->U+97ED, \\\n"
        "       U+2FB3->U+97F3, U+2FB4->U+9801, U+2FB5->U+98A8, U+2FB6->U+98DB, \\\n"
        "       U+2FB7->U+98DF, U+2FB8->U+9996, U+2FB9->U+9999, U+2FBA->U+99AC, \\\n"
        "       U+2FBB->U+9AA8, U+2FBC->U+9AD8, U+2FBD->U+9ADF, U+2FBE->U+9B25, \\\n"
        "       U+2FBF->U+9B2F, U+2FC0->U+9B32, U+2FC1->U+9B3C, U+2FC2->U+9B5A, \\\n"
        "       U+2FC3->U+9CE5, U+2FC4->U+9E75, U+2FC5->U+9E7F, U+2FC6->U+9EA5, \\\n"
        "       U+2FC7->U+9EBB, U+2FC8->U+9EC3, U+2FC9->U+9ECD, U+2FCA->U+9ED1, \\\n"
        "       U+2FCB->U+9EF9, U+2FCC->U+9EFD, U+2FCD->U+9F0E, U+2FCE->U+9F13, \\\n"
        "       U+2FCF->U+9F20, U+2FD0->U+9F3B, U+2FD1->U+9F4A, U+2FD2->U+9F52, \\\n"
        "       U+2FD3->U+9F8D, U+2FD4->U+9F9C, U+2FD5->U+9FA0, U+3042->U+3041, \\\n"
        "       U+3044->U+3043, U+3046->U+3045, U+3048->U+3047, U+304A->U+3049, \\\n"
        "       U+304C->U+304B, U+304E->U+304D, U+3050->U+304F, U+3052->U+3051, \\\n"
        "       U+3054->U+3053, U+3056->U+3055, U+3058->U+3057, U+305A->U+3059, \\\n"
        "       U+305C->U+305B, U+305E->U+305D, U+3060->U+305F, U+3062->U+3061, \\\n"
        "       U+3064->U+3063, U+3065->U+3063, U+3067->U+3066, U+3069->U+3068, \\\n"
        "       U+3070->U+306F, U+3071->U+306F, U+3073->U+3072, U+3074->U+3072, \\\n"
        "       U+3076->U+3075, U+3077->U+3075, U+3079->U+3078, U+307A->U+3078, \\\n"
        "       U+307C->U+307B, U+307D->U+307B, U+3084->U+3083, U+3086->U+3085, \\\n"
        "       U+3088->U+3087, U+308F->U+308E, U+3094->U+3046, U+3095->U+304B, \\\n"
        "       U+3096->U+3051, U+30A2->U+30A1, U+30A4->U+30A3, U+30A6->U+30A5, \\\n"
        "       U+30A8->U+30A7, U+30AA->U+30A9, U+30AC->U+30AB, U+30AE->U+30AD, \\\n"
        "       U+30B0->U+30AF, U+30B2->U+30B1, U+30B4->U+30B3, U+30B6->U+30B5, \\\n"
        "       U+30B8->U+30B7, U+30BA->U+30B9, U+30BC->U+30BB, U+30BE->U+30BD, \\\n"
        "       U+30C0->U+30BF, U+30C2->U+30C1, U+30C5->U+30C4, U+30C7->U+30C6, \\\n"
        "       U+30C9->U+30C8, U+30D0->U+30CF, U+30D1->U+30CF, U+30D3->U+30D2, \\\n"
        "       U+30D4->U+30D2, U+30D6->U+30D5, U+30D7->U+30D5, U+30D9->U+30D8, \\\n"
        "       U+30DA->U+30D8, U+30DC->U+30DB, U+30DD->U+30DB, U+30E4->U+30E3, \\\n"
        "       U+30E6->U+30E5, U+30E8->U+30E7, U+30EF->U+30EE, U+30F4->U+30A6, \\\n"
        "       U+30AB->U+30F5, U+30B1->U+30F6, U+30F7->U+30EF, U+30F8->U+30F0, \\\n"
        "       U+30F9->U+30F1, U+30FA->U+30F2, U+30AF->U+31F0, U+30B7->U+31F1, \\\n"
        "       U+30B9->U+31F2, U+30C8->U+31F3, U+30CC->U+31F4, U+30CF->U+31F5, \\\n"
        "       U+30D2->U+31F6, U+30D5->U+31F7, U+30D8->U+31F8, U+30DB->U+31F9, \\\n"
        "       U+30E0->U+31FA, U+30E9->U+31FB, U+30EA->U+31FC, U+30EB->U+31FD, \\\n"
        "       U+30EC->U+31FE, U+30ED->U+31FF, U+FF66->U+30F2, U+FF67->U+30A1, \\\n"
        "       U+FF68->U+30A3, U+FF69->U+30A5, U+FF6A->U+30A7, U+FF6B->U+30A9, \\\n"
        "       U+FF6C->U+30E3, U+FF6D->U+30E5, U+FF6E->U+30E7, U+FF6F->U+30C3, \\\n"
        "       U+FF71->U+30A1, U+FF72->U+30A3, U+FF73->U+30A5, U+FF74->U+30A7, \\\n"
        "       U+FF75->U+30A9, U+FF76->U+30AB, U+FF77->U+30AD, U+FF78->U+30AF, \\\n"
        "       U+FF79->U+30B1, U+FF7A->U+30B3, U+FF7B->U+30B5, U+FF7C->U+30B7, \\\n"
        "       U+FF7D->U+30B9, U+FF7E->U+30BB, U+FF7F->U+30BD, U+FF80->U+30BF, \\\n"
        "       U+FF81->U+30C1, U+FF82->U+30C3, U+FF83->U+30C6, U+FF84->U+30C8, \\\n"
        "       U+FF85->U+30CA, U+FF86->U+30CB, U+FF87->U+30CC, U+FF88->U+30CD, \\\n"
        "       U+FF89->U+30CE, U+FF8A->U+30CF, U+FF8B->U+30D2, U+FF8C->U+30D5, \\\n"
        "       U+FF8D->U+30D8, U+FF8E->U+30DB, U+FF8F->U+30DE, U+FF90->U+30DF, \\\n"
        "       U+FF91->U+30E0, U+FF92->U+30E1, U+FF93->U+30E2, U+FF94->U+30E3, \\\n"
        "       U+FF95->U+30E5, U+FF96->U+30E7, U+FF97->U+30E9, U+FF98->U+30EA, \\\n"
        "       U+FF99->U+30EB, U+FF9A->U+30EC, U+FF9B->U+30ED, U+FF9C->U+30EF, \\\n"
        "       U+FF9D->U+30F3, U+FFA0->U+3164, U+FFA1->U+3131, U+FFA2->U+3132, \\\n"
        "       U+FFA3->U+3133, U+FFA4->U+3134, U+FFA5->U+3135, U+FFA6->U+3136, \\\n"
        "       U+FFA7->U+3137, U+FFA8->U+3138, U+FFA9->U+3139, U+FFAA->U+313A, \\\n"
        "       U+FFAB->U+313B, U+FFAC->U+313C, U+FFAD->U+313D, U+FFAE->U+313E, \\\n"
        "       U+FFAF->U+313F, U+FFB0->U+3140, U+FFB1->U+3141, U+FFB2->U+3142, \\\n"
        "       U+FFB3->U+3143, U+FFB4->U+3144, U+FFB5->U+3145, U+FFB6->U+3146, \\\n"
        "       U+FFB7->U+3147, U+FFB8->U+3148, U+FFB9->U+3149, U+FFBA->U+314A, \\\n"
        "       U+FFBB->U+314B, U+FFBC->U+314C, U+FFBD->U+314D, U+FFBE->U+314E, \\\n"
        "       U+FFC2->U+314F, U+FFC3->U+3150, U+FFC4->U+3151, U+FFC5->U+3152, \\\n"
        "       U+FFC6->U+3153, U+FFC7->U+3154, U+FFCA->U+3155, U+FFCB->U+3156, \\\n"
        "       U+FFCC->U+3157, U+FFCD->U+3158, U+FFCE->U+3159, U+FFCF->U+315A, \\\n"
        "       U+FFD2->U+315B, U+FFD3->U+315C, U+FFD4->U+315D, U+FFD5->U+315E, \\\n"
        "       U+FFD6->U+315F, U+FFD7->U+3160, U+FFDA->U+3161, U+FFDB->U+3162, \\\n"
        "       U+FFDC->U+3163, U+3131->U+1100, U+3132->U+1101, U+3133->U+11AA, \\\n"
        "       U+3134->U+1102, U+3135->U+11AC, U+3136->U+11AD, U+3137->U+1103, \\\n"
        "       U+3138->U+1104, U+3139->U+1105, U+313A->U+11B0, U+313B->U+11B1, \\\n"
        "       U+313C->U+11B2, U+313D->U+11B3, U+313E->U+11B4, U+313F->U+11B5, \\\n"
        "       U+3140->U+111A, U+3141->U+1106, U+3142->U+1107, U+3143->U+1108, \\\n"
        "       U+3144->U+1121, U+3145->U+1109, U+3146->U+110A, U+3147->U+110B, \\\n"
        "       U+3148->U+110C, U+3149->U+110D, U+314A->U+110E, U+314B->U+110F, \\\n"
        "       U+314C->U+1110, U+314D->U+1111, U+314E->U+1112, U+314F->U+1161, \\\n"
        "       U+3150->U+1162, U+3151->U+1163, U+3152->U+1164, U+3153->U+1165, \\\n"
        "       U+3154->U+1166, U+3155->U+1167, U+3156->U+1168, U+3157->U+1169, \\\n"
        "       U+3158->U+116A, U+3159->U+116B, U+315A->U+116C, U+315B->U+116D, \\\n"
        "       U+315C->U+116E, U+315D->U+116F, U+315E->U+1170, U+315F->U+1171, \\\n"
        "       U+3160->U+1172, U+3161->U+1173, U+3162->U+1174, U+3163->U+1175, \\\n"
        "       U+3165->U+1114, U+3166->U+1115, U+3167->U+11C7, U+3168->U+11C8, \\\n"
        "       U+3169->U+11CC, U+316A->U+11CE, U+316B->U+11D3, U+316C->U+11D7, \\\n"
        "       U+316D->U+11D9, U+316E->U+111C, U+316F->U+11DD, U+3170->U+11DF, \\\n"
        "       U+3171->U+111D, U+3172->U+111E, U+3173->U+1120, U+3174->U+1122, \\\n"
        "       U+3175->U+1123, U+3176->U+1127, U+3177->U+1129, U+3178->U+112B, \\\n"
        "       U+3179->U+112C, U+317A->U+112D, U+317B->U+112E, U+317C->U+112F, \\\n"
        "       U+317D->U+1132, U+317E->U+1136, U+317F->U+1140, U+3180->U+1147, \\\n"
        "       U+3181->U+114C, U+3182->U+11F1, U+3183->U+11F2, U+3184->U+1157, \\\n"
        "       U+3185->U+1158, U+3186->U+1159, U+3187->U+1184, U+3188->U+1185, \\\n"
        "       U+3189->U+1188, U+318A->U+1191, U+318B->U+1192, U+318C->U+1194, \\\n"
        "       U+318D->U+119E, U+318E->U+11A1, U+A490->U+A408, U+A491->U+A1B9, \\\n"
        "       U+4E00..U+9FBB, U+3400..U+4DB5, U+20000..U+2A6D6, U+FA0E, U+FA0F, \\\n"
        "       U+FA11, U+FA13, U+FA14, U+FA1F, U+FA21, U+FA23, U+FA24, U+FA27, U+FA28, \\\n"
        "       U+FA29, U+3105..U+312C, U+31A0..U+31B7, U+3041, U+3043, U+3045, U+3047, \\\n"
        "       U+3049, U+304B, U+304D, U+304F, U+3051, U+3053, U+3055, U+3057, U+3059, \\\n"
        "       U+305B, U+305D, U+305F, U+3061, U+3063, U+3066, U+3068, U+306A..U+306F, \\\n"
        "       U+3072, U+3075, U+3078, U+307B, U+307E..U+3083, U+3085, U+3087, \\\n"
        "       U+3089..U+308E, U+3090..U+3093, U+30A1, U+30A3, U+30A5, U+30A7, U+30A9, \\\n"
        "       U+30AD, U+30AF, U+30B3, U+30B5, U+30BB, U+30BD, U+30BF, U+30C1, U+30C3, \\\n"
        "       U+30C4, U+30C6, U+30CA, U+30CB, U+30CD, U+30CE, U+30DE, U+30DF, U+30E1, \\\n"
        "       U+30E2, U+30E3, U+30E5, U+30E7, U+30EE, U+30F0..U+30F3, U+30F5, U+30F6, \\\n"
        "       U+31F0, U+31F1, U+31F2, U+31F3, U+31F4, U+31F5, U+31F6, U+31F7, U+31F8, \\\n"
        "       U+31F9, U+31FA, U+31FB, U+31FC, U+31FD, U+31FE, U+31FF, U+AC00..U+D7A3, \\\n"
        "       U+1100..U+1159, U+1161..U+11A2, U+11A8..U+11F9, U+A000..U+A48C, \\\n"
        "       U+A492..U+A4C6\n"
        "       ngram_chars = \\\n"
        /* Support for Chinese/Japanese/Korean, from
         * http://sphinxsearch.com/wiki/doku.php?id=charset_tables#cjk */
        "       U+4E00..U+9FBB, U+3400..U+4DB5, U+20000..U+2A6D6, U+FA0E, U+FA0F, \\\n"
        "       U+FA11, U+FA13, U+FA14, U+FA1F, U+FA21, U+FA23, U+FA24, U+FA27, U+FA28, \\\n"
        "       U+FA29, U+3105..U+312C, U+31A0..U+31B7, U+3041, U+3043, U+3045, U+3047, \\\n"
        "       U+3049, U+304B, U+304D, U+304F, U+3051, U+3053, U+3055, U+3057, U+3059, \\\n"
        "       U+305B, U+305D, U+305F, U+3061, U+3063, U+3066, U+3068, U+306A..U+306F, \\\n"
        "       U+3072, U+3075, U+3078, U+307B, U+307E..U+3083, U+3085, U+3087, \\\n"
        "       U+3089..U+308E, U+3090..U+3093, U+30A1, U+30A3, U+30A5, U+30A7, U+30A9, \\\n"
        "       U+30AD, U+30AF, U+30B3, U+30B5, U+30BB, U+30BD, U+30BF, U+30C1, U+30C3, \\\n"
        "       U+30C4, U+30C6, U+30CA, U+30CB, U+30CD, U+30CE, U+30DE, U+30DF, U+30E1, \\\n"
        "       U+30E2, U+30E3, U+30E5, U+30E7, U+30EE, U+30F0..U+30F3, U+30F5, U+30F6, \\\n"
        "       U+31F0, U+31F1, U+31F2, U+31F3, U+31F4, U+31F5, U+31F6, U+31F7, U+31F8, \\\n"
        "       U+31F9, U+31FA, U+31FB, U+31FC, U+31FD, U+31FE, U+31FF, U+AC00..U+D7A3, \\\n"
        "       U+1100..U+1159, U+1161..U+11A2, U+11A8..U+11F9, U+A000..U+A48C, \\\n"
        "       U+A492..U+A4C6\n"
        "       ngram_len = 1\n"
        "\n"
        "    rt_attr_string = cyrusid\n"
        "    rt_field = header_from\n"
        "    rt_field = header_to\n"
        "    rt_field = header_cc\n"
        "    rt_field = header_bcc\n"
        "    rt_field = header_subject\n"
        "    rt_field = header_listid\n"
        "    rt_field = header_type\n"
        "    rt_field = headers\n"
        "    rt_field = body\n"
        "}\n"
        "\n";
    static const char global_config[] =
        "searchd\n"
        "{\n"
        "    listen = $socket:mysql41\n"
        "    log = syslog\n"
        "    pid_file = $pidfile\n"
        "    binlog_path = $rootdir/binlog\n"
        "    compat_sphinxql_magics = 0\n"
        "    workers = threads\n"
        "    max_matches = " SPHINX_MAX_MATCHES "\n"
        "}\n"
        /* This index exists only to allow the searchd to start before
         * any real user indexes have been added */
        "index dummy\n"
        "{\n"
        "    type = rt\n"
        "    path = $rootdir/dummy\n"
        "    rt_field = dummy\n"
        "}\n";
    const char *rootdir = NULL;
    char *basedir = NULL;
    char *binlogdir = NULL;
    char *indexname = NULL;
    const char *config_file = NULL;
    char *new_config_file = NULL;
    int fd = -1;
    struct buf bits = BUF_INITIALIZER;
    struct buf user = BUF_INITIALIZER;
    struct buf global = BUF_INITIALIZER;
    int changed = 0;
    int r;

    config_file = sphinx_config_file();
    if (verbose) {
        if (mailbox)
            syslog(LOG_INFO, "Sphinx setting up config file %s for mailbox %s",
                    config_file, mailbox->name);
        else
            syslog(LOG_INFO, "Sphinx setting up config file %s",
                    config_file);
    }

    /* Read the old file */
    fd = open(config_file, O_RDONLY, 0);
    if (fd >= 0) {
        struct stat sb;

        r = fstat(fd, &sb);
        if (r < 0) {
            syslog(LOG_ERR, "IOERROR: unable to fstat %s: %m",
                   config_file);
            r = IMAP_IOERROR;
            goto out;
        }
        buf_ensure(&bits, sb.st_size+1);
        bits.len = sb.st_size;
        r = retry_read(fd, bits.s, bits.len);
        buf_cstring(&bits);
        if (r < 0) {
            syslog(LOG_ERR, "IOERROR: unable to read %s: %m",
                   config_file);
            r = IMAP_IOERROR;
            goto out;
        }
        close(fd);
        fd = -1;

        if (verbose)
            syslog(LOG_INFO, "Sphinx read %ld bytes of config file %s",
                    bits.len, config_file);
    }
    else if (fd != ENOENT) {
        /* it's ok to be missing the file - we build it from scratch */
        syslog(LOG_ERR, "IOERROR: unable to open %s for reading: %m",
               config_file);
        r = IMAP_IOERROR;
        goto out;
    }

    /* adjust the file contents */
    if (buf_findline(&bits, global_config) < 0) {

        /* See if the root directory already exists */
        rootdir = sphinx_rootdir(NULL);
        binlogdir = strconcat(rootdir, "/binlog", (char *)NULL);
        r = check_directory(binlogdir, verbose, create);
        if (r) goto out;

        buf_init_ro_cstr(&global, global_config);
        buf_replace_all(&global, "$socket", config_getstring(IMAPOPT_SPHINX_SOCKET));
        buf_replace_all(&global, "$pidfile", config_getstring(IMAPOPT_SPHINX_PIDFILE));
        buf_replace_all(&global, "$rootdir", rootdir);
        buf_append(&bits, &global);
        changed++;
        if (verbose)
            syslog(LOG_INFO, "Sphinx adding section \"searchd\"");
    }

    if (mailbox) {

        r = sphinx_basedir(mailbox, &basedir, &indexname);
        if (r) {
            syslog(LOG_ERR, "IOERROR: unable to name sphinx basedir for %s: %s",
                   mailbox->name, error_message(r));
            r = IMAP_IOERROR;
            goto out;
        }

        /* see if the base directory already exists */
        r = check_directory(basedir, verbose, create);
        if (r) goto out;

        buf_init_ro_cstr(&user, user_config);
        buf_replace_all(&user, "$indexname", indexname);
        if (buf_findline(&bits, user.s) < 0) {
            buf_replace_all(&user, "$basedir", basedir);
            buf_append(&bits, &user);
            changed++;
            if (verbose)
                syslog(LOG_INFO, "Sphinx adding section \"index %s\"", indexname);
        }
    }

    if (changed) {

        /* Write the new config file back */
        /* TODO: locking so multiple imapds don't all try to do this
         * at the same time */

        new_config_file = strconcat(config_file, ".new", (char *)NULL);
        if (verbose)
            syslog(LOG_INFO, "Sphinx writing new config file %s", new_config_file);

        fd = open(new_config_file, O_WRONLY|O_CREAT|O_TRUNC, 0600);
        if (fd < 0) {
            /* build the directory and retry */
            cyrus_mkdir(new_config_file, 0700);
            fd = open(new_config_file, O_WRONLY|O_CREAT|O_TRUNC, 0600);
        }
        if (fd < 0) {
            syslog(LOG_ERR, "IOERROR: unable to open %s for writing: %m",
                   new_config_file);
            r = IMAP_IOERROR;
            goto out;
        }

        r = retry_write(fd, bits.s, bits.len);
        if (r < 0) {
            syslog(LOG_ERR, "IOERROR: unable to write %s: %m",
                   new_config_file);
            r = IMAP_IOERROR;
            goto out;
        }

        close(fd);
        fd = -1;

        if (verbose)
            syslog(LOG_INFO, "Sphinx renaming config file %s into place", config_file);

        r = rename(new_config_file, config_file);
        if (r < 0) {
            syslog(LOG_ERR, "IOERROR: unable to rename %s to %s: %m",
                   new_config_file, config_file);
            r = IMAP_IOERROR;
            goto out;
        }
    }

    r = sphinx_signal(changed ? SIGHUP : 0, verbose);
    if (r == IMAP_NOTFOUND) {
        /* searchd not running, start it */
        if (verbose)
            syslog(LOG_INFO, "Sphinx starting searchd");

        r = run_command(SEARCHD, "--config", config_file,
                        "--syslog-prefix", sphinx_syslog_prefix(), (char *)NULL);
        if (r)
            syslog(LOG_ERR, "Failed to start searchd: %s", error_message(r));
    }

out:
    if (fd >= 0) close(fd);
    if (new_config_file) unlink(new_config_file);
    free(new_config_file);
    free(basedir);
    free(indexname);
    free(binlogdir);
    buf_free(&bits);
    buf_free(&user);
    buf_free(&global);
    return r;
}


static int open_latest(struct mailbox *mailbox, struct latestdb *ldb)
{
    char *basedir = NULL;
    char *path = NULL;
    int r;

    r = sphinx_basedir(mailbox, &basedir, NULL);
    if (r) return r;
    path = strconcat(basedir, LATESTDB_FNAME, NULL);
    free(basedir);

    if (!strcmpsafe(path, ldb->path)) {
        free(path);
        return 0;
    }

    /* need to open a new DB */

    close_latest(ldb);

    r = cyrusdb_open(config_getstring(IMAPOPT_SEARCH_INDEXED_DB),
                     path, CYRUSDB_CREATE, &ldb->db);
    if (r) {
        syslog(LOG_ERR, "DBERROR: opening %s: %s",
               path, cyrusdb_strerror(r));
        r = IMAP_IOERROR;
    }

    if (r) {
        free(path);
    }
    else {
        ldb->path = path;
    }
    return r;
}

static void close_latest(struct latestdb *ldb)
{
    free(ldb->path);
    ldb->path = NULL;

    if (ldb->db) {
        cyrusdb_close(ldb->db);
        ldb->db = NULL;
    }
}

/*
 * Read the most recently indexed UID for the current mailboxfrom the
 * 'latest' DB in the Sphinx directory.
 * Returns 0 on success or an IMAP error code.
 */
static int read_latest(struct latestdb *ldb,
                       struct mailbox *mailbox,
                       uint32_t *latestp,
                       int verbose)
{
    struct buf key = BUF_INITIALIZER;
    struct buf buf = BUF_INITIALIZER;
    const char *data = NULL;
    size_t datalen = 0;
    int r = 0;
    unsigned int version = 0;
    unsigned int uid = 0;

    *latestp = 0;
    if (verbose > 1)
        syslog(LOG_INFO, "read_latest db=%s mailbox=%s uidvalidity=%u",
               ldb->path, mailbox->name, mailbox->i.uidvalidity);

    buf_printf(&key, "%s.%u", mailbox->name, mailbox->i.uidvalidity);

    r = cyrusdb_fetch(ldb->db,
                      key.s, key.len,
                      &data, &datalen,
                      (struct txn **)NULL);
    if (r == CYRUSDB_NOTFOUND) {
        if (verbose > 1) syslog(LOG_INFO, "read_latest defaults to 0");
        r = 0;
        goto out;
    }
    if (r) goto out;
    buf_init_ro(&buf, data, datalen);
    buf_cstring(&buf);

    r = sscanf(buf.s, "%u %u", &version, &uid);
    if (r != 2 || version != LATESTDB_VERSION) {
        r = IMAP_MAILBOX_BADFORMAT;
        goto out;
    }

    if (verbose > 1) syslog(LOG_INFO, "read_latest uid=%u", uid);
    *latestp = uid;
    r = 0;

out:
    buf_free(&key);
    buf_free(&buf);
    return r;
}

static int write_latest(struct latestdb *ldb,
                        struct mailbox *mailbox,
                        uint32_t uid,
                        int verbose)
{
    struct buf key = BUF_INITIALIZER;
    struct buf data = BUF_INITIALIZER;
    struct txn *txn = NULL;
    int r = 0;

    if (verbose)
        syslog(LOG_INFO, "write_latest db=%s mailbox=%s uidvalidity=%u uid=%u",
               ldb->path, mailbox->name, mailbox->i.uidvalidity, uid);

    buf_printf(&key, "%s.%u", mailbox->name, mailbox->i.uidvalidity);
    buf_printf(&data, "%u %u", LATESTDB_VERSION, uid);

    do {
        r = cyrusdb_store(ldb->db,
                          key.s, key.len,
                          data.s, data.len,
                          &txn);
    } while (r == CYRUSDB_AGAIN);
    if (!r)
        r = cyrusdb_commit(ldb->db, txn);
    else
        cyrusdb_abort(ldb->db, txn);

    buf_free(&data);
    buf_free(&key);
    return r;
}

/*
 * Read the last document ID from the "latest" DB.  This is a
 * temporary measure until we can get autoincrement behaviour
 * working for Sphinx.
 *
 * Returns: 0 on success or an IMAP error code.
 */
static int read_lastid(struct latestdb *ldb,
                       uint32_t *lastidp,
                       int verbose)
{
    struct buf key = BUF_INITIALIZER;
    struct buf buf = BUF_INITIALIZER;
    const char *data = NULL;
    size_t datalen = 0;
    int r = 0;
    unsigned int version = 0;
    unsigned int id = 0;

    *lastidp = 0;
    if (verbose > 1) syslog(LOG_INFO, "read_lastid db=%s", ldb->path);

    buf_init_ro_cstr(&key, LATESTDB_LASTID_KEY);

    r = cyrusdb_fetch(ldb->db,
                      key.s, key.len,
                      &data, &datalen,
                      (struct txn **)NULL);
    if (r == CYRUSDB_NOTFOUND) {
        if (verbose > 1) syslog(LOG_INFO, "read_lastid defaults to 0");
        r = 0;
        goto out;
    }
    if (r) goto out;
    buf_init_ro(&buf, data, datalen);
    buf_cstring(&buf);

    r = sscanf(buf.s, "%u %u", &version, &id);
    if (r != 2 || version != LATESTDB_VERSION) {
        r = IMAP_MAILBOX_BADFORMAT;
        goto out;
    }

    if (verbose > 1) syslog(LOG_INFO, "read_lastid id=%u", id);
    *lastidp = id;
    r = 0;

out:
    buf_free(&key);
    buf_free(&buf);
    return r;
}

static int write_lastid(struct latestdb *ldb,
                        uint32_t lastid,
                        int verbose)
{
    struct buf key = BUF_INITIALIZER;
    struct buf data = BUF_INITIALIZER;
    struct txn *txn = NULL;
    int r = 0;

    if (verbose > 1)
        syslog(LOG_INFO, "write_lastid db=%s lastid=%u", ldb->path, lastid);

    buf_init_ro_cstr(&key, LATESTDB_LASTID_KEY);
    buf_printf(&data, "%u %u", LATESTDB_VERSION, lastid);

    do {
        r = cyrusdb_store(ldb->db,
                          key.s, key.len,
                          data.s, data.len,
                          &txn);
    } while (r == CYRUSDB_AGAIN);
    if (!r)
        r = cyrusdb_commit(ldb->db, txn);
    else
        cyrusdb_abort(ldb->db, txn);


    buf_free(&data);
    buf_free(&key);
    return r;
}

static int flush(search_text_receiver_t *rx)
{
    sphinx_update_receiver_t *tr = (sphinx_update_receiver_t *)rx;
    int r = 0;

    if (!tr->uncommitted) return 0;

    /* We write the lastid out first, to avoid a future instance
     * allocating a duplicate Sphinx document id should we crash */
    r = write_lastid(&tr->latestdb, tr->lastid, tr->super.verbose);
    if (r) return r;

    if (tr->super.verbose > 1)
        syslog(LOG_NOTICE, "Sphinx committing %u updates",
                tr->uncommitted);

    r = mysql_commit(tr->conn.mysql);
    if (r) {
        syslog(LOG_ERR, "IOERROR: Sphinx COMMIT failed for "
                        "mailbox %s, %u messages ending at uid %u: %s",
                        tr->super.mailbox->name,
                        tr->uncommitted,
                        tr->super.uid,
                        mysql_error(tr->conn.mysql));
        r = IMAP_IOERROR;
        goto out;
    }

    /* We write out the latestid for the mailbox only after successfully
     * updating the index, to avoid a future instance not realising that
     * there are unindexed messages should we fail to index */
    r = write_latest(&tr->latestdb, tr->super.mailbox, tr->latest,
                     tr->super.verbose);
    if (r) goto out;

    tr->uncommitted = 0;
    buf_reset(&tr->super.tmp);

out:
    return r;
}

static int begin_message(search_text_receiver_t *rx, message_t *msg)
{
    sphinx_receiver_t *tr = (sphinx_receiver_t *)rx;
    int i;

    message_get_uid(msg, &tr->uid);
    for (i = 0 ; i < SEARCH_NUM_PARTS ; i++)
        buf_reset(&tr->parts[i]);
    tr->parts_total = 0;
    tr->truncate_warning = 0;
}

static void begin_part(search_text_receiver_t *rx, int part)
{
    sphinx_receiver_t *tr = (sphinx_receiver_t *)rx;

    tr->part = part;
}

static void append_text(search_text_receiver_t *rx,
                        const struct buf *text)
{
    sphinx_receiver_t *tr = (sphinx_receiver_t *)rx;

    if (tr->part) {
        unsigned len = text->len;
        if (tr->parts_total + len > MAX_PARTS_SIZE) {
            if (!tr->truncate_warning++)
                syslog(LOG_ERR, "Sphinx: truncating text from "
                                "message mailbox %s uid %u",
                                tr->mailbox->name, tr->uid);
            len = MAX_PARTS_SIZE - tr->parts_total;
        }
        if (len) {
            tr->parts_total += len;
            buf_appendmap(&tr->parts[tr->part], text->s, len);
        }
    }
}

static void end_part(search_text_receiver_t *rx,
                     int part __attribute__((unused)))
{
    sphinx_receiver_t *tr = (sphinx_receiver_t *)rx;

    if (tr->verbose > 1)
        syslog(LOG_NOTICE, "Sphinx: %ld bytes in part %s",
               tr->parts[tr->part].len, search_part_as_string(tr->part));

    tr->part = 0;
}

static void log_keywords(sphinx_update_receiver_t *tr)
{
    int i;
    int r;
    struct buf query = BUF_INITIALIZER;
    MYSQL_RES *res = NULL;
    MYSQL_ROW row = NULL;

    for (i = 0 ; i < SEARCH_NUM_PARTS ; i++) {
        if (!tr->super.parts[i].len) continue;
        buf_reset(&query);
        buf_appendcstr(&query, "CALL KEYWORDS(");
        append_escaped(&query, &tr->super.parts[i], '\'');
        buf_appendcstr(&query, ", ");
        append_escaped_cstr(&query, tr->super.indexname, '\'');
        buf_appendcstr(&query, ",0)");
        r = doquery(&tr->conn, tr->super.verbose, &query);

        res = mysql_use_result(tr->conn.mysql);
        if (!res) continue;

        while ((row = mysql_fetch_row(res)))
            syslog(LOG_INFO, "keyword %s\n", row[0]);
        mysql_free_result(res);
    }
    buf_free(&query);
}

static int end_message_update(search_text_receiver_t *rx)
{
    sphinx_update_receiver_t *tr = (sphinx_update_receiver_t *)rx;
    struct buf *query = &tr->super.tmp;
    int i;
    int r;

    if (!tr->conn.mysql) return IMAP_INTERNAL;

    buf_reset(query);
    buf_printf(query, "INSERT INTO %s (id,"COL_CYRUSID, tr->super.indexname);
    for (i = 0 ; i < SEARCH_NUM_PARTS ; i++) {
        if (tr->super.parts[i].len) {
            buf_appendcstr(query, ",");
            buf_appendcstr(query, column_by_part[i]);
        }
    }
    buf_appendcstr(query, ") VALUES (");
    buf_printf(query, "%u,", ++tr->lastid);
    append_escaped(query, make_cyrusid(tr->super.mailbox, tr->super.uid), '\'');
    for (i = 0 ; i < SEARCH_NUM_PARTS ; i++) {
        if (tr->super.parts[i].len) {
            buf_appendcstr(query, ",");
            append_escaped(query, &tr->super.parts[i], '\'');
        }
    }
    /* apparently Sphinx doesn't let you explicitly INSERT a NULL */
    buf_appendcstr(query, ")");

    r = doquery(&tr->conn, tr->super.verbose, query);
    if (r) goto out; /* TODO: propagate error to the user */

    if (tr->super.verbose > 3) log_keywords(tr);

    ++tr->uncommitted;
    tr->latest = tr->super.uid;

out:
    tr->super.uid = 0;
    return r;
}

static const char *indexing_lockpath(struct mailbox *mailbox)
{
    char *userid = mboxname_to_userid(mailbox->name);
    char *usermbox = mboxname_user_mbox(userid, NULL);
    const char *lockpath = mboxname_lockpath_suffix(usermbox, INDEXING_LOCK_SUFFIX);
    free(usermbox);
    free(userid);
    return lockpath;
}

static void indexing_unlock(int *fdp)
{
    if (*fdp >= 0) {
        close(*fdp);
        *fdp = -1;
    }
}

static int indexing_lock(struct mailbox *mailbox, int *fdp)
{
    const char *lockpath = indexing_lockpath(mailbox);
    int fd;
    int r;

    indexing_unlock(fdp);

    fd = open(lockpath, O_WRONLY|O_CREAT, 0600);
    if (fd < 0) {
        if (cyrus_mkdir(lockpath, 0755) < 0) {
            syslog(LOG_ERR, "IOERROR: unable to cyrus_mkdir %s: %m", lockpath);
            return IMAP_IOERROR;
        }
        fd = open(lockpath, O_WRONLY|O_CREAT, 0600);
    }
    if (fd < 0) {
        syslog(LOG_ERR, "IOERROR: unable to create %s: %m", lockpath);
        return IMAP_IOERROR;
    }

    r = lock_blocking(fd, lockpath);
    if (r < 0) {
        syslog(LOG_ERR, "IOERROR: unable to lock %s: %m",
                lockpath);
        close(fd);
        return IMAP_IOERROR;
    }

    *fdp = fd;
    return 0;
}

static int begin_mailbox_update(search_text_receiver_t *rx,
                                struct mailbox *mailbox,
                                int incremental __attribute__((unused)))
{
    sphinx_update_receiver_t *tr = (sphinx_update_receiver_t *)rx;
    int r;

    r = sphinx_setup(mailbox, tr->super.verbose, /*create*/1);
    if (r) return r;

    r = sphinx_basedir(mailbox, NULL, &tr->super.indexname);
    if (r) return r;

    r = get_connection(&tr->conn);
    if (r) return r;

    tr->super.mailbox = mailbox;

    r = indexing_lock(mailbox, &tr->indexing_lock_fd);
    if (r) return r;

    r = open_latest(mailbox, &tr->latestdb);
    if (r) return r;

    r = read_lastid(&tr->latestdb, &tr->lastid, tr->super.verbose);
    if (r) return r;

    r = read_latest(&tr->latestdb, mailbox, &tr->latest, tr->super.verbose);
    if (r) return r;

    return 0;
}

static uint32_t first_unindexed_uid(search_text_receiver_t *rx)
{
    sphinx_update_receiver_t *tr = (sphinx_update_receiver_t *)rx;

    return tr->latest+1;
}

static int is_indexed(search_text_receiver_t *rx, uint32_t uid)
{
    sphinx_update_receiver_t *tr = (sphinx_update_receiver_t *)rx;

    return (uid <= tr->latest);
}

static int end_mailbox_update(search_text_receiver_t *rx,
                              struct mailbox *mailbox
                            __attribute__((unused)))
{
    sphinx_update_receiver_t *tr = (sphinx_update_receiver_t *)rx;
    int r = 0;

    if (tr->conn.mysql)
        r = flush(rx);

    tr->super.mailbox = NULL;
    free(tr->super.indexname);
    tr->super.indexname = NULL;

    return r;
}

static search_text_receiver_t *begin_update(int verbose)
{
    sphinx_update_receiver_t *tr;

    tr = xzmalloc(sizeof(sphinx_update_receiver_t));
    tr->super.super.begin_mailbox = begin_mailbox_update;
    tr->super.super.first_unindexed_uid = first_unindexed_uid;
    tr->super.super.is_indexed = is_indexed;
    tr->super.super.begin_message = begin_message;
    tr->super.super.begin_part = begin_part;
    tr->super.super.append_text = append_text;
    tr->super.super.end_part = end_part;
    tr->super.super.end_message = end_message_update;
    tr->super.super.end_mailbox = end_mailbox_update;
    tr->super.super.flush = flush;

    tr->indexing_lock_fd = -1;
    tr->super.verbose = verbose;

    return &tr->super.super;
}

static int free_receiver(sphinx_receiver_t *tr)
{
    int i;
    int r = 0;

    for (i = 0 ; i < SEARCH_NUM_PARTS ; i++)
        buf_free(&tr->parts[i]);
    buf_free(&tr->tmp);
    free(tr->indexname);

    free(tr);

    return r;
}

static int end_update(search_text_receiver_t *rx)
{
    sphinx_update_receiver_t *tr = (sphinx_update_receiver_t *)rx;

    close_latest(&tr->latestdb);
    close_connection(&tr->conn);
    indexing_unlock(&tr->indexing_lock_fd);
    return free_receiver(&tr->super);
}

static int begin_mailbox_snippets(search_text_receiver_t *rx,
                                  struct mailbox *mailbox,
                                  int incremental __attribute__((unused)))
{
    sphinx_snippet_receiver_t *tr = (sphinx_snippet_receiver_t *)rx;
    int r;

    r = get_connection(&tr->conn);
    if (r) return r;

    r = sphinx_basedir(mailbox, NULL, &tr->super.indexname);
    if (r) return r;

    tr->super.mailbox = mailbox;

    return 0;
}

/* Generate Sphinx extended query syntax, not SphinxQL, customised for
 * using CALL SNIPPETS.  This differs from the regular query because
 * CALL SNIPPETS will silently ignore @field modifiers instead of
 * enforcing them.  So we have to emit one CALL SNIPPETS for every
 * field that may be mentioned in the original query, with a munged
 * query that contains a disjunction of all the search terms that
 * might apply to that part.  Thanks heaps Sphinx. [IRIS-2038] */
static void generate_snippet_query(struct buf *query,
                                   int part,
                                   struct opnode *on)
{
    struct opnode *child;
    struct buf arg = BUF_INITIALIZER;

    buf_init_ro_cstr(&arg, on->arg);

    switch (on->op) {

    case SEARCH_OP_NOT:
    case SEARCH_OP_OR:
    case SEARCH_OP_AND:
        for (child = on->children ; child ; child = child->next)
            generate_snippet_query(query, part, child);
        break;

    case SEARCH_PART_ANY:
        assert(on->children == NULL);
        if (part != SEARCH_PART_HEADERS ||
            !config_getswitch(IMAPOPT_SPHINX_TEXT_EXCLUDES_ODD_HEADERS)) {
            if (query->len) buf_putc(query, '|');
            append_escaped(query, &arg, '"');
        }
        break;

    default:
        /* other SEARCH_PART_* constants */
        assert(on->op >= 0 && on->op < SEARCH_NUM_PARTS);
        assert(on->children == NULL);
        if (part == on->op) {
            if (query->len) buf_putc(query, '|');
            append_escaped(query, &arg, '"');
        }
        break;
    }

    buf_free(&arg);
}

static int end_message_snippets(search_text_receiver_t *rx)
{
    sphinx_snippet_receiver_t *tr = (sphinx_snippet_receiver_t *)rx;
    struct buf query = BUF_INITIALIZER;
    struct buf inner_query = BUF_INITIALIZER;
    MYSQL_RES *res = NULL;
    MYSQL_ROW row;
    int i;
    int r;

    if (!tr->conn.mysql) {
        r = IMAP_INTERNAL;          /* need to call begin_mailbox() */
        goto out;
    }
    if (!tr->root) {
        r = 0;
        goto out;
    }

    for (i = 0 ; i < SEARCH_NUM_PARTS ; i++) {
        if (res) {
            mysql_free_result(res);
            res = NULL;
        }
        if (i == SEARCH_PART_ANY) continue;

        if (!tr->super.parts[i].len) continue;

        buf_reset(&inner_query);
        generate_snippet_query(&inner_query, i, tr->root);
        if (!inner_query.len) continue;

        buf_reset(&query);
        buf_appendcstr(&query, "CALL SNIPPETS(");
        append_escaped(&query, &tr->super.parts[i], '\'');
        buf_appendcstr(&query, ", ");
        append_escaped_cstr(&query, tr->super.indexname, '\'');
        buf_appendcstr(&query, ", ");
        append_escaped(&query, &inner_query, '\'');
        buf_appendcstr(&query, ", 1 AS query_mode, 1 AS allow_empty)");

        r = doquery(&tr->conn, tr->super.verbose, &query);
        if (r) goto out;

        res = mysql_store_result(tr->conn.mysql);
        if (!res) continue;

        row = mysql_fetch_row(res);
        if (!row) continue;

        if (tr->super.verbose > 1)
            syslog(LOG_ERR, "snippet [%d] \"%s\"", i, row[0]);

        if (!row[0][0]) continue;
        r = tr->proc(tr->super.mailbox, tr->super.uid, i, row[0], tr->rock);
        if (r) break;
    }

out:
    if (res) mysql_free_result(res);
    buf_free(&query);
    buf_free(&inner_query);
    return r;
}

static int end_mailbox_snippets(search_text_receiver_t *rx,
                                struct mailbox *mailbox
                                    __attribute__((unused)))
{
    sphinx_snippet_receiver_t *tr = (sphinx_snippet_receiver_t *)rx;

    tr->super.mailbox = NULL;
    free(tr->super.indexname);
    tr->super.indexname = NULL;

    return 0;
}

static search_text_receiver_t *begin_snippets(void *internalised,
                                              int verbose,
                                              search_snippet_markup_t *m __attribute__((unused)),
                                              search_snippet_cb_t proc,
                                              void *rock)
{
    sphinx_snippet_receiver_t *tr;

    tr = xzmalloc(sizeof(sphinx_snippet_receiver_t));
    tr->super.super.begin_mailbox = begin_mailbox_snippets;
    tr->super.super.begin_message = begin_message;
    tr->super.super.begin_part = begin_part;
    tr->super.super.append_text = append_text;
    tr->super.super.end_part = end_part;
    tr->super.super.end_message = end_message_snippets;
    tr->super.super.end_mailbox = end_mailbox_snippets;

    tr->super.verbose = verbose;
    tr->root = (struct opnode *)internalised;
    tr->proc = proc;
    tr->rock = rock;

    return &tr->super.super;
}

static int end_snippets(search_text_receiver_t *rx)
{
    sphinx_snippet_receiver_t *tr = (sphinx_snippet_receiver_t *)rx;

    close_connection(&tr->conn);
    return free_receiver(&tr->super);
}

static int start_daemon(int verbose)
{
    return sphinx_setup(NULL, verbose, /*create*/1);
}

static int stop_daemon(int verbose)
{
    int r;

    if (verbose)
        syslog(LOG_INFO, "Sphinx stopping searchd");

    r = run_command(SEARCHD, "--config", sphinx_config_file(),
                    "--syslog-prefix", sphinx_syslog_prefix(),
                    "--stopwait", (char *)NULL);
    if (r)
        syslog(LOG_ERR, "Failed to stop searchd: %s", error_message(r));
    return r;
}

const struct search_engine sphinx_search_engine = {
    "Sphinx",
    SEARCH_FLAG_CAN_BATCH,
    begin_search,
    end_search,
    begin_update,
    end_update,
    begin_snippets,
    end_snippets,
    describe_internalised,
    free_internalised,
    start_daemon,
    stop_daemon,
    /* list_files */NULL,   /* XXX: fixme */
    /* compact */NULL,
    /* deluser */NULL   /* XXX: fixme */
};

