+#define SE_CHARS ";,.()-/?<> \r\n\t"
+
+static int tokenize_simple(zebra_map_t zm,
+ const char **result_buf, size_t *result_len)
+{
+ char *buf = wrbuf_buf(zm->simple_buf);
+ size_t len = wrbuf_len(zm->simple_buf);
+ size_t i = zm->simple_off;
+ size_t start;
+
+ while (i < len && strchr(SE_CHARS, buf[i]))
+ i++;
+ start = i;
+ while (i < len && !strchr(SE_CHARS, buf[i]))
+ {
+ if (buf[i] > 32 && buf[i] < 127)
+ buf[i] = tolower(buf[i]);
+ i++;
+ }
+
+ zm->simple_off = i;
+ if (start != i)
+ {
+ *result_buf = buf + start;
+ *result_len = i - start;
+ return 1;
+ }
+ return 0;
+ }
+
+int zebra_map_tokenize(zebra_map_t zm,
+ const char *buf, size_t len,
+ const char **result_buf, size_t *result_len)
+{
+ assert(zm->use_chain);
+ if (!zm->icu_chain)
+ {
+ if (buf)
+ {
+ wrbuf_rewind(zm->simple_buf);
+ wrbuf_write(zm->simple_buf, buf, len);
+ zm->simple_off = 0;
+ }
+ return tokenize_simple(zm, result_buf, result_len);
+ }
+ return 0;
+}
+
+int zebra_maps_is_icu(zebra_map_t zm)
+{
+#if HAVE_ICU
+ return zm->use_chain;
+#else
+ return 0;
+#endif
+}
+
+