- for (token = zm->replace_tokens; !replace_in && token;
- token = token->next)
- {
- int j = 0;
- int replace_done = 0;
- replace_out = 0;
- for (;; j++)
- {
- int c;
- if (!token->token_from[j])
- {
- replace_in = j;
- break;
- }
- if (ex_list && strchr (ex_list, token->token_from[j]))
- break;
- if (i+j < 0 || j+i >= input_len)
- c = ' ';
- else
- c = input_str[j+i] & 255;
- if (token->token_from[j] == ZEBRA_REPLACE_ANY)
- {
- if (c == ' ')
- break;
- replace_string[replace_out++] = c;
- }
- else
- {
- if (c != token->token_from[j])
- {
- break;
- }
- if (!replace_done)
- {
- const char *cp = token->token_to;
- replace_done = 1;
- for (; cp && *cp; cp++)
- replace_string[replace_out++] = *cp;
- }
- }
- }
- }
- if (!replace_in)
- {
- if (i >= 0 && i < input_len)
- wrbuf_putc(wrbuf, input_str[i]);
- i++;
- }
- else
- {
- no_replaces++;
- if (replace_out)
- wrbuf_write(wrbuf, replace_string, replace_out);
- i += replace_in;
- }
+ *result_len = strlen(*result_buf);
+
+ if (display_buf)
+ {
+ *display_buf = icu_chain_token_display(zm->icu_chain);
+ if (display_len)
+ *display_len = strlen(*display_buf);
+ }
+ if (zm->debug)
+ {
+ wrbuf_rewind(zm->print_str);
+ wrbuf_write_escaped(zm->print_str, *result_buf, *result_len);
+ yaz_log(YLOG_LOG, "output %s", wrbuf_cstr(zm->print_str));
+ }
+
+ if (**result_buf != '\0')
+ return 1;
+ }
+ }
+ return 0;
+#else
+ return tokenize_simple(zm, result_buf, result_len);
+#endif
+}
+
+int zebra_map_tokenize_start(zebra_map_t zm,
+ const char *buf, size_t len)
+{
+ assert(zm->use_chain);
+
+ wrbuf_rewind(zm->input_str);
+ wrbuf_write(zm->input_str, buf, len);
+ zm->simple_off = 0;
+#if YAZ_HAVE_ICU
+ if (zm->icu_chain)
+ {
+ UErrorCode status;
+ if (zm->debug)
+ {
+ wrbuf_rewind(zm->print_str);
+ wrbuf_write_escaped(zm->print_str, wrbuf_buf(zm->input_str),
+ wrbuf_len(zm->input_str));
+
+ yaz_log(YLOG_LOG, "input %s",
+ wrbuf_cstr(zm->print_str));
+ }
+ icu_chain_assign_cstr(zm->icu_chain,
+ wrbuf_cstr(zm->input_str),
+ &status);
+ if (!U_SUCCESS(status))
+ {
+ if (zm->debug)
+ {
+ yaz_log(YLOG_WARN, "bad encoding for input");
+ }
+ return -1;
+ }