/* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2013 Index Data
+ * Copyright (C) Index Data
* See the file LICENSE for details.
*/
struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
UErrorCode *status)
{
- struct icu_tokenizer * tokenizer
+ struct icu_tokenizer *tokenizer
= (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
icu_tokenizer_reset(tokenizer, action);
return 0;
}
-void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
+void icu_tokenizer_destroy(struct icu_tokenizer *tokenizer)
{
if (tokenizer)
{
}
}
-int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
- struct icu_buf_utf16 * src16,
+int icu_tokenizer_attach(struct icu_tokenizer *tokenizer,
+ struct icu_buf_utf16 *src16,
UErrorCode *status)
{
if (!tokenizer || !tokenizer->bi || !src16)
return 1;
}
-int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
- struct icu_buf_utf16 * tkn16,
- UErrorCode *status)
+int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
+ struct icu_buf_utf16 *tkn16,
+ UErrorCode *status,
+ size_t *start, size_t *len)
{
int32_t tkn_start = 0;
int32_t tkn_end = 0;
tokenizer->token_start = tkn_start;
tokenizer->token_end = tkn_end;
+ *start = tkn_start;
+ *len = tkn_end - tkn_start;
+
/* copying into token buffer if it exists */
if (tkn16)
{
return tkn_len;
}
-int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
+int32_t icu_tokenizer_token_count(struct icu_tokenizer *tokenizer)
{
return tokenizer->token_count;
}