X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ficu_tokenizer.c;h=7e2fc3f0656b275fa57e814fb3b20a031b9795a1;hp=67246eaf0dbcff6c38ca01e287750b5c238e4bb2;hb=94b1547e5951e1e01bf5180159e74095cd0527f4;hpb=05e94adc1a717e85ba5b775468c3c9e2c8a26034;ds=inline

diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c
index 67246ea..7e2fc3f 100644
--- a/src/icu_tokenizer.c
+++ b/src/icu_tokenizer.c
@@ -155,7 +155,8 @@ int icu_tokenizer_attach(struct icu_tokenizer *tokenizer,
 
 int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
                                  struct icu_buf_utf16 *tkn16,
-                                 UErrorCode *status)
+                                 UErrorCode *status,
+                                 size_t *start, size_t *len)
 {
     int32_t tkn_start = 0;
     int32_t tkn_end = 0;
@@ -202,6 +203,9 @@ int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
     tokenizer->token_start = tkn_start;
     tokenizer->token_end = tkn_end;
 
+    *start = tkn_start;
+    *len = tkn_end - tkn_start;
+
     /* copying into token buffer if it exists */
     if (tkn16)
     {