First go at returning start+offset

[yaz-moved-to-github.git] / src / icu_tokenizer.c

diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c
index 67246ea..7e2fc3f 100644 (file)
--- a/src/icu_tokenizer.c
+++ b/src/icu_tokenizer.c
@@ -155,7 +155,8 @@ int icu_tokenizer_attach(struct icu_tokenizer *tokenizer,
 
 int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
                                  struct icu_buf_utf16 *tkn16,
-                                 UErrorCode *status)
+                                 UErrorCode *status,
+                                 size_t *start, size_t *len)
 {
     int32_t tkn_start = 0;
     int32_t tkn_end = 0;
@@ -202,6 +203,9 @@ int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
     tokenizer->token_start = tkn_start;
     tokenizer->token_end = tkn_end;
 
+    *start = tkn_start;
+    *len = tkn_end - tkn_start;
+
     /* copying into token buffer if it exists */
     if (tkn16)
     {