projects
/
yaz-moved-to-github.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch 'master' into sru_2_0
[yaz-moved-to-github.git]
/
src
/
icu_tokenizer.c
diff --git
a/src/icu_tokenizer.c
b/src/icu_tokenizer.c
index
67246ea
..
7e2fc3f
100644
(file)
--- a/
src/icu_tokenizer.c
+++ b/
src/icu_tokenizer.c
@@
-155,7
+155,8
@@
int icu_tokenizer_attach(struct icu_tokenizer *tokenizer,
int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
struct icu_buf_utf16 *tkn16,
int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
struct icu_buf_utf16 *tkn16,
- UErrorCode *status)
+ UErrorCode *status,
+ size_t *start, size_t *len)
{
int32_t tkn_start = 0;
int32_t tkn_end = 0;
{
int32_t tkn_start = 0;
int32_t tkn_end = 0;
@@
-202,6
+203,9
@@
int32_t icu_tokenizer_next_token(struct icu_tokenizer *tokenizer,
tokenizer->token_start = tkn_start;
tokenizer->token_end = tkn_end;
tokenizer->token_start = tkn_start;
tokenizer->token_end = tkn_end;
+ *start = tkn_start;
+ *len = tkn_end - tkn_start;
+
/* copying into token buffer if it exists */
if (tkn16)
{
/* copying into token buffer if it exists */
if (tkn16)
{