/* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2009 Index Data.
+ * Copyright (C) 1995-2010 Index Data.
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
struct icu_buf_utf16 * src16,
UErrorCode * status);
-struct icu_tokenizer
-{
- char action;
- UBreakIterator* bi;
- struct icu_buf_utf16 * buf16;
- int32_t token_count;
- int32_t token_id;
- int32_t token_start;
- int32_t token_end;
-/*
- keep always invariant
- 0 <= token_start
- <= token_end
- <= buf16->utf16_len
- and invariant
- 0 <= token_id <= token_count
-*/
-};
-
+struct icu_tokenizer;
struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
UErrorCode *status);