-
- /* either an old token not finished yet, or a new token, thus
- perform the work, eventually put this steps output in
- step->buf16 or the chains UTF8 output buffers */
-
- switch (step->type)
- {
- case ICU_chain_step_type_display:
- icu_utf16_to_utf8(chain->display8, src16, status);
- break;
- case ICU_chain_step_type_casemap:
- icu_casemap_casemap(step->u.casemap,
- step->buf16, src16, status,
- chain->locale);
- break;
- case ICU_chain_step_type_transform:
- case ICU_chain_step_type_transliterate:
- icu_transform_trans(step->u.transform,
- step->buf16, src16, status);
- break;
- case ICU_chain_step_type_tokenize:
- /* attach to new src16 token only first time during splitting */
- if (step->need_new_token)
- {
- icu_tokenizer_attach(step->u.tokenizer, src16, status);
- step->need_new_token = 0;
- }
-
- /* splitting one src16 token into multiple buf16 tokens */
- step->more_tokens
- = icu_tokenizer_next_token(step->u.tokenizer,
- step->buf16, status);
-
- /* make sure to get new previous token if this one had been used up
- by recursive call to _same_ step */
-
- if (!step->more_tokens)
- {
- step->more_tokens = icu_chain_step_next_token(chain, step, status);
- return step->more_tokens; /* avoid one token count too much! */
- }
- break;
- default:
- return 0;
- break;
- }
-
- if (U_FAILURE(*status))
- return 0;
-
- /* if token disappered into thin air, tell caller */
- /* if (!step->buf16->utf16_len && !step->more_tokens) */
- /* return 0; */
-
- return 1;