X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fcharsets.c;h=32d5f4dad3b78a160f0ca6f7df8abe7ca3d2fdf2;hb=77682ed3596450ab7b3caff707999d2f7977d614;hp=fe914204e3b271f327d98978797e2d1b0496cba2;hpb=7118da8de283be0e294bbaf4501ba002cca22d89;p=pazpar2-moved-to-github.git

diff --git a/src/charsets.c b/src/charsets.c
index fe91420..32d5f4d 100644
--- a/src/charsets.c
+++ b/src/charsets.c
@@ -1,5 +1,5 @@
 /* This file is part of Pazpar2.
-   Copyright (C) 2006-2012 Index Data
+   Copyright (C) Index Data
 
 Pazpar2 is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free
@@ -53,6 +53,8 @@ struct pp2_charset_s {
     const char *(*token_next_handler)(pp2_charset_token_t prt);
     const char *(*get_sort_handler)(pp2_charset_token_t prt);
     const char *(*get_display_handler)(pp2_charset_token_t prt);
+    void (*get_org_handler)(pp2_charset_token_t ptr,
+                            size_t *start, size_t *len);
 #if YAZ_HAVE_ICU
     struct icu_chain * icu_chn;
     UErrorCode icu_sts;
@@ -63,11 +65,15 @@ static const char *pp2_charset_token_null(pp2_charset_token_t prt);
 static const char *pp2_charset_token_a_to_z(pp2_charset_token_t prt);
 static const char *pp2_get_sort_ascii(pp2_charset_token_t prt);
 static const char *pp2_get_display_ascii(pp2_charset_token_t prt);
+static void pp2_get_org_ascii(pp2_charset_token_t prt,
+                              size_t *start, size_t *len);
 
 #if YAZ_HAVE_ICU
 static const char *pp2_charset_token_icu(pp2_charset_token_t prt);
 static const char *pp2_get_sort_icu(pp2_charset_token_t prt);
 static const char *pp2_get_display_icu(pp2_charset_token_t prt);
+static void pp2_get_org_icu(pp2_charset_token_t prt,
+                            size_t *start, size_t *len);
 #endif
 
 /* tokenzier handle */
@@ -80,6 +86,9 @@ struct pp2_charset_token_s {
 #if YAZ_HAVE_ICU
     yaz_icu_iter_t iter;
 #endif
+    const char *cp0;
+    size_t start;
+    size_t len;
 };
 
 struct pp2_charset_fact_s {
@@ -194,15 +203,9 @@ pp2_charset_t pp2_charset_create_xml(xmlNode *xml_node)
 #if YAZ_HAVE_ICU
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain *chain = 0;
-    while (xml_node && xml_node->type != XML_ELEMENT_NODE)
-        xml_node = xml_node->next;
     chain = icu_chain_xml_config(xml_node, 1, &status);
-    if (!chain || U_FAILURE(status)){
-        //xmlDocPtr icu_doc = 0;
-        //xmlChar *xmlstr = 0;
-                //int size = 0;
-                //xmlDocDumpMemory(icu_doc, size);
-
+    if (!chain || U_FAILURE(status))
+    {
         yaz_log(YLOG_FATAL, "Could not parse ICU chain config:\n"
                 "<%s>\n ... \n</%s>",
                 xml_node->name, xml_node->name);
@@ -226,6 +229,7 @@ pp2_charset_t pp2_charset_create(void)
     pct->token_next_handler = pp2_charset_token_null;
     pct->get_sort_handler  = pp2_get_sort_ascii;
     pct->get_display_handler  = pp2_get_display_ascii;
+    pct->get_org_handler = pp2_get_org_ascii;
 #if YAZ_HAVE_ICU
     pct->icu_chn = 0;
 #endif // YAZ_HAVE_ICU
@@ -250,6 +254,7 @@ pp2_charset_t pp2_charset_create_icu(struct icu_chain *icu_chn)
         pct->token_next_handler = pp2_charset_token_icu;
         pct->get_sort_handler = pp2_get_sort_icu;
         pct->get_display_handler = pp2_get_display_icu;
+        pct->get_org_handler = pp2_get_org_icu;
     }
     return pct;
 }
@@ -290,6 +295,8 @@ pp2_charset_token_t pp2_charset_tokenize(pp2_charset_t pct)
     if (pct->icu_chn)
         prt->iter = icu_iter_create(pct->icu_chn);
 #endif
+    prt->start = 0;
+    prt->len = 0;
     return prt;
 }
 
@@ -313,6 +320,7 @@ void pp2_charset_token_first(pp2_charset_token_t prt,
 
     wrbuf_rewind(prt->norm_str);
     wrbuf_rewind(prt->sort_str);
+    prt->cp0 = buf;
     prt->cp = buf;
     prt->last_cp = 0;
 
@@ -354,6 +362,12 @@ const char *pp2_get_display(pp2_charset_token_t prt)
     return prt->pct->get_display_handler(prt);
 }
 
+void pp2_get_org(pp2_charset_token_t prt, size_t *start, size_t *len)
+{
+    prt->pct->get_org_handler(prt, start, len);
+}
+
+
 #define raw_char(c) (((c) >= 'a' && (c) <= 'z') ? (c) : -1)
 /* original tokenizer with our tokenize interface, but we
    add +1 to ensure no '\0' are in our string (except for EOF)
@@ -363,6 +377,7 @@ static const char *pp2_charset_token_a_to_z(pp2_charset_token_t prt)
     const char *cp = prt->cp;
     int c;
 
+    prt->start = cp - prt->cp0;
     /* skip white space */
     while (*cp && (c = raw_char(tolower(*(const unsigned char *)cp))) < 0)
         cp++;
@@ -381,6 +396,7 @@ static const char *pp2_charset_token_a_to_z(pp2_charset_token_t prt)
         wrbuf_putc(prt->norm_str, c);
         cp++;
     }
+    prt->len = (cp - prt->cp0) - prt->start;
     prt->cp = cp;
     return wrbuf_cstr(prt->norm_str);
 }
@@ -412,6 +428,13 @@ static const char *pp2_get_display_ascii(pp2_charset_token_t prt)
     }
 }
 
+static void pp2_get_org_ascii(pp2_charset_token_t prt,
+                              size_t *start, size_t *len)
+{
+    *start = prt->start;
+    *len = prt->len;
+}
+
 static const char *pp2_charset_token_null(pp2_charset_token_t prt)
 {
     const char *cp = prt->cp;
@@ -420,6 +443,7 @@ static const char *pp2_charset_token_null(pp2_charset_token_t prt)
     while (*cp)
         cp++;
     prt->cp = cp;
+    prt->len = cp - prt->cp0;
     return prt->last_cp;
 }
 
@@ -443,6 +467,11 @@ static const char *pp2_get_display_icu(pp2_charset_token_t prt)
     return icu_iter_get_display(prt->iter);
 }
 
+static void pp2_get_org_icu(pp2_charset_token_t prt, size_t *start, size_t *len)
+{
+    icu_iter_get_org_info(prt->iter, start, len);
+}
+
 #endif // YAZ_HAVE_ICU