X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fsession.c;h=9867d8cdf21b903d5715c64d151f46962babbcdb;hb=b92f5f5d4bd3fb1cf21910047ba2f95ba9c9a4a8;hp=c6b1f214ceb530e099949ae2a275b4dd4758b4e1;hpb=b6d7a5bde6efed3fd29a23e8493b22f4596db8ae;p=pazpar2-moved-to-github.git

diff --git a/src/session.c b/src/session.c
index c6b1f21..9867d8c 100644
--- a/src/session.c
+++ b/src/session.c
@@ -97,7 +97,6 @@ struct client_list {
 /* session counting (1) , disable client counting (0) */
 static YAZ_MUTEX g_session_mutex = 0;
 static int no_sessions = 0;
-static int no_session_total = 0;
 
 static int session_use(int delta)
 {
@@ -106,11 +105,10 @@ static int session_use(int delta)
         yaz_mutex_create(&g_session_mutex);
     yaz_mutex_enter(g_session_mutex);
     no_sessions += delta;
-    if (delta > 0)
-        no_session_total += delta;
     sessions = no_sessions;
     yaz_mutex_leave(g_session_mutex);
-    yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : (delta > 0 ? "INC" : "DEC"), no_sessions);
+    yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" :
+            (delta > 0 ? "INC" : "DEC"), no_sessions);
     return sessions;
 }
 
@@ -119,17 +117,6 @@ int sessions_count(void)
     return session_use(0);
 }
 
-int session_count_total(void)
-{
-    int total = 0;
-    if (!g_session_mutex)
-        return 0;
-    yaz_mutex_enter(g_session_mutex);
-    total = no_session_total;
-    yaz_mutex_leave(g_session_mutex);
-    return total;
-}
-
 static void log_xml_doc(xmlDoc *doc)
 {
     FILE *lf = yaz_log_file();
@@ -162,10 +149,9 @@ static void session_leave(struct session *s, const char *caller)
         session_log(s, YLOG_DEBUG, "Session unlock by %s", caller);
 }
 
-static void session_normalize_facet(struct session *s, const char *type,
-                                    const char *value,
-                                    WRBUF display_wrbuf,
-                                    WRBUF facet_wrbuf)
+static void session_normalize_facet(struct session *s,
+                                    const char *type, const char *value,
+                                    WRBUF display_wrbuf, WRBUF facet_wrbuf)
 {
     struct conf_service *service = s->service;
     pp2_charset_token_t prt;
@@ -311,7 +297,8 @@ static void insert_settings_parameters(struct session_database *sdb,
 
 // Add static values from session database settings if applicable
 static void insert_settings_values(struct session_database *sdb, xmlDoc *doc,
-    struct conf_service *service)
+                                   xmlNode *root,
+                                   struct conf_service *service)
 {
     int i;
 
@@ -326,8 +313,7 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc,
             const char *val = session_setting_oneval(sdb, offset);
             if (val)
             {
-                xmlNode *r = xmlDocGetRootElement(doc);
-                xmlNode *n = xmlNewTextChild(r, 0, (xmlChar *) "metadata",
+                xmlNode *n = xmlNewTextChild(root, 0, (xmlChar *) "metadata",
                                              (xmlChar *) val);
                 xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name);
             }
@@ -352,17 +338,6 @@ static xmlDoc *normalize_record(struct session *se,
         {
             session_log(se, YLOG_WARN, "Normalize failed");
         }
-        else
-        {
-            insert_settings_values(sdb, rdoc, service);
-
-            if (global_parameters.dump_records)
-            {
-                session_log(se, YLOG_LOG, "Normalized record from %s",
-                            sdb->database->id);
-                log_xml_doc(rdoc);
-            }
-        }
     }
     return rdoc;
 }
@@ -648,7 +623,8 @@ static void session_clear_set(struct session *se, struct reclist_sortparms *sp)
 
 static void session_sort_unlocked(struct session *se,
                                   struct reclist_sortparms *sp,
-                                  const char *mergekey)
+                                  const char *mergekey,
+                                  const char *rank)
 {
     struct client_list *l;
     const char *field = sp->name;
@@ -659,14 +635,24 @@ static void session_sort_unlocked(struct session *se,
     session_log(se, YLOG_DEBUG, "session_sort field=%s increasing=%d type=%d",
                 field, increasing, type);
 
-    if (mergekey && strcmp(se->mergekey, mergekey))
+    if (rank && (!se->rank || strcmp(se->rank, rank)))
+    {
+        /* new rank must research/reingest anyway */
+        assert(rank);
+        xfree(se->rank);
+        se->rank = *rank ? xstrdup(rank) : 0;
+        clients_research = 1;
+        session_log(se, YLOG_DEBUG, "session_sort: new rank = %s",
+                    rank);
+    }
+    if (mergekey && (!se->mergekey || strcmp(se->mergekey, mergekey)))
     {
         /* new mergekey must research/reingest anyway */
         assert(mergekey);
         xfree(se->mergekey);
         se->mergekey = *mergekey ? xstrdup(mergekey) : 0;
         clients_research = 1;
-        session_log(se, YLOG_DEBUG, "search_sort: new mergekey = %s",
+        session_log(se, YLOG_DEBUG, "session_sort: new mergekey = %s",
                     mergekey);
     }
     if (clients_research == 0)
@@ -677,12 +663,12 @@ static void session_sort_unlocked(struct session *se,
                 break;
         if (sr)
         {
-            session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d type=%d already fetched",
+            session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d already fetched",
                         field, increasing, type);
             return;
         }
     }
-    session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d type=%d must fetch",
+    session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d must fetch",
                 field, increasing, type);
 
     // We need to reset reclist on every sort that changes the records, not just for position
@@ -699,7 +685,7 @@ static void session_sort_unlocked(struct session *se,
     if (clients_research)
     {
         session_log(se, YLOG_DEBUG,
-                    "Reset results due to %d clients researching",
+                    "session_sort: reset results due to %d clients researching",
                     clients_research);
         session_clear_set(se, sp);
     }
@@ -712,7 +698,7 @@ static void session_sort_unlocked(struct session *se,
         sr->type = type;
         sr->next = se->sorted_results;
         se->sorted_results = sr;
-        session_log(se, YLOG_DEBUG, "No research/ingesting done");
+        session_log(se, YLOG_DEBUG, "session_sort: no research/ingesting done");
         return ;
     }
     session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order");
@@ -725,20 +711,21 @@ static void session_sort_unlocked(struct session *se,
             client_get_state(cl) == Client_Working) {
             client_start_search(cl);
         }
-        else {
+        else
+        {
             session_log(se, YLOG_DEBUG,
-                        "Client %s: No re-start/ingest in show. Wrong client state: %d",
+                        "session_sort: %s: No re-start/ingest in show. "
+                        "Wrong client state: %d",
                         client_get_id(cl), client_get_state(cl));
         }
-
     }
 }
 
 void session_sort(struct session *se, struct reclist_sortparms *sp,
-                  const char *mergekey)
+                  const char *mergekey, const char *rank)
 {
     //session_enter(se, "session_sort");
-    session_sort_unlocked(se, sp, mergekey);
+    session_sort_unlocked(se, sp, mergekey, rank);
     //session_leave(se, "session_sort");
 }
 
@@ -750,8 +737,10 @@ enum pazpar2_error_code session_search(struct session *se,
                                        const char *filter,
                                        const char *limit,
                                        const char **addinfo,
+                                       const char **addinfo2,
                                        struct reclist_sortparms *sp,
-                                       const char *mergekey)
+                                       const char *mergekey,
+                                       const char *rank)
 {
     int live_channels = 0;
     int no_working = 0;
@@ -759,6 +748,11 @@ enum pazpar2_error_code session_search(struct session *se,
     int no_failed_limit = 0;
     struct client_list *l, *l0;
 
+    session_alert_watch(se, SESSION_WATCH_SHOW);
+    session_alert_watch(se, SESSION_WATCH_BYTARGET);
+    session_alert_watch(se, SESSION_WATCH_TERMLIST);
+    session_alert_watch(se, SESSION_WATCH_SHOW_PREF);
+
     session_log(se, YLOG_DEBUG, "Search");
 
     *addinfo = 0;
@@ -777,6 +771,11 @@ enum pazpar2_error_code session_search(struct session *se,
         xfree(se->mergekey);
         se->mergekey = *mergekey ? xstrdup(mergekey) : 0;
     }
+    if (rank)
+    {
+        xfree(se->rank);
+        se->rank = *rank ? xstrdup(rank) : 0;
+    }
 
     session_clear_set(se, sp);
     relevance_destroy(&se->relevance);
@@ -809,7 +808,7 @@ enum pazpar2_error_code session_search(struct session *se,
         if (prepare_map(se, client_get_database(cl)) < 0)
             continue;
 
-        parse_ret = client_parse_query(cl, query, se->facet_limits);
+        parse_ret = client_parse_query(cl, query, se->facet_limits, addinfo2);
         if (parse_ret == -1)
             no_failed_query++;
         else if (parse_ret == -2)
@@ -841,7 +840,6 @@ enum pazpar2_error_code session_search(struct session *se,
         else
             return PAZPAR2_NO_TARGETS;
     }
-    session_log(se, YLOG_LOG, "session_start_search done");
     return PAZPAR2_NO_ERROR;
 }
 
@@ -942,7 +940,7 @@ void session_apply_setting(struct session *se, char *dbname, char *setting,
 void session_destroy(struct session *se)
 {
     struct session_database *sdb;
-    session_log(se, YLOG_DEBUG, "Destroying");
+    session_log(se, YLOG_LOG, "destroy");
     session_use(-1);
     session_remove_cached_clients(se);
 
@@ -952,6 +950,7 @@ void session_destroy(struct session *se)
     relevance_destroy(&se->relevance);
     reclist_destroy(se->reclist);
     xfree(se->mergekey);
+    xfree(se->rank);
     if (nmem_total(se->nmem))
         session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem));
     if (nmem_total(se->session_nmem))
@@ -1001,6 +1000,7 @@ struct session *new_session(NMEM nmem, struct conf_service *service,
     session->sorted_results = 0;
     session->facet_limits = 0;
     session->mergekey = 0;
+    session->rank = 0;
 
     for (i = 0; i <= SESSION_WATCH_MAX; i++)
     {
@@ -1010,6 +1010,8 @@ struct session *new_session(NMEM nmem, struct conf_service *service,
     session->normalize_cache = normalize_cache_create();
     session->session_mutex = 0;
     pazpar2_mutex_create(&session->session_mutex, tmp_str);
+    session_log(session, YLOG_LOG, "create");
+
     session_use(1);
     return session;
 }
@@ -1234,7 +1236,7 @@ struct record_cluster *show_single_start(struct session *se, const char *id,
     session_enter(se, "show_single_start");
     *prev_r = 0;
     *next_r = 0;
-    reclist_limit(se->reclist, se);
+    reclist_limit(se->reclist, se, 1);
 
     reclist_enter(se->reclist);
     while ((r = reclist_read_record(se->reclist)))
@@ -1308,7 +1310,7 @@ struct record_cluster **show_range_start(struct session *se,
     *sumhits = 0;
     *approx_hits = 0;
     *total = 0;
-    reclist_limit(se->reclist, se);
+    reclist_limit(se->reclist, se, 0);
     if (se->relevance)
     {
         for (spp = sp; spp; spp = spp->next)
@@ -1461,6 +1463,7 @@ static struct record_metadata *record_metadata_init(
 
         rec_md->data.text.disp = p;
         rec_md->data.text.sort = 0;
+        rec_md->data.text.snippet = 0;
     }
     else if (type == Metadata_type_year || type == Metadata_type_date)
     {
@@ -1480,6 +1483,26 @@ static struct record_metadata *record_metadata_init(
     return rec_md;
 }
 
+static void mergekey_norm_wr(pp2_charset_fact_t charsets,
+                             WRBUF norm_wr, const char *value)
+{
+    const char *norm_str;
+    pp2_charset_token_t prt =
+        pp2_charset_token_create(charsets, "mergekey");
+
+    pp2_charset_token_first(prt, value, 0);
+    while ((norm_str = pp2_charset_token_next(prt)))
+    {
+        if (*norm_str)
+        {
+            if (wrbuf_len(norm_wr))
+                wrbuf_puts(norm_wr, " ");
+            wrbuf_puts(norm_wr, norm_str);
+        }
+    }
+    pp2_charset_token_destroy(prt);
+}
+
 static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name,
                                  struct conf_service *service, WRBUF norm_wr)
 {
@@ -1500,24 +1523,11 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name,
                 xmlChar *value = xmlNodeListGetString(doc, n->children, 1);
                 if (value && *value)
                 {
-                    const char *norm_str;
-                    pp2_charset_token_t prt =
-                        pp2_charset_token_create(service->charsets, "mergekey");
-
-                    pp2_charset_token_first(prt, (const char *) value, 0);
                     if (wrbuf_len(norm_wr) > 0)
                         wrbuf_puts(norm_wr, " ");
                     wrbuf_puts(norm_wr, name);
-                    while ((norm_str =
-                            pp2_charset_token_next(prt)))
-                    {
-                        if (*norm_str)
-                        {
-                            wrbuf_puts(norm_wr, " ");
-                            wrbuf_puts(norm_wr, norm_str);
-                        }
-                    }
-                    pp2_charset_token_destroy(prt);
+                    mergekey_norm_wr(service->charsets, norm_wr,
+                                     (const char *) value);
                     no_found++;
                 }
                 if (value)
@@ -1529,12 +1539,12 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name,
     return no_found;
 }
 
-static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no,
+static const char *get_mergekey(xmlDoc *doc, xmlNode *root, 
+                                struct client *cl, int record_no,
                                 struct conf_service *service, NMEM nmem,
                                 const char *session_mergekey)
 {
     char *mergekey_norm = 0;
-    xmlNode *root = xmlDocGetRootElement(doc);
     WRBUF norm_wr = wrbuf_alloc();
     xmlChar *mergekey;
 
@@ -1550,21 +1560,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no,
     }
     else if ((mergekey = xmlGetProp(root, (xmlChar *) "mergekey")))
     {
-        const char *norm_str;
-        pp2_charset_token_t prt =
-            pp2_charset_token_create(service->charsets, "mergekey");
-
-        pp2_charset_token_first(prt, (const char *) mergekey, 0);
-        while ((norm_str = pp2_charset_token_next(prt)))
-        {
-            if (*norm_str)
-            {
-                if (wrbuf_len(norm_wr))
-                    wrbuf_puts(norm_wr, " ");
-                wrbuf_puts(norm_wr, norm_str);
-            }
-        }
-        pp2_charset_token_destroy(prt);
+        mergekey_norm_wr(service->charsets, norm_wr, (const char *) mergekey);
         xmlFree(mergekey);
     }
     else
@@ -1667,12 +1663,37 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb)
     return match;
 }
 
-
 static int ingest_to_cluster(struct client *cl,
                              xmlDoc *xdoc,
                              xmlNode *root,
                              int record_no,
-                             const char *mergekey_norm);
+                             struct record_metadata_attr *mergekey);
+
+static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root,
+                             int record_no, NMEM nmem,
+                             struct session_database *sdb,
+                             struct record_metadata_attr *mergekeys)
+{
+    int ret = 0;
+    struct session *se = client_get_session(cl);
+    struct conf_service *service = se->service;
+
+    insert_settings_values(sdb, xdoc, root, service);
+
+    if (!check_record_filter(root, sdb))
+    {
+        session_log(se, YLOG_LOG,
+                    "Filtered out record no %d from %s",
+                    record_no, sdb->database->id);
+        return 0;
+    }
+    session_enter(se, "ingest_sub_record");
+    if (client_get_session(cl) == se && se->relevance)
+        ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekeys);
+    session_leave(se, "ingest_sub_record");
+
+    return ret;
+}
 
 /** \brief ingest XML record
     \param cl client holds the result set for record
@@ -1687,50 +1708,110 @@ int ingest_record(struct client *cl, const char *rec,
                   int record_no, NMEM nmem)
 {
     struct session *se = client_get_session(cl);
-    int ret = 0;
     struct session_database *sdb = client_get_database(cl);
     struct conf_service *service = se->service;
     xmlDoc *xdoc = normalize_record(se, sdb, service, rec, nmem);
+    int r = 0;
     xmlNode *root;
-    const char *mergekey_norm;
 
     if (!xdoc)
         return -1;
 
+    if (global_parameters.dump_records)
+    {
+        session_log(se, YLOG_LOG, "Normalized record from %s",
+                    sdb->database->id);
+        log_xml_doc(xdoc);
+    }
+
     root = xmlDocGetRootElement(xdoc);
 
-    if (!check_record_filter(root, sdb))
+    if (!strcmp((const char *) root->name, "cluster"))
     {
-        session_log(se, YLOG_LOG, "Filtered out record no %d from %s", record_no, sdb->database->id);
-        xmlFreeDoc(xdoc);
-        return -2;
+        int no_merge_keys = 0;
+        int no_merge_dups = 0;
+        xmlNode *sroot;
+        struct record_metadata_attr *mk = 0;
+
+        for (sroot = root->children; sroot; sroot = sroot->next)
+            if (sroot->type == XML_ELEMENT_NODE &&
+                !strcmp((const char *) sroot->name, "record"))
+            {
+                struct record_metadata_attr **mkp;
+                const char *mergekey_norm =
+                    get_mergekey(xdoc, sroot, cl, record_no, service, nmem,
+                                 se->mergekey);
+                if (!mergekey_norm)
+                {
+                    r = -1;
+                    break;
+                }
+                for (mkp = &mk; *mkp; mkp = &(*mkp)->next)
+                    if (!strcmp((*mkp)->value, mergekey_norm))
+                        break;
+                if (!*mkp)
+                {
+                    *mkp = (struct record_metadata_attr*)
+                        nmem_malloc(nmem, sizeof(**mkp));
+                    (*mkp)->name = 0;
+                    (*mkp)->value = nmem_strdup(nmem, mergekey_norm);
+                    (*mkp)->next = 0;
+                    no_merge_keys++;
+                }
+                else
+                    no_merge_dups++;
+            }
+        if (no_merge_keys > 1 || no_merge_dups > 0)
+        {
+            yaz_log(YLOG_LOG, "Got %d mergekeys, %d dups for position %d",
+                    no_merge_keys, no_merge_dups, record_no);
+        }
+        for (sroot = root->children; !r && sroot; sroot = sroot->next)
+            if (sroot->type == XML_ELEMENT_NODE &&
+                !strcmp((const char *) sroot->name, "record"))
+            {
+                r = ingest_sub_record(cl, xdoc, sroot, record_no, nmem, sdb,
+                                      mk);
+            }
     }
+    else if (!strcmp((const char *) root->name, "record"))
+    {
+        const char *mergekey_norm =
+            get_mergekey(xdoc, root, cl, record_no, service, nmem,
+                         se->mergekey);
+        if (mergekey_norm)
+        {
+            struct record_metadata_attr *mk = (struct record_metadata_attr*)
+                nmem_malloc(nmem, sizeof(*mk));
+            mk->name = 0;
+            mk->value = nmem_strdup(nmem, mergekey_norm);
+            mk->next = 0;
 
-    mergekey_norm = get_mergekey(xdoc, cl, record_no, service, nmem,
-        se->mergekey);
-    if (!mergekey_norm)
+            r = ingest_sub_record(cl, xdoc, root, record_no, nmem, sdb, mk);
+        }
+    }
+    else
     {
-        session_log(se, YLOG_WARN, "Got no mergekey");
-        xmlFreeDoc(xdoc);
-        return -1;
+        session_log(se, YLOG_WARN, "Bad pz root element: %s",
+                    (const char *) root->name);
+        r = -1;
     }
-    session_enter(se, "ingest_record");
-    if (client_get_session(cl) == se && se->relevance)
-        ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm);
-    session_leave(se, "ingest_record");
-
     xmlFreeDoc(xdoc);
-    return ret;
+    return r;
 }
 
+
 //    struct conf_metadata *ser_md = &service->metadata[md_field_id];
 //    struct record_metadata *rec_md = record->metadata[md_field_id];
-static int match_metadata_local(struct conf_metadata *ser_md,
+static int match_metadata_local(struct conf_service *service,
+                                struct conf_metadata *ser_md,
                                 struct record_metadata *rec_md0,
                                 char **values, int num_v)
 {
     int i;
     struct record_metadata *rec_md = rec_md0;
+    WRBUF val_wr = 0;
+    WRBUF text_wr = wrbuf_alloc();
     for (i = 0; i < num_v; )
     {
         if (rec_md)
@@ -1745,21 +1826,29 @@ static int match_metadata_local(struct conf_metadata *ser_md,
             }
             else
             {
-                yaz_log(YLOG_DEBUG, "cmp: '%s' '%s'", rec_md->data.text.disp, values[i]);
-                if (!strcmp(rec_md->data.text.disp, values[i]))
+                if (!val_wr)
                 {
-                    // Value equals, should not be filtered.
-                    break;
+                    val_wr = wrbuf_alloc();
+                    mergekey_norm_wr(service->charsets, val_wr, values[i]);
                 }
+                wrbuf_rewind(text_wr);
+                mergekey_norm_wr(service->charsets, text_wr,
+                                 rec_md->data.text.disp);
+                if (!strcmp(wrbuf_cstr(val_wr), wrbuf_cstr(text_wr)))
+                    break;
             }
             rec_md = rec_md->next;
         }
         else
         {
             rec_md = rec_md0;
+            wrbuf_destroy(val_wr);
+            val_wr = 0;
             i++;
         }
     }
+    wrbuf_destroy(val_wr);
+    wrbuf_destroy(text_wr);
     return i < num_v ? 1 : 0;
 }
 
@@ -1795,7 +1884,8 @@ int session_check_cluster_limit(struct session *se, struct record_cluster *rec)
                 nmem_strsplit_escape2(nmem_tmp, "|", value, &values,
                                       &num, 1, '\\', 1);
 
-                if (!match_metadata_local(&service->metadata[md_field_id],
+                if (!match_metadata_local(service,
+                                          &service->metadata[md_field_id],
                                           rec->metadata[md_field_id],
                                           values, num))
                 {
@@ -1837,6 +1927,7 @@ static int check_limit_local(struct client *cl,
                  md_field_id++)
             {
                 if (match_metadata_local(
+                        service,
                         &service->metadata[md_field_id],
                         record->metadata[md_field_id],
                         values, num_v))
@@ -1854,6 +1945,7 @@ static int check_limit_local(struct client *cl,
                 break;
             }
             if (!match_metadata_local(
+                    service,
                     &service->metadata[md_field_id],
                     record->metadata[md_field_id],
                     values, num_v))
@@ -1870,7 +1962,7 @@ static int ingest_to_cluster(struct client *cl,
                              xmlDoc *xdoc,
                              xmlNode *root,
                              int record_no,
-                             const char *mergekey_norm)
+                             struct record_metadata_attr *merge_keys)
 {
     xmlNode *n;
     xmlChar *type = 0;
@@ -1881,6 +1973,9 @@ static int ingest_to_cluster(struct client *cl,
     struct record_cluster *cluster;
     struct record_metadata **metadata0;
     struct session_database *sdb = client_get_database(cl);
+    NMEM ingest_nmem = 0;
+    char **rank_values = 0;
+    int rank_num = 0;
     struct record *record = record_create(se->nmem,
                                           service->num_metadata,
                                           service->num_sortkeys, cl,
@@ -1940,6 +2035,18 @@ static int ingest_to_cluster(struct client *cl,
                             "for element '%s'", value, type);
                 continue;
             }
+
+            if (ser_md->type == Metadata_type_generic)
+            {
+                WRBUF w = wrbuf_alloc();
+                if (relevance_snippet(se->relevance,
+                                      (char*) value, ser_md->name, w))
+                    rec_md->data.text.snippet = nmem_strdup(se->nmem,
+                                                            wrbuf_cstr(w));
+                wrbuf_destroy(w);
+            }
+
+
             wheretoput = &record->metadata[md_field_id];
             while (*wheretoput)
                 wheretoput = &(*wheretoput)->next;
@@ -1957,10 +2064,16 @@ static int ingest_to_cluster(struct client *cl,
             xmlFree(value);
         return -2;
     }
-    cluster = reclist_insert(se->reclist, service, record,
-                             mergekey_norm, &se->total_merged);
+    cluster = reclist_insert(se->reclist, se->relevance, service, record,
+                             merge_keys, &se->total_merged);
     if (!cluster)
+    {
+        if (type)
+            xmlFree(type);
+        if (value)
+            xmlFree(value);
         return 0; // complete match with existing record
+    }
 
     {
         const char *use_term_factor_str =
@@ -1980,14 +2093,19 @@ static int ingest_to_cluster(struct client *cl,
         session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid,
                     sdb->database->id, record_no);
 
-
-    relevance_newrec(se->relevance, cluster);
-
     // original metadata, to check if first existence of a field
     metadata0 = xmalloc(sizeof(*metadata0) * service->num_metadata);
     memcpy(metadata0, cluster->metadata,
            sizeof(*metadata0) * service->num_metadata);
 
+    ingest_nmem = nmem_create();
+    if (se->rank)
+    {
+        yaz_log(YLOG_LOG, "local in sort : %s", se->rank);
+        nmem_strsplit_escape2(ingest_nmem, ",", se->rank, &rank_values,
+                              &rank_num, 1, '\\', 1);
+    }
+
     // now parsing XML record and adding data to cluster or record metadata
     for (n = root->children; n; n = n->next)
     {
@@ -2008,8 +2126,8 @@ static int ingest_to_cluster(struct client *cl,
             struct record_metadata *rec_md = 0;
             int md_field_id = -1;
             int sk_field_id = -1;
-            const char *rank;
-            xmlChar *xml_rank;
+            const char *rank = 0;
+            xmlChar *xml_rank = 0;
 
             type = xmlGetProp(n, (xmlChar *) "type");
             value = xmlNodeListGetString(xdoc, n->children, 1);
@@ -2039,8 +2157,28 @@ static int ingest_to_cluster(struct client *cl,
             if (!rec_md)
                 continue;
 
-            xml_rank = xmlGetProp(n, (xmlChar *) "rank");
-            rank = xml_rank ? (const char *) xml_rank : ser_md->rank;
+            if (rank_num)
+            {
+                int i;
+                for (i = 0; i < rank_num; i++)
+                {
+                    const char *val = rank_values[i];
+                    const char *cp = strchr(val, '=');
+                    if (!cp)
+                        continue;
+                    if ((cp - val) == strlen((const char *) type)
+                        && !memcmp(val, type, cp - val))
+                    {
+                        rank = cp + 1;
+                        break;
+                    }
+                }
+            }
+            else
+            {
+                xml_rank = xmlGetProp(n, (xmlChar *) "rank");
+                rank = xml_rank ? (const char *) xml_rank : ser_md->rank;
+            }
 
             wheretoput = &cluster->metadata[md_field_id];
 
@@ -2139,7 +2277,6 @@ static int ingest_to_cluster(struct client *cl,
                 relevance_countwords(se->relevance, cluster,
                                      (char *) value, rank, ser_md->name);
             }
-
             // construct facets ... unless the client already has reported them
             if (ser_md->termlist && !client_has_facet(cl, (char *) type))
             {
@@ -2179,6 +2316,7 @@ static int ingest_to_cluster(struct client *cl,
     if (value)
         xmlFree(value);
 
+    nmem_destroy(ingest_nmem);
     xfree(metadata0);
     relevance_donerecord(se->relevance, cluster);
     se->total_records++;