X-Git-Url: http://git.indexdata.com/?p=pazpar2-moved-to-github.git;a=blobdiff_plain;f=src%2Freclists.c;h=e96ee929cfcf687a445095ecfa850bf2eec3acdf;hp=738fdc481621d340533d4abc3949ec8d394840aa;hb=HEAD;hpb=d7dc14dcdfbd1ecdc805a0d649203f3b9888749c diff --git a/src/reclists.c b/src/reclists.c index 738fdc4..e96ee92 100644 --- a/src/reclists.c +++ b/src/reclists.c @@ -1,7 +1,5 @@ -/* $Id: reclists.c,v 1.23 2007-09-10 16:25:50 adam Exp $ - Copyright (c) 2006-2007, Index Data. - -This file is part of Pazpar2. +/* This file is part of Pazpar2. + Copyright (C) Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -14,92 +12,67 @@ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with Pazpar2; see the file LICENSE. If not, write to the -Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA -02111-1307, USA. - */ +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ #include #if HAVE_CONFIG_H -#include +#include #endif #include -#include "pazpar2.h" +#include "ppmutex.h" +#include "session.h" #include "reclists.h" +#include "jenkins_hash.h" -extern struct parameters global_parameters; - -// Not threadsafe -static struct reclist_sortparms *sortparms = 0; - -struct reclist_bucket +struct reclist { - struct record_cluster *record; - struct reclist_bucket *next; + struct reclist_bucket **hashtable; + unsigned hash_size; + + int num_records; + struct record_cluster *sorted_list; + struct record_cluster *sorted_ptr; + NMEM nmem; + YAZ_MUTEX mutex; }; -#if 0 -struct reclist_sortparms * -reclist_sortparms_insert_field_id(NMEM nmem, - struct reclist_sortparms **sortparms, - int field_id, - enum conf_sortkey_type type, - int increasing) +struct reclist_bucket { - struct reclist_sortparms * tmp_rlsp = 0; - // assert(nmem); - - if(!sortparms || field_id < 0) - return 0; - - // construct new reclist_sortparms - tmp_rlsp = nmem_malloc(nmem, sizeof(struct reclist_sortparms)); - tmp_rlsp->offset = field_id; - tmp_rlsp->type = type; - tmp_rlsp->increasing = increasing; - - - // insert in *sortparms place, moving *sortparms one down the list - tmp_rlsp->next = *sortparms; - *sortparms = tmp_rlsp; - - return *sortparms; + struct record_cluster *record; + struct reclist_bucket *hash_next; }; -#endif -#if 0 -struct reclist_sortparms * -reclist_sortparms_insert(NMEM nmem, - struct reclist_sortparms **sortparms, - struct conf_service * service, - const char * name, - int increasing) +static void append_merge_keys(struct record_metadata_attr **p, + const struct record_metadata_attr *a, + NMEM nmem) { - int field_id = 0; - - if (!sortparms || !service || !name) - return 0; - - field_id = conf_service_sortkey_field_id(service, name); - - if (-1 == field_id) - return 0; - - return reclist_sortparms_insert_field_id(nmem, sortparms, field_id, - service->sortkeys[field_id].type, - increasing); -}; -#endif - + for (; a; a = a->next) + { + struct record_metadata_attr **pi = p; + for (; *pi; pi = &(*pi)->next) + if (!strcmp((*pi)->value, a->value)) + break; + if (!*pi) + { + *pi = (struct record_metadata_attr *) nmem_malloc(nmem, sizeof(**p)); + (*pi)->name = nmem_strdup_null(nmem, a->name); + (*pi)->value = nmem_strdup_null(nmem, a->value); + (*pi)->next = 0; + } + } +} -struct reclist_sortparms *reclist_parse_sortparms(NMEM nmem, const char *parms) +struct reclist_sortparms *reclist_parse_sortparms(NMEM nmem, const char *parms, + struct conf_service *service) { struct reclist_sortparms *res = 0; struct reclist_sortparms **rp = &res; - struct conf_service *service = config->servers->service; if (strlen(parms) > 256) return 0; @@ -108,54 +81,84 @@ struct reclist_sortparms *reclist_parse_sortparms(NMEM nmem, const char *parms) char parm[256]; char *pp; const char *cpp; - int increasing; + int increasing = 0; int i; int offset = 0; - enum conf_sortkey_type type; + enum conf_metadata_type type = Metadata_type_generic; struct reclist_sortparms *new; if (!(cpp = strchr(parms, ','))) cpp = parms + strlen(parms); - strncpy(parm, parms, cpp - parms); + strncpy(parm, parms, cpp - parms); parm[cpp-parms] = '\0'; if ((pp = strchr(parm, ':'))) { - increasing = pp[1] == '1' ? 1 : 0; + if (pp[1] == '1') + increasing = 1; + else if (pp[1] == '0') + increasing = 0; + else + { + yaz_log(YLOG_FATAL, "Bad sortkey modifier: %s", parm); + return 0; + } + + if (pp[2]) + { + if (pp[2] == 'p') + type = Metadata_type_position; + else + yaz_log(YLOG_FATAL, "Bad sortkey modifier: %s", parm); + } *pp = '\0'; } - else - increasing = 0; - if (!strcmp(parm, "relevance")) + if (type != Metadata_type_position) { - type = Metadata_sortkey_relevance; - } - else - { - for (i = 0; i < service->num_sortkeys; i++) + if (!strcmp(parm, "relevance")) { - struct conf_sortkey *sk = &service->sortkeys[i]; - if (!strcmp(sk->name, parm)) + type = Metadata_type_relevance; + } + else if (!strcmp(parm, "position")) + { + type = Metadata_type_position; + } + else if (!strcmp(parm, "retrieval")) + { + type = Metadata_type_retrieval; + } + else if (service) + { + for (i = 0; i < service->num_sortkeys; i++) { - type = sk->type; - if (type == Metadata_sortkey_skiparticle) - type = Metadata_sortkey_string; - break; + struct conf_sortkey *sk = &service->sortkeys[i]; + if (!strcmp(sk->name, parm)) + { + type = sk->type; + break; + } } + if (i >= service->num_sortkeys) + { + yaz_log(YLOG_FATAL, "Sortkey not defined in service: %s", + parm); + return 0; + } + offset = i; } - if (i >= service->num_sortkeys) + else { - yaz_log(YLOG_FATAL, "Bad sortkey: %s", parm); + yaz_log(YLOG_FATAL, "Sortkey not defined in service: %s", + parm); return 0; } - else - offset = i; } new = *rp = nmem_malloc(nmem, sizeof(struct reclist_sortparms)); new->next = 0; new->offset = offset; new->type = type; new->increasing = increasing; + new->name = nmem_strdup(nmem, parm); rp = &new->next; if (*(parms = cpp)) parms++; @@ -167,6 +170,7 @@ static int reclist_cmp(const void *p1, const void *p2) { struct record_cluster *r1 = (*(struct record_cluster**) p1); struct record_cluster *r2 = (*(struct record_cluster**) p2); + struct reclist_sortparms *sortparms = r1->sort_parms; struct reclist_sortparms *s; int res = 0; @@ -174,175 +178,405 @@ static int reclist_cmp(const void *p1, const void *p2) { union data_types *ut1 = r1->sortkeys[s->offset]; union data_types *ut2 = r2->sortkeys[s->offset]; + const char *s1, *s2; switch (s->type) { - const char *s1, *s2; - - case Metadata_sortkey_relevance: - res = r2->relevance - r1->relevance; - break; - case Metadata_sortkey_string: - s1 = ut1 ? ut1->text.sort : ""; - s2 = ut2 ? ut2->text.sort : ""; - res = strcmp(s2, s1); - if (res) - { - if (s->increasing) - res *= -1; - } - break; - case Metadata_sortkey_numeric: - if (ut1 && ut2) - { - if (s->increasing) - res = ut1->number.min - ut2->number.min; - else - res = ut2->number.max - ut1->number.max; - } - else if (ut1 && !ut2) - res = -1; - else if (!ut1 && ut2) - res = 1; + case Metadata_type_relevance: + res = r1->relevance_score - r2->relevance_score; + break; + case Metadata_type_generic: + case Metadata_type_skiparticle: + s1 = ut1 ? ut1->text.sort : ""; + s2 = ut2 ? ut2->text.sort : ""; + res = strcmp(s1, s2); + break; + case Metadata_type_year: + case Metadata_type_date: + if (ut1 && ut2) + { + if (s->increasing) + res = ut1->number.min - ut2->number.min; else + res = ut1->number.max - ut2->number.max; + } + else if (ut1 && !ut2) + { + res = -1; /* without date/year: last! */ + continue; + } + else if (!ut1 && ut2) + { + res = 1; /* without date/year: last! */ + continue; + } + else + res = 0; + break; + case Metadata_type_position: + if (r1->records && r2->records) + { + int pos1 = 0, pos2 = 0; + struct record *rec; + for (rec = r1->records; rec; rec = rec->next) + if (pos1 == 0 || rec->position < pos1) + pos1 = rec->position; + for (rec = r2->records; rec; rec = rec->next) + if (pos2 == 0 || rec->position < pos2) + pos2 = rec->position; + res = pos1 - pos2; + continue; + } + break; + case Metadata_type_float: + if (ut1 && ut2) + { + if (ut1->fnumber == ut2->fnumber) res = 0; - break; - default: - yaz_log(YLOG_WARN, "Bad sort type: %d", s->type); + else if (ut1->fnumber > ut2->fnumber) + res = 1; + else + res = -1; + } + else if (ut1) + res = 1; + else if (ut2) + res = -1; + else res = 0; + break; + case Metadata_type_retrieval: + res = r1->retrieval_position - r2->retrieval_position; + break; } + if (res && !s->increasing) + res *= -1; } + if (res == 0) + res = strcmp(r1->recid, r2->recid); return res; } +void reclist_limit(struct reclist *l, struct session *se, int lazy) +{ + unsigned i; + int num = 0; + struct record_cluster **pp = &l->sorted_list; + + reclist_enter(l); + + if (!lazy || !*pp) + { + for (i = 0; i < l->hash_size; i++) + { + struct reclist_bucket *p; + for (p = l->hashtable[i]; p; p = p->hash_next) + p->record->sorted_next = 0; + } + for (i = 0; i < l->hash_size; i++) + { + struct reclist_bucket *p; + for (p = l->hashtable[i]; p; p = p->hash_next) + { + if (session_check_cluster_limit(se, p->record)) + { + if (!p->record->sorted_next) + { + *pp = p->record; + pp = &p->record->sorted_next; + *pp = p->record; /* signal already in use */ + num++; + } + } + } + } + *pp = 0; + } + l->num_records = num; + reclist_leave(l); +} + void reclist_sort(struct reclist *l, struct reclist_sortparms *parms) { - sortparms = parms; - qsort(l->flatlist, l->num_records, - sizeof(struct record_cluster*), reclist_cmp); - reclist_rewind(l); + struct record_cluster **flatlist = xmalloc(sizeof(*flatlist) * l->num_records); + struct record_cluster *ptr; + struct record_cluster **prev; + int i = 0; + + reclist_enter(l); + + ptr = l->sorted_list; + prev = &l->sorted_list; + while (ptr) + { + ptr->sort_parms = parms; + flatlist[i] = ptr; + ptr = ptr->sorted_next; + i++; + } + assert(i == l->num_records); + + qsort(flatlist, l->num_records, sizeof(*flatlist), reclist_cmp); + for (i = 0; i < l->num_records; i++) + { + *prev = flatlist[i]; + prev = &flatlist[i]->sorted_next; + } + *prev = 0; + + xfree(flatlist); + + reclist_leave(l); } struct record_cluster *reclist_read_record(struct reclist *l) { - if (l && l->pointer < l->num_records) - return l->flatlist[l->pointer++]; + if (l && l->sorted_ptr) + { + struct record_cluster *t = l->sorted_ptr; + l->sorted_ptr = l->sorted_ptr->sorted_next; + return t; + } else return 0; } +void reclist_enter(struct reclist *l) +{ + yaz_mutex_enter(l->mutex); + l->sorted_ptr = l->sorted_list; +} + + +void reclist_leave(struct reclist *l) +{ + yaz_mutex_leave(l->mutex); + l->sorted_ptr = l->sorted_list; +} + void reclist_rewind(struct reclist *l) { - if (l) - l->pointer = 0; + l->sorted_ptr = l->sorted_list; } -// Jenkins one-at-a-time hash (from wikipedia) -static unsigned int hash(const unsigned char *key) + +struct reclist *reclist_create(NMEM nmem) { - unsigned int hash = 0; + struct reclist *res = nmem_malloc(nmem, sizeof(struct reclist)); + res->hash_size = 399; + res->hashtable + = nmem_malloc(nmem, res->hash_size * sizeof(struct reclist_bucket*)); + memset(res->hashtable, 0, res->hash_size * sizeof(struct reclist_bucket*)); + res->nmem = nmem; + + res->sorted_ptr = 0; + res->sorted_list = 0; + + res->num_records = 0; + res->mutex = 0; + pazpar2_mutex_create(&res->mutex, "reclist"); + return res; +} - while (*key) +void reclist_destroy(struct reclist *l) +{ + if (l) { - hash += *(key++); - hash += (hash << 10); - hash ^= (hash >> 6); + unsigned i; + for (i = 0; i < l->hash_size; i++) + { + struct reclist_bucket *p; + for (p = l->hashtable[i]; p; p = p->hash_next) + { + wrbuf_destroy(p->record->relevance_explain1); + wrbuf_destroy(p->record->relevance_explain2); + p->record->relevance_explain1 = 0; + p->record->relevance_explain2 = 0; + } + } + yaz_mutex_destroy(&l->mutex); } - hash += (hash << 3); - hash ^= (hash >> 11); - hash += (hash << 15); - return hash; } -struct reclist *reclist_create(NMEM nmem, int numrecs) +int reclist_get_num_records(struct reclist *l) { - int hashsize = 1; - struct reclist *res; - - assert(numrecs); - while (hashsize < numrecs) - hashsize <<= 1; - res = nmem_malloc(nmem, sizeof(struct reclist)); - res->hashtable - = nmem_malloc(nmem, hashsize * sizeof(struct reclist_bucket*)); - memset(res->hashtable, 0, hashsize * sizeof(struct reclist_bucket*)); - res->hashtable_size = hashsize; - res->nmem = nmem; - res->hashmask = hashsize - 1; // Creates a bitmask + if (l) + return l->num_records; + return 0; +} - res->num_records = 0; - res->flatlist = nmem_malloc(nmem, numrecs * sizeof(struct record_cluster*)); - res->flatlist_size = numrecs; +static void merge_cluster(struct reclist *l, + struct relevance *r, + struct record_cluster *dst, + struct record_cluster *src) +{ + struct record_metadata_attr *mkl; + struct record **rp = &dst->records; + for (; *rp; rp = &(*rp)->next) + ; + *rp = src->records; - return res; + for (mkl = src->merge_keys; mkl; mkl = mkl->next) + { + const char *merge_key = mkl->value; + unsigned int bucket = + jenkins_hash((unsigned char*) merge_key) % l->hash_size; + struct reclist_bucket *p; + + for (p = l->hashtable[bucket]; p; p = p->hash_next) + if (p->record == src) + p->record = dst; + } + + /* not merging metadata and sortkeys yet */ + + relevance_mergerec(r, dst, src); + + wrbuf_puts(dst->relevance_explain1, wrbuf_cstr(src->relevance_explain1)); + wrbuf_puts(dst->relevance_explain2, wrbuf_cstr(src->relevance_explain2)); + + wrbuf_destroy(src->relevance_explain1); + src->relevance_explain1 = 0; + wrbuf_destroy(src->relevance_explain2); + src->relevance_explain2 = 0; + + append_merge_keys(&dst->merge_keys, src->merge_keys, l->nmem); +} + +static struct record_cluster *new_cluster( + struct reclist *l, + struct relevance *r, + struct conf_service *service, + struct record *record, + struct record_metadata_attr *merge_keys + ) +{ + struct record_cluster *cluster; + cluster = nmem_malloc(l->nmem, sizeof(*cluster)); + + record->next = 0; + cluster->retrieval_position = l->num_records; + cluster->records = record; + cluster->merge_keys = 0; + append_merge_keys(&cluster->merge_keys, merge_keys, l->nmem); + cluster->relevance_score = 0; + cluster->recid = cluster->merge_keys->value; + cluster->metadata = + nmem_malloc(l->nmem, + sizeof(struct record_metadata*) * service->num_metadata); + memset(cluster->metadata, 0, + sizeof(struct record_metadata*) * service->num_metadata); + cluster->sortkeys = + nmem_malloc(l->nmem, sizeof(struct record_metadata*) * service->num_sortkeys); + memset(cluster->sortkeys, 0, + sizeof(union data_types*) * service->num_sortkeys); + relevance_newrec(r, cluster); + cluster->relevance_explain1 = wrbuf_alloc(); + cluster->relevance_explain2 = wrbuf_alloc(); + /* attach to hash list */ + l->num_records++; + l->sorted_list = l->sorted_ptr = 0; + return cluster; } // Insert a record. Return record cluster (newly formed or pre-existing) -struct record_cluster *reclist_insert( struct reclist *l, - struct conf_service *service, - struct record *record, - char *merge_key, int *total) +struct record_cluster *reclist_insert(struct reclist *l, + struct relevance *r, + struct conf_service *service, + struct record *record, + struct record_metadata_attr *merge_keys, + int *total) { - unsigned int bucket; - struct reclist_bucket **p; struct record_cluster *cluster = 0; - + struct record_metadata_attr *mkl = merge_keys; + assert(service); assert(l); assert(record); - assert(merge_key); + assert(merge_keys); assert(total); - bucket = hash((unsigned char*) merge_key) & l->hashmask; + yaz_mutex_enter(l->mutex); - for (p = &l->hashtable[bucket]; *p; p = &(*p)->next) + for (; mkl; mkl = mkl->next) { - // We found a matching record. Merge them - if (!strcmp(merge_key, (*p)->record->merge_key)) + const char *merge_key = mkl->value; + unsigned int bucket = + jenkins_hash((unsigned char*) merge_key) % l->hash_size; + struct reclist_bucket **p; + struct reclist_bucket *rb = 0; + + for (p = &l->hashtable[bucket]; *p; p = &(*p)->hash_next) { - struct record_cluster *existing = (*p)->record; - record->next = existing->records; - existing->records = record; - cluster = existing; - break; + struct record_metadata_attr *mkr = (*p)->record->merge_keys; + for (; mkr; mkr = mkr->next) + { + // We found a matching record. Merge them + if (!strcmp(merge_key, mkr->value)) + { + rb = *p; + if (!cluster) + { + struct record **re; + for (re = &rb->record->records; *re; re = &(*re)->next) + { + if ((*re)->client == record->client && + record_compare(record, *re, service)) + { + yaz_mutex_leave(l->mutex); + return 0; + } + } + cluster = rb->record; + *re = record; + record->next = 0; + } + else + { + if (cluster != rb->record) + { + assert(rb->record->relevance_explain1); + merge_cluster(l, r, cluster, rb->record); + (*total)--; + } + } + } + } + } + if (!cluster) + { + (*total)++; + cluster = new_cluster(l, r, service, record, merge_keys); + } + + if (!rb) + { + rb = nmem_malloc(l->nmem, sizeof(*rb)); + rb->record = cluster; + rb->hash_next = 0; + + *p = rb; } } - if (!cluster && l->num_records < l->flatlist_size) - { - struct reclist_bucket *new = - nmem_malloc(l->nmem, sizeof(struct reclist_bucket)); - struct record_cluster *newc = - nmem_malloc(l->nmem, sizeof(struct record_cluster)); - - record->next = 0; - new->record = newc; - new->next = 0; - newc->records = record; - newc->merge_key = merge_key; - newc->relevance = 0; - newc->term_frequency_vec = 0; - newc->recid = merge_key; - (*total)++; - newc->metadata = nmem_malloc(l->nmem, - sizeof(struct record_metadata*) * service->num_metadata); - memset(newc->metadata, 0, - sizeof(struct record_metadata*) * service->num_metadata); - newc->sortkeys = nmem_malloc(l->nmem, - sizeof(struct record_metadata*) * service->num_sortkeys); - memset(newc->sortkeys, 0, - sizeof(union data_types*) * service->num_sortkeys); - - *p = new; - l->flatlist[l->num_records++] = newc; - cluster = newc; - } + yaz_mutex_leave(l->mutex); return cluster; } - +int reclist_sortparms_cmp(struct reclist_sortparms *sort1, struct reclist_sortparms *sort2) +{ + int rc; + if (sort1 == sort2) + return 0; + if (sort1 == 0 || sort2 == 0) + return 1; + rc = strcmp(sort1->name, sort2->name) || sort1->increasing != sort2->increasing || sort1->type != sort2->type; + return rc; +} /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +