--- /dev/null
+/*
+
+The University of Liverpool
+
+Modifications to Zebra 1.1 / YAZ 1.7 to enable ranking
+by attribute weight.
+
+Copyright (c) 2001-2002 The University of Liverpool. All
+rights reserved.
+
+Licensed under the Academic Free License version 1.1.
+http://opensource.org/licenses/academic.php
+
+$Id: livcode.c,v 1.1 2003-03-26 16:41:48 adam Exp $
+
+*/
+
+#include <stdlib.h>
+#include <stdio.h>
+#ifdef WIN32
+#include <process.h>
+#else
+#include <unistd.h>
+#endif
+#include <assert.h>
+
+#include "index.h"
+#include "zserver.h"
+
+/*
+** These functions/routines
+** 1. reads in and builds a linked list of rank attr/rank score pairs
+** 2. expand a simple query into a paired list of complex/simple nodes.
+*/
+
+typedef struct rstype
+{
+ struct rstype *next_rsnode ;
+ int rank ;
+ int score ;
+ char *rankstr ;
+} rsnode, *refrsnode ;
+
+refrsnode start_rsnode = NULL ;
+
+/*
+** Function/Routine prototypes
+*/
+static int search_for_score( char *rankstr ) ;
+static char *search_for_rankstr( int rank ) ;
+static int search_for_rank( int rank ) ;
+static refrsnode set_rsnode( int rank, int score ) ;
+static int read_zrank_file(ZebraHandle zh) ;
+
+static void convert_simple2complex(ZebraHandle zh, Z_RPNStructure *rpnstruct ) ;
+static void walk_complex_query(ZebraHandle zh, Z_RPNStructure *rpnstruct ) ;
+static Z_Complex *expand_query(ZebraHandle zh, Z_Operand *thisop ) ;
+static Z_Complex *set_1complex_1operand( Z_Complex *comp,Z_Operand *simp ) ;
+static Z_Complex *set_2operands( Z_Operand *sim1,Z_Operand *sim2 ) ;
+static Z_Operand *set_operand( Z_Operand *thisop, int newattr ) ;
+static int check_operand_attrs( Z_Operand *thisop ) ;
+
+/*
+** search_for_score()
+** given a rank-string traverse down the linked list ;
+** return its score if found otherwise return -1.
+*/
+int search_for_score( char *rankstr )
+{
+ refrsnode node = start_rsnode ;
+ int rank ;
+
+ if ( sscanf( rankstr,"%d",&rank ) )
+ {
+ while ( node )
+ {
+ if ( node->rank == rank ) return node->score ;
+ node = node->next_rsnode ;
+ }
+ }
+ return -1 ;
+}
+
+/*
+** search_for_rankstr()
+** given a rank traverse down the linked list ;
+** return its string if found otherwise return NULL.
+*/
+char *search_for_rankstr( int rank )
+{
+ refrsnode node = start_rsnode ;
+
+ while ( node )
+ {
+ if ( node->rank == rank ) return node->rankstr ;
+ node = node->next_rsnode ;
+ }
+ return "rank" ;
+}
+
+/*
+** search_for_rank()
+** given a rank traverse down the linked list ;
+** return 1 if found otherwise return 0.
+*/
+int search_for_rank( int rank )
+{
+ refrsnode node = start_rsnode ;
+
+ while ( node )
+ {
+ if ( node->rank == rank ) return 1 ;
+ node = node->next_rsnode ;
+ }
+ return 0 ;
+}
+
+/*
+** set_rsnode()
+** given a rank and a score, build the rest of the rsnode structure.
+*/
+refrsnode set_rsnode( int rank, int score )
+{
+#define BUFFMAX 128
+ refrsnode node = (refrsnode)malloc( sizeof(rsnode) ) ;
+ char buff[BUFFMAX] ;
+
+ node->next_rsnode = NULL ;
+ node->rank = rank ;
+ node->score = score ;
+
+ sprintf( buff,"%d",rank ) ;
+ node->rankstr = (char *)malloc( strlen(buff)+1 ) ;
+ strcpy( node->rankstr, buff ) ;
+
+ return node ;
+}
+
+/*
+** read_zrank_file(zh)
+** read in the rankfile and build the rank/score linked list ;
+** return 0 : can't open the zebra config. file
+** return 0 : can't find the rankfile entry in the zebra config. file
+** return 0 : can't open the rankfile itself
+** return the number of distinct ranks read in.
+*/
+int read_zrank_file(ZebraHandle zh)
+{
+#define LINEMAX 256
+ char line[ LINEMAX ] ;
+ char rname[ LINEMAX ] ;
+ char *lineptr ;
+ FILE *ifd ;
+ int rank = 0 ;
+ int score = 0 ;
+ int numranks = 0 ;
+
+ /*
+ ** open the zebra configuration file and look for the "rankfile:"
+ ** entry which contains the path/name of the rankfile
+ */
+
+ const char *rankfile = res_get_def(zh->res, "rankfile", 0);
+ const char *profilePath = res_get_def(zh->res, "profilePath",
+ DEFAULT_PROFILE_PATH);
+
+ if (!rankfile)
+ {
+ yaz_log(LOG_LOG, "rankfile entry not found in config file" ) ;
+ return 0 ;
+ }
+ ifd = yaz_path_fopen(profilePath, rankfile, "r" ) ;
+ if ( ifd )
+ {
+ while ( (lineptr = fgets( line,LINEMAX,ifd )) )
+ {
+ if ( sscanf( lineptr,"rankfile: %s", rname ) == 1 )
+ rankfile = rname ;
+ }
+
+ /*
+ ** open the rankfile and read the rank/score pairs
+ ** ignore 1016
+ ** ignore duplicate ranks
+ ** ignore ranks without +ve scores
+ */
+ if ( rankfile )
+ {
+ if ( !(ifd = fopen( rankfile, "r" )) )
+ {
+ logf( LOG_LOG, "unable to open rankfile %s",rankfile ) ;
+ return 0;
+ }
+
+ while ( (lineptr = fgets( line,LINEMAX,ifd )) )
+ {
+ sscanf( lineptr,"%d : %d", &rank,&score ) ;
+ if ( ( score > 0 ) && ( rank != 1016 ) )
+ {
+ refrsnode new_rsnode ;
+
+ if ( search_for_rank( rank ) == 0 )
+ {
+ new_rsnode = set_rsnode( rank,score ) ;
+ new_rsnode->next_rsnode = start_rsnode ;
+ start_rsnode = new_rsnode ;
+ numranks++ ;
+ }
+ }
+ }
+ }
+ else
+ {
+ yaz_log(LOG_WARN|LOG_ERRNO, "unable to open config file (%s)",
+ rankfile);
+ }
+ }
+ return numranks ;
+}
+
+/*
+** set_operand()
+** build an operand "node" - hav to make a complete copy of thisop and
+** then insert newattr in the appropriate place
+**
+*/
+Z_Operand *set_operand( Z_Operand *thisop, int newattr )
+{
+ Z_Operand *operand ;
+ Z_AttributesPlusTerm *attributesplusterm ;
+ Z_AttributeList *attributelist ;
+ Z_AttributeElement *attributeelement ;
+ Z_AttributeElement *attrptr ;
+ Z_AttributeElement **attrptrptr ;
+ Z_Term *term ;
+ Odr_oct *general ;
+ int i ;
+
+ operand = (Z_Operand *)
+ malloc( sizeof( Z_Operand ) ) ;
+ attributesplusterm = (Z_AttributesPlusTerm *)
+ malloc( sizeof( Z_AttributesPlusTerm ) ) ;
+ attributelist = (Z_AttributeList *)
+ malloc( sizeof( Z_AttributeList ) ) ;
+ attributeelement = (Z_AttributeElement *)
+ malloc( sizeof( Z_AttributeElement ) ) ;
+ term = (Z_Term *)
+ malloc( sizeof( Z_Term ) ) ;
+ general = (Odr_oct *)
+ malloc( sizeof( Odr_oct ) ) ;
+
+ operand->which = Z_Operand_APT ;
+ operand->u.attributesPlusTerm = attributesplusterm ;
+
+ attributesplusterm->attributes = attributelist ;
+ attributesplusterm->term = term ;
+
+ attributelist->num_attributes = thisop->u.attributesPlusTerm->
+ attributes->num_attributes ;
+
+ attrptr = (Z_AttributeElement *) malloc( sizeof(Z_AttributeElement) *
+ attributelist->num_attributes ) ;
+ attrptrptr = (Z_AttributeElement **) malloc( sizeof(Z_AttributeElement) *
+ attributelist->num_attributes ) ;
+
+ attributelist->attributes = attrptrptr ;
+
+ for ( i = 0 ; i < attributelist->num_attributes ; i++ )
+ {
+ *attrptr = *thisop->u.attributesPlusTerm->attributes->attributes[i] ;
+
+ attrptr->attributeType = (int *)malloc( sizeof(int *) ) ;
+ *attrptr->attributeType = *thisop->u.attributesPlusTerm->attributes->
+ attributes[i]->attributeType;
+
+ attrptr->value.numeric = (int *)malloc( sizeof(int *) ) ;
+ *attrptr->value.numeric = *thisop->u.attributesPlusTerm->attributes->
+ attributes[i]->value.numeric;
+
+ if ( (*attrptr->attributeType == 1) &&
+ (*attrptr->value.numeric == 1016) )
+ {
+ *attrptr->value.numeric = newattr ;
+ }
+ *attrptrptr++ = attrptr++ ;
+ }
+
+ term->which = Z_Term_general ;
+ term->u.general = general ;
+
+ general->len = thisop->u.attributesPlusTerm->term->u.general->len ;
+ general->size = thisop->u.attributesPlusTerm->term->u.general->size ;
+ general->buf = malloc( general->size ) ;
+ strcpy( general->buf,
+ thisop->u.attributesPlusTerm->term->u.general->buf ) ;
+
+ return operand ;
+}
+
+/*
+** set_2operands()
+** build a complex "node" with two (simple) operand "nodes" as branches
+*/
+Z_Complex *set_2operands( Z_Operand *sim1,Z_Operand *sim2 )
+{
+ Z_Complex *top ;
+ Z_RPNStructure *s1 ;
+ Z_RPNStructure *s2 ;
+ Z_Operator *roperator ;
+
+ top = (Z_Complex *) malloc( sizeof( Z_Complex ) ) ;
+ s1 = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
+ s2 = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
+ roperator = (Z_Operator *) malloc( sizeof( Z_Operator ) ) ;
+
+ top->roperator = roperator ;
+ top->roperator->which = Z_Operator_or ;
+ top->roperator->u.op_or = odr_nullval() ;
+
+ top->s1 = s1 ;
+ top->s1->which = Z_RPNStructure_simple ;
+ top->s1->u.simple = sim1 ;
+
+ top->s2 = s2 ;
+ top->s2->which = Z_RPNStructure_simple ;
+ top->s2->u.simple = sim2 ;
+
+ return top ;
+}
+
+/*
+** set_1complex_1operand()
+** build a complex "node" with a complex "node" branch and an
+** operand "node" branch
+*/
+Z_Complex *set_1complex_1operand( Z_Complex *comp,Z_Operand *simp )
+{
+ Z_Complex *top ;
+ Z_RPNStructure *s1 ;
+ Z_RPNStructure *s2 ;
+ Z_Operator *roperator ;
+
+ top = (Z_Complex *) malloc( sizeof( Z_Complex ) ) ;
+ s1 = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
+ s2 = (Z_RPNStructure *)malloc( sizeof( Z_RPNStructure ) ) ;
+ roperator = (Z_Operator *) malloc( sizeof( Z_Operator ) ) ;
+
+ top->roperator = roperator ;
+ top->roperator->which = Z_Operator_or ;
+ top->roperator->u.op_or = odr_nullval() ;
+
+ top->s1 = s1 ;
+ top->s1->which = Z_RPNStructure_complex ;
+ top->s1->u.complex = comp ;
+
+ top->s2 = s2 ;
+ top->s2->which = Z_RPNStructure_simple ;
+ top->s2->u.simple = simp ;
+
+ return top ;
+}
+
+/*
+** expand_query()
+** expand a simple query into a number of complex queries
+*/
+Z_Complex *expand_query(ZebraHandle zh, Z_Operand *thisop )
+{
+ Z_Complex *top ;
+ int numattrs = 0 ;
+
+ /*
+ ** start_rsnode will be set if we have already read the rankfile
+ ** so don't bother again but we need to know the number of attributes
+ ** in the linked list so traverse it again to find out how many.
+ */
+ if ( start_rsnode )
+ {
+ refrsnode node = start_rsnode ;
+ while ( node )
+ {
+ numattrs++ ;
+ node = node->next_rsnode ;
+ }
+ }
+
+ /*
+ ** only expand the query if there are 2 or more attributes
+ */
+ if ( numattrs >= 2 )
+ {
+ refrsnode node = start_rsnode ;
+ int attr1 ;
+ int attr2 ;
+
+ attr1 = node->rank ; node = node->next_rsnode ;
+ attr2 = node->rank ; node = node->next_rsnode ;
+
+ /*
+ ** this is the special case and has to be done first because the
+ ** last complex node in the linear list has two simple nodes whereas
+ ** all the others have a complex and a simple.
+ */
+ top = set_2operands( set_operand( thisop,attr1 ),
+ set_operand( thisop,attr2 ) ) ;
+
+ /*
+ ** do the rest as complex/simple pairs
+ */
+ while ( node )
+ {
+ attr1 = node->rank ; node = node->next_rsnode ;
+ top = set_1complex_1operand( top,set_operand( thisop,attr1 ) ) ;
+ }
+ /*
+ ** finally add the 1016 rank attribute at the top of the tree
+ */
+ top = set_1complex_1operand( top,set_operand( thisop,1016 ) ) ;
+
+ return top ;
+ }
+ else return NULL ;
+}
+
+/*
+** check_operand_attrs()
+** loop through the attributes of a particular operand
+** return 1 if (type==1 && value==1016) && (type==2 && value==102)
+** otherwise return 0
+*/
+int check_operand_attrs( Z_Operand *thisop )
+{
+ Z_AttributeElement *attrptr ;
+ int cond1 = 0 ;
+ int cond2 = 0 ;
+ int numattrs ;
+ int i ;
+
+ numattrs = thisop->u.attributesPlusTerm->attributes->num_attributes ;
+
+ for ( i = 0 ; i < numattrs ; i++ )
+ {
+ attrptr = thisop->u.attributesPlusTerm->attributes->attributes[i] ;
+
+ if ( (*attrptr->attributeType == 1) &&
+ (*attrptr->value.numeric == 1016) )
+ cond1 = 1 ;
+
+ if ( (*attrptr->attributeType == 2) &&
+ (*attrptr->value.numeric == 102) )
+ cond2 = 1 ;
+ }
+
+ return (cond1 & cond2) ;
+}
+
+/*
+** convert_simple2complex()
+**
+*/
+void convert_simple2complex(ZebraHandle zh, Z_RPNStructure *rpnstruct )
+{
+ Z_Complex *complex = NULL ;
+ Z_Operand *operand = rpnstruct->u.simple ;
+
+ if ( check_operand_attrs( operand ) )
+ {
+ complex = expand_query(zh, operand ) ;
+
+ if ( complex )
+ {
+ /*
+ ** Everything is complete so replace the original
+ ** operand with the newly built complex structure
+ ** This is it ... no going back!!
+ */
+ rpnstruct->which = Z_RPNStructure_complex ;
+ rpnstruct->u.complex = complex ;
+ }
+ }
+}
+
+/*
+** walk_complex_query()
+** recursively traverse the tree expanding any simple queries we find
+*/
+void walk_complex_query(ZebraHandle zh, Z_RPNStructure *rpnstruct )
+{
+ if ( rpnstruct->which == Z_RPNStructure_simple )
+ {
+ convert_simple2complex(zh, rpnstruct ) ;
+ }
+ else
+ {
+ walk_complex_query(zh, rpnstruct->u.complex->s1 ) ;
+ walk_complex_query(zh, rpnstruct->u.complex->s2 ) ;
+ }
+}
+
+void zebra_livcode_transform(ZebraHandle zh, Z_RPNQuery *query)
+{
+ /*
+ ** Got a search request,
+ ** 1. if it is a simple query, see if it suitable for expansion
+ ** i.e. the attributes are of the form ...
+ ** (type==1 && value==1016) && (type==2 && value==102)
+ ** or
+ ** 2. if it is complex, traverse the complex query tree and expand
+ ** any simples simples as above
+ */
+#if LIV_CODE
+ Z_RPNStructure *rpnstruct = query->RPNStructure ;
+
+ if ( rpnstruct->which == Z_RPNStructure_simple )
+ {
+ convert_simple2complex(zh, rpnstruct ) ;
+ }
+ else if ( rpnstruct->which == Z_RPNStructure_complex )
+ {
+ walk_complex_query(zh, rpnstruct ) ;
+ }
+#endif
+}
+
+
+struct rank_class_info {
+ int dummy;
+};
+
+struct rank_term_info {
+ int local_occur;
+ int global_occur;
+ int global_inv;
+ int rank_flag;
+};
+
+struct rank_set_info {
+ int last_pos;
+ int no_entries;
+ int no_rank_entries;
+ struct rank_term_info *entries;
+};
+
+static int log2_int (unsigned g)
+{
+ int n = 0;
+ while ((g = g>>1))
+ n++;
+ return n;
+}
+
+/*
+ * create: Creates/Initialises this rank handler. This routine is
+ * called exactly once. The routine returns the class_handle.
+ */
+static void *create (ZebraHandle zh)
+{
+ struct rank_class_info *ci = (struct rank_class_info *)
+ xmalloc (sizeof(*ci));
+
+ logf (LOG_DEBUG, "livrank create");
+
+ read_zrank_file(zh) ;
+
+ return ci;
+}
+
+/*
+ * destroy: Destroys this rank handler. This routine is called
+ * when the handler is no longer needed - i.e. when the server
+ * dies. The class_handle was previously returned by create.
+ */
+static void destroy (struct zebra_register *reg, void *class_handle)
+{
+ struct rank_class_info *ci = (struct rank_class_info *) class_handle;
+
+ logf (LOG_DEBUG, "livrank destroy");
+ xfree (ci);
+}
+
+
+/*
+ * begin: Prepares beginning of "real" ranking. Called once for
+ * each result set. The returned handle is a "set handle" and
+ * will be used in each of the handlers below.
+ */
+static void *begin (struct zebra_register *reg, void *class_handle, RSET rset)
+{
+ struct rank_set_info *si = (struct rank_set_info *) xmalloc (sizeof(*si));
+ int i;
+
+ logf (LOG_DEBUG, "livrank begin");
+ si->no_entries = rset->no_rset_terms;
+ si->no_rank_entries = 0;
+ si->entries = (struct rank_term_info *)
+ xmalloc (sizeof(*si->entries)*si->no_entries);
+ for (i = 0; i < si->no_entries; i++)
+ {
+ const char *flags = rset->rset_terms[i]->flags;
+ int g = rset->rset_terms[i]->nn;
+ const char *cp = strstr(flags, ",u=");
+
+ si->entries[i].rank_flag = 1;
+ if (cp)
+ {
+ char *t = search_for_rankstr(atoi(cp+3));
+ if (t)
+ si->entries[i].rank_flag = search_for_score(t) ;
+ }
+ if ( si->entries[i].rank_flag )
+ (si->no_rank_entries)++;
+
+ si->entries[i].local_occur = 0;
+ si->entries[i].global_occur = g;
+ si->entries[i].global_inv = 32 - log2_int (g);
+ logf (LOG_DEBUG, "-------- %d ------", 32 - log2_int (g));
+ }
+ return si;
+}
+
+/*
+ * end: Terminates ranking process. Called after a result set
+ * has been ranked.
+ */
+static void end (struct zebra_register *reg, void *set_handle)
+{
+ struct rank_set_info *si = (struct rank_set_info *) set_handle;
+ logf (LOG_DEBUG, "livrank end");
+ xfree (si->entries);
+ xfree (si);
+}
+
+/*
+ * add: Called for each word occurence in a result set. This routine
+ * should be as fast as possible. This routine should "incrementally"
+ * update the score.
+ */
+static void add (void *set_handle, int seqno, int term_index)
+{
+ struct rank_set_info *si = (struct rank_set_info *) set_handle;
+ logf (LOG_DEBUG, "rank-1 add seqno=%d term_index=%d", seqno, term_index);
+ si->last_pos = seqno;
+ si->entries[term_index].local_occur++;
+}
+
+/*
+ * calc: Called for each document in a result. This handler should
+ * produce a score based on previous call(s) to the add handler. The
+ * score should be between 0 and 1000. If score cannot be obtained
+ * -1 should be returned.
+ */
+static int calc (void *set_handle, int sysno)
+{
+ int i, lo, divisor, score = 0;
+ struct rank_set_info *si = (struct rank_set_info *) set_handle;
+
+ logf (LOG_DEBUG, "livrank calc sysno=%d", sysno);
+
+ if (!si->no_rank_entries)
+ return -1;
+ for (i = 0; i < si->no_entries; i++)
+ {
+ score += si->entries[i].local_occur * si->entries[i].rank_flag ;
+ }
+ for (i = 0; i < si->no_entries; i++)
+ if (si->entries[i].rank_flag && (lo = si->entries[i].local_occur))
+ score += (8+log2_int (lo)) * si->entries[i].global_inv;
+ score *= 34;
+ divisor = si->no_rank_entries * (8+log2_int (si->last_pos/si->no_entries));
+ score = score / divisor;
+ if (score > 1000)
+ score = 1000;
+ for (i = 0; i < si->no_entries; i++)
+ si->entries[i].local_occur = 0;
+ return score;
+}
+
+/*
+ * Pseudo-meta code with sequence of calls as they occur in a
+ * server. Handlers are prefixed by --:
+ *
+ * server init
+ * -- create
+ * foreach search
+ * rank result set
+ * -- begin
+ * foreach record
+ * foreach word
+ * -- add
+ * -- calc
+ * -- end
+ * -- destroy
+ * server close
+ */
+
+static struct rank_control rank_control = {
+ "livrank",
+ create,
+ destroy,
+ begin,
+ end,
+ calc,
+ add,
+};
+
+struct rank_control *rankliv_class = &rank_control;
projects / idzebra-moved-to-github.git / commitdiff