/* Copyright (C) 2006, Index Data ApS
* See the file LICENSE for details.
*
- * $Id: nfa.c,v 1.1 2006-05-03 09:04:33 heikki Exp $
+ * $Id: nfa.c,v 1.7 2006-05-05 14:02:27 heikki Exp $
+ */
+
+/**
+ * \file nfa.c
+ * \brief NFA for character set normalizing
*
* This is a simple NFA-based system for character set normalizing
* in yaz and zebra. Unlike standard NFA's, this operates on ranges of
yaz_nfa_state *firststate; /* the initial state */
yaz_backref_info *curr_backrefs; /* the backrefs we are matching */
yaz_backref_info *best_backrefs; /* backrefs of the best match so far*/
+ int lastmatch; /* the result of last match */
};
struct yaz_nfa_state {
#define LOOP_LIMIT 100
+typedef enum {
+ conv_none,
+ conv_string,
+ conv_backref,
+ conv_range
+} yaz_nfa_converter_type;
+
+struct yaz_nfa_converter {
+ yaz_nfa_converter *next;
+ yaz_nfa_converter_type type;
+ yaz_nfa_char *string;
+ size_t strlen;
+ int backref_no;
+ int char_diff;
+};
/* * * * * * *
yaz_nfa *yaz_nfa_init() {
NMEM my_nmem = nmem_create();
- yaz_nfa *n=nmem_malloc(my_nmem, sizeof(yaz_nfa));
- n->nmem=my_nmem;
- n->nstates=0;
- n->laststate=0;
+ yaz_nfa *n = nmem_malloc(my_nmem, sizeof(yaz_nfa));
+ n->nmem = my_nmem;
+ n->nstates = 0;
+ n->laststate = 0;
+ n->firststate = 0;
+ n->nbackrefs = 0;
+ n->curr_backrefs = 0;
+ n->best_backrefs = 0;
+ n->lastmatch = YAZ_NFA_NOMATCH;
return n;
}
* * * * * */
yaz_nfa_state *yaz_nfa_add_state(yaz_nfa *n) {
- yaz_nfa_state *s = nmem_malloc(n->nmem,sizeof(yaz_nfa_state));
+ yaz_nfa_state *s = nmem_malloc(n->nmem, sizeof(yaz_nfa_state));
s->num = (n->nstates)++;
- s->result=0;
- s->lasttrans=0;
- s->backref_start=0;
- s->backref_end=0;
+ s->result = 0;
+ s->lasttrans = 0;
+ s->backref_start = 0;
+ s->backref_end = 0;
if (n->laststate) {
- s->next=n->laststate->next;
- n->laststate->next=s;
- n->laststate=s;
+ s->next = n->laststate->next;
+ n->laststate->next = s;
+ n->laststate = s;
} else { /* first state */
- n->laststate=s;
- n->firststate=s;
- s->next=s;
+ n->laststate = s;
+ n->firststate = s;
+ s->next = s;
}
return s;
}
-int yaz_nfa_set_result(yaz_nfa *n, yaz_nfa_state *s,void *result) {
+int yaz_nfa_set_result(yaz_nfa *n, yaz_nfa_state *s, void *result) {
if ((s->result)&&result)
return 1;
- s->result=result;
+ s->result = result;
return 0;
}
return s->result;
}
-int yaz_nfa_set_backref(yaz_nfa *n, yaz_nfa_state *s,
+int yaz_nfa_set_backref_point(yaz_nfa *n, yaz_nfa_state *s,
int backref_number,
int is_start ){
if (is_start) {
if (s->backref_start)
return 1;
- s->backref_start=backref_number;
- if (n->nbackrefs<backref_number)
- n->nbackrefs=backref_number;
+ s->backref_start = backref_number;
+ if (n->nbackrefs<=backref_number) {
+ n->nbackrefs = backref_number+1;
+ n->curr_backrefs = 0;
+ n->best_backrefs = 0;
+ /* clear them just in case we have already matched on */
+ /* with this nfa, and created a too small backref table */
+ /* we will reallocate when matching. */
+ }
} else {
if (s->backref_end)
return 1;
- s->backref_end=backref_number;
+ if (n->nbackrefs<backref_number)
+ return 2; /* can't end a backref that has not been started */
+ s->backref_end = backref_number;
}
return 0; /* ok */
}
-int yaz_nfa_get_backref(yaz_nfa *n, yaz_nfa_state *s,
+int yaz_nfa_get_backref_point(yaz_nfa *n, yaz_nfa_state *s,
int is_start ) {
if (!s)
return 0;
yaz_nfa_state *to_state,
yaz_nfa_char range_start,
yaz_nfa_char range_end) {
- yaz_nfa_transition *t=nmem_malloc(n->nmem,sizeof(yaz_nfa_transition));
- t->range_start=range_start;
- t->range_end=range_end;
- t->to_state=to_state;
+ yaz_nfa_transition *t = nmem_malloc(n->nmem, sizeof(yaz_nfa_transition));
+ if (!from_state)
+ from_state = n->firststate;
+ t->range_start = range_start;
+ t->range_end = range_end;
+ t->to_state = to_state;
if (from_state->lasttrans) {
t->next= from_state->lasttrans->next;
- from_state->lasttrans->next=t;
- from_state->lasttrans=t;
+ from_state->lasttrans->next = t;
+ from_state->lasttrans = t;
} else { /* first trans */
- from_state->lasttrans=t;
- t->next=t;
+ from_state->lasttrans = t;
+ t->next = t;
}
}
void yaz_nfa_add_empty_transition( yaz_nfa *n,
yaz_nfa_state *from_state,
yaz_nfa_state *to_state) {
- yaz_nfa_add_transition(n,from_state,to_state,
- EMPTY_START,EMPTY_END);
+ yaz_nfa_add_transition(n, from_state, to_state,
+ EMPTY_START, EMPTY_END);
}
/* * * * * * * *
yaz_nfa_state *s,
yaz_nfa_char range_start,
yaz_nfa_char range_end) {
- yaz_nfa_transition *t=s->lasttrans;
+ yaz_nfa_transition *t = s->lasttrans;
if (!t)
return 0;
do {
- t=t->next;
+ t = t->next;
if ( ( t->range_start == range_start ) && ( t->range_end == range_end) )
return t->to_state;
} while (t != s->lasttrans );
yaz_nfa_char range_end) {
yaz_nfa_state *nextstate;
if (!s) /* default to top-level of the nfa */
- s=n->firststate;
- nextstate=find_single_trans(s,range_start,range_end);
+ s = n->firststate;
+ nextstate = find_single_trans(s, range_start, range_end);
if (!nextstate) {
nextstate = yaz_nfa_add_state(n);
yaz_nfa_add_transition(n, s, nextstate, range_start, range_end);
yaz_nfa_char *seq ){
yaz_nfa_state *nextstate;
if (!s) /* default to top-level of the nfa */
- s=n->firststate;
- nextstate=find_single_trans(s,*seq,*seq);
+ s = n->firststate;
+ nextstate = find_single_trans(s, *seq, *seq);
if (nextstate) {
seq++;
if (!*seq) /* whole sequence matched */
return yaz_nfa_add_sequence(n, nextstate, seq);
} else { /* no next state, build the rest */
while (*seq) {
- s=yaz_nfa_add_range(n,s, *seq, *seq);
+ s = yaz_nfa_add_range(n, s, *seq, *seq);
seq++;
}
return s;
* * * * * * */
struct matcher {
+ yaz_nfa *n;
yaz_nfa_char *longest;
- int bestnode;
+ int bestnode;
void *result;
int errorcode;
int empties; /* count how many consecutive empty transitions */
};
/* Finds the longest match. In case of ties, chooses the one with the
- * lowest numbered state */
-static void match_state(yaz_nfa_state *s,
- yaz_nfa_char *inchar,
- size_t incharsleft,
- struct matcher *m ) {
- yaz_nfa_transition *t=s->lasttrans;
-
+ * lowest numbered state. Keep track of the back references. Recursively
+ * traverses the NFA. Keeps track of the best hit it has found. */
+
+static void match_state(
+ yaz_nfa_state *s,
+ yaz_nfa_char *prevmatch,
+ yaz_nfa_char *inchar,
+ size_t incharsleft,
+ struct matcher *m ) {
+ yaz_nfa_transition *t = s->lasttrans;
+ if (s->backref_start)
+ m->n->curr_backrefs[s->backref_start].start = inchar;
+ if (s->backref_end)
+ m->n->curr_backrefs[s->backref_end].end = prevmatch;
if (t) {
if (incharsleft) {
do {
- t=t->next;
+ t = t->next;
if ( (( t->range_start <= *inchar ) && ( t->range_end >= *inchar )) ){
- m->empties=0;
- match_state(t->to_state, inchar+1,incharsleft-1,m);
- /* yes, descent to all matching nodes, even if overrun, */
- /* since we can find a better one later */
- } else if (( t->range_start==EMPTY_START) && (t->range_end==EMPTY_END)) {
+ m->empties = 0;
+ if (t->range_start!=t->range_end){
+ /* backref 0 is special: the last range operation */
+ m->n->curr_backrefs[0].start = inchar;
+ m->n->curr_backrefs[0].end = inchar;
+ }
+ match_state(t->to_state, inchar,
+ inchar+1, incharsleft-1, m);
+ } else if (( t->range_start==EMPTY_START) &&
+ (t->range_end==EMPTY_END)) {
if ( m->empties++ > LOOP_LIMIT )
m->errorcode= YAZ_NFA_LOOP;
else
- match_state(t->to_state, inchar,incharsleft,m);
+ match_state(t->to_state, prevmatch,
+ inchar, incharsleft, m);
}
} while (t != s->lasttrans );
} else {
- m->errorcode=YAZ_NFA_OVERRUN;
+ m->errorcode = YAZ_NFA_OVERRUN;
}
}
if (s->result) { /* terminal node */
if ( (m->longest < inchar) || /* longer result */
((m->longest == inchar)&&(m->bestnode<s->num)) ){
/* or as long, but with lower node number. Still better */
- m->longest=inchar;
- m->bestnode=s->num;
- m->result=s->result;
+ int i;
+ m->longest = inchar;
+ m->bestnode = s->num;
+ m->result = s->result;
+ if (m->n->curr_backrefs)
+ for (i = 0; i<m->n->nbackrefs; i++) {
+ m->n->best_backrefs[i]=m->n->curr_backrefs[i];
+ }
}
}
+ if (s->backref_start)
+ m->n->curr_backrefs[s->backref_start].start = 0;
+ if (s->backref_end)
+ m->n->curr_backrefs[s->backref_end].end = 0;
+ m->n->curr_backrefs[0].start = 0;
+ m->n->curr_backrefs[0].end = 0;
} /* match_state */
int yaz_nfa_match(yaz_nfa *n,
yaz_nfa_char **inbuff,
- size_t incharsleft,
+ size_t *incharsleft,
void **result ){
struct matcher m;
- if (!n->firststate)
- return YAZ_NFA_NOMATCH;
+ int sz;
+ int i;
+ if (!n->firststate) {
+ n->lastmatch = YAZ_NFA_NOMATCH;
+ return n->lastmatch;
+ }
+ m.n = n;
m.longest=*inbuff;
- m.bestnode=n->nstates;
- m.result=0;
- m.errorcode=0;
+ m.bestnode = n->nstates;
+ m.result = 0;
+ m.errorcode = 0;
+ m.empties = 0;
+ sz = sizeof( struct yaz_nfa_backref_info) * n->nbackrefs;
if (!n->curr_backrefs) {
- int sz=sizeof( struct yaz_nfa_backref_info);
- n->curr_backrefs=nmem_malloc( n->nmem, n->nbackrefs* sz);
- n->best_backrefs=nmem_malloc( n->nmem, n->nbackrefs* sz);
+ n->curr_backrefs = nmem_malloc( n->nmem, sz);
+ n->best_backrefs = nmem_malloc( n->nmem, sz);
+ }
+ for (i = 0; i<n->nbackrefs; i++) {
+ n->curr_backrefs[i].start = 0;
+ n->curr_backrefs[i].end = 0;
+ n->best_backrefs[i].start = 0;
+ n->best_backrefs[i].end = 0;
}
-
- match_state(n->firststate, *inbuff, incharsleft, &m);
+ match_state(n->firststate, *inbuff, *inbuff, *incharsleft, &m);
if (m.result) {
- *result=m.result;
- *inbuff=m.longest;
+ *incharsleft -= (m.longest-*inbuff);
+ *result = m.result;
+ *inbuff = m.longest;
if (m.errorcode)
- return m.errorcode;
+ n->lastmatch = m.errorcode;
else
- return YAZ_NFA_SUCCESS;
+ n->lastmatch= YAZ_NFA_SUCCESS;
+ return n->lastmatch;
+ }
+ n->lastmatch = YAZ_NFA_NOMATCH;
+ return n->lastmatch;
+}
+
+
+int yaz_nfa_get_backref( yaz_nfa *n,
+ int backref_no,
+ yaz_nfa_char **start,
+ yaz_nfa_char **end) {
+ if (backref_no>=n->nbackrefs)
+ return 2;
+ if (backref_no<0)
+ return 2;
+ if (n->lastmatch== YAZ_NFA_NOMATCH)
+ return 1; /* accept other errors, they return partial matches*/
+
+ *start = n->best_backrefs[backref_no].start;
+ *end = n->best_backrefs[backref_no].end;
+ return 0;
+}
+
+/* * * * * * * * * * * * * *
+ * Converters
+ * * * * * * * * * * * * * */
+
+static yaz_nfa_converter *create_null_converter ( yaz_nfa *n) {
+ yaz_nfa_converter *c;
+ c=nmem_malloc(n->nmem, sizeof(yaz_nfa_converter));
+ c->next=0;
+ c->type=conv_none;
+ c->string=0;
+ c->strlen=0;
+ c->backref_no=0;
+ c->char_diff=0;
+ return c;
+}
+
+yaz_nfa_converter *yaz_nfa_create_string_converter (
+ yaz_nfa *n,
+ yaz_nfa_char *string,
+ size_t length){
+ yaz_nfa_converter *c;
+ int i;
+ c=create_null_converter(n);
+ c->type=conv_string;
+ c->string=nmem_malloc(n->nmem, length*sizeof(yaz_nfa_char));
+ for (i=0;i<length;i++)
+ c->string[i]=string[i];
+ c->strlen=length;
+ return c;
+}
+
+yaz_nfa_converter *yaz_nfa_create_backref_converter (
+ yaz_nfa *n, int backref_no ) {
+ yaz_nfa_converter *c;
+ c=create_null_converter(n);
+ c->type=conv_backref;
+ c->backref_no=backref_no;
+ return c;
+}
+
+yaz_nfa_converter *yaz_nfa_create_range_converter (
+ yaz_nfa *n, int backref_no,
+ yaz_nfa_char from_char,
+ yaz_nfa_char to_char){
+ yaz_nfa_converter *c;
+ c=create_null_converter(n);
+ c->type=conv_range;
+ c->backref_no=backref_no;
+ c->char_diff=to_char - from_char;
+ return c;
+
+}
+
+
+void yaz_nfa_append_converter (
+ yaz_nfa *n,
+ yaz_nfa_converter *startpoint,
+ yaz_nfa_converter *newconverter) {
+ while (startpoint->next)
+ startpoint=startpoint->next;
+ startpoint->next=newconverter;
+}
+
+static int string_convert (
+ yaz_nfa *n,
+ yaz_nfa_converter *c,
+ yaz_nfa_char **outbuff,
+ size_t *outcharsleft){
+ size_t sz=c->strlen;
+ yaz_nfa_char *p=c->string;
+ while (sz--) {
+ if ((*outcharsleft)-- <= 0)
+ return 2;
+ **outbuff=*p++;
+ (*outbuff)++;
+ }
+ return 0;
+}
+static int backref_convert (
+ yaz_nfa *n,
+ yaz_nfa_converter *c,
+ yaz_nfa_char **outbuff,
+ size_t *outcharsleft){
+ yaz_nfa_char *cp1,*cp2;
+ int i;
+ i=yaz_nfa_get_backref(n,c->backref_no, &cp1, &cp2);
+ if (i==2) /* no backref, produce no output, that's ok */
+ return 0;
+ if (i==1) /* no match in dfa */
+ return 1; /* should not happen */
+ while (cp2>=cp1) {
+ if ((*outcharsleft)-- <= 0)
+ return 2;
+ **outbuff=*cp1++;
+ (*outbuff)++;
}
- return YAZ_NFA_NOMATCH;
+ return 0;
}
+static int range_convert (
+ yaz_nfa *n,
+ yaz_nfa_converter *c,
+ yaz_nfa_char **outbuff,
+ size_t *outcharsleft){
+ yaz_nfa_char *cp1,*cp2;
+ int i;
+ i = yaz_nfa_get_backref(n,c->backref_no, &cp1, &cp2);
+ printf ("range_convert: i=%d d=%d, cp1=%p cp2=%p \n",i,c->char_diff,cp1,cp2);
+ if (i == 2) /* no backref, produce no output, not ok */
+ return 1; /* should not happen */
+ if (i == 1) /* no match in dfa */
+ return 1; /* should not happen */
+ while (cp2 >= cp1) {
+ if ((*outcharsleft)-- <= 0)
+ return 2;
+ printf(" range_convert: %d '%c' -> ",*cp1,*cp1);
+ **outbuff=(*cp1++) + c->char_diff ;
+ printf("%d '%c'\n",**outbuff, **outbuff);
+ (*outbuff)++;
+ }
+ return 0;
+}
+int yaz_nfa_run_converters(
+ yaz_nfa *n,
+ yaz_nfa_converter *c,
+ yaz_nfa_char **outbuff,
+ size_t *outcharsleft){
+ int rc=0;
+ while (c && !rc) {
+ switch(c->type) {
+ case conv_string:
+ rc=string_convert(n,c,outbuff,outcharsleft);
+ break;
+ case conv_backref:
+ rc=backref_convert(n,c,outbuff,outcharsleft);
+ break;
+ case conv_range:
+ rc=range_convert(n,c,outbuff,outcharsleft);
+ break;
+ default:
+ rc=3; /* internal error */
+ }
+ c=c->next;
+ }
+ return rc;
+}
/* * * * * * * * *
* Debug routines
static void dump_trans(FILE *F, yaz_nfa_transition *t ) {
char c1;
char c2;
- c1=t->range_start;
- c2=t->range_end;
- char *e="";
+ char *e;
+ c1 = t->range_start;
+ c2 = t->range_end;
+ e = "";
if ( (t->range_start <= ' ') || (t->range_start>'z'))
- c1='?';
+ c1 = '?';
if ( (t->range_end <= ' ') || (t->range_end>'z'))
- c2='?';
+ c2 = '?';
if ((t->range_start==EMPTY_START) && (t->range_end==EMPTY_END)) {
- e="(empty)";
+ e = "(empty)";
}
- fprintf(F," -> [%d] %s '%c' %x - '%c' %x \n",
+ fprintf(F, " -> [%d] %s '%c' %x - '%c' %x \n",
t->to_state->num, e,
c1, t->range_start,
c2, t->range_end );
static void dump_state(FILE *F, yaz_nfa_state *s,
char *(*strfunc)(void *) ) {
yaz_nfa_transition *t;
- char *resultstring="";
+ char *resultstring = "";
if (s->result) {
if (strfunc) {
- resultstring=(*strfunc)(s->result);
+ resultstring = (*strfunc)(s->result);
}
else
- resultstring=s->result;
+ resultstring = s->result;
}
- fprintf(F," state [%d] %s %s",
- s->num, s->result?"(FINAL)":"",resultstring );
+ fprintf(F, " state [%d] %s %s",
+ s->num, s->result?"(FINAL)":"", resultstring );
if (s->backref_start) {
- fprintf(F," start-%d",s->backref_start);
+ fprintf(F, " start-%d", s->backref_start);
}
if (s->backref_end) {
- fprintf(F," end-%d",s->backref_end);
+ fprintf(F, " end-%d", s->backref_end);
}
- fprintf(F,"\n");
- t=s->lasttrans;
+ fprintf(F, "\n");
+ t = s->lasttrans;
if (!t) {
- fprintf(F," (no transitions)\n");
+ fprintf(F, " (no transitions)\n");
} else {
do {
- t=t->next;
- dump_trans(F,t);
+ t = t->next;
+ dump_trans(F, t);
} while (t != s->lasttrans);
}
char *(*strfunc)(void *) ) {
yaz_nfa_state *s;
if (!F) /* lazy programmers can just pass 0 for F */
- F=stdout;
- fprintf(F,"The NFA has %d states \n",n->nstates);
- s=n->laststate;
+ F = stdout;
+ fprintf(F, "The NFA has %d states and %d backrefs\n",
+ n->nstates, n->nbackrefs);
+ s = n->laststate;
if (s) {
do {
- s=s->next;
- dump_state(F,s, strfunc);
+ s = s->next;
+ dump_state(F, s, strfunc);
} while (s != n->laststate);
}
}
-
/*
* Local variables:
* c-basic-offset: 4
projects / yaz-moved-to-github.git / blobdiff