X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Frpnsearch.c;h=d1b9b88b8b544b25567497aea04e73bc019be4db;hp=ca44a8139d0b9d0d53ddabac07f3ca49ad35b9b2;hb=896b30853daabb6294afe8b0a2f74fa6d6e397d8;hpb=6f7dfe3c3f09f7104c1ae7616c9d207edeab308d diff --git a/index/rpnsearch.c b/index/rpnsearch.c index ca44a81..d1b9b88 100644 --- a/index/rpnsearch.c +++ b/index/rpnsearch.c @@ -1,4 +1,4 @@ -/* $Id: rpnsearch.c,v 1.18 2007-10-30 19:17:15 adam Exp $ +/* $Id: rpnsearch.c,v 1.27 2007-12-07 14:09:09 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -65,7 +65,10 @@ void rpn_char_map_prepare(struct zebra_register *reg, zebra_map_t zm, struct rpn_char_map_info *map_info) { map_info->zm = zm; - dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler); + if (zebra_maps_is_icu(zm)) + dict_grep_cmap(reg->dict, 0, 0); + else + dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler); } #define TERM_COUNT @@ -79,7 +82,7 @@ struct grep_info { int isam_p_indx; int trunc_max; ZebraHandle zh; - int reg_type; + const char *index_type; ZebraSet termset; }; @@ -108,7 +111,7 @@ static int add_isam_p(const char *name, const char *info, if (p->isam_p_buf) { memcpy(new_isam_p_buf, p->isam_p_buf, - p->isam_p_indx * sizeof(*p->isam_p_buf)); + p->isam_p_indx * sizeof(*p->isam_p_buf)); xfree(p->isam_p_buf); } p->isam_p_buf = new_isam_p_buf; @@ -118,7 +121,7 @@ static int add_isam_p(const char *name, const char *info, if (p->term_no) { memcpy(new_term_no, p->isam_p_buf, - p->isam_p_indx * sizeof(*p->term_no)); + p->isam_p_indx * sizeof(*p->term_no)); xfree(p->term_no); } p->term_no = new_term_no; @@ -135,7 +138,7 @@ static int add_isam_p(const char *name, const char *info, const char *index_name; int len = key_SU_decode(&ord, (const unsigned char *) name); - zebra_term_untrans (p->zh, p->reg_type, term_tmp, name+len); + zebra_term_untrans (p->zh, p->index_type, term_tmp, name+len); yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp); zebraExplain_lookup_ord(p->zh->reg->zei, ord, 0 /* index_type */, &db, &index_name); @@ -232,6 +235,31 @@ static void add_non_space(const char *start, const char *end, } } + +static int term_100_icu(zebra_map_t zm, + const char **src, WRBUF term_dict, int space_split, + char *dst_term) +{ + int i; + const char *res_buf = 0; + size_t res_len = 0; + if (!zebra_map_tokenize_next(zm, &res_buf, &res_len)) + { + *src += strlen(*src); + return 0; + } + strcat(dst_term, *src); + for (i = 0; i < res_len; i++) + { + if (strchr(REGEX_CHARS, res_buf[i])) + wrbuf_putc(term_dict, '\\'); + if (res_buf[i] < 32) + wrbuf_putc(term_dict, 1); + wrbuf_putc(term_dict, res_buf[i]); + } + return 1; +} + /* term_100: handle term, where trunc = none(no operators at all) */ static int term_100(zebra_map_t zm, const char **src, WRBUF term_dict, int space_split, @@ -245,6 +273,9 @@ static int term_100(zebra_map_t zm, const char *space_start = 0; const char *space_end = 0; + if (zebra_maps_is_icu(zm)) + return term_100_icu(zm, src, term_dict, space_split, dst_term); + if (!term_pre(zm, src, NULL, NULL, !space_split)) return 0; s0 = *src; @@ -833,13 +864,14 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const char *index_type, int complete_flag, char *term_dst, const char *xpath_use, - struct ord_list **ol); - -static ZEBRA_RES term_limits_APT(ZebraHandle zh, - Z_AttributesPlusTerm *zapt, - zint *hits_limit_value, - const char **term_ref_id_str, - NMEM nmem) + struct ord_list **ol, + zebra_map_t zm); + +ZEBRA_RES zebra_term_limits_APT(ZebraHandle zh, + Z_AttributesPlusTerm *zapt, + zint *hits_limit_value, + const char **term_ref_id_str, + NMEM nmem) { AttrType term_ref_id_attr; AttrType hits_limit_attr; @@ -882,32 +914,35 @@ static ZEBRA_RES term_limits_APT(ZebraHandle zh, return ZEBRA_OK; } -static ZEBRA_RES term_trunc(ZebraHandle zh, - Z_AttributesPlusTerm *zapt, - const char **term_sub, - const Odr_oid *attributeSet, NMEM stream, - struct grep_info *grep_info, - const char *index_type, int complete_flag, - char *term_dst, - const char *rank_type, - const char *xpath_use, - NMEM rset_nmem, - RSET *rset, - struct rset_key_control *kc) +/** \brief search for term (which may be truncated) + */ +static ZEBRA_RES search_term(ZebraHandle zh, + Z_AttributesPlusTerm *zapt, + const char **term_sub, + const Odr_oid *attributeSet, NMEM stream, + struct grep_info *grep_info, + const char *index_type, int complete_flag, + char *term_dst, + const char *rank_type, + const char *xpath_use, + NMEM rset_nmem, + RSET *rset, + struct rset_key_control *kc, + zebra_map_t zm) { ZEBRA_RES res; struct ord_list *ol; zint hits_limit_value; const char *term_ref_id_str = 0; WRBUF term_dict = wrbuf_alloc(); - *rset = 0; - term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream); + zebra_term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, + stream); grep_info->isam_p_indx = 0; res = string_term(zh, zapt, term_sub, term_dict, attributeSet, stream, grep_info, index_type, complete_flag, - term_dst, xpath_use, &ol); + term_dst, xpath_use, &ol, zm); wrbuf_destroy(term_dict); if (res != ZEBRA_OK) return res; @@ -933,7 +968,8 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const char *index_type, int complete_flag, char *term_dst, const char *xpath_use, - struct ord_list **ol) + struct ord_list **ol, + zebra_map_t zm) { int r; AttrType truncation; @@ -948,8 +984,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, int relation_error; char ord_buf[32]; int ord_len, i; - zebra_map_t zm = zebra_map_get(zh->reg->zebra_maps, *index_type); - + *ol = ord_list_create(stream); rpn_char_map_prepare(zh->reg, zm, &rcmi); @@ -1095,8 +1130,13 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const char *input = wrbuf_cstr(term_dict) + prefix_len; esc_str(buf, sizeof(buf), input, strlen(input)); } - yaz_log(log_level_rpn, "dict_lookup_grep: %s", - wrbuf_cstr(term_dict) + prefix_len); + { + WRBUF pr_wr = wrbuf_alloc(); + + wrbuf_write_escaped(pr_wr, wrbuf_buf(term_dict), wrbuf_len(term_dict)); + yaz_log(YLOG_LOG, "dict_lookup_grep: %s", wrbuf_cstr(pr_wr)); + wrbuf_destroy(pr_wr); + } r = dict_lookup_grep(zh->reg->dict, wrbuf_cstr(term_dict), regex_range, grep_info, &max_pos, ord_len /* number of "exact" chars */, @@ -1123,7 +1163,7 @@ static void grep_info_delete(struct grep_info *grep_info) static ZEBRA_RES grep_info_prepare(ZebraHandle zh, Z_AttributesPlusTerm *zapt, struct grep_info *grep_info, - int reg_type) + const char *index_type) { #ifdef TERM_COUNT grep_info->term_no = 0; @@ -1132,7 +1172,7 @@ static ZEBRA_RES grep_info_prepare(ZebraHandle zh, grep_info->isam_p_size = 0; grep_info->isam_p_buf = NULL; grep_info->zh = zh; - grep_info->reg_type = reg_type; + grep_info->index_type = index_type; grep_info->termset = 0; if (zapt) { @@ -1168,7 +1208,7 @@ static ZEBRA_RES grep_info_prepare(ZebraHandle zh, termset_name = resname; } else - termset_name = termset_value_string; + termset_name = termset_value_string; yaz_log(log_level_rpn, "creating termset set %s", termset_name); grep_info->termset = resultSetAdd(zh, termset_name, 1); if (!grep_info->termset) @@ -1181,43 +1221,28 @@ static ZEBRA_RES grep_info_prepare(ZebraHandle zh, } return ZEBRA_OK; } - -/** - \brief Create result set(s) for list of terms - \param zh Zebra Handle - \param zapt Attributes Plust Term (RPN leaf) - \param termz term as used in query but converted to UTF-8 - \param attributeSet default attribute set - \param stream memory for result - \param index_type register type ("w", "p",..) - \param complete_flag whether it's phrases or not - \param rank_type term flags for ranking - \param xpath_use use attribute for X-Path (-1 for no X-path) - \param rset_nmem memory for result sets - \param result_sets output result set for each term in list (output) - \param num_result_sets number of output result sets - \param kc rset key control to be used for created result sets -*/ -static ZEBRA_RES term_list_trunc(ZebraHandle zh, - Z_AttributesPlusTerm *zapt, - const char *termz, - const Odr_oid *attributeSet, - NMEM stream, - const char *index_type, int complete_flag, - const char *rank_type, - const char *xpath_use, - NMEM rset_nmem, - RSET **result_sets, int *num_result_sets, - struct rset_key_control *kc) + +static ZEBRA_RES search_terms_chrmap(ZebraHandle zh, + Z_AttributesPlusTerm *zapt, + const char *termz, + const Odr_oid *attributeSet, + NMEM stream, + const char *index_type, int complete_flag, + const char *rank_type, + const char *xpath_use, + NMEM rset_nmem, + RSET **result_sets, int *num_result_sets, + struct rset_key_control *kc, + zebra_map_t zm) { char term_dst[IT_MAX_WORD+1]; struct grep_info grep_info; const char *termp = termz; int alloc_sets = 0; - + *num_result_sets = 0; *term_dst = 0; - if (grep_info_prepare(zh, zapt, &grep_info, *index_type) == ZEBRA_FAIL) + if (grep_info_prepare(zh, zapt, &grep_info, index_type) == ZEBRA_FAIL) return ZEBRA_FAIL; while(1) { @@ -1233,13 +1258,13 @@ static ZEBRA_RES term_list_trunc(ZebraHandle zh, alloc_sets = alloc_sets + add; *result_sets = rnew; } - res = term_trunc(zh, zapt, &termp, attributeSet, - stream, &grep_info, - index_type, complete_flag, - term_dst, rank_type, - xpath_use, rset_nmem, - &(*result_sets)[*num_result_sets], - kc); + res = search_term(zh, zapt, &termp, attributeSet, + stream, &grep_info, + index_type, complete_flag, + term_dst, rank_type, + xpath_use, rset_nmem, + &(*result_sets)[*num_result_sets], + kc, zm); if (res != ZEBRA_OK) { int i; @@ -1258,14 +1283,55 @@ static ZEBRA_RES term_list_trunc(ZebraHandle zh, grep_info_delete(&grep_info); return ZEBRA_OK; } + +/** + \brief Create result set(s) for list of terms + \param zh Zebra Handle + \param zapt Attributes Plust Term (RPN leaf) + \param termz term as used in query but converted to UTF-8 + \param attributeSet default attribute set + \param stream memory for result + \param index_type register type ("w", "p",..) + \param complete_flag whether it's phrases or not + \param rank_type term flags for ranking + \param xpath_use use attribute for X-Path (-1 for no X-path) + \param rset_nmem memory for result sets + \param result_sets output result set for each term in list (output) + \param num_result_sets number of output result sets + \param kc rset key control to be used for created result sets +*/ +static ZEBRA_RES search_terms_list(ZebraHandle zh, + Z_AttributesPlusTerm *zapt, + const char *termz, + const Odr_oid *attributeSet, + NMEM stream, + const char *index_type, int complete_flag, + const char *rank_type, + const char *xpath_use, + NMEM rset_nmem, + RSET **result_sets, int *num_result_sets, + struct rset_key_control *kc) +{ + zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, index_type); + if (zebra_maps_is_icu(zm)) + zebra_map_tokenize_start(zm, termz, strlen(termz)); + return search_terms_chrmap(zh, zapt, termz, attributeSet, + stream, index_type, complete_flag, + rank_type, xpath_use, + rset_nmem, result_sets, num_result_sets, + kc, zm); +} -static ZEBRA_RES rpn_search_APT_position(ZebraHandle zh, - Z_AttributesPlusTerm *zapt, - const Odr_oid *attributeSet, - const char *index_type, - NMEM rset_nmem, - RSET *rset, - struct rset_key_control *kc) + +/** \brief limit a search by position - returns result set + */ +static ZEBRA_RES search_position(ZebraHandle zh, + Z_AttributesPlusTerm *zapt, + const Odr_oid *attributeSet, + const char *index_type, + NMEM rset_nmem, + RSET *rset, + struct rset_key_control *kc) { int position_value; AttrType position; @@ -1275,7 +1341,7 @@ static ZEBRA_RES rpn_search_APT_position(ZebraHandle zh, int ord_len; char *val; ISAM_P isam_p; - zebra_map_t zm = zebra_map_get(zh->reg->zebra_maps, *index_type); + zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, index_type); attr_init_APT(&position, zapt, 3); position_value = attr_find(&position, NULL); @@ -1301,13 +1367,6 @@ static ZEBRA_RES rpn_search_APT_position(ZebraHandle zh, return ZEBRA_FAIL; } - if (!zh->reg->isamb && !zh->reg->isamc) - { - zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_POSITION_ATTRIBUTE, - position_value); - return ZEBRA_FAIL; - } - if (zebra_apt_get_ord(zh, zapt, index_type, 0, attributeSet, &ord) != ZEBRA_OK) { @@ -1321,23 +1380,22 @@ static ZEBRA_RES rpn_search_APT_position(ZebraHandle zh, { assert(*val == sizeof(ISAM_P)); memcpy(&isam_p, val+1, sizeof(isam_p)); - - if (zh->reg->isamb) - *rset = rsisamb_create(rset_nmem, kc, kc->scope, - zh->reg->isamb, isam_p, 0); - else if (zh->reg->isamc) - *rset = rsisamc_create(rset_nmem, kc, kc->scope, - zh->reg->isamc, isam_p, 0); + + *rset = zebra_create_rset_isam(zh, rset_nmem, kc, kc->scope, + isam_p, 0); } return ZEBRA_OK; } - + +/** \brief returns result set for phrase search + */ static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const char *termz_org, const Odr_oid *attributeSet, NMEM stream, - const char *index_type, int complete_flag, + const char *index_type, + int complete_flag, const char *rank_type, const char *xpath_use, NMEM rset_nmem, @@ -1347,24 +1405,29 @@ static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh, RSET *result_sets = 0; int num_result_sets = 0; ZEBRA_RES res = - term_list_trunc(zh, zapt, termz_org, attributeSet, - stream, index_type, complete_flag, - rank_type, xpath_use, - rset_nmem, - &result_sets, &num_result_sets, kc); - + search_terms_list(zh, zapt, termz_org, attributeSet, + stream, index_type, complete_flag, + rank_type, xpath_use, + rset_nmem, + &result_sets, &num_result_sets, kc); + if (res != ZEBRA_OK) return res; if (num_result_sets > 0) { RSET first_set = 0; - res = rpn_search_APT_position(zh, zapt, attributeSet, - index_type, - rset_nmem, &first_set, - kc); + res = search_position(zh, zapt, attributeSet, + index_type, + rset_nmem, &first_set, + kc); if (res != ZEBRA_OK) + { + int i; + for (i = 0; iscope, - num_result_sets, result_sets); + num_result_sets, result_sets); if (!*rset) return ZEBRA_FAIL; return ZEBRA_OK; @@ -1634,7 +1701,7 @@ static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, int relation_error = 0; int ord, ord_len, i; char ord_buf[32]; - zebra_map_t zm = zebra_map_get(zh->reg->zebra_maps, *index_type); + zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, index_type); *ol = ord_list_create(stream); @@ -1703,10 +1770,11 @@ static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh, zint hits_limit_value; const char *term_ref_id_str = 0; - term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream); + zebra_term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, + stream); yaz_log(log_level_rpn, "APT_numeric t='%s'", termz); - if (grep_info_prepare(zh, zapt, &grep_info, *index_type) == ZEBRA_FAIL) + if (grep_info_prepare(zh, zapt, &grep_info, index_type) == ZEBRA_FAIL) return ZEBRA_FAIL; while (1) { @@ -1827,7 +1895,7 @@ static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt, sort_sequence->num_specs = 10; sort_sequence->specs = (Z_SortKeySpec **) nmem_malloc(stream, sort_sequence->num_specs * - sizeof(*sort_sequence->specs)); + sizeof(*sort_sequence->specs)); for (i = 0; inum_specs; i++) sort_sequence->specs[i] = 0; } @@ -1835,7 +1903,7 @@ static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt, i = 0; else i = atoi_n((char *) zapt->term->u.general->buf, - zapt->term->u.general->len); + zapt->term->u.general->len); if (i >= sort_sequence->num_specs) i = 0; sprintf(termz, "%d", i); @@ -1904,7 +1972,7 @@ static RSET xpath_trunc(ZebraHandle zh, NMEM stream, int ord = zebraExplain_lookup_attr_str(zh->reg->zei, zinfo_index_category_index, index_type, xpath_use); - if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL) + if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, "0") == ZEBRA_FAIL) return rset_create_null(rset_nmem, kc, 0); if (ord < 0) @@ -1969,21 +2037,21 @@ ZEBRA_RES rpn_search_xpath(ZebraHandle zh, } /* - //a -> a/.* - //a/b -> b/a/.* - /a -> a/ - /a/b -> b/a/ + //a -> a/.* + //a/b -> b/a/.* + /a -> a/ + /a/b -> b/a/ - / -> none + / -> none - a[@attr = value]/b[@other = othervalue] + a[@attr = value]/b[@other = othervalue] - /e/@a val range(e/,range(@a,freetext(w,1015,val),@a),e/) - /a/b val range(b/a/,freetext(w,1016,val),b/a/) - /a/b/@c val range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/) - /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y) - /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y) - /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x) + /e/@a val range(e/,range(@a,freetext(w,1015,val),@a),e/) + /a/b val range(b/a/,freetext(w,1016,val),b/a/) + /a/b/@c val range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/) + /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y) + /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y) + /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x) */ @@ -2098,7 +2166,7 @@ static ZEBRA_RES rpn_search_database(ZebraHandle zh, static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const Odr_oid *attributeSet, NMEM stream, Z_SortKeySpecList *sort_sequence, - int num_bases, char **basenames, + int num_bases, const char **basenames, NMEM rset_nmem, RSET *rset, struct rset_key_control *kc) @@ -2268,7 +2336,7 @@ static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs, const Odr_oid *attributeSet, NMEM stream, NMEM rset_nmem, Z_SortKeySpecList *sort_sequence, - int num_bases, char **basenames, + int num_bases, const char **basenames, RSET **result_sets, int *num_result_sets, Z_Operator *parent_op, struct rset_key_control *kc); @@ -2308,7 +2376,7 @@ ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs, const Odr_oid *attributeSet, NMEM stream, NMEM rset_nmem, Z_SortKeySpecList *sort_sequence, - int num_bases, char **basenames, + int num_bases, const char **basenames, RSET *result_set) { RSET *result_sets = 0; @@ -2345,7 +2413,7 @@ ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs, const Odr_oid *attributeSet, NMEM stream, NMEM rset_nmem, Z_SortKeySpecList *sort_sequence, - int num_bases, char **basenames, + int num_bases, const char **basenames, RSET **result_sets, int *num_result_sets, Z_Operator *parent_op, struct rset_key_control *kc)