X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fzebraapi.c;h=d3fdab22fc183f54dfeba7ba3d27fda174e4397c;hb=9982694ca69efc2bf4db54cd5b5607e3680cd32a;hp=7b146618852e3344a3453719636999e9e8d59221;hpb=9b46631dae8ab93da1e8f87f5d6638a493135c03;p=idzebra-moved-to-github.git diff --git a/index/zebraapi.c b/index/zebraapi.c index 7b14661..d3fdab2 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -1,5 +1,5 @@ -/* $Id: zebraapi.c,v 1.237 2006-12-05 14:06:29 adam Exp $ - Copyright (C) 1995-2006 +/* $Id: zebraapi.c,v 1.251 2007-03-14 11:48:32 adam Exp $ + Copyright (C) 1995-2007 Index Data ApS This file is part of the Zebra server. @@ -143,6 +143,9 @@ ZebraHandle zebra_open(ZebraService zs, Res res) zh->m_staticrank = 0; zh->m_segment_indexing = 0; + zh->break_handler_func = 0; + zh->break_handler_data = 0; + default_encoding = res_get_def(zh->session_res, "encoding", "ISO-8859-1"); zh->iconv_to_utf8 = @@ -258,6 +261,7 @@ ZebraService zebra_start_res(const char *configName, Res def_res, Res over_res) } } + zh->timing = yaz_timing_create(); zh->path_root = res_get (zh->global_res, "root"); zh->nmem = nmem_create(); zh->record_classes = recTypeClass_create (zh->global_res, zh->nmem); @@ -314,6 +318,7 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name, const char *recordCompression = 0; const char *profilePath; char cwd[1024]; + int sort_type = ZEBRA_SORT_TYPE_FLAT; ZEBRA_RES ret = ZEBRA_OK; ASSERTZS; @@ -385,7 +390,7 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name, reg->records = 0; reg->dict = 0; - reg->sortIdx = 0; + reg->sort_index = 0; reg->isams = 0; reg->matchDict = 0; reg->isamc = 0; @@ -427,9 +432,22 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name, yaz_log (YLOG_WARN, "dict_open failed"); ret = ZEBRA_FAIL; } - if (!(reg->sortIdx = sortIdx_open (reg->bfs, rw))) + + + if (res_get_match (res, "sortindex", "f", "f")) + sort_type = ZEBRA_SORT_TYPE_FLAT; + else if (res_get_match (res, "sortindex", "i", "f")) + sort_type = ZEBRA_SORT_TYPE_ISAMB; + else { - yaz_log (YLOG_WARN, "sortIdx_open failed"); + yaz_log (YLOG_WARN, "bad_value for 'sortindex'"); + ret = ZEBRA_FAIL; + } + + + if (!(reg->sort_index = zebra_sort_open(reg->bfs, rw, sort_type))) + { + yaz_log (YLOG_WARN, "zebra_sort_open failed"); ret = ZEBRA_FAIL; } if (res_get_match (res, "isam", "s", ISAM_DEFAULT)) @@ -540,7 +558,7 @@ static void zebra_register_close(ZebraService zs, struct zebra_register *reg) dict_close (reg->dict); if (reg->matchDict) dict_close (reg->matchDict); - sortIdx_close (reg->sortIdx); + zebra_sort_close(reg->sort_index); if (reg->isams) isams_close (reg->isams); if (reg->isamc) @@ -567,8 +585,6 @@ ZEBRA_RES zebra_stop(ZebraService zs) { if (!zs) return ZEBRA_OK; - yaz_log (log_level, "zebra_stop"); - while (zs->sessions) { zebra_close (zs->sessions); @@ -582,6 +598,15 @@ ZEBRA_RES zebra_stop(ZebraService zs) recTypeClass_destroy(zs->record_classes); nmem_destroy(zs->nmem); res_close (zs->global_res); + + yaz_timing_stop(zs->timing); + yaz_log (YLOG_LOG, "zebra_stop: %4.2f %4.2f %4.2f", + yaz_timing_get_real(zs->timing), + yaz_timing_get_user(zs->timing), + yaz_timing_get_sys(zs->timing)); + + + yaz_timing_destroy(&zs->timing); xfree(zs); return ZEBRA_OK; } @@ -993,8 +1018,25 @@ ZEBRA_RES zebra_set_approx_limit(ZebraHandle zh, zint approx_limit) return ZEBRA_OK; } -ZEBRA_RES zebra_search_RPN(ZebraHandle zh, ODR o, Z_RPNQuery *query, - const char *setname, zint *hits) +void zebra_set_partial_result(ZebraHandle zh) +{ + zh->partial_result = 1; +} + + +ZEBRA_RES zebra_set_break_handler(ZebraHandle zh, + int (*f)(void *client_data), + void *client_data) +{ + zh->break_handler_func = f; + zh->break_handler_data = client_data; + return ZEBRA_OK; +} + +ZEBRA_RES zebra_search_RPN_x(ZebraHandle zh, ODR o, Z_RPNQuery *query, + const char *setname, zint *hits, + int *estimated_hit_count, + int *partial_resultset) { ZEBRA_RES r; @@ -1005,19 +1047,31 @@ ZEBRA_RES zebra_search_RPN(ZebraHandle zh, ODR o, Z_RPNQuery *query, assert(hits); assert(setname); yaz_log(log_level, "zebra_search_rpn"); - zh->hits = 0; - *hits = 0; + + zh->partial_result = 0; if (zebra_begin_read(zh) == ZEBRA_FAIL) return ZEBRA_FAIL; r = resultSetAddRPN(zh, odr_extract_mem(o), query, - zh->num_basenames, zh->basenames, setname); + zh->num_basenames, zh->basenames, setname, + hits, estimated_hit_count); + + *partial_resultset = zh->partial_result; zebra_end_read(zh); - *hits = zh->hits; return r; } +ZEBRA_RES zebra_search_RPN(ZebraHandle zh, ODR o, Z_RPNQuery *query, + const char *setname, zint *hits) +{ + int estimated_hit_count; + int partial_resultset; + return zebra_search_RPN_x(zh, o, query, setname, hits, + &estimated_hit_count, + &partial_resultset); +} + ZEBRA_RES zebra_records_retrieve(ZebraHandle zh, ODR stream, const char *setname, Z_RecordComposition *comp, @@ -1346,13 +1400,14 @@ ZEBRA_RES zebra_admin_import_segment (ZebraHandle zh, Z_Segment *segment) Odr_oct *oct = fragment->u.notExternallyTagged; sysno = 0; - if (zebra_update_record(zh, - 0, /* record Type */ - &sysno, - 0, /* match */ - 0, /* fname */ - (const char *) oct->buf, oct->len, - 0) == ZEBRA_FAIL) + if (zebra_update_record( + zh, + action_update, + 0, /* record Type */ + &sysno, + 0, /* match */ + 0, /* fname */ + (const char *) oct->buf, oct->len) == ZEBRA_FAIL) res = ZEBRA_FAIL; } } @@ -1360,99 +1415,6 @@ ZEBRA_RES zebra_admin_import_segment (ZebraHandle zh, Z_Segment *segment) return res; } -ZEBRA_RES zebra_admin_exchange_record(ZebraHandle zh, - const char *rec_buf, - size_t rec_len, - const char *recid_buf, size_t recid_len, - int action) - /* 1 = insert. Fail it already exists */ - /* 2 = replace. Fail it does not exist */ - /* 3 = delete. Fail if does not exist */ - /* 4 = update. Insert/replace */ -{ - ZEBRA_RES res; - zint sysno = 0; - char *rinfo = 0; - char recid_z[256]; - int db_ord; - ZEBRA_CHECK_HANDLE(zh); - assert(action>0 && action <=4); - assert(rec_buf); - - yaz_log(log_level, "zebra_admin_exchange_record ac=%d", action); - - if (!recid_buf || recid_len <= 0 || recid_len >= sizeof(recid_z)) - { - zebra_setError(zh, YAZ_BIB1_ES_IMMEDIATE_EXECUTION_FAILED, - "no record ID or empty record ID"); - return ZEBRA_FAIL; - } - - memcpy (recid_z, recid_buf, recid_len); - recid_z[recid_len] = 0; - - if (zebra_begin_trans(zh, 1) == ZEBRA_FAIL) - return ZEBRA_FAIL; - - db_ord = zebraExplain_get_database_ord(zh->reg->zei); - rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord, recid_z); - if (rinfo) - { - if (action == 1) /* fail if insert */ - { - if (zebra_end_trans(zh) != ZEBRA_OK) - yaz_log(YLOG_WARN, "zebra_end_trans failed"); - zebra_setError(zh, YAZ_BIB1_ES_IMMEDIATE_EXECUTION_FAILED, - "Cannot insert record: already exist"); - return ZEBRA_FAIL; - } - - memcpy (&sysno, rinfo+1, sizeof(sysno)); - } - else - { - if (action == 2 || action == 3) /* fail if delete or update */ - { - if (zebra_end_trans(zh) != ZEBRA_OK) - yaz_log(YLOG_WARN, "zebra_end_trans failed"); - zebra_setError(zh, YAZ_BIB1_ES_IMMEDIATE_EXECUTION_FAILED, - "Cannot delete/update record: does not exist"); - return ZEBRA_FAIL; - } - action = 1; /* make it an insert (if it's an update).. */ - } - res = zebra_buffer_extract_record(zh, rec_buf, rec_len, - action == 3 ? 1 : 0 /* delete flag */, - 0, /* test mode */ - 0, /* recordType */ - &sysno, - 0, /* match */ - 0, /* fname */ - 0, /* force update */ - 1 /* allow update */ - ); - if (res == ZEBRA_FAIL) - { - zebra_setError(zh, YAZ_BIB1_ES_IMMEDIATE_EXECUTION_FAILED, - "Unable to parse record"); - } - if (action == 1) - { - dict_insert_ord(zh->reg->matchDict, db_ord, recid_z, - sizeof(sysno), &sysno); - } - else if (action == 3) - { - dict_delete_ord(zh->reg->matchDict, db_ord, recid_z); - } - if (zebra_end_trans(zh) != ZEBRA_OK) - { - yaz_log(YLOG_WARN, "zebra_end_trans failed"); - res = ZEBRA_FAIL; - } - return res; -} - int delete_w_handle(const char *info, void *handle) { ZebraHandle zh = (ZebraHandle) handle; @@ -1581,9 +1543,9 @@ int zebra_string_norm (ZebraHandle zh, unsigned reg_id, \param seqno sequence number val is one of: - d=writing to shadow(dirty) - o=no writing, - c=commit + d=writing to shadow(shadow enabled); writing to register (shadow disabled) + o=reading only + c=commit (writing to register, reading from shadow, shadow mode only) */ static void zebra_set_state (ZebraHandle zh, int val, int seqno) { @@ -1662,7 +1624,7 @@ static void read_res_for_transaction(ZebraHandle zh) v = res_get_prefix(zh->res, "openRW", group, "1"); zh->m_flag_rw = atoi(v); - v = res_get_prefix(zh->res, "fileVerboseLimit", group, "100000"); + v = res_get_prefix(zh->res, "fileVerboseLimit", group, "1000"); zh->m_file_verbose_limit = atoi(v); } @@ -1721,6 +1683,7 @@ ZEBRA_RES zebra_begin_trans(ZebraHandle zh, int rw) zh->records_updated = 0; zh->records_deleted = 0; zh->records_processed = 0; + zh->records_skipped = 0; #if HAVE_SYS_TIMES_H times (&zh->tms1); @@ -1743,6 +1706,11 @@ ZEBRA_RES zebra_begin_trans(ZebraHandle zh, int rw) if (val != 'o') { /* either we didn't finish commit or shadow is dirty */ + if (!rval) + { + yaz_log(YLOG_WARN, "previous transaction did not finish " + "(shadow disabled)"); + } zebra_unlock (zh->lock_shadow); zebra_unlock (zh->lock_normal); if (zebra_commit (zh)) @@ -2028,6 +1996,7 @@ static ZEBRA_RES zebra_commit_ex(ZebraHandle zh, int clean_only) if (val == 'd') { + /* shadow area is dirty and so we must throw it away */ yaz_log(YLOG_WARN, "previous transaction didn't reach commit"); clean_only = 1; } @@ -2244,48 +2213,20 @@ void zebra_set_shadow_enable (ZebraHandle zh, int value) ZEBRA_RES zebra_add_record(ZebraHandle zh, const char *buf, int buf_size) { - return zebra_update_record(zh, 0, 0 /* sysno */, 0, 0, buf, buf_size, 0); -} - -ZEBRA_RES zebra_insert_record(ZebraHandle zh, - const char *recordType, - zint *sysno, const char *match, - const char *fname, - const char *buf, int buf_size, int force_update) -{ - ZEBRA_RES res; - ASSERTZH; - assert(sysno); - assert(buf); - yaz_log(log_level, "zebra_insert_record sysno=" ZINT_FORMAT, *sysno); - - if (buf_size < 1) - buf_size = strlen(buf); - - if (zebra_begin_trans(zh, 1) == ZEBRA_FAIL) - return ZEBRA_FAIL; - res = zebra_buffer_extract_record(zh, buf, buf_size, - 0, /* delete_flag */ - 0, /* test_mode */ - recordType, - sysno, - match, fname, - 0, - 0); /* allow_update */ - if (zebra_end_trans(zh) != ZEBRA_OK) - { - yaz_log(YLOG_WARN, "zebra_end_trans failed"); - res = ZEBRA_FAIL; - } - return res; + return zebra_update_record(zh, action_update, + 0 /* record type */, + 0 /* sysno */ , + 0 /* match */, + 0 /* fname */, + buf, buf_size); } ZEBRA_RES zebra_update_record(ZebraHandle zh, + enum zebra_recctrl_action_t action, const char *recordType, zint *sysno, const char *match, const char *fname, - const char *buf, int buf_size, - int force_update) + const char *buf, int buf_size) { ZEBRA_RES res; @@ -2297,18 +2238,18 @@ ZEBRA_RES zebra_update_record(ZebraHandle zh, if (sysno) yaz_log(log_level, " sysno=" ZINT_FORMAT, *sysno); - if (buf_size < 1) buf_size = strlen(buf); + if (buf_size < 1) + buf_size = strlen(buf); if (zebra_begin_trans(zh, 1) == ZEBRA_FAIL) return ZEBRA_FAIL; res = zebra_buffer_extract_record(zh, buf, buf_size, - 0, /* delete_flag */ + action, 0, /* test_mode */ recordType, sysno, - match, fname, - force_update, - 1); /* allow_update */ + match, + fname); if (zebra_end_trans(zh) != ZEBRA_OK) { yaz_log(YLOG_WARN, "zebra_end_trans failed"); @@ -2317,42 +2258,6 @@ ZEBRA_RES zebra_update_record(ZebraHandle zh, return res; } -ZEBRA_RES zebra_delete_record(ZebraHandle zh, - const char *recordType, - zint *sysno, const char *match, - const char *fname, - const char *buf, int buf_size, - int force_update) -{ - ZEBRA_RES res; - - ZEBRA_CHECK_HANDLE(zh); - - assert(buf); - yaz_log(log_level, "zebra_delete_record"); - if (sysno) - yaz_log(log_level, " sysno=" ZINT_FORMAT, *sysno); - - if (buf_size < 1) buf_size = strlen(buf); - - if (zebra_begin_trans(zh, 1) == ZEBRA_FAIL) - return ZEBRA_FAIL; - res = zebra_buffer_extract_record(zh, buf, buf_size, - 1, /* delete_flag */ - 0, /* test_mode */ - recordType, - sysno, - match,fname, - force_update, - 1); /* allow_update */ - if (zebra_end_trans(zh) != ZEBRA_OK) - { - yaz_log(YLOG_WARN, "zebra_end_trans failed"); - res = ZEBRA_FAIL; - } - return res; -} - /* --------------------------------------------------------------------------- Searching */ @@ -2365,6 +2270,7 @@ ZEBRA_RES zebra_search_PQF(ZebraHandle zh, const char *pqf_query, Z_RPNQuery *query; ODR odr; + ZEBRA_CHECK_HANDLE(zh); odr = odr_createmem(ODR_ENCODE);