X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fkinput.c;h=9a2d6c1b0b667800bea961b5e9d9f5dc860b8ba7;hb=3b985da49d20315701f5bb2db353d242ff1e85a6;hp=07addbe8f3abf97202bec99c186799e7f7418da7;hpb=bceca936710690858ea162e7bc22de819f0e280f;p=idzebra-moved-to-github.git diff --git a/index/kinput.c b/index/kinput.c index 07addbe..9a2d6c1 100644 --- a/index/kinput.c +++ b/index/kinput.c @@ -1,121 +1,12 @@ /* * Copyright (C) 1994-1999, Index Data * All rights reserved. - * Sebastian Hammer, Adam Dickmeiss - * - * $Log: kinput.c,v $ - * Revision 1.32 1999-05-12 13:08:06 adam - * First version of ISAMS. - * - * Revision 1.31 1999/02/02 14:50:56 adam - * Updated WIN32 code specific sections. Changed header. - * - * Revision 1.30 1998/10/28 10:53:57 adam - * Added type cast to prevent warning. - * - * Revision 1.29 1998/06/11 15:41:39 adam - * Minor changes. - * - * Revision 1.28 1998/03/05 08:45:12 adam - * New result set model and modular ranking system. Moved towards - * descent server API. System information stored as "SGML" records. - * - * Revision 1.27 1998/02/17 10:32:52 adam - * Fixed bug: binary files weren't opened with flag b on NT. - * - * Revision 1.26 1998/01/29 13:39:13 adam - * Compress ISAM is default. - * - * Revision 1.25 1997/09/17 12:19:14 adam - * Zebra version corresponds to YAZ version 1.4. - * Changed Zebra server so that it doesn't depend on global common_resource. - * - * Revision 1.24 1997/09/09 13:38:07 adam - * Partial port to WIN95/NT. - * - * Revision 1.23 1997/09/04 13:57:39 adam - * Added O_BINARY for open calls. - * - * Revision 1.22 1997/02/12 20:39:45 adam - * Implemented options -f that limits the log to the first - * records. - * Changed some log messages also. - * - * Revision 1.21 1996/11/08 11:10:23 adam - * Buffers used during file match got bigger. - * Compressed ISAM support everywhere. - * Bug fixes regarding masking characters in queries. - * Redesigned Regexp-2 queries. - * - * Revision 1.20 1996/11/01 08:58:41 adam - * Interface to isamc system now includes update and delete. - * - * Revision 1.19 1996/10/29 14:09:46 adam - * Use of cisam system - enabled if setting isamc is 1. - * - * Revision 1.18 1996/06/04 10:18:59 adam - * Minor changes - removed include of ctype.h. - * - * Revision 1.17 1996/05/14 15:47:07 adam - * Cleanup of various buffer size entities. - * - * Revision 1.16 1996/04/09 10:05:20 adam - * Bug fix: prev_name buffer possibly too small; allocated in key_file_init. - * - * Revision 1.15 1996/03/21 14:50:09 adam - * File update uses modify-time instead of change-time. - * - * Revision 1.14 1996/02/07 14:06:37 adam - * Better progress report during register merge. - * New command: clean - removes temporary shadow files. - * - * Revision 1.13 1996/02/05 12:30:00 adam - * Logging reduced a bit. - * The remaining running time is estimated during register merge. - * - * Revision 1.12 1995/12/06 17:49:19 adam - * Uses dict_delete now. - * - * Revision 1.11 1995/12/06 16:06:43 adam - * Better diagnostics. Work on 'real' dictionary deletion. - * - * Revision 1.10 1995/12/06 12:41:22 adam - * New command 'stat' for the index program. - * Filenames can be read from stdin by specifying '-'. - * Bug fix/enhancement of the transformation from terms to regular - * expressons in the search engine. - * - * Revision 1.9 1995/10/10 12:24:39 adam - * Temporary sort files are compressed. - * - * Revision 1.8 1995/10/04 16:57:19 adam - * Key input and merge sort in one pass. - * - * Revision 1.7 1995/10/02 15:18:52 adam - * New member in recRetrieveCtrl: diagnostic. - * - * Revision 1.6 1995/09/29 15:51:56 adam - * First work on multi-way read. - * - * Revision 1.5 1995/09/29 14:01:43 adam - * Bug fixes. - * - * Revision 1.4 1995/09/28 14:22:57 adam - * Sort uses smaller temporary files. - * - * Revision 1.3 1995/09/06 16:11:17 adam - * Option: only one word key per file. - * - * Revision 1.2 1995/09/04 12:33:42 adam - * Various cleanup. YAZ util used instead. - * - * Revision 1.1 1995/09/04 09:10:37 adam - * More work on index add/del/update. - * Merge sort implemented. - * Initial work on z39 server. + * Sebastian Hammer, Adam Dickmeiss, Heikki Levanto * + * (log at the end) + */ - + #include #ifdef WIN32 #include @@ -211,7 +102,7 @@ struct key_file *key_file_init (int no, int chunk) { struct key_file *f; - f = xmalloc (sizeof(*f)); + f = (struct key_file *) xmalloc (sizeof(*f)); f->sysno = 0; f->seqno = 0; f->no = no; @@ -219,8 +110,8 @@ struct key_file *key_file_init (int no, int chunk) f->offset = 0; f->length = 0; f->readHandler = NULL; - f->buf = xmalloc (f->chunk); - f->prev_name = xmalloc (INP_NAME_MAX); + f->buf = (unsigned char *) xmalloc (f->chunk); + f->prev_name = (char *) xmalloc (INP_NAME_MAX); *f->prev_name = '\0'; key_file_chunk_read (f); return f; @@ -316,6 +207,8 @@ struct heap_info { ISAM isam; ISAMC isamc; ISAMS isams; + ISAMH isamh; + ISAMD isamd; }; struct heap_info *key_heap_init (int nkeys, @@ -324,16 +217,17 @@ struct heap_info *key_heap_init (int nkeys, struct heap_info *hi; int i; - hi = xmalloc (sizeof(*hi)); - hi->info.file = xmalloc (sizeof(*hi->info.file) * (1+nkeys)); - hi->info.buf = xmalloc (sizeof(*hi->info.buf) * (1+nkeys)); + hi = (struct heap_info *) xmalloc (sizeof(*hi)); + hi->info.file = (struct key_file **) + xmalloc (sizeof(*hi->info.file) * (1+nkeys)); + hi->info.buf = (char **) xmalloc (sizeof(*hi->info.buf) * (1+nkeys)); hi->heapnum = 0; - hi->ptr = xmalloc (sizeof(*hi->ptr) * (1+nkeys)); + hi->ptr = (int *) xmalloc (sizeof(*hi->ptr) * (1+nkeys)); hi->cmp = cmp; for (i = 0; i<= nkeys; i++) { hi->ptr[i] = i; - hi->info.buf[i] = xmalloc (INP_NAME_MAX); + hi->info.buf[i] = (char *) xmalloc (INP_NAME_MAX); } return hi; } @@ -423,7 +317,7 @@ struct heap_cread_info { int heap_cread_item (void *vp, char **dst, int *insertMode) { - struct heap_cread_info *p = vp; + struct heap_cread_info *p = (struct heap_cread_info *) vp; struct heap_info *hi = p->hi; if (p->mode == 1) @@ -451,9 +345,9 @@ int heap_cread_item (void *vp, char **dst, int *insertMode) int heap_inpc (struct heap_info *hi) { struct heap_cread_info hci; - ISAMC_I isamc_i = xmalloc (sizeof(*isamc_i)); + ISAMC_I isamc_i = (ISAMC_I) xmalloc (sizeof(*isamc_i)); - hci.key = xmalloc (KEY_SIZE); + hci.key = (char *) xmalloc (KEY_SIZE); hci.mode = 1; hci.hi = hi; hci.more = heap_read_one (hi, hci.cur_name, hci.key); @@ -502,9 +396,9 @@ int heap_inpc (struct heap_info *hi) int heap_inps (struct heap_info *hi) { struct heap_cread_info hci; - ISAMS_I isams_i = xmalloc (sizeof(*isams_i)); + ISAMS_I isams_i = (ISAMS_I) xmalloc (sizeof(*isams_i)); - hci.key = xmalloc (KEY_SIZE); + hci.key = (char *) xmalloc (KEY_SIZE); hci.mode = 1; hci.hi = hi; hci.more = heap_read_one (hi, hci.cur_name, hci.key); @@ -534,6 +428,110 @@ int heap_inps (struct heap_info *hi) return 0; } +int heap_inph (struct heap_info *hi) +{ + struct heap_cread_info hci; + ISAMH_I isamh_i = (ISAMH_I) xmalloc (sizeof(*isamh_i)); + + hci.key = (char *) xmalloc (KEY_SIZE); + hci.mode = 1; + hci.hi = hi; + hci.more = heap_read_one (hi, hci.cur_name, hci.key); + + isamh_i->clientData = &hci; + isamh_i->read_item = heap_cread_item; + + while (hci.more) + { + char this_name[INP_NAME_MAX]; + ISAMH_P isamh_p, isamh_p2; + char *dict_info; + + strcpy (this_name, hci.cur_name); + assert (hci.cur_name[1]); + no_diffs++; + if ((dict_info = dict_lookup (hi->dict, hci.cur_name))) + { + memcpy (&isamh_p, dict_info+1, sizeof(ISAMH_P)); + isamh_p2 = isamh_append (hi->isamh, isamh_p, isamh_i); + if (!isamh_p2) + { + no_deletions++; + if (!dict_delete (hi->dict, this_name)) + abort(); + } + else + { + no_updates++; + if (isamh_p2 != isamh_p) + dict_insert (hi->dict, this_name, + sizeof(ISAMH_P), &isamh_p2); + } + } + else + { + isamh_p = isamh_append (hi->isamh, 0, isamh_i); + no_insertions++; + dict_insert (hi->dict, this_name, sizeof(ISAMH_P), &isamh_p); + } + } + xfree (isamh_i); + return 0; +} + +int heap_inpd (struct heap_info *hi) +{ + struct heap_cread_info hci; + ISAMD_I isamd_i = (ISAMD_I) xmalloc (sizeof(*isamd_i)); + + hci.key = (char *) xmalloc (KEY_SIZE); + hci.mode = 1; + hci.hi = hi; + hci.more = heap_read_one (hi, hci.cur_name, hci.key); + + isamd_i->clientData = &hci; + isamd_i->read_item = heap_cread_item; + + while (hci.more) + { + char this_name[INP_NAME_MAX]; + ISAMD_P isamd_p, isamd_p2; + char *dict_info; + + strcpy (this_name, hci.cur_name); + assert (hci.cur_name[1]); + no_diffs++; + if ((dict_info = dict_lookup (hi->dict, hci.cur_name))) + { + memcpy (&isamd_p, dict_info+1, sizeof(ISAMD_P)); + isamd_p2 = isamd_append (hi->isamd, isamd_p, isamd_i); + if (!isamd_p2) + { + no_deletions++; + if (!dict_delete (hi->dict, this_name)) + abort(); + } + else + { + no_updates++; + if (isamd_p2 != isamd_p) + dict_insert (hi->dict, this_name, + sizeof(ISAMD_P), &isamd_p2); + } + } + else + { + isamd_p = isamd_append (hi->isamd, 0, isamd_i); + no_insertions++; + dict_insert (hi->dict, this_name, sizeof(ISAMD_P), &isamd_p); + } + } + xfree (isamd_i); + return 0; +} + + + int heap_inp (struct heap_info *hi) { @@ -546,8 +544,8 @@ int heap_inp (struct heap_info *hi) char *key_buf; int more; - next_key = xmalloc (KEY_SIZE); - key_buf = xmalloc (key_buf_size); + next_key = (char *) xmalloc (KEY_SIZE); + key_buf = (char *) xmalloc (key_buf_size); more = heap_read_one (hi, cur_name, key_buf); while (more) /* EOF ? */ { @@ -564,7 +562,7 @@ int heap_inp (struct heap_info *hi) if (key_buf_ptr+(int) KEY_SIZE >= key_buf_size) { char *new_key_buf; - new_key_buf = xmalloc (key_buf_size + INP_BUF_ADD); + new_key_buf = (char *) xmalloc (key_buf_size + INP_BUF_ADD); memcpy (new_key_buf, key_buf, key_buf_size); key_buf_size += INP_BUF_ADD; xfree (key_buf); @@ -614,7 +612,7 @@ struct progressInfo { void progressFunc (struct key_file *keyp, void *info) { - struct progressInfo *p = info; + struct progressInfo *p = (struct progressInfo *) info; time_t now, remaining; if (keyp->buf_size <= 0 || p->totalBytes <= 0) @@ -647,6 +645,8 @@ void key_input (BFiles bfs, int nkeys, int cache) ISAM isam = NULL; ISAMC isamc = NULL; ISAMS isams = NULL; + ISAMH isamh = NULL; + ISAMD isamd = NULL; struct key_file **kf; char rbuf[1024]; int i, r; @@ -667,7 +667,7 @@ void key_input (BFiles bfs, int nkeys, int cache) if (!nkeys) return ; } - dict = dict_open (bfs, FNAME_DICT, cache, 1); + dict = dict_open (bfs, FNAME_DICT, cache, 1, 0); if (!dict) { logf (LOG_FATAL, "dict_open fail"); @@ -675,8 +675,9 @@ void key_input (BFiles bfs, int nkeys, int cache) } if (res_get_match (common_resource, "isam", "s", NULL)) { + struct ISAMS_M_s isams_m; isams = isams_open (bfs, FNAME_ISAMS, 1, - key_isams_m (common_resource)); + key_isams_m (common_resource, &isams_m)); if (!isams) { logf (LOG_FATAL, "isams_open fail"); @@ -693,17 +694,39 @@ void key_input (BFiles bfs, int nkeys, int cache) exit (1); } } + else if (res_get_match (common_resource, "isam", "h", NULL)) + { + isamh = isamh_open (bfs, FNAME_ISAMH, 1, + key_isamh_m (common_resource)); + if (!isamh) + { + logf (LOG_FATAL, "isamh_open fail"); + exit (1); + } + } + else if (res_get_match (common_resource, "isam", "d", NULL)) + { + struct ISAMD_M_s isamd_m; + isamd = isamd_open (bfs, FNAME_ISAMD, 1, + key_isamd_m (common_resource,&isamd_m)); + if (!isamd) + { + logf (LOG_FATAL, "isamd_open fail"); + exit (1); + } + } else { + struct ISAMC_M_s isamc_m; isamc = isc_open (bfs, FNAME_ISAMC, 1, - key_isamc_m (common_resource)); + key_isamc_m (common_resource, &isamc_m)); if (!isamc) { logf (LOG_FATAL, "isc_open fail"); exit (1); } } - kf = xmalloc ((1+nkeys) * sizeof(*kf)); + kf = (struct key_file **) xmalloc ((1+nkeys) * sizeof(*kf)); progressInfo.totalBytes = 0; progressInfo.totalOffset = 0; time (&progressInfo.startTime); @@ -721,7 +744,9 @@ void key_input (BFiles bfs, int nkeys, int cache) hi->isam = isam; hi->isamc = isamc; hi->isams = isams; - + hi->isamh = isamh; + hi->isamd = isamd; + for (i = 1; i<=nkeys; i++) if ((r = key_file_read (kf[i], rbuf))) key_heap_insert (hi, rbuf, r, kf[i]); @@ -731,6 +756,11 @@ void key_input (BFiles bfs, int nkeys, int cache) heap_inps (hi); else if (isam) heap_inp (hi); + else if (isamh) + heap_inph (hi); + else if (isamd) + heap_inpd (hi); + dict_close (dict); if (isam) is_close (isam); @@ -738,6 +768,10 @@ void key_input (BFiles bfs, int nkeys, int cache) isc_close (isamc); if (isams) isams_close (isams); + if (isamh) + isamh_close (isamh); + if (isamd) + isamd_close (isamd); for (i = 1; i<=nkeys; i++) { @@ -752,3 +786,134 @@ void key_input (BFiles bfs, int nkeys, int cache) } + +/* + * $Log: kinput.c,v $ + * Revision 1.37 1999-07-14 13:21:34 heikki + * Added isam-d files. Compiles (almost) clean. Doesn't work at all + * + * Revision 1.36 1999/07/14 10:59:26 adam + * Changed functions isc_getmethod, isams_getmethod. + * Improved fatal error handling (such as missing EXPLAIN schema). + * + * Revision 1.35 1999/06/30 15:07:23 heikki + * Adding isamh stuff + * + * Revision 1.34 1999/05/26 07:49:13 adam + * C++ compilation. + * + * Revision 1.33 1999/05/15 14:36:38 adam + * Updated dictionary. Implemented "compression" of dictionary. + * + * Revision 1.32 1999/05/12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.31 1999/02/02 14:50:56 adam + * Updated WIN32 code specific sections. Changed header. + * + * Revision 1.30 1998/10/28 10:53:57 adam + * Added type cast to prevent warning. + * + * Revision 1.29 1998/06/11 15:41:39 adam + * Minor changes. + * + * Revision 1.28 1998/03/05 08:45:12 adam + * New result set model and modular ranking system. Moved towards + * descent server API. System information stored as "SGML" records. + * + * Revision 1.27 1998/02/17 10:32:52 adam + * Fixed bug: binary files weren't opened with flag b on NT. + * + * Revision 1.26 1998/01/29 13:39:13 adam + * Compress ISAM is default. + * + * Revision 1.25 1997/09/17 12:19:14 adam + * Zebra version corresponds to YAZ version 1.4. + * Changed Zebra server so that it doesn't depend on global common_resource. + * + * Revision 1.24 1997/09/09 13:38:07 adam + * Partial port to WIN95/NT. + * + * Revision 1.23 1997/09/04 13:57:39 adam + * Added O_BINARY for open calls. + * + * Revision 1.22 1997/02/12 20:39:45 adam + * Implemented options -f that limits the log to the first + * records. + * Changed some log messages also. + * + * Revision 1.21 1996/11/08 11:10:23 adam + * Buffers used during file match got bigger. + * Compressed ISAM support everywhere. + * Bug fixes regarding masking characters in queries. + * Redesigned Regexp-2 queries. + * + * Revision 1.20 1996/11/01 08:58:41 adam + * Interface to isamc system now includes update and delete. + * + * Revision 1.19 1996/10/29 14:09:46 adam + * Use of cisam system - enabled if setting isamc is 1. + * + * Revision 1.18 1996/06/04 10:18:59 adam + * Minor changes - removed include of ctype.h. + * + * Revision 1.17 1996/05/14 15:47:07 adam + * Cleanup of various buffer size entities. + * + * Revision 1.16 1996/04/09 10:05:20 adam + * Bug fix: prev_name buffer possibly too small; allocated in key_file_init. + * + * Revision 1.15 1996/03/21 14:50:09 adam + * File update uses modify-time instead of change-time. + * + * Revision 1.14 1996/02/07 14:06:37 adam + * Better progress report during register merge. + * New command: clean - removes temporary shadow files. + * + * Revision 1.13 1996/02/05 12:30:00 adam + * Logging reduced a bit. + * The remaining running time is estimated during register merge. + * + * Revision 1.12 1995/12/06 17:49:19 adam + * Uses dict_delete now. + * + * Revision 1.11 1995/12/06 16:06:43 adam + * Better diagnostics. Work on 'real' dictionary deletion. + * + * Revision 1.10 1995/12/06 12:41:22 adam + * New command 'stat' for the index program. + * Filenames can be read from stdin by specifying '-'. + * Bug fix/enhancement of the transformation from terms to regular + * expressons in the search engine. + * + * Revision 1.9 1995/10/10 12:24:39 adam + * Temporary sort files are compressed. + * + * Revision 1.8 1995/10/04 16:57:19 adam + * Key input and merge sort in one pass. + * + * Revision 1.7 1995/10/02 15:18:52 adam + * New member in recRetrieveCtrl: diagnostic. + * + * Revision 1.6 1995/09/29 15:51:56 adam + * First work on multi-way read. + * + * Revision 1.5 1995/09/29 14:01:43 adam + * Bug fixes. + * + * Revision 1.4 1995/09/28 14:22:57 adam + * Sort uses smaller temporary files. + * + * Revision 1.3 1995/09/06 16:11:17 adam + * Option: only one word key per file. + * + * Revision 1.2 1995/09/04 12:33:42 adam + * Various cleanup. YAZ util used instead. + * + * Revision 1.1 1995/09/04 09:10:37 adam + * More work on index add/del/update. + * Merge sort implemented. + * Initial work on z39 server. + * + */ +