X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fkinput.c;h=ee8b87c6a610f07cdc62a8bead871ca555766173;hb=d960df9d3f91d209797047fca679027e365e8d02;hp=ac03e13b986857eae09c6e7759cc5dee3ea9bdc4;hpb=f0dbdd21e0227559e4407a5e25ce3ead47ede825;p=idzebra-moved-to-github.git diff --git a/index/kinput.c b/index/kinput.c index ac03e13..ee8b87c 100644 --- a/index/kinput.c +++ b/index/kinput.c @@ -1,10 +1,59 @@ /* - * Copyright (C) 1994-1996, Index Data I/S + * Copyright (C) 1994-1999, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: kinput.c,v $ - * Revision 1.20 1996-11-01 08:58:41 adam + * Revision 1.34 1999-05-26 07:49:13 adam + * C++ compilation. + * + * Revision 1.33 1999/05/15 14:36:38 adam + * Updated dictionary. Implemented "compression" of dictionary. + * + * Revision 1.32 1999/05/12 13:08:06 adam + * First version of ISAMS. + * + * Revision 1.31 1999/02/02 14:50:56 adam + * Updated WIN32 code specific sections. Changed header. + * + * Revision 1.30 1998/10/28 10:53:57 adam + * Added type cast to prevent warning. + * + * Revision 1.29 1998/06/11 15:41:39 adam + * Minor changes. + * + * Revision 1.28 1998/03/05 08:45:12 adam + * New result set model and modular ranking system. Moved towards + * descent server API. System information stored as "SGML" records. + * + * Revision 1.27 1998/02/17 10:32:52 adam + * Fixed bug: binary files weren't opened with flag b on NT. + * + * Revision 1.26 1998/01/29 13:39:13 adam + * Compress ISAM is default. + * + * Revision 1.25 1997/09/17 12:19:14 adam + * Zebra version corresponds to YAZ version 1.4. + * Changed Zebra server so that it doesn't depend on global common_resource. + * + * Revision 1.24 1997/09/09 13:38:07 adam + * Partial port to WIN95/NT. + * + * Revision 1.23 1997/09/04 13:57:39 adam + * Added O_BINARY for open calls. + * + * Revision 1.22 1997/02/12 20:39:45 adam + * Implemented options -f that limits the log to the first + * records. + * Changed some log messages also. + * + * Revision 1.21 1996/11/08 11:10:23 adam + * Buffers used during file match got bigger. + * Compressed ISAM support everywhere. + * Bug fixes regarding masking characters in queries. + * Redesigned Regexp-2 queries. + * + * Revision 1.20 1996/11/01 08:58:41 adam * Interface to isamc system now includes update and delete. * * Revision 1.19 1996/10/29 14:09:46 adam @@ -74,7 +123,11 @@ */ #include +#ifdef WIN32 +#include +#else #include +#endif #include #include #include @@ -119,10 +172,10 @@ void getFnameTmp (char *fname, int no) void key_file_chunk_read (struct key_file *f) { - int nr = 0, r, fd; + int nr = 0, r = 0, fd; char fname[1024]; getFnameTmp (fname, f->no); - fd = open (fname, O_RDONLY); + fd = open (fname, O_BINARY|O_RDONLY); if (fd == -1) { logf (LOG_FATAL|LOG_ERRNO, "cannot open %s", fname); @@ -164,7 +217,7 @@ struct key_file *key_file_init (int no, int chunk) { struct key_file *f; - f = xmalloc (sizeof(*f)); + f = (struct key_file *) xmalloc (sizeof(*f)); f->sysno = 0; f->seqno = 0; f->no = no; @@ -172,8 +225,8 @@ struct key_file *key_file_init (int no, int chunk) f->offset = 0; f->length = 0; f->readHandler = NULL; - f->buf = xmalloc (f->chunk); - f->prev_name = xmalloc (INP_NAME_MAX); + f->buf = (unsigned char *) xmalloc (f->chunk); + f->prev_name = (char *) xmalloc (INP_NAME_MAX); *f->prev_name = '\0'; key_file_chunk_read (f); return f; @@ -268,6 +321,7 @@ struct heap_info { Dict dict; ISAM isam; ISAMC isamc; + ISAMS isams; }; struct heap_info *key_heap_init (int nkeys, @@ -276,16 +330,17 @@ struct heap_info *key_heap_init (int nkeys, struct heap_info *hi; int i; - hi = xmalloc (sizeof(*hi)); - hi->info.file = xmalloc (sizeof(*hi->info.file) * (1+nkeys)); - hi->info.buf = xmalloc (sizeof(*hi->info.buf) * (1+nkeys)); + hi = (struct heap_info *) xmalloc (sizeof(*hi)); + hi->info.file = (struct key_file **) + xmalloc (sizeof(*hi->info.file) * (1+nkeys)); + hi->info.buf = (char **) xmalloc (sizeof(*hi->info.buf) * (1+nkeys)); hi->heapnum = 0; - hi->ptr = xmalloc (sizeof(*hi->ptr) * (1+nkeys)); + hi->ptr = (int *) xmalloc (sizeof(*hi->ptr) * (1+nkeys)); hi->cmp = cmp; for (i = 0; i<= nkeys; i++) { hi->ptr[i] = i; - hi->info.buf[i] = xmalloc (INP_NAME_MAX); + hi->info.buf[i] = (char *) xmalloc (INP_NAME_MAX); } return hi; } @@ -375,7 +430,7 @@ struct heap_cread_info { int heap_cread_item (void *vp, char **dst, int *insertMode) { - struct heap_cread_info *p = vp; + struct heap_cread_info *p = (struct heap_cread_info *) vp; struct heap_info *hi = p->hi; if (p->mode == 1) @@ -403,9 +458,9 @@ int heap_cread_item (void *vp, char **dst, int *insertMode) int heap_inpc (struct heap_info *hi) { struct heap_cread_info hci; - ISAMC_I isamc_i = xmalloc (sizeof(*isamc_i)); + ISAMC_I isamc_i = (ISAMC_I) xmalloc (sizeof(*isamc_i)); - hci.key = xmalloc (KEY_SIZE); + hci.key = (char *) xmalloc (KEY_SIZE); hci.mode = 1; hci.hi = hi; hci.more = heap_read_one (hi, hci.cur_name, hci.key); @@ -420,7 +475,8 @@ int heap_inpc (struct heap_info *hi) char *dict_info; strcpy (this_name, hci.cur_name); - logf (LOG_DEBUG, "inserting %s", 1+hci.cur_name); + assert (hci.cur_name[1]); + no_diffs++; if ((dict_info = dict_lookup (hi->dict, hci.cur_name))) { memcpy (&isamc_p, dict_info+1, sizeof(ISAMC_P)); @@ -437,7 +493,6 @@ int heap_inpc (struct heap_info *hi) if (isamc_p2 != isamc_p) dict_insert (hi->dict, this_name, sizeof(ISAMC_P), &isamc_p2); - } } else @@ -451,6 +506,42 @@ int heap_inpc (struct heap_info *hi) return 0; } +int heap_inps (struct heap_info *hi) +{ + struct heap_cread_info hci; + ISAMS_I isams_i = (ISAMS_I) xmalloc (sizeof(*isams_i)); + + hci.key = (char *) xmalloc (KEY_SIZE); + hci.mode = 1; + hci.hi = hi; + hci.more = heap_read_one (hi, hci.cur_name, hci.key); + + isams_i->clientData = &hci; + isams_i->read_item = heap_cread_item; + + while (hci.more) + { + char this_name[INP_NAME_MAX]; + ISAMS_P isams_p; + char *dict_info; + + strcpy (this_name, hci.cur_name); + assert (hci.cur_name[1]); + no_diffs++; + if (!(dict_info = dict_lookup (hi->dict, hci.cur_name))) + { + isams_p = isams_merge (hi->isams, isams_i); + no_insertions++; + dict_insert (hi->dict, this_name, sizeof(ISAMS_P), &isams_p); + } + else + abort(); + } + xfree (isams_i); + return 0; +} + + int heap_inp (struct heap_info *hi) { char *info; @@ -462,8 +553,8 @@ int heap_inp (struct heap_info *hi) char *key_buf; int more; - next_key = xmalloc (KEY_SIZE); - key_buf = xmalloc (key_buf_size); + next_key = (char *) xmalloc (KEY_SIZE); + key_buf = (char *) xmalloc (key_buf_size); more = heap_read_one (hi, cur_name, key_buf); while (more) /* EOF ? */ { @@ -477,10 +568,10 @@ int heap_inp (struct heap_info *hi) break; memcpy (key_buf + key_buf_ptr, next_key, KEY_SIZE); key_buf_ptr += KEY_SIZE; - if (key_buf_ptr+KEY_SIZE >= key_buf_size) + if (key_buf_ptr+(int) KEY_SIZE >= key_buf_size) { char *new_key_buf; - new_key_buf = xmalloc (key_buf_size + INP_BUF_ADD); + new_key_buf = (char *) xmalloc (key_buf_size + INP_BUF_ADD); memcpy (new_key_buf, key_buf, key_buf_size); key_buf_size += INP_BUF_ADD; xfree (key_buf); @@ -489,11 +580,10 @@ int heap_inp (struct heap_info *hi) } no_diffs++; nmemb = key_buf_ptr / KEY_SIZE; - assert (nmemb*KEY_SIZE == key_buf_ptr); + assert (nmemb * (int) KEY_SIZE == key_buf_ptr); if ((info = dict_lookup (hi->dict, cur_name))) { ISAM_P isam_p, isam_p2; - logf (LOG_DEBUG, "updating %s", 1+cur_name); memcpy (&isam_p, info+1, sizeof(ISAM_P)); isam_p2 = is_merge (hi->isam, isam_p, nmemb, key_buf); if (!isam_p2) @@ -512,7 +602,6 @@ int heap_inp (struct heap_info *hi) else { ISAM_P isam_p; - logf (LOG_DEBUG, "inserting %s", 1+cur_name); no_insertions++; isam_p = is_merge (hi->isam, 0, nmemb, key_buf); dict_insert (hi->dict, cur_name, sizeof(ISAM_P), &isam_p); @@ -532,7 +621,7 @@ struct progressInfo { void progressFunc (struct key_file *keyp, void *info) { - struct progressInfo *p = info; + struct progressInfo *p = (struct progressInfo *) info; time_t now, remaining; if (keyp->buf_size <= 0 || p->totalBytes <= 0) @@ -542,8 +631,8 @@ void progressFunc (struct key_file *keyp, void *info) if (now >= p->lastTime+10) { p->lastTime = now; - remaining = (now - p->startTime)* - ((double) p->totalBytes/p->totalOffset - 1.0); + remaining = (time_t) ((now - p->startTime)* + ((double) p->totalBytes/p->totalOffset - 1.0)); if (remaining <= 130) logf (LOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining", (100.0*p->totalOffset) / p->totalBytes, (long) remaining); @@ -554,12 +643,17 @@ void progressFunc (struct key_file *keyp, void *info) p->totalOffset += keyp->buf_size; } -void key_input (int nkeys, int cache) +#ifndef R_OK +#define R_OK 4 +#endif + +void key_input (BFiles bfs, int nkeys, int cache) { Dict dict; ISAM isam = NULL; ISAMC isamc = NULL; + ISAMS isams = NULL; struct key_file **kf; char rbuf[1024]; int i, r; @@ -580,31 +674,43 @@ void key_input (int nkeys, int cache) if (!nkeys) return ; } - dict = dict_open (FNAME_DICT, cache, 1); + dict = dict_open (bfs, FNAME_DICT, cache, 1, 0); if (!dict) { logf (LOG_FATAL, "dict_open fail"); exit (1); } - if (res_get_match (common_resource, "isam", "c", NULL)) + if (res_get_match (common_resource, "isam", "s", NULL)) { - isamc = isc_open (FNAME_ISAMC, 1, key_isamc_m ()); - if (!isamc) + isams = isams_open (bfs, FNAME_ISAMS, 1, + key_isams_m (common_resource)); + if (!isams) { - logf (LOG_FATAL, "isc_open fail"); - exit (1); + logf (LOG_FATAL, "isams_open fail"); + exit (1); } } - else + else if (res_get_match (common_resource, "isam", "i", NULL)) { - isam = is_open (FNAME_ISAM, key_compare, 1, sizeof(struct it_key)); + isam = is_open (bfs, FNAME_ISAM, key_compare, 1, + sizeof(struct it_key), common_resource); if (!isam) { logf (LOG_FATAL, "is_open fail"); exit (1); } } - kf = xmalloc ((1+nkeys) * sizeof(*kf)); + else + { + isamc = isc_open (bfs, FNAME_ISAMC, 1, + key_isamc_m (common_resource)); + if (!isamc) + { + logf (LOG_FATAL, "isc_open fail"); + exit (1); + } + } + kf = (struct key_file **) xmalloc ((1+nkeys) * sizeof(*kf)); progressInfo.totalBytes = 0; progressInfo.totalOffset = 0; time (&progressInfo.startTime); @@ -621,20 +727,25 @@ void key_input (int nkeys, int cache) hi->dict = dict; hi->isam = isam; hi->isamc = isamc; + hi->isams = isams; for (i = 1; i<=nkeys; i++) if ((r = key_file_read (kf[i], rbuf))) key_heap_insert (hi, rbuf, r, kf[i]); if (isamc) heap_inpc (hi); - else - heap_inp (hi); + else if (isams) + heap_inps (hi); + else if (isam) + heap_inp (hi); dict_close (dict); if (isam) is_close (isam); if (isamc) isc_close (isamc); - + if (isams) + isams_close (isams); + for (i = 1; i<=nkeys; i++) { getFnameTmp (rbuf, i);