From c41c84a497ae744aa825a90f144c85b54f1cd4bb Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 20 Mar 2000 19:08:35 +0000 Subject: [PATCH] Added remote record import using Z39.50 extended services and Segment Requests. --- bfile/cfile.c | 18 +- bfile/cfile.h | 13 +- bfile/mfile.c | 9 +- include/isamd.h | 12 +- include/mfile.h | 6 +- include/recctrl.h | 7 +- include/zebra-lock.h | 12 + index/Makefile.in | 6 +- index/extract.c | 63 +-- index/index.h | 25 +- index/kinput.c | 144 +++++- index/main.c | 8 +- index/retrieve.c | 125 ++--- index/zebraapi.c | 1278 ++++++++++++++++++++++++++++++++++++++++++-------- index/zebraapi.h | 16 +- index/zinfo.c | 32 +- index/zinfo.h | 20 +- index/zrpn.c | 8 +- index/zserver.c | 46 +- index/zserver.h | 63 ++- index/zsets.c | 12 +- util/zebra-lock.c | 34 ++ 22 files changed, 1504 insertions(+), 453 deletions(-) diff --git a/bfile/cfile.c b/bfile/cfile.c index d7626b6..5b27f49 100644 --- a/bfile/cfile.c +++ b/bfile/cfile.c @@ -1,10 +1,13 @@ /* - * Copyright (C) 1995-1999, Index Data ApS + * Copyright (C) 1995-2000, Index Data ApS * All rights reserved. - * Sebastian Hammer, Adam Dickmeiss * * $Log: cfile.c,v $ - * Revision 1.25 1999-05-26 07:49:12 adam + * Revision 1.26 2000-03-20 19:08:35 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.25 1999/05/26 07:49:12 adam * C++ compilation. * * Revision 1.24 1999/05/12 13:08:06 adam @@ -206,6 +209,7 @@ CFile cf_open (MFile mf, MFile_area area, const char *fname, memset (cf->iobuf, 0, cf->head.block_size); cf->no_hits = 0; cf->no_miss = 0; + zebra_mutex_init (&cf->mutex); return cf; } @@ -515,8 +519,13 @@ int cf_read (CFile cf, int no, int offset, int nbytes, void *buf) int block; assert (cf); + zebra_mutex_lock (&cf->mutex); if (!(block = cf_lookup (cf, no))) + { + zebra_mutex_unlock (&cf->mutex); return -1; + } + zebra_mutex_unlock (&cf->mutex); if (!mf_read (cf->block_mf, block, offset, nbytes, buf)) { logf (LOG_FATAL|LOG_ERRNO, "cf_read no=%d, block=%d", no, block); @@ -530,6 +539,7 @@ int cf_write (CFile cf, int no, int offset, int nbytes, const void *buf) int block; assert (cf); + zebra_mutex_lock (&cf->mutex); if (!(block = cf_lookup (cf, no))) { block = cf_new (cf, no); @@ -542,6 +552,7 @@ int cf_write (CFile cf, int no, int offset, int nbytes, const void *buf) nbytes = 0; } } + zebra_mutex_unlock (&cf->mutex); if (mf_write (cf->block_mf, block, offset, nbytes, buf)) { logf (LOG_FATAL|LOG_ERRNO, "cf_write no=%d, block=%d", no, block); @@ -566,6 +577,7 @@ int cf_close (CFile cf) xfree (cf->array); xfree (cf->parray); xfree (cf->iobuf); + zebra_mutex_destroy (&cf->mutex); xfree (cf); return 0; } diff --git a/bfile/cfile.h b/bfile/cfile.h index 0f22c29..ef0a692 100644 --- a/bfile/cfile.h +++ b/bfile/cfile.h @@ -3,15 +3,15 @@ * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - * $Id: cfile.h,v 1.12 1999-05-26 07:49:12 adam Exp $ + * $Id: cfile.h,v 1.13 2000-03-20 19:08:35 adam Exp $ */ #ifndef CFILE_H #define CFILE_H -#ifdef __cplusplus -extern "C" { -#endif +#include + +YAZ_BEGIN_CDECL #define HASH_BUCKET 15 @@ -56,6 +56,7 @@ typedef struct CFile_struct MFile rmf; int no_hits; int no_miss; + Zebra_mutex mutex; } *CFile; int cf_close (CFile cf); @@ -66,8 +67,6 @@ int cf_write (CFile cf, int no, int offset, int nbytes, const void *buf); void cf_unlink (CFile cf); void cf_commit (CFile cf); -#ifdef __cplusplus -} -#endif +YAZ_END_CDECL #endif diff --git a/bfile/mfile.c b/bfile/mfile.c index 684d6b0..6942bbd 100644 --- a/bfile/mfile.c +++ b/bfile/mfile.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: mfile.c,v $ - * Revision 1.37 2000-03-15 15:00:30 adam + * Revision 1.38 2000-03-20 19:08:35 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.37 2000/03/15 15:00:30 adam * First work on threaded version. * * Revision 1.36 1999/12/08 15:03:11 adam @@ -527,7 +531,10 @@ int mf_read(MFile mf, int no, int offset, int nbytes, void *buf) if ((rd = file_position(mf, no, offset)) < 0) { if (rd == -2) + { + zebra_mutex_unlock (&mf->mutex); return 0; + } else exit(1); } diff --git a/include/isamd.h b/include/isamd.h index 9dc95fe..3228ad3 100644 --- a/include/isamd.h +++ b/include/isamd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995-1997, Index Data. + * Copyright (c) 1995-2000, Index Data. * See the file LICENSE for details. * * IsamH is a simple ISAM that can only append to the end of the list. @@ -17,9 +17,7 @@ #include -#ifdef __cplusplus -extern "C" { -#endif +YAZ_BEGIN_CDECL typedef struct ISAMD_s *ISAMD; typedef int ISAMD_P; @@ -77,11 +75,7 @@ int isamd_block_size (ISAMD is, int type); void isamd_buildfirstblock(ISAMD_PP pp); void isamd_buildlaterblock(ISAMD_PP pp); - - -#ifdef __cplusplus -} -#endif +YAZ_END_CDECL #endif /* ISAMD_H */ diff --git a/include/mfile.h b/include/mfile.h index dc62499..26c9300 100644 --- a/include/mfile.h +++ b/include/mfile.h @@ -3,7 +3,7 @@ * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * - * $Id: mfile.h,v 1.13 2000-03-15 15:00:30 adam Exp $ + * $Id: mfile.h,v 1.14 2000-03-20 19:08:36 adam Exp $ */ #ifndef MFILE_H @@ -55,8 +55,8 @@ typedef struct meta_file int no_files; int cur_file; int open; /* is this file open? */ - int blocksize; - int min_bytes_creat; /* minimum bytes required to enter directory */ + off_t blocksize; + off_t min_bytes_creat; /* minimum bytes required to enter directory */ MFile_area ma; int wr; Zebra_mutex mutex; diff --git a/include/recctrl.h b/include/recctrl.h index 394dbba..1b3cce8 100644 --- a/include/recctrl.h +++ b/include/recctrl.h @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recctrl.h,v $ - * Revision 1.33 2000-02-25 13:24:49 adam + * Revision 1.34 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.33 2000/02/25 13:24:49 adam * Fixed bug regarding pointer conversion that showed up on OSF V5. * * Revision 1.32 1999/11/30 13:48:03 adam @@ -160,6 +164,7 @@ struct recExtractCtrl { int seqno[256]; void (*schemaAdd)(struct recExtractCtrl *p, Odr_oid *oid); data1_handle dh; + void *handle; }; /* Retrieve record control */ diff --git a/include/zebra-lock.h b/include/zebra-lock.h index b7d6164..01f338d 100644 --- a/include/zebra-lock.h +++ b/include/zebra-lock.h @@ -33,5 +33,17 @@ YAZ_EXPORT int zebra_lock_rdwr_wlock (Zebra_lock_rdwr *p); YAZ_EXPORT int zebra_lock_rdwr_runlock (Zebra_lock_rdwr *p); YAZ_EXPORT int zebra_lock_rdwr_wunlock (Zebra_lock_rdwr *p); +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; +} Zebra_mutex_cond; + +YAZ_EXPORT int zebra_mutex_cond_init (Zebra_mutex_cond *p); +YAZ_EXPORT int zebra_mutex_cond_destroy (Zebra_mutex_cond *p); +YAZ_EXPORT int zebra_mutex_cond_lock (Zebra_mutex_cond *p); +YAZ_EXPORT int zebra_mutex_cond_unlock (Zebra_mutex_cond *p); +YAZ_EXPORT int zebra_mutex_cond_wait (Zebra_mutex_cond *p); +YAZ_EXPORT int zebra_mutex_cond_signal (Zebra_mutex_cond *p); + YAZ_END_CDECL #endif diff --git a/index/Makefile.in b/index/Makefile.in index e126644..a519e7d 100644 --- a/index/Makefile.in +++ b/index/Makefile.in @@ -1,7 +1,7 @@ # Copyright (C) 1995-1999, Index Data # All rights reserved. # Sebastian Hammer, Adam Dickmeiss -# $Id: Makefile.in,v 1.10 1999-12-01 13:30:30 adam Exp $ +# $Id: Makefile.in,v 1.11 2000-03-20 19:08:36 adam Exp $ SHELL=/bin/sh @@ -29,7 +29,7 @@ O1 = main.o dir.o dirs.o trav.o extract.o kinput.o kcompare.o \ O2 = kdump.o O3 = zserver.o kcompare.o zrpn.o zsets.o attribute.o recindex.o \ lockutil.o locksrv.o zinfo.o trunc.o sortidx.o rank1.o zebraapi.o \ - retrieve.o + retrieve.o kinput.o O4 = hlvl.o hlvltest.o kcompare.o O5 = apitest.o kcompare.o zrpn.o zsets.o attribute.o recindex.o \ lockutil.o locksrv.o zinfo.o trunc.o sortidx.o rank1.o zebraapi.o \ @@ -62,7 +62,7 @@ $(TPROG5): $(O5) @SUBLIBS@ $(CC) -c $(DEFS) $(CFLAGS) $< clean: - rm -f *.[oa] $(TPROG1) $(TPROG2) $(TPROG3) + rm -f *.[oa] $(TPROG1) $(TPROG2) $(TPROG3) $(TPROG4) $(TPROG5) rm -f core mon.out gmon.out errlist depend: diff --git a/index/extract.c b/index/extract.c index fa7f652..7c325ca 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.99 2000-02-24 10:57:02 adam + * Revision 1.100 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.99 2000/02/24 10:57:02 adam * Sequence number incremented after each incomplete-field. * * Revision 1.98 1999/09/07 07:19:21 adam @@ -562,7 +566,7 @@ void key_flush (void) logf (LOG_LOG, "sorting section %d", key_file_no); #if !SORT_EXTRA qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_qsort_compare); - getFnameTmp (out_fname, key_file_no); + getFnameTmp (common_resource, out_fname, key_file_no); if (!(outf = fopen (out_fname, "wb"))) { @@ -637,7 +641,7 @@ int key_close (struct recordGroup *rGroup) int rw = rGroup->flagRw; if (rw) zebraExplain_runNumberIncrement (zti, 1); - zebraExplain_close (zti, rw, 0); + zebraExplain_close (zti, rw); key_flush (); xfree (key_buf); rec_close (&records); @@ -1681,59 +1685,6 @@ int fileExtract (SYSNO *sysno, const char *fname, if (rGroup->flagStoreKeys == -1) rGroup->flagStoreKeys = 0; -#if ZEBRASDR - if (rGroup->useSDR) - { - ZebraSdrHandle h; - char xname[128], *xp; - - strncpy (xname, fname, 127); - if (!(xp = strchr (xname, '.'))) - return 0; - *xp = '\0'; - if (strcmp (xp+1, "sdr.bits")) - return 0; - - h = zebraSdr_open (xname); - if (!h) - { - logf (LOG_WARN, "sdr open %s", xname); - return 0; - } - for (;;) - { - unsigned char *buf; - char sdr_name[128]; - int r, segmentno; - - segmentno = zebraSdr_segment (h, 0); - sprintf (sdr_name, "%%%s.%d", xname, segmentno); - -#if 0 - if (segmentno > 20) - break; -#endif - r = zebraSdr_read (h, &buf); - - if (!r) - break; - - fi = file_read_start (0); - fi->sdrbuf = buf; - fi->sdrmax = r; - do - { - file_begin (fi); - r = recordExtract (sysno, sdr_name, rGroup, deleteFlag, fi, - recType, subType); - } while (r && !sysno && fi->file_more); - file_read_stop (fi); - free (buf); - } - zebraSdr_close (h); - return 1; - } -#endif if (sysno && deleteFlag) fd = -1; else diff --git a/index/index.h b/index/index.h index 219ab38..3fabd51 100644 --- a/index/index.h +++ b/index/index.h @@ -4,6 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss, Heikki Levanto * (log at the end) */ + +#ifndef INDEX_H +#define INDEX_H + #include #include #include @@ -22,9 +26,7 @@ #include #include -#ifdef __cplusplus -extern "C" { -#endif +YAZ_BEGIN_CDECL #define IT_MAX_WORD 256 #define IT_KEY_HAVE_SEQNO 1 @@ -73,7 +75,7 @@ struct recordGroup { RecTypes recTypes; }; -void getFnameTmp (char *fname, int no); +void getFnameTmp (Res res, char *fname, int no); struct dirs_info *dirs_open (Dict dict, const char *rep, int rw); struct dirs_info *dirs_fopen (Dict dict, const char *path); @@ -103,7 +105,7 @@ int key_qsort_compare (const void *p1, const void *p2); void key_logdump (int mask, const void *p); void inv_prstat (BFiles bfs); void inv_compact (BFiles bfs); -void key_input (BFiles bfs, int nkeys, int cache); +void key_input (BFiles bfs, int nkeys, int cache, Res res); ISAMS_M key_isams_m (Res res, ISAMS_M me); #if ZMBOL ISAMC_M key_isamc_m (Res res, ISAMC_M me); @@ -158,15 +160,16 @@ void zebra_load_atts (data1_handle dh, Res res); extern Res common_resource; -#ifdef __cplusplus -} -#endif - - +YAZ_END_CDECL +#endif /* * $Log: index.h,v $ - * Revision 1.68 2000-02-24 11:00:07 adam + * Revision 1.69 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.68 2000/02/24 11:00:07 adam * Fixed bug: indexer would run forever when lock dir was non-existant. * * Revision 1.67 1999/11/30 13:48:03 adam diff --git a/index/kinput.c b/index/kinput.c index f7d7dae..13f3685 100644 --- a/index/kinput.c +++ b/index/kinput.c @@ -23,6 +23,7 @@ #include #include "index.h" +#include "zserver.h" #define KEY_SIZE (1+sizeof(struct it_key)) #define INP_NAME_MAX 768 @@ -49,13 +50,22 @@ struct key_file { /* handler invoked in each read */ void (*readHandler)(struct key_file *keyp, void *rinfo); void *readInfo; + Res res; }; -void getFnameTmp (char *fname, int no) +void getFnameTmp (Res res, char *fname, int no) { const char *pre; - pre = res_get_def (common_resource, "keyTmpDir", "."); + pre = res_get_def (res, "keyTmpDir", "."); + sprintf (fname, "%s/key%d.tmp", pre, no); +} + +void extract_get_fname_tmp (ZebraHandle zh, char *fname, int no) +{ + const char *pre; + + pre = res_get_def (zh->service->res, "keyTmpDir", "."); sprintf (fname, "%s/key%d.tmp", pre, no); } @@ -63,25 +73,30 @@ void key_file_chunk_read (struct key_file *f) { int nr = 0, r = 0, fd; char fname[1024]; - getFnameTmp (fname, f->no); + getFnameTmp (f->res, fname, f->no); fd = open (fname, O_BINARY|O_RDONLY); + + f->buf_ptr = 0; + f->buf_size = 0; if (fd == -1) { - logf (LOG_FATAL|LOG_ERRNO, "cannot open %s", fname); - exit (1); + logf (LOG_WARN|LOG_ERRNO, "cannot open %s", fname); + return ; } if (!f->length) { if ((f->length = lseek (fd, 0L, SEEK_END)) == (off_t) -1) { - logf (LOG_FATAL|LOG_ERRNO, "cannot seek %s", fname); - exit (1); + logf (LOG_WARN|LOG_ERRNO, "cannot seek %s", fname); + close (fd); + return ; } } if (lseek (fd, f->offset, SEEK_SET) == -1) { - logf (LOG_FATAL|LOG_ERRNO, "cannot seek %s", fname); - exit (1); + logf (LOG_WARN|LOG_ERRNO, "cannot seek %s", fname); + close(fd); + return ; } while (f->chunk - nr > 0) { @@ -92,21 +107,22 @@ void key_file_chunk_read (struct key_file *f) } if (r == -1) { - logf (LOG_FATAL|LOG_ERRNO, "read of %s", fname); - exit (1); + logf (LOG_WARN|LOG_ERRNO, "read of %s", fname); + close (fd); + return; } f->buf_size = nr; - f->buf_ptr = 0; if (f->readHandler) (*f->readHandler)(f, f->readInfo); close (fd); } -struct key_file *key_file_init (int no, int chunk) +struct key_file *key_file_init (int no, int chunk, Res res) { struct key_file *f; f = (struct key_file *) xmalloc (sizeof(*f)); + f->res = res; f->sysno = 0; f->seqno = 0; f->no = no; @@ -595,7 +611,79 @@ void progressFunc (struct key_file *keyp, void *info) #define R_OK 4 #endif -void key_input (BFiles bfs, int nkeys, int cache) +void zebra_index_merge (ZebraHandle zh) +{ + struct key_file **kf; + char rbuf[1024]; + int i, r; + struct heap_info *hi; + struct progressInfo progressInfo; + int nkeys = zh->key_file_no; + + if (nkeys < 0) + { + char fname[1024]; + nkeys = 0; + while (1) + { + extract_get_fname_tmp (zh, fname, nkeys+1); + if (access (fname, R_OK) == -1) + break; + nkeys++; + } + if (!nkeys) + return ; + } + kf = (struct key_file **) xmalloc ((1+nkeys) * sizeof(*kf)); + progressInfo.totalBytes = 0; + progressInfo.totalOffset = 0; + time (&progressInfo.startTime); + time (&progressInfo.lastTime); + for (i = 1; i<=nkeys; i++) + { + kf[i] = key_file_init (i, 32768, zh->service->res); + kf[i]->readHandler = progressFunc; + kf[i]->readInfo = &progressInfo; + progressInfo.totalBytes += kf[i]->length; + progressInfo.totalOffset += kf[i]->buf_size; + } + hi = key_heap_init (nkeys, key_qsort_compare); + hi->dict = zh->service->dict; + hi->isams = zh->service->isams; +#if ZMBOL + hi->isam = zh->service->isam; + hi->isamc = zh->service->isamc; + hi->isamd = zh->service->isamd; +#endif + + for (i = 1; i<=nkeys; i++) + if ((r = key_file_read (kf[i], rbuf))) + key_heap_insert (hi, rbuf, r, kf[i]); + if (zh->service->isams) + heap_inps (hi); +#if ZMBOL + else if (zh->service->isamc) + heap_inpc (hi); + else if (zh->service->isam) + heap_inp (hi); + else if (zh->service->isamd) + heap_inpd (hi); +#endif + + for (i = 1; i<=nkeys; i++) + { + extract_get_fname_tmp (zh, rbuf, i); + unlink (rbuf); + } + logf (LOG_LOG, "Iterations . . .%7d", no_iterations); + logf (LOG_LOG, "Distinct words .%7d", no_diffs); + logf (LOG_LOG, "Updates. . . . .%7d", no_updates); + logf (LOG_LOG, "Deletions. . . .%7d", no_deletions); + logf (LOG_LOG, "Insertions . . .%7d", no_insertions); + zh->key_file_no = 0; +} + +void key_input (BFiles bfs, int nkeys, int cache, Res res) { Dict dict; @@ -617,7 +705,7 @@ void key_input (BFiles bfs, int nkeys, int cache) nkeys = 0; while (1) { - getFnameTmp (fname, nkeys+1); + getFnameTmp (res, fname, nkeys+1); if (access (fname, R_OK) == -1) break; nkeys++; @@ -631,11 +719,11 @@ void key_input (BFiles bfs, int nkeys, int cache) logf (LOG_FATAL, "dict_open fail"); exit (1); } - if (res_get_match (common_resource, "isam", "s", ISAM_DEFAULT)) + if (res_get_match (res, "isam", "s", ISAM_DEFAULT)) { struct ISAMS_M_s isams_m; isams = isams_open (bfs, FNAME_ISAMS, 1, - key_isams_m (common_resource, &isams_m)); + key_isams_m (res, &isams_m)); if (!isams) { logf (LOG_FATAL, "isams_open fail"); @@ -644,32 +732,32 @@ void key_input (BFiles bfs, int nkeys, int cache) logf (LOG_LOG, "isams opened"); } #if ZMBOL - else if (res_get_match (common_resource, "isam", "i", ISAM_DEFAULT)) + else if (res_get_match (res, "isam", "i", ISAM_DEFAULT)) { isam = is_open (bfs, FNAME_ISAM, key_compare, 1, - sizeof(struct it_key), common_resource); + sizeof(struct it_key), res); if (!isam) { logf (LOG_FATAL, "is_open fail"); exit (1); } } - else if (res_get_match (common_resource, "isam", "d", ISAM_DEFAULT)) + else if (res_get_match (res, "isam", "d", ISAM_DEFAULT)) { struct ISAMD_M_s isamd_m; isamd = isamd_open (bfs, FNAME_ISAMD, 1, - key_isamd_m (common_resource,&isamd_m)); + key_isamd_m (res,&isamd_m)); if (!isamd) { logf (LOG_FATAL, "isamd_open fail"); exit (1); } } - else if (res_get_match (common_resource, "isam", "c", ISAM_DEFAULT)) + else if (res_get_match (res, "isam", "c", ISAM_DEFAULT)) { struct ISAMC_M_s isamc_m; isamc = isc_open (bfs, FNAME_ISAMC, 1, - key_isamc_m (common_resource, &isamc_m)); + key_isamc_m (res, &isamc_m)); if (!isamc) { logf (LOG_FATAL, "isc_open fail"); @@ -684,7 +772,7 @@ void key_input (BFiles bfs, int nkeys, int cache) time (&progressInfo.lastTime); for (i = 1; i<=nkeys; i++) { - kf[i] = key_file_init (i, 32768); + kf[i] = key_file_init (i, 32768, res); kf[i]->readHandler = progressFunc; kf[i]->readInfo = &progressInfo; progressInfo.totalBytes += kf[i]->length; @@ -727,7 +815,7 @@ void key_input (BFiles bfs, int nkeys, int cache) for (i = 1; i<=nkeys; i++) { - getFnameTmp (rbuf, i); + getFnameTmp (res, rbuf, i); unlink (rbuf); } logf (LOG_LOG, "Iterations . . .%7d", no_iterations); @@ -743,7 +831,11 @@ void key_input (BFiles bfs, int nkeys, int cache) /* * $Log: kinput.c,v $ - * Revision 1.42 1999-12-01 21:58:48 adam + * Revision 1.43 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.42 1999/12/01 21:58:48 adam * Proper handle of illegal use of isams. * * Revision 1.41 1999/11/30 13:48:03 adam diff --git a/index/main.c b/index/main.c index 00a35f1..79104a8 100644 --- a/index/main.c +++ b/index/main.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: main.c,v $ - * Revision 1.75 1999-12-08 22:44:45 adam + * Revision 1.76 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.75 1999/12/08 22:44:45 adam * Zebra/Z'mbol dependencies added. * * Revision 1.74 1999/12/08 15:03:11 adam @@ -567,7 +571,7 @@ int main (int argc, char **argv) if (nsections) { logf (LOG_LOG, "merging with index"); - key_input (rGroup.bfs, nsections, 60); + key_input (rGroup.bfs, nsections, 60, common_resource); #ifndef WIN32 sync (); #endif diff --git a/index/retrieve.c b/index/retrieve.c index 34f0dc0..2eb556a 100644 --- a/index/retrieve.c +++ b/index/retrieve.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: retrieve.c,v $ - * Revision 1.12 2000-03-15 15:00:30 adam + * Revision 1.13 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.12 2000/03/15 15:00:30 adam * First work on threaded version. * * Revision 1.11 1999/10/29 10:00:00 adam @@ -65,47 +69,39 @@ #include "zebrasdr.h" #endif -struct fetch_control { - int record_offset; - int record_int_pos; - char *record_int_buf; - int record_int_len; - int fd; -}; - -static int record_ext_read (void *fh, char *buf, size_t count) +int zebra_record_ext_read (void *fh, char *buf, size_t count) { - struct fetch_control *fc = (struct fetch_control *) fh; + struct zebra_fetch_control *fc = (struct zebra_fetch_control *) fh; return read (fc->fd, buf, count); } -static off_t record_ext_seek (void *fh, off_t offset) +off_t zebra_record_ext_seek (void *fh, off_t offset) { - struct fetch_control *fc = (struct fetch_control *) fh; + struct zebra_fetch_control *fc = (struct zebra_fetch_control *) fh; return lseek (fc->fd, offset + fc->record_offset, SEEK_SET); } -static off_t record_ext_tell (void *fh) +off_t zebra_record_ext_tell (void *fh) { - struct fetch_control *fc = (struct fetch_control *) fh; + struct zebra_fetch_control *fc = (struct zebra_fetch_control *) fh; return lseek (fc->fd, 0, SEEK_CUR) - fc->record_offset; } -static off_t record_int_seek (void *fh, off_t offset) +off_t zebra_record_int_seek (void *fh, off_t offset) { - struct fetch_control *fc = (struct fetch_control *) fh; + struct zebra_fetch_control *fc = (struct zebra_fetch_control *) fh; return (off_t) (fc->record_int_pos = offset); } -static off_t record_int_tell (void *fh) +off_t zebra_record_int_tell (void *fh) { - struct fetch_control *fc = (struct fetch_control *) fh; + struct zebra_fetch_control *fc = (struct zebra_fetch_control *) fh; return (off_t) fc->record_int_pos; } -static int record_int_read (void *fh, char *buf, size_t count) +int zebra_record_int_read (void *fh, char *buf, size_t count) { - struct fetch_control *fc = (struct fetch_control *) fh; + struct zebra_fetch_control *fc = (struct zebra_fetch_control *) fh; int l = fc->record_int_len - fc->record_int_pos; if (l <= 0) return 0; @@ -115,6 +111,12 @@ static int record_int_read (void *fh, char *buf, size_t count) return l; } +void zebra_record_int_end (void *fh, off_t off) +{ + struct zebra_fetch_control *fc = (struct zebra_fetch_control *) fh; + fc->offset_end = off; +} + int zebra_record_fetch (ZebraHandle zh, int sysno, int score, ODR stream, oid_value input_format, Z_RecordComposition *comp, oid_value *output_format, char **rec_bufp, @@ -125,7 +127,7 @@ int zebra_record_fetch (ZebraHandle zh, int sysno, int score, ODR stream, RecType rt; struct recRetrieveCtrl retrieveCtrl; char subType[128]; - struct fetch_control fc; + struct zebra_fetch_control fc; RecordAttr *recordAttr; void *clientData; @@ -155,79 +157,14 @@ int zebra_record_fetch (ZebraHandle zh, int sysno, int score, ODR stream, fc.fd = -1; if (rec->size[recInfo_storeData] > 0) { - retrieveCtrl.readf = record_int_read; - retrieveCtrl.seekf = record_int_seek; - retrieveCtrl.tellf = record_int_tell; + retrieveCtrl.readf = zebra_record_int_read; + retrieveCtrl.seekf = zebra_record_int_seek; + retrieveCtrl.tellf = zebra_record_int_tell; fc.record_int_len = rec->size[recInfo_storeData]; fc.record_int_buf = rec->info[recInfo_storeData]; fc.record_int_pos = 0; logf (LOG_DEBUG, "Internal retrieve. %d bytes", fc.record_int_len); } -#if ZEBRASDR - else if (*fname == '%') - { - ZebraSdrHandle h; - int segment = 0, r; - char *cp, xname[128]; - unsigned char *buf; - - logf (LOG_DEBUG, "SDR"); - strcpy (xname, fname+1); - if ((cp = strrchr (xname, '.'))) - { - *cp++ = 0; - segment = atoi(cp); - } - h = zebraSdr_open (xname); - if (!h) - { - logf (LOG_WARN, "sdr open %s", xname); - return 0; - } - if (zebraSdr_segment (h, &segment) < 0) - { - logf (LOG_WARN, "zebraSdr_segment fail segment=%d", - segment); - rec_rm (&rec); - return 14; - } - r = zebraSdr_read (h, &buf); - if (r < 1) - { - logf (LOG_WARN, "zebraSdr_read fail segment=%d", - segment); - rec_rm (&rec); - return 14; - } - zebraSdr_close (h); - - fc.record_int_len = recordAttr->recordSize; - fc.record_int_buf = buf + recordAttr->recordOffset; - fc.record_int_pos = 0; - - logf (LOG_LOG, "segment = %d len=%d off=%d", - segment, - recordAttr->recordSize, - recordAttr->recordOffset); - if (fc.record_int_len > 180) - { - logf (LOG_LOG, "%.70s", fc.record_int_buf); - logf (LOG_LOG, "%.70s", fc.record_int_buf + - (fc.record_int_len - 70)); - } - else - logf (LOG_LOG, "%.*s", - fc.record_int_len, fc.record_int_buf); - - /* the following two lines makes rec_rm delete buf */ - rec->size[recInfo_storeData] = r; - rec->info[recInfo_storeData] = buf; - - retrieveCtrl.readf = record_int_read; - retrieveCtrl.seekf = record_int_seek; - retrieveCtrl.tellf = record_int_tell; - } -#endif else { if ((fc.fd = open (fname, O_BINARY|O_RDONLY)) == -1) @@ -239,11 +176,11 @@ int zebra_record_fetch (ZebraHandle zh, int sysno, int score, ODR stream, } fc.record_offset = recordAttr->recordOffset; - retrieveCtrl.readf = record_ext_read; - retrieveCtrl.seekf = record_ext_seek; - retrieveCtrl.tellf = record_ext_tell; + retrieveCtrl.readf = zebra_record_ext_read; + retrieveCtrl.seekf = zebra_record_ext_seek; + retrieveCtrl.tellf = zebra_record_ext_tell; - record_ext_seek (retrieveCtrl.fh, 0); + zebra_record_ext_seek (retrieveCtrl.fh, 0); } retrieveCtrl.subType = subType; retrieveCtrl.localno = sysno; diff --git a/index/zebraapi.c b/index/zebraapi.c index 1bb461b..7062e17 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -1,10 +1,13 @@ /* - * Copyright (C) 1995-1999, Index Data + * Copyright (C) 1995-2000, Index Data * All rights reserved. - * Sebastian Hammer, Adam Dickmeiss * * $Log: zebraapi.c,v $ - * Revision 1.28 2000-03-15 15:00:30 adam + * Revision 1.29 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.28 2000/03/15 15:00:30 adam * First work on threaded version. * * Revision 1.27 2000/02/24 12:31:17 adam @@ -99,6 +102,7 @@ * */ +#include #include #ifdef WIN32 #include @@ -110,6 +114,7 @@ #include #include "zserver.h" +#include static void zebra_chdir (ZebraService zh) { @@ -124,205 +129,99 @@ static void zebra_chdir (ZebraService zh) #endif } -static void zebra_register_unlock (ZebraService zh); - -static int zebra_register_lock (ZebraService zh) -{ - time_t lastChange; - int state; - int errCode = 0; +static int extract_rec_in_mem (ZebraHandle zh, const char *recordType, + const char *buf, size_t buf_size, + const char *databaseName, int delete_flag, + int test_mode, int *sysno, + int store_keys, int store_data, + const char *match_criteria); - errCode = 0; +static int explain_extract (void *handle, Record rec, data1_node *n); +static void extract_index (ZebraHandle zh); - zebra_chdir (zh); - - state = zebra_server_lock_get_state(zh, &lastChange); - - switch (state) - { - case 'c': - state = 1; - break; - default: - state = 0; - } - zebra_server_lock (zh, state); -#if HAVE_SYS_TIMES_H - times (&zh->tms1); -#endif - if (zh->registerState == state) - { - if (zh->registerChange >= lastChange) - return 0; - logf (LOG_LOG, "Register completely updated since last access"); - } - else if (zh->registerState == -1) - logf (LOG_LOG, "Reading register using state %d pid=%ld", state, - (long) getpid()); - else - logf (LOG_LOG, "Register has changed state from %d to %d", - zh->registerState, state); - zh->registerChange = lastChange; - if (zh->records) - { - zebraExplain_close (zh->zei, 0, 0); - if (zh->dict) - dict_close (zh->dict); - if (zh->sortIdx) - sortIdx_close (zh->sortIdx); - if (zh->isams) - isams_close (zh->isams); -#if ZMBOL - if (zh->isam) - is_close (zh->isam); - if (zh->isamc) - isc_close (zh->isamc); -#endif - rec_close (&zh->records); - } - bf_cache (zh->bfs, state ? res_get (zh->res, "shadow") : NULL); - zh->registerState = state; - - zh->isams = NULL; -#if ZMBOL - zh->isam = NULL; - zh->isamc = NULL; -#endif - zh->dict = NULL; - zh->sortIdx = NULL; - zh->zei = NULL; +static void zebra_register_unlock (ZebraHandle zh); - if (!(zh->records = rec_open (zh->bfs, 0, 0))) - { - logf (LOG_WARN, "rec_open"); - errCode = 2; - } - else - { - if (!(zh->dict = dict_open (zh->bfs, FNAME_DICT, 40, 0, 0))) - { - logf (LOG_WARN, "dict_open"); - errCode = 2; - } - if (!(zh->sortIdx = sortIdx_open (zh->bfs, 0))) - { - logf (LOG_WARN, "sortIdx_open"); - errCode = 2; - } - if (res_get_match (zh->res, "isam", "s", ISAM_DEFAULT)) - { - struct ISAMS_M_s isams_m; - if (!(zh->isams = isams_open (zh->bfs, FNAME_ISAMS, 0, - key_isams_m(zh->res, &isams_m)))) - { - logf (LOG_WARN, "isams_open"); - errCode = 2; - } - } -#if ZMBOL - else if (res_get_match (zh->res, "isam", "i", ISAM_DEFAULT)) - { - if (!(zh->isam = is_open (zh->bfs, FNAME_ISAM, key_compare, 0, - sizeof (struct it_key), zh->res))) - { - logf (LOG_WARN, "is_open"); - errCode = 2; - } - } - else if (res_get_match (zh->res, "isam", "c", ISAM_DEFAULT)) - { - struct ISAMC_M_s isamc_m; - if (!(zh->isamc = isc_open (zh->bfs, FNAME_ISAMC, - 0, key_isamc_m(zh->res, &isamc_m)))) - { - logf (LOG_WARN, "isc_open"); - errCode = 2; - } - } -#endif - zh->zei = zebraExplain_open (zh->records, zh->dh, - zh->res, 0, 0, 0); - if (!zh->zei) - { - logf (LOG_WARN, "Cannot obtain EXPLAIN information"); - errCode = 2; - } - } - if (errCode) +static int zebra_register_lock (ZebraHandle zh) +{ + if (!zh->service->active) { - zebra_register_unlock (zh); - zh->registerState = -1; - return -1; + zh->errCode = 1019; + return 1; } - return errCode; + return 0; } -static void zebra_register_unlock (ZebraService zh) +static void zebra_register_unlock (ZebraHandle zh) { - static int waitSec = -1; - -#if HAVE_SYS_TIMES_H - times (&zh->tms2); - logf (LOG_LOG, "user/system: %ld/%ld", - (long) (zh->tms2.tms_utime - zh->tms1.tms_utime), - (long) (zh->tms2.tms_stime - zh->tms1.tms_stime)); -#endif - if (waitSec == -1) - { - char *s = res_get (zh->res, "debugRequestWait"); - if (s) - waitSec = atoi (s); - else - waitSec = 0; - } -#ifdef WIN32 -#else - if (waitSec > 0) - sleep (waitSec); -#endif - if (zh->registerState != -1) - zebra_server_unlock (zh, zh->registerState); } -ZebraHandle zebra_open (ZebraService service) +ZebraHandle zebra_open (ZebraService zs) { ZebraHandle zh; + assert (zs); + if (zs->stop_flag) + { + zh->errCode = 1019; + return 0; + } + zh = (ZebraHandle) xmalloc (sizeof(*zh)); - zh->service = service; + zh->service = zs; zh->sets = 0; + zh->destroyed = 0; + zh->errCode = 0; + zh->errString = 0; + + zh->key_buf = 0; + + zebra_mutex_cond_lock (&zs->session_lock); + + zh->next = zs->sessions; + zs->sessions = zh; + zebra_mutex_cond_unlock (&zs->session_lock); return zh; } +static int zebra_register_activate (ZebraService zh); +static int zebra_register_deactivate (ZebraService zh); + ZebraService zebra_start (const char *configName) { ZebraService zh = xmalloc (sizeof(*zh)); + + yaz_log (LOG_LOG, "zebra_start %s", configName); + zh->configName = xstrdup(configName); zh->sessions = 0; - yaz_log (LOG_LOG, "zebra_start %s", configName); - if (!(zh->res = res_open (configName))) + zh->stop_flag = 0; + zh->active = 0; + zebra_mutex_cond_init (&zh->session_lock); + zebra_register_activate (zh); + return zh; +} + +static int zebra_register_activate (ZebraService zh) +{ + if (zh->active) + return 0; + yaz_log (LOG_LOG, "zebra_register_activate"); + if (!(zh->res = res_open (zh->configName))) { - logf (LOG_WARN, "Failed to read resources `%s'", configName); - return NULL; + logf (LOG_WARN, "Failed to read resources `%s'", zh->configName); + return -1; } zebra_chdir (zh); - zebra_server_lock_init (zh); zh->dh = data1_create (); if (!zh->dh) - { - zebra_server_lock_destroy (zh); - xfree (zh); - return 0; - } + return -1; zh->bfs = bfs_create (res_get (zh->res, "register")); if (!zh->bfs) { - zebra_server_lock_destroy (zh); data1_destroy(zh->dh); - xfree (zh); - return 0; + return -1; } bf_lockDir (zh->bfs, res_get (zh->res, "lockDir")); data1_set_tabpath (zh->dh, res_get(zh->res, "profilePath")); @@ -334,6 +233,15 @@ ZebraService zebra_start (const char *configName) zh->records = NULL; zh->zebra_maps = zebra_maps_open (zh->res); zh->rank_classes = NULL; + + zh->records = 0; + zh->dict = 0; + zh->sortIdx = 0; + zh->isams = 0; + zh->isam = 0; + zh->isamc = 0; + zh->isamd = 0; + zh->zei = 0; zebraRankInstall (zh, rank1_class); @@ -347,21 +255,109 @@ ZebraService zebra_start (const char *configName) else passwd_db_file (zh->passwd_db, res_get (zh->res, "passwd")); } - return zh; + + if (!(zh->records = rec_open (zh->bfs, 1, 0))) + { + logf (LOG_WARN, "rec_open"); + return -1; + } + if (!(zh->dict = dict_open (zh->bfs, FNAME_DICT, 40, 1, 0))) + { + logf (LOG_WARN, "dict_open"); + return -1; + } + if (!(zh->sortIdx = sortIdx_open (zh->bfs, 0))) + { + logf (LOG_WARN, "sortIdx_open"); + return -1; + } + if (res_get_match (zh->res, "isam", "s", ISAM_DEFAULT)) + { + struct ISAMS_M_s isams_m; + if (!(zh->isams = isams_open (zh->bfs, FNAME_ISAMS, 1, + key_isams_m(zh->res, &isams_m)))) + { + logf (LOG_WARN, "isams_open"); + return -1; + } + } +#if ZMBOL + else if (res_get_match (zh->res, "isam", "i", ISAM_DEFAULT)) + { + if (!(zh->isam = is_open (zh->bfs, FNAME_ISAM, key_compare, 1, + sizeof (struct it_key), zh->res))) + { + logf (LOG_WARN, "is_open"); + return -1; + } + } + else if (res_get_match (zh->res, "isam", "c", ISAM_DEFAULT)) + { + struct ISAMC_M_s isamc_m; + if (!(zh->isamc = isc_open (zh->bfs, FNAME_ISAMC, + 1, key_isamc_m(zh->res, &isamc_m)))) + { + logf (LOG_WARN, "isc_open"); + return -1; + } + } + else if (res_get_match (zh->res, "isam", "d", ISAM_DEFAULT)) + { + struct ISAMD_M_s isamd_m; + + if (!(zh->isamd = isamd_open (zh->bfs, FNAME_ISAMD, + 1, key_isamd_m(zh->res, &isamd_m)))) + { + logf (LOG_WARN, "isamd_open"); + return -1; + } + } +#endif + zh->zei = zebraExplain_open (zh->records, zh->dh, + zh->res, 1, 0 /* rGroup */, + explain_extract); + if (!zh->zei) + { + logf (LOG_WARN, "Cannot obtain EXPLAIN information"); + return -1; + } + zh->active = 1; + yaz_log (LOG_LOG, "zebra_register_activate ok"); + return 0; } -void zebra_stop(ZebraService zh) +void zebra_admin_shutdown (ZebraHandle zh) { - if (!zh) - return ; - yaz_log (LOG_LOG, "zebra_stop"); + zebraExplain_flush (zh->service->zei, 1, zh); + extract_index (zh); - /* wait for sessions to die ...... */ + zebra_mutex_cond_lock (&zh->service->session_lock); + zh->service->stop_flag = 1; + if (!zh->service->sessions) + zebra_register_deactivate(zh->service); + zebra_mutex_cond_unlock (&zh->service->session_lock); +} + +void zebra_admin_start (ZebraHandle zh) +{ + ZebraService zs = zh->service; + zh->errCode = 0; + zebra_mutex_cond_lock (&zs->session_lock); + if (!zs->stop_flag) + zebra_register_activate(zs); + zebra_mutex_cond_unlock (&zs->session_lock); +} +static int zebra_register_deactivate (ZebraService zh) +{ + zh->stop_flag = 0; + if (!zh->active) + return 0; + yaz_log(LOG_LOG, "zebra_register_deactivate"); zebra_chdir (zh); if (zh->records) { - zebraExplain_close (zh->zei, 0, 0); + zebraExplain_close (zh->zei, 1); dict_close (zh->dict); sortIdx_close (zh->sortIdx); if (zh->isams) @@ -371,31 +367,67 @@ void zebra_stop(ZebraService zh) is_close (zh->isam); if (zh->isamc) isc_close (zh->isamc); + if (zh->isamd) + isamd_close (zh->isamd); #endif rec_close (&zh->records); - zebra_register_unlock (zh); } recTypes_destroy (zh->recTypes); zebra_maps_close (zh->zebra_maps); zebraRankDestroy (zh); bfs_destroy (zh->bfs); data1_destroy (zh->dh); - zebra_server_lock_destroy (zh); if (zh->passwd_db) passwd_db_close (zh->passwd_db); res_close (zh->res); + zh->active = 0; + return 0; +} +void zebra_stop(ZebraService zh) +{ + if (!zh) + return ; + yaz_log (LOG_LOG, "zebra_stop"); + + assert (!zh->sessions); + + zebra_mutex_cond_destroy (&zh->session_lock); + + zebra_register_deactivate(zh); xfree (zh->configName); xfree (zh); } void zebra_close (ZebraHandle zh) { + ZebraService zs = zh->service; + struct zebra_session **sp; if (!zh) return ; resultSetDestroy (zh, -1, 0, 0); + if (zh->key_buf) + { + xfree (zh->key_buf); + zh->key_buf = 0; + } + zebra_mutex_cond_lock (&zs->session_lock); + sp = &zs->sessions; + while (1) + { + assert (*sp); + if (*sp == zh) + { + *sp = (*sp)->next; + break; + } + sp = &(*sp)->next; + } + if (!zs->sessions && zs->stop_flag) + zebra_register_deactivate(zs); + zebra_mutex_cond_unlock (&zs->session_lock); xfree (zh); } @@ -472,12 +504,12 @@ void zebra_search_rpn (ZebraHandle zh, ODR stream, ODR decode, const char *setname) { zh->hits = 0; - if (zebra_register_lock (zh->service)) + if (zebra_register_lock (zh)) return; map_basenames (zh, stream, &num_bases, &basenames); resultSetAddRPN (zh, stream, decode, query, num_bases, basenames, setname); - zebra_register_unlock (zh->service); + zebra_register_unlock (zh); } void zebra_records_retrieve (ZebraHandle zh, ODR stream, @@ -488,8 +520,9 @@ void zebra_records_retrieve (ZebraHandle zh, ODR stream, ZebraPosSet poset; int i, *pos_array; - if (zebra_register_lock (zh->service)) + if (zebra_register_lock (zh)) return; + zh->errCode = 0; pos_array = (int *) xmalloc (num_recs * sizeof(*pos_array)); for (i = 0; iservice); + zebra_register_unlock (zh); xfree (pos_array); } @@ -536,7 +569,7 @@ void zebra_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt, int *position, int *num_entries, ZebraScanEntry **entries, int *is_partial) { - if (zebra_register_lock (zh->service)) + if (zebra_register_lock (zh)) { *entries = 0; *num_entries = 0; @@ -546,7 +579,7 @@ void zebra_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt, rpn_scan (zh, stream, zapt, attributeset, num_bases, basenames, position, num_entries, entries, is_partial); - zebra_register_unlock (zh->service); + zebra_register_unlock (zh); } void zebra_sort (ZebraHandle zh, ODR stream, @@ -554,11 +587,11 @@ void zebra_sort (ZebraHandle zh, ODR stream, const char *output_setname, Z_SortKeySpecList *sort_sequence, int *sort_status) { - if (zebra_register_lock (zh->service)) + if (zebra_register_lock (zh)) return; resultSetSort (zh, stream->mem, num_input_setnames, input_setnames, output_setname, sort_sequence, sort_status); - zebra_register_unlock (zh->service); + zebra_register_unlock (zh); } int zebra_deleleResultSet(ZebraHandle zh, int function, @@ -566,7 +599,7 @@ int zebra_deleleResultSet(ZebraHandle zh, int function, int *statuses) { int i, status; - if (zebra_register_lock (zh->service)) + if (zebra_register_lock (zh)) return Z_DeleteStatus_systemProblemAtTarget; switch (function) { @@ -577,7 +610,7 @@ int zebra_deleleResultSet(ZebraHandle zh, int function, resultSetDestroy (zh, -1, 0, statuses); break; } - zebra_register_unlock (zh->service); + zebra_register_unlock (zh); status = Z_DeleteStatus_success; for (i = 0; inum_segmentRecords == 0) + { + zebraExplain_flush (zh->service->zei, 1, zh); + extract_index (zh); + } + else + { + int sysno; + int i; + for (i = 0; inum_segmentRecords; i++) + { + Z_NamePlusRecord *npr = segment->segmentRecords[i]; + printf ("--------------%d--------------------\n", i); + if (npr->which == Z_NamePlusRecord_intermediateFragment) + { + Z_FragmentSyntax *fragment = npr->u.intermediateFragment; + if (fragment->which == Z_FragmentSyntax_notExternallyTagged) + { + Odr_oct *oct = fragment->u.notExternallyTagged; + printf ("%.*s", (oct->len > 100 ? 100 : oct->len) , + oct->buf); + + sysno = 0; + extract_rec_in_mem (zh, "grs.sgml", + oct->buf, oct->len, + "Default", 0 /* delete_flag */, + 0 /* test_mode */, + &sysno /* sysno */, + 1 /* store_keys */, + 1 /* store_data */, + 0 /* match criteria */); + } + } + } + } + zebra_register_unlock(zh); } void zebra_admin_create (ZebraHandle zh, const char *database) { - + ZebraService zs = zh->service; + if (zebra_register_lock(zh)) + { + zh->errCode = 1019; + return; + } + /* announce database */ + if (zebraExplain_newDatabase (zs->zei, database, 0 /* explainDatabase */)) + { + zh->errCode = 224; + zh->errString = "Database already exist"; + } + zebra_register_unlock(zh); } int zebra_string_norm (ZebraHandle zh, unsigned reg_id, @@ -650,3 +729,808 @@ int zebra_string_norm (ZebraHandle zh, unsigned reg_id, output_str[wrbuf_len(wrbuf)] = '\0'; return wrbuf_len(wrbuf); } + +static void extract_init (struct recExtractCtrl *p, RecWord *w) +{ + w->zebra_maps = p->zebra_maps; + w->seqnos = p->seqno; + w->attrSet = VAL_BIB1; + w->attrUse = 1016; + w->reg_type = 'w'; + w->extractCtrl = p; +} + +static void extract_add_index_string (RecWord *p, const char *string, + int length) +{ + char *dst; + unsigned char attrSet; + unsigned short attrUse; + int lead = 0; + int diff = 0; + int *pseqno = &p->seqnos[p->reg_type]; + ZebraHandle zh = p->extractCtrl->handle; + struct recKeys *keys = &zh->keys; + + if (keys->buf_used+1024 > keys->buf_max) + { + char *b; + + b = (char *) xmalloc (keys->buf_max += 128000); + if (keys->buf_used > 0) + memcpy (b, keys->buf, keys->buf_used); + xfree (keys->buf); + keys->buf = b; + } + dst = keys->buf + keys->buf_used; + + attrSet = p->attrSet; + if (keys->buf_used > 0 && keys->prevAttrSet == attrSet) + lead |= 1; + else + keys->prevAttrSet = attrSet; + attrUse = p->attrUse; + if (keys->buf_used > 0 && keys->prevAttrUse == attrUse) + lead |= 2; + else + keys->prevAttrUse = attrUse; +#if 1 + diff = 1 + *pseqno - keys->prevSeqNo; + if (diff >= 1 && diff <= 15) + lead |= (diff << 2); + else + diff = 0; +#endif + keys->prevSeqNo = *pseqno; + + *dst++ = lead; + + if (!(lead & 1)) + { + memcpy (dst, &attrSet, sizeof(attrSet)); + dst += sizeof(attrSet); + } + if (!(lead & 2)) + { + memcpy (dst, &attrUse, sizeof(attrUse)); + dst += sizeof(attrUse); + } + *dst++ = p->reg_type; + memcpy (dst, string, length); + dst += length; + *dst++ = '\0'; + + if (!diff) + { + memcpy (dst, pseqno, sizeof(*pseqno)); + dst += sizeof(*pseqno); + } + keys->buf_used = dst - keys->buf; + if (*pseqno) + (*pseqno)++; +} + +static void extract_add_sort_string (RecWord *p, const char *string, + int length) +{ + struct sortKey *sk; + ZebraHandle zh = p->extractCtrl->handle; + struct sortKey *sortKeys = zh->sortKeys; + + for (sk = sortKeys; sk; sk = sk->next) + if (sk->attrSet == p->attrSet && sk->attrUse == p->attrUse) + return; + + sk = (struct sortKey *) xmalloc (sizeof(*sk)); + sk->next = sortKeys; + sortKeys = sk; + + sk->string = (char *) xmalloc (length); + sk->length = length; + memcpy (sk->string, string, length); + + sk->attrSet = p->attrSet; + sk->attrUse = p->attrUse; +} + +static void extract_add_string (RecWord *p, const char *string, int length) +{ + assert (length > 0); + if (zebra_maps_is_sort (p->zebra_maps, p->reg_type)) + extract_add_sort_string (p, string, length); + else + extract_add_index_string (p, string, length); +} + +static void extract_add_incomplete_field (RecWord *p) +{ + const char *b = p->string; + int remain = p->length; + const char **map = 0; + + if (remain > 0) + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + + while (map) + { + char buf[IT_MAX_WORD+1]; + int i, remain; + + /* Skip spaces */ + while (map && *map && **map == *CHR_SPACE) + { + remain = p->length - (b - p->string); + if (remain > 0) + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + else + map = 0; + } + if (!map) + break; + i = 0; + while (map && *map && **map != *CHR_SPACE) + { + const char *cp = *map; + + while (i < IT_MAX_WORD && *cp) + buf[i++] = *(cp++); + remain = p->length - (b - p->string); + if (remain > 0) + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + else + map = 0; + } + if (!i) + return; + extract_add_string (p, buf, i); + } + (p->seqnos[p->reg_type])++; /* to separate this from next one */ +} + +static void extract_add_complete_field (RecWord *p) +{ + const char *b = p->string; + char buf[IT_MAX_WORD+1]; + const char **map = 0; + int i = 0, remain = p->length; + + if (remain > 0) + map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, remain); + + while (remain > 0 && i < IT_MAX_WORD) + { + while (map && *map && **map == *CHR_SPACE) + { + remain = p->length - (b - p->string); + if (remain > 0) + map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain); + else + map = 0; + } + if (!map) + break; + + if (i && i < IT_MAX_WORD) + buf[i++] = *CHR_SPACE; + while (map && *map && **map != *CHR_SPACE) + { + const char *cp = *map; + + if (i >= IT_MAX_WORD) + break; + while (i < IT_MAX_WORD && *cp) + buf[i++] = *(cp++); + remain = p->length - (b - p->string); + if (remain > 0) + map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, + remain); + else + map = 0; + } + } + if (!i) + return; + extract_add_string (p, buf, i); +} + +static void extract_token_add (RecWord *p) +{ + WRBUF wrbuf; + if ((wrbuf = zebra_replace(p->zebra_maps, p->reg_type, 0, + p->string, p->length))) + { + p->string = wrbuf_buf(wrbuf); + p->length = wrbuf_len(wrbuf); + } + if (zebra_maps_is_complete (p->zebra_maps, p->reg_type)) + extract_add_complete_field (p); + else + extract_add_incomplete_field(p); +} + +static void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid) +{ + ZebraHandle zh = (ZebraHandle) (p->handle); + zebraExplain_addSchema (zh->service->zei, oid); +} + +static void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, + int cmd, struct sortKey **skp) +{ + struct sortKey *sk = *skp; + SortIdx sortIdx = zh->service->sortIdx; + + sortIdx_sysno (sortIdx, sysno); + while (sk) + { + struct sortKey *sk_next = sk->next; + sortIdx_type (sortIdx, sk->attrUse); + sortIdx_add (sortIdx, sk->string, sk->length); + xfree (sk->string); + xfree (sk); + sk = sk_next; + } + *skp = 0; +} + +struct encode_info { + int sysno; + int seqno; + int cmd; + char buf[768]; +}; + +void encode_key_init (struct encode_info *i) +{ + i->sysno = 0; + i->seqno = 0; + i->cmd = -1; +} + +char *encode_key_int (int d, char *bp) +{ + if (d <= 63) + *bp++ = d; + else if (d <= 16383) + { + *bp++ = 64 + (d>>8); + *bp++ = d & 255; + } + else if (d <= 4194303) + { + *bp++ = 128 + (d>>16); + *bp++ = (d>>8) & 255; + *bp++ = d & 255; + } + else + { + *bp++ = 192 + (d>>24); + *bp++ = (d>>16) & 255; + *bp++ = (d>>8) & 255; + *bp++ = d & 255; + } + return bp; +} + +void encode_key_write (char *k, struct encode_info *i, FILE *outf) +{ + struct it_key key; + char *bp = i->buf; + + while ((*bp++ = *k++)) + ; + memcpy (&key, k+1, sizeof(struct it_key)); + bp = encode_key_int ( (key.sysno - i->sysno) * 2 + *k, bp); + if (i->sysno != key.sysno) + { + i->sysno = key.sysno; + i->seqno = 0; + } + else if (!i->seqno && !key.seqno && i->cmd == *k) + return; + bp = encode_key_int (key.seqno - i->seqno, bp); + i->seqno = key.seqno; + i->cmd = *k; + if (fwrite (i->buf, bp - i->buf, 1, outf) != 1) + { + logf (LOG_FATAL|LOG_ERRNO, "fwrite"); + exit (1); + } +} + +static void extract_flushWriteKeys (ZebraHandle zh) +{ + FILE *outf; + char out_fname[200]; + char *prevcp, *cp; + struct encode_info encode_info; + int ptr_i = zh->ptr_i; +#if SORT_EXTRA + int i; +#endif + if (!zh->key_buf || ptr_i <= 0) + return; + + (zh->key_file_no)++; + logf (LOG_LOG, "sorting section %d", (zh->key_file_no)); +#if !SORT_EXTRA + qsort (zh->key_buf + zh->ptr_top - ptr_i, ptr_i, sizeof(char*), + key_qsort_compare); + extract_get_fname_tmp (zh, out_fname, zh->key_file_no); + + if (!(outf = fopen (out_fname, "wb"))) + { + logf (LOG_FATAL|LOG_ERRNO, "fopen %s", out_fname); + exit (1); + } + logf (LOG_LOG, "writing section %d", zh->key_file_no); + prevcp = cp = (zh->key_buf)[zh->ptr_top - ptr_i]; + + encode_key_init (&encode_info); + encode_key_write (cp, &encode_info, outf); + + while (--ptr_i > 0) + { + cp = (zh->key_buf)[zh->ptr_top - ptr_i]; + if (strcmp (cp, prevcp)) + { + encode_key_init (&encode_info); + encode_key_write (cp, &encode_info, outf); + prevcp = cp; + } + else + encode_key_write (cp + strlen(cp), &encode_info, outf); + } +#else + qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_x_compare); + extract_get_fname_tmp (out_fname, key_file_no); + + if (!(outf = fopen (out_fname, "wb"))) + { + logf (LOG_FATAL|LOG_ERRNO, "fopen %s", out_fname); + exit (1); + } + logf (LOG_LOG, "writing section %d", key_file_no); + i = ptr_i; + prevcp = key_buf[ptr_top-i]; + while (1) + if (!--i || strcmp (prevcp, key_buf[ptr_top-i])) + { + key_y_len = strlen(prevcp)+1; +#if 0 + logf (LOG_LOG, "key_y_len: %2d %02x %02x %s", + key_y_len, prevcp[0], prevcp[1], 2+prevcp); +#endif + qsort (key_buf + ptr_top-ptr_i, ptr_i - i, + sizeof(char*), key_y_compare); + cp = key_buf[ptr_top-ptr_i]; + --key_y_len; + encode_key_init (&encode_info); + encode_key_write (cp, &encode_info, outf); + while (--ptr_i > i) + { + cp = key_buf[ptr_top-ptr_i]; + encode_key_write (cp+key_y_len, &encode_info, outf); + } + if (!i) + break; + prevcp = key_buf[ptr_top-ptr_i]; + } +#endif + if (fclose (outf)) + { + logf (LOG_FATAL|LOG_ERRNO, "fclose %s", out_fname); + exit (1); + } + logf (LOG_LOG, "finished section %d", zh->key_file_no); + zh->ptr_i = 0; + zh->key_buf_used = 0; +} + +static void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, + int cmd, struct recKeys *reckeys) +{ + unsigned char attrSet = (unsigned char) -1; + unsigned short attrUse = (unsigned short) -1; + int seqno = 0; + int off = 0; + ZebraExplainInfo zei = zh->service->zei; + + if (!zh->key_buf) + { + int mem = 8*1024*1024; + zh->key_buf = (char**) xmalloc (mem); + zh->ptr_top = mem/sizeof(char*); + zh->ptr_i = 0; + zh->key_buf_used = 0; + zh->key_file_no = 0; + } + zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); + while (off < reckeys->buf_used) + { + const char *src = reckeys->buf + off; + struct it_key key; + int lead, ch; + + lead = *src++; + + if (!(lead & 1)) + { + memcpy (&attrSet, src, sizeof(attrSet)); + src += sizeof(attrSet); + } + if (!(lead & 2)) + { + memcpy (&attrUse, src, sizeof(attrUse)); + src += sizeof(attrUse); + } + if (zh->key_buf_used + 1024 > (zh->ptr_top-zh->ptr_i)*sizeof(char*)) + extract_flushWriteKeys (zh); + ++(zh->ptr_i); + (zh->key_buf)[zh->ptr_top - zh->ptr_i] = + (char*)zh->key_buf + zh->key_buf_used; + + ch = zebraExplain_lookupSU (zei, attrSet, attrUse); + if (ch < 0) + ch = zebraExplain_addSU (zei, attrSet, attrUse); + assert (ch > 0); + zh->key_buf_used += + key_SU_code (ch,((char*)zh->key_buf) + zh->key_buf_used); + + while (*src) + ((char*)zh->key_buf) [(zh->key_buf_used)++] = *src++; + src++; + ((char*)(zh->key_buf))[(zh->key_buf_used)++] = '\0'; + ((char*)(zh->key_buf))[(zh->key_buf_used)++] = cmd; + + if (lead & 60) + seqno += ((lead>>2) & 15)-1; + else + { + memcpy (&seqno, src, sizeof(seqno)); + src += sizeof(seqno); + } + key.seqno = seqno; + key.sysno = sysno; + memcpy ((char*)zh->key_buf + zh->key_buf_used, &key, sizeof(key)); + (zh->key_buf_used) += sizeof(key); + off = src - reckeys->buf; + } + assert (off == reckeys->buf_used); +} + +static void extract_index (ZebraHandle zh) +{ + extract_flushWriteKeys (zh); + zebra_index_merge (zh); +} + +static int explain_extract (void *handle, Record rec, data1_node *n) +{ + ZebraHandle zh = (ZebraHandle) handle; + struct recExtractCtrl extractCtrl; + int i; + + if (zebraExplain_curDatabase (zh->service->zei, + rec->info[recInfo_databaseName])) + { + abort(); + if (zebraExplain_newDatabase (zh->service->zei, + rec->info[recInfo_databaseName], 0)) + abort (); + } + + zh->keys.buf_used = 0; + zh->keys.prevAttrUse = -1; + zh->keys.prevAttrSet = -1; + zh->keys.prevSeqNo = 0; + zh->sortKeys = 0; + + extractCtrl.init = extract_init; + extractCtrl.tokenAdd = extract_token_add; + extractCtrl.schemaAdd = extract_schema_add; + extractCtrl.dh = zh->service->dh; + for (i = 0; i<256; i++) + extractCtrl.seqno[i] = 0; + extractCtrl.zebra_maps = zh->service->zebra_maps; + extractCtrl.flagShowRecords = 0; + extractCtrl.handle = handle; + + grs_extract_tree(&extractCtrl, n); + + logf (LOG_LOG, "flush explain record, sysno=%d", rec->sysno); + + if (rec->size[recInfo_delKeys]) + { + struct recKeys delkeys; + struct sortKey *sortKeys = 0; + + delkeys.buf_used = rec->size[recInfo_delKeys]; + delkeys.buf = rec->info[recInfo_delKeys]; + extract_flushSortKeys (zh, rec->sysno, 0, &sortKeys); + extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys); + } + extract_flushRecordKeys (zh, rec->sysno, 1, &zh->keys); + extract_flushSortKeys (zh, rec->sysno, 1, &zh->sortKeys); + + xfree (rec->info[recInfo_delKeys]); + rec->size[recInfo_delKeys] = zh->keys.buf_used; + rec->info[recInfo_delKeys] = zh->keys.buf; + zh->keys.buf = NULL; + zh->keys.buf_max = 0; + return 0; +} + +static int extract_rec_in_mem (ZebraHandle zh, const char *recordType, + const char *buf, size_t buf_size, + const char *databaseName, int delete_flag, + int test_mode, int *sysno, + int store_keys, int store_data, + const char *match_criteria) +{ + RecordAttr *recordAttr; + struct recExtractCtrl extractCtrl; + int i, r; + RecType recType; + char subType[1024]; + void *clientData; + const char *fname = ""; + Record rec; + long recordOffset = 0; + struct zebra_fetch_control fc; + + fc.fd = -1; + fc.record_int_buf = buf; + fc.record_int_len = buf_size; + fc.record_int_pos = 0; + fc.offset_end = 0; + fc.record_offset = 0; + + extractCtrl.offset = 0; + extractCtrl.readf = zebra_record_int_read; + extractCtrl.seekf = zebra_record_int_seek; + extractCtrl.tellf = zebra_record_int_tell; + extractCtrl.endf = zebra_record_int_end; + extractCtrl.fh = &fc; + + /* announce database */ + if (zebraExplain_curDatabase (zh->service->zei, databaseName)) + { + if (zebraExplain_newDatabase (zh->service->zei, databaseName, 0)) + return 0; + } + if (!(recType = + recType_byName (zh->service->recTypes, recordType, subType, + &clientData))) + { + logf (LOG_WARN, "No such record type: %s", recordType); + return 0; + } + + zh->keys.buf_used = 0; + zh->keys.prevAttrUse = -1; + zh->keys.prevAttrSet = -1; + zh->keys.prevSeqNo = 0; + zh->sortKeys = 0; + + + extractCtrl.subType = subType; + extractCtrl.init = extract_init; + extractCtrl.tokenAdd = extract_token_add; + extractCtrl.schemaAdd = extract_schema_add; + extractCtrl.dh = zh->service->dh; + extractCtrl.handle = zh; + extractCtrl.zebra_maps = zh->service->zebra_maps; + extractCtrl.flagShowRecords = 0; + for (i = 0; i<256; i++) + { + if (zebra_maps_is_positioned(zh->service->zebra_maps, i)) + extractCtrl.seqno[i] = 1; + else + extractCtrl.seqno[i] = 0; + } + + r = (*recType->extract)(clientData, &extractCtrl); + + if (r == RECCTRL_EXTRACT_EOF) + return 0; + else if (r == RECCTRL_EXTRACT_ERROR) + { + /* error occured during extraction ... */ +#if 1 + yaz_log (LOG_WARN, "extract error"); +#else + if (rGroup->flagRw && + records_processed < rGroup->fileVerboseLimit) + { + logf (LOG_WARN, "fail %s %s %ld", rGroup->recordType, + fname, (long) recordOffset); + } +#endif + return 0; + } + if (zh->keys.buf_used == 0) + { + /* the extraction process returned no information - the record + is probably empty - unless flagShowRecords is in use */ + if (test_mode) + return 1; + logf (LOG_WARN, "No keys generated for record"); + logf (LOG_WARN, " The file is probably empty"); + return 1; + } + /* match criteria */ + + if (! *sysno) + { + /* new record */ + if (delete_flag) + { + logf (LOG_LOG, "delete %s %s %ld", recordType, + fname, (long) recordOffset); + logf (LOG_WARN, "cannot delete record above (seems new)"); + return 1; + } + logf (LOG_LOG, "add %s %s %ld", recordType, fname, + (long) recordOffset); + rec = rec_new (zh->service->records); + + *sysno = rec->sysno; + + recordAttr = rec_init_attr (zh->service->zei, rec); + +#if 0 + if (matchStr) + { + dict_insert (matchDict, matchStr, sizeof(*sysno), sysno); + } +#endif + extract_flushRecordKeys (zh, *sysno, 1, &zh->keys); + extract_flushSortKeys (zh, *sysno, 1, &zh->sortKeys); + } + else + { + /* record already exists */ + struct recKeys delkeys; + + rec = rec_get (zh->service->records, *sysno); + assert (rec); + + recordAttr = rec_init_attr (zh->service->zei, rec); + + if (recordAttr->runNumber == + zebraExplain_runNumberIncrement (zh->service->zei, 0)) + { + logf (LOG_LOG, "skipped %s %s %ld", recordType, + fname, (long) recordOffset); + rec_rm (&rec); + return 1; + } + delkeys.buf_used = rec->size[recInfo_delKeys]; + delkeys.buf = rec->info[recInfo_delKeys]; + extract_flushSortKeys (zh, *sysno, 0, &zh->sortKeys); + extract_flushRecordKeys (zh, *sysno, 0, &delkeys); + if (delete_flag) + { + /* record going to be deleted */ + if (!delkeys.buf_used) + { + logf (LOG_LOG, "delete %s %s %ld", recordType, + fname, (long) recordOffset); + logf (LOG_WARN, "cannot delete file above, storeKeys false"); + } + else + { + logf (LOG_LOG, "delete %s %s %ld", recordType, + fname, (long) recordOffset); +#if 0 + if (matchStr) + dict_delete (matchDict, matchStr); +#endif + rec_del (zh->service->records, &rec); + } + rec_rm (&rec); + return 1; + } + else + { + /* record going to be updated */ + if (!delkeys.buf_used) + { + logf (LOG_LOG, "update %s %s %ld", recordType, + fname, (long) recordOffset); + logf (LOG_WARN, "cannot update file above, storeKeys false"); + } + else + { + logf (LOG_LOG, "update %s %s %ld", recordType, + fname, (long) recordOffset); + extract_flushRecordKeys (zh, *sysno, 1, &zh->keys); + } + } + } + /* update file type */ + xfree (rec->info[recInfo_fileType]); + rec->info[recInfo_fileType] = + rec_strdup (recordType, &rec->size[recInfo_fileType]); + + /* update filename */ + xfree (rec->info[recInfo_filename]); + rec->info[recInfo_filename] = + rec_strdup (fname, &rec->size[recInfo_filename]); + + /* update delete keys */ + xfree (rec->info[recInfo_delKeys]); + if (zh->keys.buf_used > 0 && store_keys == 1) + { + rec->size[recInfo_delKeys] = zh->keys.buf_used; + rec->info[recInfo_delKeys] = zh->keys.buf; + zh->keys.buf = NULL; + zh->keys.buf_max = 0; + } + else + { + rec->info[recInfo_delKeys] = NULL; + rec->size[recInfo_delKeys] = 0; + } + + /* save file size of original record */ + zebraExplain_recordBytesIncrement (zh->service->zei, + - recordAttr->recordSize); +#if 0 + recordAttr->recordSize = fi->file_moffset - recordOffset; + if (!recordAttr->recordSize) + recordAttr->recordSize = fi->file_max - recordOffset; +#else + recordAttr->recordSize = buf_size; +#endif + zebraExplain_recordBytesIncrement (zh->service->zei, + recordAttr->recordSize); + + /* set run-number for this record */ + recordAttr->runNumber = + zebraExplain_runNumberIncrement (zh->service->zei, 0); + + /* update store data */ + xfree (rec->info[recInfo_storeData]); + if (store_data == 1) + { + rec->size[recInfo_storeData] = recordAttr->recordSize; + rec->info[recInfo_storeData] = (char *) + xmalloc (recordAttr->recordSize); +#if 1 + memcpy (rec->info[recInfo_storeData], buf, recordAttr->recordSize); +#else + if (lseek (fi->fd, recordOffset, SEEK_SET) < 0) + { + logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", + (long) recordOffset, fname); + exit (1); + } + if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize) + < recordAttr->recordSize) + { + logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s", + recordAttr->recordSize, fname); + exit (1); + } +#endif + } + else + { + rec->info[recInfo_storeData] = NULL; + rec->size[recInfo_storeData] = 0; + } + /* update database name */ + xfree (rec->info[recInfo_databaseName]); + rec->info[recInfo_databaseName] = + rec_strdup (databaseName, &rec->size[recInfo_databaseName]); + + /* update offset */ + recordAttr->recordOffset = recordOffset; + + /* commit this record */ + rec_put (zh->service->records, &rec); + + return 0; +} diff --git a/index/zebraapi.h b/index/zebraapi.h index d5168ac..60adace 100644 --- a/index/zebraapi.h +++ b/index/zebraapi.h @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zebraapi.h,v $ - * Revision 1.10 2000-03-15 15:00:31 adam + * Revision 1.11 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.10 2000/03/15 15:00:31 adam * First work on threaded version. * * Revision 1.9 2000/02/24 12:31:17 adam @@ -119,4 +123,14 @@ YAZ_EXPORT void zebra_admin_create (ZebraHandle zh, const char *db); YAZ_EXPORT ZebraService zebra_start (const char *configName); YAZ_EXPORT void zebra_stop (ZebraService zs); +YAZ_EXPORT void zebra_admin_shutdown (ZebraHandle zh); +YAZ_EXPORT void zebra_admin_start (ZebraHandle zh); + +YAZ_EXPORT void zebra_shutdown (ZebraService zs); + +YAZ_EXPORT void zebra_admin_import_begin (ZebraHandle zh, const char *database); + +YAZ_EXPORT void zebra_admin_import_segment (ZebraHandle zh, + Z_Segment *segment); + YAZ_END_CDECL diff --git a/index/zinfo.c b/index/zinfo.c index cceff98..5c1b7df 100644 --- a/index/zinfo.c +++ b/index/zinfo.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zinfo.c,v $ - * Revision 1.17 1999-07-14 10:53:51 adam + * Revision 1.18 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.17 1999/07/14 10:53:51 adam * Updated various routines to handle missing explain schema. * * Revision 1.16 1999/05/26 07:49:13 adam @@ -331,17 +335,14 @@ static Record createRecord (Records records, int *sysno) return rec; } -void zebraExplain_close (ZebraExplainInfo zei, int writeFlag, - int (*updateH)(Record drec, data1_node *n)) +void zebraExplain_flush (ZebraExplainInfo zei, int writeFlag, void *handle) { - struct zebDatabaseInfoB *zdi; - - logf (LOG_DEBUG, "zebraExplain_close wr=%d", writeFlag); - if (!zei) - return; + zei->updateHandle = handle; if (writeFlag) { + struct zebDatabaseInfoB *zdi; zebAccessObject o; + /* write each database info record */ for (zdi = zei->databaseInfo; zdi; zdi = zdi->next) { @@ -370,8 +371,17 @@ void zebraExplain_close (ZebraExplainInfo zei, int writeFlag, zdi->databaseName, 0); } zebraExplain_writeTarget (zei, 0); - } +} + +void zebraExplain_close (ZebraExplainInfo zei, int writeFlag) +{ +#if ZINFO_DEBUG + logf (LOG_LOG, "zebraExplain_close wr=%d", writeFlag); +#endif + if (!zei) + return; + zebraExplain_flush (zei, writeFlag, zei->updateHandle); nmem_destroy (zei->nmem); } @@ -454,7 +464,9 @@ ZebraExplainInfo zebraExplain_open ( struct tm *tm; NMEM nmem = nmem_create (); - logf (LOG_DEBUG, "zebraExplain_open wr=%d", writeFlag); +#if ZINFO_DEBUG + logf (LOG_LOG, "zebraExplain_open wr=%d", writeFlag); +#endif zei = (ZebraExplainInfo) nmem_malloc (nmem, sizeof(*zei)); zei->updateHandle = updateHandle; zei->updateFunc = updateFunc; diff --git a/index/zinfo.h b/index/zinfo.h index 30a764a..b1ade76 100644 --- a/index/zinfo.h +++ b/index/zinfo.h @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zinfo.h,v $ - * Revision 1.8 1999-11-30 13:48:03 adam + * Revision 1.9 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.8 1999/11/30 13:48:03 adam * Improved installation. Updated for inclusion of YAZ header files. * * Revision 1.7 1999/05/26 07:49:13 adam @@ -39,9 +43,7 @@ #include #include "recindex.h" -#ifdef __cplusplus -extern "C" { -#endif +YAZ_BEGIN_CDECL typedef struct zebraExplainInfo *ZebraExplainInfo; typedef struct zebDatabaseInfo ZebDatabaseInfo; @@ -52,8 +54,8 @@ ZebraExplainInfo zebraExplain_open (Records records, data1_handle dh, int (*updateFunc)(void *handle, Record drec, data1_node *n)); -void zebraExplain_close (ZebraExplainInfo zei, int writeFlag, - int (*updateH)(Record drec, data1_node *n)); +void zebraExplain_close (ZebraExplainInfo zei, int writeFlag /*, + int (*updateH)(Record drec, data1_node *n)*/); int zebraExplain_curDatabase (ZebraExplainInfo zei, const char *database); int zebraExplain_newDatabase (ZebraExplainInfo zei, const char *database, int explain_database); @@ -64,6 +66,8 @@ void zebraExplain_recordCountIncrement (ZebraExplainInfo zei, int adjust_num); void zebraExplain_recordBytesIncrement (ZebraExplainInfo zei, int adjust_num); int zebraExplain_runNumberIncrement (ZebraExplainInfo zei, int adjust_num); void zebraExplain_loadAttsets (data1_handle dh, Res res); +void zebraExplain_flush (ZebraExplainInfo zei, int writeFlag, + void *updateHandle); typedef struct { int recordSize; @@ -72,8 +76,6 @@ typedef struct { } RecordAttr; RecordAttr *rec_init_attr (ZebraExplainInfo zei, Record rec); -#ifdef __cplusplus -} -#endif +YAZ_END_CDECL #endif diff --git a/index/zrpn.c b/index/zrpn.c index 4e1acd9..5b7e7ea 100644 --- a/index/zrpn.c +++ b/index/zrpn.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zrpn.c,v $ - * Revision 1.102 2000-03-15 15:00:31 adam + * Revision 1.103 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.102 2000/03/15 15:00:31 adam * First work on threaded version. * * Revision 1.101 2000/03/02 14:35:03 adam @@ -1557,7 +1561,7 @@ static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no, } for (i = 0; ibend_sort = bend_sort; q->bend_delete = bend_delete; q->bend_esrequest = bend_esrequest; + q->bend_segment = bend_segment; q->implementation_name = "Z'mbol Information Server"; q->implementation_version = "Z'mbol 1.0"; @@ -499,6 +505,14 @@ static int es_admin_request (ZebraHandle zh, Z_AdminEsRequest *r) case Z_ESAdminOriginPartToKeep_commit: yaz_log(LOG_LOG, "adm-commit"); break; + case Z_ESAdminOriginPartToKeep_shutdown: + yaz_log(LOG_LOG, "shutdown"); + zebra_admin_shutdown(zh); + break; + case Z_ESAdminOriginPartToKeep_start: + yaz_log(LOG_LOG, "start"); + zebra_admin_start(zh); + break; default: yaz_log(LOG_LOG, "unknown admin"); zh->errCode = 1001; @@ -528,6 +542,14 @@ static int es_admin (ZebraHandle zh, Z_Admin *r) return 0; } +int bend_segment (void *handle, bend_segment_rr *rr) +{ + ZebraHandle zh = (ZebraHandle) handle; + + zebra_admin_import_segment (zh, rr->segment); + return 0; +} + int bend_esrequest (void *handle, bend_esrequest_rr *rr) { ZebraHandle zh = (ZebraHandle) handle; @@ -675,19 +697,25 @@ int bend_esrequest (void *handle, bend_esrequest_rr *rr) switch (*toKeep->action) { case Z_IUOriginPartToKeep_recordInsert: - yaz_log (LOG_LOG, " recordInsert"); + yaz_log (LOG_LOG, "recordInsert"); break; case Z_IUOriginPartToKeep_recordReplace: - yaz_log (LOG_LOG, " recordUpdate"); + yaz_log (LOG_LOG, "recordUpdate"); break; case Z_IUOriginPartToKeep_recordDelete: - yaz_log (LOG_LOG, " recordDelete"); + yaz_log (LOG_LOG, "recordDelete"); break; case Z_IUOriginPartToKeep_elementUpdate: - yaz_log (LOG_LOG, " elementUpdate"); + yaz_log (LOG_LOG, "elementUpdate"); break; case Z_IUOriginPartToKeep_specialUpdate: - yaz_log (LOG_LOG, " specialUpdate"); + yaz_log (LOG_LOG, "specialUpdate"); + break; + case Z_ESAdminOriginPartToKeep_shutdown: + yaz_log (LOG_LOG, "shutDown"); + break; + case Z_ESAdminOriginPartToKeep_start: + yaz_log (LOG_LOG, "start"); break; default: yaz_log (LOG_LOG, " unknown (%d)", *toKeep->action); @@ -783,8 +811,10 @@ static void bend_start (struct statserv_options_block *sob) static void bend_stop(struct statserv_options_block *sob) { if (sob->handle) - zebra_stop(sob->handle); - sob->handle = 0; + { + ZebraService service = sob->handle; + zebra_stop(service); + } } int main (int argc, char **argv) diff --git a/index/zserver.h b/index/zserver.h index 823ae27..be38062 100644 --- a/index/zserver.h +++ b/index/zserver.h @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zserver.h,v $ - * Revision 1.46 2000-03-15 15:00:31 adam + * Revision 1.47 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.46 2000/03/15 15:00:31 adam * First work on threaded version. * * Revision 1.45 1999/11/30 13:48:04 adam @@ -202,6 +206,7 @@ struct zebra_service { #if ZMBOL ISAM isam; ISAMC isamc; + ISAMD isamd; #endif Dict dict; SortIdx sortIdx; @@ -223,21 +228,46 @@ struct zebra_service { ZebraRankClass rank_classes; RecTypes recTypes; Passwd_db passwd_db; + Zebra_mutex_cond session_lock; + int stop_flag; + int active; +}; + +struct recKeys { + int buf_used; + int buf_max; + char *buf; + char prevAttrSet; + short prevAttrUse; + int prevSeqNo; +}; + +struct sortKey { + char *string; + int length; + int attrSet; + int attrUse; + struct sortKey *next; }; struct zebra_session { -#if HAVE_PTHREADS_H - pthread_t *pthread_session; -#endif struct zebra_session *next; - struct zebra_info *info; struct zebra_service *service; + struct recKeys keys; + struct sortKey *sortKeys; + + char **key_buf; + size_t ptr_top; + size_t ptr_i; + size_t key_buf_used; + int key_file_no; + + int destroyed; ZebraSet sets; int errCode; int hits; char *errString; - }; struct rank_control { @@ -315,4 +345,25 @@ int zebra_record_fetch (ZebraHandle zh, int sysno, int score, ODR stream, oid_value *output_format, char **rec_bufp, int *rec_lenp, char **basenamep); +void extract_get_fname_tmp (ZebraHandle zh, char *fname, int no); +void zebra_index_merge (ZebraHandle zh); + + +struct zebra_fetch_control { + int offset_end; + int record_offset; + int record_int_pos; + const char *record_int_buf; + int record_int_len; + int fd; +}; + +int zebra_record_ext_read (void *fh, char *buf, size_t count); +off_t zebra_record_ext_seek (void *fh, off_t offset); +off_t zebra_record_ext_tell (void *fh); +off_t zebra_record_int_seek (void *fh, off_t offset); +off_t zebra_record_int_tell (void *fh); +int zebra_record_int_read (void *fh, char *buf, size_t count); +void zebra_record_int_end (void *fh, off_t offset); + YAZ_END_CDECL diff --git a/index/zsets.c b/index/zsets.c index c6a03c2..07d4f03 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -4,7 +4,11 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zsets.c,v $ - * Revision 1.25 2000-03-15 15:00:31 adam + * Revision 1.26 2000-03-20 19:08:36 adam + * Added remote record import using Z39.50 extended services and Segment + * Requests. + * + * Revision 1.25 2000/03/15 15:00:31 adam * First work on threaded version. * * Revision 1.24 1999/11/04 15:00:45 adam @@ -358,13 +362,13 @@ void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num) xfree (records); } -struct sortKey { +struct sortKeyInfo { int relation; int attrUse; }; void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, - struct sortKey *criteria, int num_criteria, + struct sortKeyInfo *criteria, int num_criteria, int sysno) { struct zset_sort_entry this_entry; @@ -512,7 +516,7 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, { int i, psysno = 0; struct it_key key; - struct sortKey sort_criteria[3]; + struct sortKeyInfo sort_criteria[3]; int num_criteria; int term_index; RSFD rfd; diff --git a/util/zebra-lock.c b/util/zebra-lock.c index 6739a18..0e08cde 100644 --- a/util/zebra-lock.c +++ b/util/zebra-lock.c @@ -100,3 +100,37 @@ int zebra_lock_rdwr_wunlock (Zebra_lock_rdwr *p) return 0; } +int zebra_mutex_cond_init (Zebra_mutex_cond *p) +{ + pthread_cond_init (&p->cond, 0); + pthread_mutex_init (&p->mutex, 0); + return 0; +} + +int zebra_mutex_cond_destroy (Zebra_mutex_cond *p) +{ + pthread_cond_destroy (&p->cond); + pthread_mutex_destroy (&p->mutex); + return 0; +} + +int zebra_mutex_cond_lock (Zebra_mutex_cond *p) +{ + return pthread_mutex_lock (&p->mutex); +} + +int zebra_mutex_cond_unlock (Zebra_mutex_cond *p) +{ + return pthread_mutex_unlock (&p->mutex); +} + +int zebra_mutex_cond_wait (Zebra_mutex_cond *p) +{ + return pthread_cond_wait (&p->cond, &p->mutex); +} + +int zebra_mutex_cond_signal (Zebra_mutex_cond *p) +{ + return pthread_cond_signal (&p->cond); +} + -- 1.7.10.4