X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=20509804f87344aee144971aec6dec27971c6480;hb=9765f689118107e45f5fba7261363980d378f7b3;hp=34cb68b2424e2ece028a38c5c20f70d269e34d5a;hpb=51a52e1b014d5237581f74379916b8ce92cafef7;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 34cb68b..2050980 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,10 +1,27 @@ /* - * Copyright (C) 1995, Index Data I/S + * Copyright (C) 1994-1995, Index Data I/S * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.1 1995-09-01 14:06:35 adam + * Revision 1.6 1995-09-08 14:52:27 adam + * Minor changes. Dictionary is lower case now. + * + * Revision 1.5 1995/09/06 16:11:16 adam + * Option: only one word key per file. + * + * Revision 1.4 1995/09/05 15:28:39 adam + * More work on search engine. + * + * Revision 1.3 1995/09/04 12:33:41 adam + * Various cleanup. YAZ util used instead. + * + * Revision 1.2 1995/09/04 09:10:34 adam + * More work on index add/del/update. + * Merge sort implemented. + * Initial work on z39 server. + * + * Revision 1.1 1995/09/01 14:06:35 adam * Split of work into more files. * */ @@ -14,7 +31,7 @@ #include #include -#include +#include #include "index.h" #define KEY_BUF_SIZE 100000 @@ -22,6 +39,7 @@ static Dict file_idx; static SYSNO sysno_next; static int key_fd = -1; +static int sys_idx_fd = -1; static char *key_buf; static int key_offset; @@ -32,18 +50,19 @@ void key_open (const char *fname) return; if ((key_fd = open (fname, O_RDWR|O_CREAT, 0666)) == -1) { - log (LOG_FATAL|LOG_ERRNO, "Creat %s", fname); + logf (LOG_FATAL|LOG_ERRNO, "open %s", fname); exit (1); } + logf (LOG_DEBUG, "key_open of %s", fname); if (!(key_buf = malloc (KEY_BUF_SIZE))) { - log (LOG_FATAL|LOG_ERRNO, "malloc"); + logf (LOG_FATAL|LOG_ERRNO, "malloc"); exit (1); } key_offset = 0; - if (!(file_idx = dict_open ("fileidx", 10, 1))) + if (!(file_idx = dict_open (FNAME_FILE_DICT, 40, 1))) { - log (LOG_FATAL, "dict_open fail of %s", "fileidx"); + logf (LOG_FATAL, "dict_open fail of %s", "fileidx"); exit (1); } file_key = dict_lookup (file_idx, "."); @@ -51,29 +70,42 @@ void key_open (const char *fname) memcpy (&sysno_next, (char*)file_key+1, sizeof(sysno_next)); else sysno_next = 1; + if ((sys_idx_fd = open (FNAME_SYS_IDX, O_RDWR|O_CREAT, 0666)) == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "open %s", FNAME_SYS_IDX); + exit (1); + } } -void key_close (void) +int key_close (void) { if (key_fd == -1) - return; + { + logf (LOG_DEBUG, "key_close - but no file"); + return 0; + } close (key_fd); + close (sys_idx_fd); dict_insert (file_idx, ".", sizeof(sysno_next), &sysno_next); dict_close (file_idx); key_fd = -1; + logf (LOG_DEBUG, "key close - key file exist"); + return 1; } void key_flush (void) { size_t i = 0; int w; - + + if (key_fd == -1) + return; while (i < key_offset) { w = write (key_fd, key_buf + i, key_offset - i); if (w == -1) { - log (LOG_FATAL|LOG_ERRNO, "Write key fail"); + logf (LOG_FATAL|LOG_ERRNO, "Write key fail"); exit (1); } i += w; @@ -83,33 +115,54 @@ void key_flush (void) void key_write (int cmd, struct it_key *k, const char *str) { - char x = cmd; + char x; size_t slen = strlen(str); if (key_offset + sizeof(*k) + slen >= KEY_BUF_SIZE - 2) key_flush (); + x = (cmd == 'a') ? 1 : 0; + memcpy (key_buf + key_offset, str, slen+1); + key_offset += slen+1; memcpy (key_buf + key_offset, &x, 1); key_offset++; memcpy (key_buf + key_offset, k, sizeof(*k)); key_offset += sizeof(*k); - memcpy (key_buf + key_offset, str, slen+1); - key_offset += slen+1; } -void text_extract (SYSNO sysno, int cmd, const char *fname) +void key_write_x (struct strtab *t, int cmd, struct it_key *k, const char *str) +{ + void **oldinfo; + + if (strtab_src (t, str, &oldinfo)) + ((struct it_key *) *oldinfo)->seqno++; + else + { + *oldinfo = xmalloc (sizeof(*k)); + memcpy (*oldinfo, k, sizeof(*k)); + ((struct it_key *) *oldinfo)->seqno = 1; + } +} + +void key_rec_flush (const char *str, void *info, void *data) +{ + key_write (*((int*) data), (struct it_key *)info, str); + xfree (info); +} + +void text_extract (struct strtab *t, SYSNO sysno, int cmd, const char *fname) { FILE *inf; struct it_key k; int seqno = 1; int c; - char w[256]; + char w[IT_MAX_WORD]; - log (LOG_DEBUG, "Text extract of %d", sysno); + logf (LOG_DEBUG, "Text extract of %d", sysno); k.sysno = sysno; inf = fopen (fname, "r"); if (!inf) { - log (LOG_WARN|LOG_ERRNO, "open %s", fname); + logf (LOG_WARN|LOG_ERRNO, "open %s", fname); return; } while ((c=getc (inf)) != EOF) @@ -117,7 +170,7 @@ void text_extract (SYSNO sysno, int cmd, const char *fname) int i = 0; while (i < 254 && c != EOF && isalnum(c)) { - w[i++] = c; + w[i++] = index_char_cvt (c); c = getc (inf); } if (i) @@ -125,8 +178,10 @@ void text_extract (SYSNO sysno, int cmd, const char *fname) w[i] = 0; k.seqno = seqno++; +#if IT_KEY_HAVE_FIELD k.field = 0; - key_write (cmd, &k, w); +#endif + key_write_x (t, cmd, &k, w); } if (c == EOF) break; @@ -142,8 +197,9 @@ void file_extract (int cmd, const char *fname, const char *kname) char ext_res[128]; const char *file_type; void *file_info; + struct strtab *t; - log (LOG_DEBUG, "%c %s k=%s", cmd, fname, kname); + logf (LOG_DEBUG, "%c %s k=%s", cmd, fname, kname); for (i = strlen(fname); --i >= 0; ) if (fname[i] == '/') { @@ -164,11 +220,15 @@ void file_extract (int cmd, const char *fname, const char *kname) { sysno = sysno_next++; dict_insert (file_idx, kname, sizeof(sysno), &sysno); + lseek (sys_idx_fd, sysno * SYS_IDX_ENTRY_LEN, SEEK_SET); + write (sys_idx_fd, kname, strlen(kname)+1); } else memcpy (&sysno, (char*) file_info+1, sizeof(sysno)); + t = strtab_mk (); if (!strcmp (file_type, "text")) - text_extract (sysno, cmd, fname); + text_extract (t, sysno, cmd, fname); + strtab_del (t, key_rec_flush, &cmd); }