X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=0d9afd5b171a701ef5dcd008e85607bd81a78bf5;hb=eb4e33573a00795f2b1c0dfebb51ab59ffd5a0d9;hp=20509804f87344aee144971aec6dec27971c6480;hpb=9765f689118107e45f5fba7261363980d378f7b3;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 2050980..0d9afd5 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,30 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.6 1995-09-08 14:52:27 adam + * Revision 1.13 1995-09-29 14:01:39 adam + * Bug fixes. + * + * Revision 1.12 1995/09/28 14:22:56 adam + * Sort uses smaller temporary files. + * + * Revision 1.11 1995/09/28 12:10:31 adam + * Bug fixes. Field prefix used in queries. + * + * Revision 1.10 1995/09/28 09:19:41 adam + * xfree/xmalloc used everywhere. + * Extract/retrieve method seems to work for text records. + * + * Revision 1.9 1995/09/27 12:22:28 adam + * More work on extract in record control. + * Field name is not in isam keys but in prefix in dictionary words. + * + * Revision 1.8 1995/09/14 07:48:22 adam + * Record control management. + * + * Revision 1.7 1995/09/11 13:09:32 adam + * More work on relevance feedback. + * + * Revision 1.6 1995/09/08 14:52:27 adam * Minor changes. Dictionary is lower case now. * * Revision 1.5 1995/09/06 16:11:16 adam @@ -32,34 +55,33 @@ #include #include +#include #include "index.h" -#define KEY_BUF_SIZE 100000 - static Dict file_idx; static SYSNO sysno_next; -static int key_fd = -1; static int sys_idx_fd = -1; -static char *key_buf; -static int key_offset; -void key_open (const char *fname) +static int key_cmd; +static int key_sysno; +static char **key_buf; +static size_t ptr_top; +static size_t ptr_i; +static size_t kused; +static int key_file_no; + +void key_open (int mem) { void *file_key; - if (key_fd != -1) - return; - if ((key_fd = open (fname, O_RDWR|O_CREAT, 0666)) == -1) - { - logf (LOG_FATAL|LOG_ERRNO, "open %s", fname); - exit (1); - } - logf (LOG_DEBUG, "key_open of %s", fname); - if (!(key_buf = malloc (KEY_BUF_SIZE))) - { - logf (LOG_FATAL|LOG_ERRNO, "malloc"); - exit (1); - } - key_offset = 0; + + if (mem < 50000) + mem = 50000; + key_buf = xmalloc (mem); + ptr_top = mem/sizeof(char*); + ptr_i = 0; + kused = 0; + key_file_no = 0; + if (!(file_idx = dict_open (FNAME_FILE_DICT, 40, 1))) { logf (LOG_FATAL, "dict_open fail of %s", "fileidx"); @@ -76,117 +98,110 @@ void key_open (const char *fname) exit (1); } } - -int key_close (void) -{ - if (key_fd == -1) - { - logf (LOG_DEBUG, "key_close - but no file"); - return 0; - } - close (key_fd); - close (sys_idx_fd); - dict_insert (file_idx, ".", sizeof(sysno_next), &sysno_next); - dict_close (file_idx); - key_fd = -1; - logf (LOG_DEBUG, "key close - key file exist"); - return 1; -} - + void key_flush (void) { - size_t i = 0; - int w; + FILE *outf; + char out_fname[200]; + char *prevcp, *cp; + + if (ptr_i <= 0) + return; + + key_file_no++; + logf (LOG_LOG, "sorting section %d", key_file_no); + qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_qsort_compare); + sprintf (out_fname, TEMP_FNAME, key_file_no); - if (key_fd == -1) - return; - while (i < key_offset) + if (!(outf = fopen (out_fname, "w"))) + { + logf (LOG_FATAL|LOG_ERRNO, "fopen (4) %s", out_fname); + exit (1); + } + logf (LOG_LOG, "writing section %d", key_file_no); + prevcp = cp = key_buf[ptr_top-ptr_i]; + + if (fwrite (cp, strlen (cp)+2+sizeof(struct it_key), 1, outf) != 1) { - w = write (key_fd, key_buf + i, key_offset - i); - if (w == -1) + logf (LOG_FATAL|LOG_ERRNO, "fwrite %s", out_fname); + exit (1); + } + while (--ptr_i > 0) + { + cp = key_buf[ptr_top-ptr_i]; + if (strcmp (cp, prevcp)) + { + if (fwrite (cp, strlen (cp)+2+sizeof(struct it_key), 1, + outf) != 1) + { + logf (LOG_FATAL|LOG_ERRNO, "fwrite %s", out_fname); + exit (1); + } + prevcp = cp; + } + else { - logf (LOG_FATAL|LOG_ERRNO, "Write key fail"); - exit (1); + cp = strlen (cp) + cp; + if (fwrite (cp, 2+sizeof(struct it_key), 1, outf) != 1) + { + logf (LOG_FATAL|LOG_ERRNO, "fwrite %s", out_fname); + exit (1); + } } - i += w; } - key_offset = 0; -} - -void key_write (int cmd, struct it_key *k, const char *str) -{ - char x; - size_t slen = strlen(str); - - if (key_offset + sizeof(*k) + slen >= KEY_BUF_SIZE - 2) - key_flush (); - x = (cmd == 'a') ? 1 : 0; - memcpy (key_buf + key_offset, str, slen+1); - key_offset += slen+1; - memcpy (key_buf + key_offset, &x, 1); - key_offset++; - memcpy (key_buf + key_offset, k, sizeof(*k)); - key_offset += sizeof(*k); + if (fclose (outf)) + { + logf (LOG_FATAL|LOG_ERRNO, "fclose %s", out_fname); + exit (1); + } + logf (LOG_LOG, "finished section %d", key_file_no); + ptr_i = 0; + kused = 0; } -void key_write_x (struct strtab *t, int cmd, struct it_key *k, const char *str) +int key_close (void) { - void **oldinfo; - - if (strtab_src (t, str, &oldinfo)) - ((struct it_key *) *oldinfo)->seqno++; - else - { - *oldinfo = xmalloc (sizeof(*k)); - memcpy (*oldinfo, k, sizeof(*k)); - ((struct it_key *) *oldinfo)->seqno = 1; - } + key_flush (); + xfree (key_buf); + close (sys_idx_fd); + dict_insert (file_idx, ".", sizeof(sysno_next), &sysno_next); + dict_close (file_idx); + return key_file_no; } -void key_rec_flush (const char *str, void *info, void *data) +static void wordInit (RecWord *p) { - key_write (*((int*) data), (struct it_key *)info, str); - xfree (info); + p->attrSet = 1; + p->attrUse = 1016; + p->which = Word_String; } -void text_extract (struct strtab *t, SYSNO sysno, int cmd, const char *fname) +static void wordAdd (const RecWord *p) { - FILE *inf; - struct it_key k; - int seqno = 1; - int c; - char w[IT_MAX_WORD]; + struct it_key key; + size_t i; - logf (LOG_DEBUG, "Text extract of %d", sysno); - k.sysno = sysno; - inf = fopen (fname, "r"); - if (!inf) - { - logf (LOG_WARN|LOG_ERRNO, "open %s", fname); - return; - } - while ((c=getc (inf)) != EOF) + if (kused + 1024 > (ptr_top-ptr_i)*sizeof(char*)) + key_flush (); + ++ptr_i; + key_buf[ptr_top-ptr_i] = (char*)key_buf + kused; + kused += index_word_prefix ((char*)key_buf + kused, + p->attrSet, p->attrUse); + switch (p->which) { - int i = 0; - while (i < 254 && c != EOF && isalnum(c)) - { - w[i++] = index_char_cvt (c); - c = getc (inf); - } - if (i) - { - w[i] = 0; - - k.seqno = seqno++; -#if IT_KEY_HAVE_FIELD - k.field = 0; -#endif - key_write_x (t, cmd, &k, w); - } - if (c == EOF) - break; + case Word_String: + for (i = 0; p->u.string[i]; i++) + ((char*)key_buf) [kused++] = index_char_cvt (p->u.string[i]); + ((char*)key_buf) [kused++] = '\0'; + break; + default: + return ; } - fclose (inf); + ((char*) key_buf)[kused++] = ((key_cmd == 'a') ? 1 : 0); + key.sysno = key_sysno; + key.seqno = p->seqno; + memcpy ((char*)key_buf + kused, &key, sizeof(key)); + kused += sizeof(key); } void file_extract (int cmd, const char *fname, const char *kname) @@ -197,7 +212,9 @@ void file_extract (int cmd, const char *fname, const char *kname) char ext_res[128]; const char *file_type; void *file_info; - struct strtab *t; + FILE *inf; + struct recExtractCtrl extractCtrl; + RecType rt; logf (LOG_DEBUG, "%c %s k=%s", cmd, fname, kname); for (i = strlen(fname); --i >= 0; ) @@ -214,21 +231,31 @@ void file_extract (int cmd, const char *fname, const char *kname) sprintf (ext_res, "fileExtension.%s", ext); if (!(file_type = res_get (common_resource, ext_res))) return; - + if (!(rt = recType_byName (file_type))) + return; file_info = dict_lookup (file_idx, kname); if (!file_info) { sysno = sysno_next++; dict_insert (file_idx, kname, sizeof(sysno), &sysno); lseek (sys_idx_fd, sysno * SYS_IDX_ENTRY_LEN, SEEK_SET); + write (sys_idx_fd, file_type, strlen (file_type)+1); write (sys_idx_fd, kname, strlen(kname)+1); } else memcpy (&sysno, (char*) file_info+1, sizeof(sysno)); - t = strtab_mk (); - if (!strcmp (file_type, "text")) - text_extract (t, sysno, cmd, fname); - strtab_del (t, key_rec_flush, &cmd); -} - + if (!(inf = fopen (fname, "r"))) + { + logf (LOG_WARN|LOG_ERRNO, "open %s", fname); + return; + } + extractCtrl.inf = inf; + extractCtrl.subType = ""; + extractCtrl.init = wordInit; + extractCtrl.add = wordAdd; + key_sysno = sysno; + key_cmd = cmd; + (*rt->extract)(&extractCtrl); + fclose (inf); +}