-/* $Id: extract.c,v 1.124 2002-10-16 09:30:57 heikki Exp $
+/* $Id: extract.c,v 1.129 2002-10-24 21:54:29 adam Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002
Index Data Aps
zh->reg->keys.prevAttrUse = -1;
zh->reg->keys.prevAttrSet = -1;
zh->reg->keys.prevSeqNo = 0;
- zh->reg->sortKeys = 0;
+ zh->reg->sortKeys.buf_used = 0;
+ zh->reg->sortKeys.buf_max = 0;
+ zh->reg->sortKeys.buf = 0;
recordOffset = fi->file_moffset;
extractCtrl.offset = fi->file_moffset;
{
dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno);
}
- extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
+ extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
zh->records_inserted++;
}
{
/* record already exists */
struct recKeys delkeys;
+ struct sortKeys sortKeys;
rec = rec_get (zh->reg->records, *sysno);
assert (rec);
}
delkeys.buf_used = rec->size[recInfo_delKeys];
delkeys.buf = rec->info[recInfo_delKeys];
- extract_flushSortKeys (zh, *sysno, 0, &zh->reg->sortKeys);
+
+ sortKeys.buf_used = rec->size[recInfo_sortKeys];
+ sortKeys.buf = rec->info[recInfo_sortKeys];
+
+ extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
extract_flushRecordKeys (zh, *sysno, 0, &delkeys);
if (deleteFlag)
{
if (zh->records_processed < rGroup->fileVerboseLimit)
logf (LOG_LOG, "update %s %s " PRINTF_OFF_T,
rGroup->recordType, fname, recordOffset);
+ extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
zh->records_updated++;
}
xfree (rec->info[recInfo_delKeys]);
if (zh->reg->keys.buf_used > 0 && rGroup->flagStoreKeys == 1)
{
-#if 1
rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
rec->info[recInfo_delKeys] = zh->reg->keys.buf;
zh->reg->keys.buf = NULL;
zh->reg->keys.buf_max = 0;
-#else
- rec->info[recInfo_delKeys] = xmalloc (reckeys.buf_used);
- rec->size[recInfo_delKeys] = reckeys.buf_used;
- memcpy (rec->info[recInfo_delKeys], reckeys.buf,
- rec->size[recInfo_delKeys]);
-#endif
}
else
{
rec->size[recInfo_delKeys] = 0;
}
+ /* update sort keys */
+ xfree (rec->info[recInfo_sortKeys]);
+
+ rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
+ rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
+ zh->reg->sortKeys.buf = NULL;
+ zh->reg->sortKeys.buf_max = 0;
+
/* save file size of original record */
zebraExplain_recordBytesIncrement (zh->reg->zei,
- recordAttr->recordSize);
zh->reg->keys.prevAttrUse = -1;
zh->reg->keys.prevAttrSet = -1;
zh->reg->keys.prevSeqNo = 0;
- zh->reg->sortKeys = 0;
+ zh->reg->sortKeys.buf_used = 0;
extractCtrl.subType = subType;
extractCtrl.init = extract_init;
dict_insert (zh->reg->matchDict, matchStr,
sizeof(*sysno), sysno);
}
- extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
+ extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
}
else
{
{
logf (LOG_LOG, "update %s %s %ld", recordType,
fname, (long) recordOffset);
+ extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
}
}
zh->reg->keys.prevAttrUse = -1;
zh->reg->keys.prevAttrSet = -1;
zh->reg->keys.prevSeqNo = 0;
- zh->reg->sortKeys = 0;
+ zh->reg->sortKeys.buf_used = 0;
extractCtrl.init = extract_init;
extractCtrl.tokenAdd = extract_token_add;
if (rec->size[recInfo_delKeys])
{
struct recKeys delkeys;
- struct sortKey *sortKeys = 0;
+ struct sortKeys sortkeys;
delkeys.buf_used = rec->size[recInfo_delKeys];
delkeys.buf = rec->info[recInfo_delKeys];
- extract_flushSortKeys (zh, rec->sysno, 0, &sortKeys);
+
+ sortkeys.buf_used = rec->size[recInfo_sortKeys];
+ sortkeys.buf = rec->info[recInfo_sortKeys];
+
+ extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys);
extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys);
}
extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys);
rec->info[recInfo_delKeys] = zh->reg->keys.buf;
zh->reg->keys.buf = NULL;
zh->reg->keys.buf_max = 0;
+
+ xfree (rec->info[recInfo_sortKeys]);
+ rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
+ rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
+ zh->reg->sortKeys.buf = NULL;
+ zh->reg->sortKeys.buf_max = 0;
+
return 0;
}
if (!zh->reg->key_buf)
{
- int mem = 8*1024*1024;
+ int mem= 1024*1024* atoi( res_get_def( zh->res, "memmax", "8"));
+ if (mem <= 0)
+ {
+ logf(LOG_WARN, "Invalid memory setting, using default 8 MB");
+ mem= 1024*1024*8;
+ }
+ /* FIXME: That "8" should be in a default settings include */
+ /* not hard-coded here! -H */
zh->reg->key_buf = (char**) xmalloc (mem);
zh->reg->ptr_top = mem/sizeof(char*);
zh->reg->ptr_i = 0;
static void extract_add_sort_string (RecWord *p, const char *string,
int length)
{
- struct sortKey *sk;
ZebraHandle zh = p->extractCtrl->handle;
+ struct sortKeys *sk = &zh->reg->sortKeys;
+ size_t off = 0;
- for (sk = zh->reg->sortKeys; sk; sk = sk->next)
- if (sk->attrSet == p->attrSet && sk->attrUse == p->attrUse)
- return;
-
- sk = (struct sortKey *) xmalloc (sizeof(*sk));
- sk->next = zh->reg->sortKeys;
- zh->reg->sortKeys = sk;
-
- sk->string = (char *) xmalloc (length);
- sk->length = length;
- memcpy (sk->string, string, length);
+ while (off < sk->buf_used)
+ {
+ int set, use, slen;
- sk->attrSet = p->attrSet;
- sk->attrUse = p->attrUse;
+ off += key_SU_decode(&set, sk->buf + off);
+ off += key_SU_decode(&use, sk->buf + off);
+ off += key_SU_decode(&slen, sk->buf + off);
+ off += slen;
+ if (p->attrSet == set && p->attrUse == use)
+ return;
+ }
+ assert (off == sk->buf_used);
+
+ if (sk->buf_used + IT_MAX_WORD > sk->buf_max)
+ {
+ char *b;
+
+ b = (char *) xmalloc (sk->buf_max += 128000);
+ if (sk->buf_used > 0)
+ memcpy (b, sk->buf, sk->buf_used);
+ xfree (sk->buf);
+ sk->buf = b;
+ }
+ off += key_SU_encode(p->attrSet, sk->buf + off);
+ off += key_SU_encode(p->attrUse, sk->buf + off);
+ off += key_SU_encode(length, sk->buf + off);
+ memcpy (sk->buf + off, string, length);
+ sk->buf_used = off + length;
}
void extract_add_string (RecWord *p, const char *string, int length)
}
void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno,
- int cmd, struct sortKey **skp)
+ int cmd, struct sortKeys *sk)
{
- struct sortKey *sk = *skp;
SortIdx sortIdx = zh->reg->sortIdx;
+ size_t off = 0;
sortIdx_sysno (sortIdx, sysno);
- while (sk)
+
+ while (off < sk->buf_used)
{
- struct sortKey *sk_next = sk->next;
- sortIdx_type (sortIdx, sk->attrUse);
- sortIdx_add (sortIdx, sk->string, sk->length);
- xfree (sk->string);
- xfree (sk);
- sk = sk_next;
+ int set, use, slen, l;
+
+ off += key_SU_decode(&set, sk->buf + off);
+ off += key_SU_decode(&use, sk->buf + off);
+ off += key_SU_decode(&slen, sk->buf + off);
+
+ sortIdx_type(sortIdx, use);
+ if (cmd == 1)
+ sortIdx_add(sortIdx, sk->buf + off, slen);
+ else
+ sortIdx_add(sortIdx, "", 1);
+ off += slen;
}
- *skp = 0;
}
void encode_key_init (struct encode_info *i)
while ((*bp++ = *k++))
;
i->keylen= bp - i->buf -1;
+ assert(i->keylen+1+sizeof(struct it_key) < ENCODE_BUFLEN);
}
else
{
i->prevseq=key.seqno;
i->prevcmd=*k;
}
-#ifdef SKIPTHIS_OLDCODE
- bp = encode_key_int ( (key.sysno - i->sysno) * 2 + *k, bp);
- if (i->sysno != key.sysno)
- {
- i->sysno = key.sysno;
- i->seqno = 0;
- }
- else if (!i->seqno && !key.seqno && i->cmd == *k)
- return;
- bp = encode_key_int (key.seqno - i->seqno, bp);
- i->seqno = key.seqno;
- i->cmd = *k;
- if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
- {
- logf (LOG_FATAL|LOG_ERRNO, "fwrite");
- exit (1);
- }
- i->keylen=0; /* ok, it's written, forget it */
-#endif
}
void encode_key_flush (struct encode_info *i, FILE *outf)