From: Adam Dickmeiss Date: Wed, 22 Nov 1995 17:19:16 +0000 (+0000) Subject: Record management uses the bfile system. X-Git-Tag: ZEBRA.1.0~669 X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=commitdiff_plain;h=dd5f8f31bc5018a7b4a00accdda256ce1eaef14b Record management uses the bfile system. --- diff --git a/index/extract.c b/index/extract.c index 76e4938..56bc949 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.29 1995-11-21 15:01:14 adam + * Revision 1.30 1995-11-22 17:19:16 adam + * Record management uses the bfile system. + * + * Revision 1.29 1995/11/21 15:01:14 adam * New general match criteria implemented. * New feature: document groups. * @@ -732,6 +735,11 @@ int fileExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, /* new record ? */ if (! *sysno) { + if (deleteFlag) + { + logf (LOG_LOG, "? record %s", fname); + return 1; + } logf (LOG_LOG, "add record %s", fname); rec = rec_new (records); *sysno = rec->sysno; @@ -745,15 +753,25 @@ int fileExtract (SYSNO *sysno, const char *fname, struct recordGroup *rGroup, else { struct recKeys delkeys; - + rec = rec_get (records, *sysno); delkeys.buf_used = rec->size[2]; delkeys.buf = rec->info[2]; flushRecordKeys (*sysno, 0, &delkeys, rec->info[3]); - flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); - - records_updated++; + if (deleteFlag) + { + logf (LOG_LOG, "delete record %s", fname); + records_deleted++; + rec_del (records, &rec); + return 1; + } + else + { + logf (LOG_LOG, "update record %s", fname); + flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName); + records_updated++; + } } free (rec->info[0]); rec->info[0] = rec_strdup (file_type, &rec->size[0]); diff --git a/index/index.h b/index/index.h index a018b7e..6fdee19 100644 --- a/index/index.h +++ b/index/index.h @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: index.h,v $ - * Revision 1.25 1995-11-21 15:29:12 adam + * Revision 1.26 1995-11-22 17:19:17 adam + * Record management uses the bfile system. + * + * Revision 1.25 1995/11/21 15:29:12 adam * Config file 'base' read by default by both indexer and server. * * Revision 1.24 1995/11/21 15:01:15 adam @@ -133,7 +136,8 @@ void dir_sort (struct dir_entry *e); void dir_free (struct dir_entry **e_p); void repositoryUpdate (struct recordGroup *rGroup); -void repositoryExtract (struct recordGroup *rGroup); +void repositoryAdd (struct recordGroup *rGroup); +void repositoryDelete (struct recordGroup *rGroup); void key_open (int mem); int key_close (void); diff --git a/index/main.c b/index/main.c index 1650899..8e68139 100644 --- a/index/main.c +++ b/index/main.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: main.c,v $ - * Revision 1.17 1995-11-21 15:01:16 adam + * Revision 1.18 1995-11-22 17:19:17 adam + * Record management uses the bfile system. + * + * Revision 1.17 1995/11/21 15:01:16 adam * New general match criteria implemented. * New feature: document groups. * @@ -141,12 +144,9 @@ int main (int argc, char **argv) if (cmd == 'u') repositoryUpdate (&rGroup); else if (cmd == 'a') - repositoryExtract (&rGroup); + repositoryAdd (&rGroup); else if (cmd == 'd') - { - logf (LOG_FATAL, "Not implemented yet."); - exit (1); - } + repositoryDelete (&rGroup); cmd = 0; } } diff --git a/index/recindex.c b/index/recindex.c index bdd134c..1955b98 100644 --- a/index/recindex.c +++ b/index/recindex.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindex.c,v $ - * Revision 1.4 1995-11-20 16:59:46 adam + * Revision 1.5 1995-11-22 17:19:18 adam + * Record management uses the bfile system. + * + * Revision 1.4 1995/11/20 16:59:46 adam * New update method: the 'old' keys are saved for each records. * * Revision 1.3 1995/11/16 15:34:55 adam @@ -27,6 +30,577 @@ #include "recindex.h" +#define USE_BF 1 + +#if USE_BF +#include + +#define REC_BLOCK_TYPES 2 +#define REC_HEAD_MAGIC "recindx" + +struct records_info { + int rw; + + char *index_fname; + BFile index_BFile; + + + char *data_fname[REC_BLOCK_TYPES]; + BFile data_BFile[REC_BLOCK_TYPES]; + + char *tmp_buf; + int tmp_size; + + struct record_cache_entry *record_cache; + int cache_size; + int cache_cur; + int cache_max; + + struct records_head { + char magic[8]; + int block_size[REC_BLOCK_TYPES]; + int block_free[REC_BLOCK_TYPES]; + int block_last[REC_BLOCK_TYPES]; + int block_used[REC_BLOCK_TYPES]; + int block_move[REC_BLOCK_TYPES]; + + int index_last; + int index_free; + int no_records; + + } head; +}; + +enum recordCacheFlag { recordFlagNop, recordFlagWrite, recordFlagDelete }; + +struct record_cache_entry { + Record rec; + enum recordCacheFlag flag; +}; + +struct record_index_entry { + union { + struct { + int next; + int size; + } used; + struct { + int next; + } free; + } u; +}; + + +static void rec_write_head (Records p) +{ + int r; + + assert (p); + assert (p->index_BFile); + + r = bf_write (p->index_BFile, 0, 0, sizeof(p->head), &p->head); + if (r) + { + logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->index_fname); + exit (1); + } +} + +static void rec_tmp_expand (Records p, int size, int dst_type) +{ + if (p->tmp_size < size + 256 || + p->tmp_size < p->head.block_size[dst_type]*2) + { + free (p->tmp_buf); + p->tmp_size = size + p->head.block_size[dst_type]*2 + + 256; + if (!(p->tmp_buf = malloc (p->tmp_size))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + } +} + +static int read_indx (Records p, int sysno, void *buf, int itemsize, + int ignoreError) +{ + int r; + int pos = (sysno-1)*itemsize; + + r = bf_read (p->index_BFile, 1+pos/128, pos%128, itemsize, buf); + if (r != 1 && !ignoreError) + { + logf (LOG_FATAL|LOG_ERRNO, "read in %s at pos %ld", + p->index_fname, (long) pos); + abort (); + exit (1); + } + return r; +} + +static void write_indx (Records p, int sysno, void *buf, int itemsize) +{ + int pos = (sysno-1)*itemsize; + + bf_write (p->index_BFile, 1+pos/128, pos%128, itemsize, buf); +} + +static void rec_release_blocks (Records p, int sysno) +{ + struct record_index_entry entry; + int freeblock, freenext; + int dst_type; + + if (read_indx (p, sysno, &entry, sizeof(entry), 1) != 1) + return ; + freeblock = entry.u.used.next; + assert (freeblock > 0); + dst_type = freeblock & 7; + freeblock = freeblock / 8; + while (freeblock) + { + if (bf_read (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext), + &freenext) != 1) + { + logf (LOG_FATAL|LOG_ERRNO, "read in rec_del_single"); + exit (1); + } + if (bf_write (p->data_BFile[dst_type], freeblock, 0, sizeof(freenext), + &p->head.block_free[dst_type])) + { + logf (LOG_FATAL|LOG_ERRNO, "write in rec_del_single"); + exit (1); + } + p->head.block_free[dst_type] = freeblock; + freeblock = freenext; + p->head.block_used[dst_type]--; + } +} + +static void rec_delete_single (Records p, Record rec) +{ + struct record_index_entry entry; + + rec_release_blocks (p, rec->sysno); + + entry.u.free.next = p->head.index_free; + p->head.index_free = rec->sysno; + write_indx (p, rec->sysno, &entry, sizeof(entry)); +} + +static void rec_write_single (Records p, Record rec) +{ + int i, size = 0; + char *cptr; + int dst_type = 0; + int no_written = 0; + int block_prev = -1, block_free; + struct record_index_entry entry; + + rec_release_blocks (p, rec->sysno); + + for (i = 0; i < REC_NO_INFO; i++) + if (!rec->info[i]) + size += sizeof(*rec->size); + else + size += sizeof(*rec->size) + rec->size[i]; + + for (i = 1; i= p->head.block_move[i]) + dst_type = i; + + rec_tmp_expand (p, size, dst_type); + + cptr = p->tmp_buf + sizeof(int); /* a hack! */ + for (i = 0; i < REC_NO_INFO; i++) + { + memcpy (cptr, &rec->size[i], sizeof(*rec->size)); + cptr += sizeof(*rec->size); + if (rec->info[i]) + { + memcpy (cptr, rec->info[i], rec->size[i]); + cptr += rec->size[i]; + } + } + cptr = p->tmp_buf; + while (no_written < size) + { + block_free = p->head.block_free[dst_type]; + if (block_free) + { + if (bf_read (p->data_BFile[dst_type], + block_free, 0, sizeof(*p->head.block_free), + &p->head.block_free[dst_type]) != 1) + { + logf (LOG_FATAL|LOG_ERRNO, "read in %s at free block %d", + p->data_fname[dst_type], block_free); + } + } + else + block_free = p->head.block_last[dst_type]++; + if (block_prev == -1) + { + entry.u.used.next = block_free*8 + dst_type; + entry.u.used.size = size; + + write_indx (p, rec->sysno, &entry, sizeof(entry)); + } + else + { + memcpy (cptr, &block_free, sizeof(int)); + bf_write (p->data_BFile[dst_type], block_prev, 0, 0, cptr); + cptr = p->tmp_buf + no_written; + } + block_prev = block_free; + no_written += p->head.block_size[dst_type] - sizeof(int); + p->head.block_used[dst_type]++; + } + assert (block_prev != -1); + block_free = 0; + memcpy (cptr, &block_free, sizeof(int)); + bf_write (p->data_BFile[dst_type], block_prev, 0, + sizeof(int) + (p->tmp_buf+size) - cptr, cptr); +} + + +Records rec_open (int rw) +{ + Records p; + int i, r; + + if (!(p = malloc (sizeof(*p)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + p->rw = rw; + p->tmp_size = 1024; + p->tmp_buf = malloc (p->tmp_size); + if (!p->tmp_buf) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + p->index_fname = "recindex"; + p->index_BFile = bf_open (p->index_fname, 128, rw); + if (p->index_BFile == NULL) + { + logf (LOG_FATAL|LOG_ERRNO, "open %s", p->index_fname); + exit (1); + } + r = bf_read (p->index_BFile, 0, 0, 0, p->tmp_buf); + switch (r) + { + case 0: + memcpy (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)); + p->head.index_free = 0; + p->head.index_last = 1; + p->head.no_records = 0; + for (i = 0; ihead.block_free[i] = 0; + p->head.block_last[i] = 1; + p->head.block_used[i] = 0; + } + p->head.block_size[0] = 128; + p->head.block_move[0] = 0; + for (i = 1; ihead.block_size[i] = p->head.block_size[i-1] * 4; + p->head.block_move[i] = p->head.block_size[i] * 3; + } + if (rw) + rec_write_head (p); + break; + case 1: + memcpy (&p->head, p->tmp_buf, sizeof(p->head)); + if (memcmp (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic))) + { + logf (LOG_FATAL, "read %s. bad header", p->index_fname); + exit (1); + } + break; + } + for (i = 0; idata_fname[i] = malloc (strlen(str)+1); + strcpy (p->data_fname[i], str); + p->data_BFile[i] = NULL; + } + for (i = 0; idata_BFile[i] = bf_open (p->data_fname[i], + p->head.block_size[i], + rw))) + { + logf (LOG_FATAL|LOG_ERRNO, "bf_open %s", p->data_fname[i]); + exit (1); + } + } + p->cache_max = 10; + p->cache_cur = 0; + if (!(p->record_cache = malloc (sizeof(*p->record_cache)*p->cache_max))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + return p; +} + +static void rec_cache_flush (Records p) +{ + int i; + for (i = 0; icache_cur; i++) + { + struct record_cache_entry *e = p->record_cache + i; + switch (e->flag) + { + case recordFlagNop: + break; + case recordFlagWrite: + rec_write_single (p, e->rec); + break; + case recordFlagDelete: + rec_delete_single (p, e->rec); + break; + } + rec_rm (&e->rec); + } + p->cache_cur = 0; +} + +static Record *rec_cache_lookup (Records p, int sysno, + enum recordCacheFlag flag) +{ + int i; + for (i = 0; icache_cur; i++) + { + struct record_cache_entry *e = p->record_cache + i; + if (e->rec->sysno == sysno) + { + if (flag != recordFlagNop) + e->flag = flag; + return &e->rec; + } + } + return NULL; +} + +static void rec_cache_insert (Records p, Record rec, enum recordCacheFlag flag) +{ + struct record_cache_entry *e; + + if (p->cache_cur == p->cache_max) + rec_cache_flush (p); + assert (p->cache_cur < p->cache_max); + + e = p->record_cache + (p->cache_cur)++; + e->flag = flag; + e->rec = rec_cp (rec); +} + +void rec_close (Records *pp) +{ + Records p = *pp; + int i; + + assert (p); + + rec_cache_flush (p); + free (p->record_cache); + + if (p->rw) + rec_write_head (p); + + if (p->index_BFile) + bf_close (p->index_BFile); + + for (i = 0; idata_BFile[i]) + bf_close (p->data_BFile[i]); + free (p->data_fname[i]); + } + free (p->tmp_buf); + free (p); + *pp = NULL; +} + + +Record rec_get (Records p, int sysno) +{ + int i; + Record rec, *recp; + struct record_index_entry entry; + int freeblock, dst_type; + char *nptr, *cptr; + + assert (sysno > 0); + assert (p); + + if ((recp = rec_cache_lookup (p, sysno, recordFlagNop))) + return rec_cp (*recp); + + read_indx (p, sysno, &entry, sizeof(entry), 0); + + dst_type = entry.u.used.next & 7; + freeblock = entry.u.used.next / 8; + + assert (freeblock > 0); + + if (!(rec = malloc (sizeof(*rec)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + rec_tmp_expand (p, entry.u.used.size, dst_type); + + cptr = p->tmp_buf; + bf_read (p->data_BFile[dst_type], freeblock, 0, 0, cptr); + memcpy (&freeblock, cptr, sizeof(freeblock)); + + while (freeblock) + { + int tmp; + + cptr += p->head.block_size[dst_type] - sizeof(freeblock); + + memcpy (&tmp, cptr, sizeof(tmp)); + bf_read (p->data_BFile[dst_type], freeblock, 0, 0, cptr); + memcpy (&freeblock, cptr, sizeof(freeblock)); + memcpy (cptr, &tmp, sizeof(tmp)); + } + + rec->sysno = sysno; + nptr = p->tmp_buf + sizeof(freeblock); + for (i = 0; i < REC_NO_INFO; i++) + { + memcpy (&rec->size[i], nptr, sizeof(*rec->size)); + nptr += sizeof(*rec->size); + if (rec->size[i]) + { + rec->info[i] = malloc (rec->size[i]); + memcpy (rec->info[i], nptr, rec->size[i]); + nptr += rec->size[i]; + } + else + rec->info[i] = NULL; + } + rec_cache_insert (p, rec, recordFlagNop); + return rec; +} + +Record rec_new (Records p) +{ + int sysno, i; + Record rec; + + assert (p); + if (!(rec = malloc (sizeof(*rec)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + if (p->head.index_free == 0) + sysno = (p->head.index_last)++; + else + { + struct record_index_entry entry; + + read_indx (p, p->head.index_free, &entry, sizeof(entry), 0); + sysno = p->head.index_free; + p->head.index_free = entry.u.free.next; + } + (p->head.no_records)++; + rec->sysno = sysno; + for (i = 0; i < REC_NO_INFO; i++) + { + rec->info[i] = NULL; + rec->size[i] = 0; + } + rec_cache_insert (p, rec, recordFlagWrite); + return rec; +} + +void rec_del (Records p, Record *recpp) +{ + Record *recp; + + if ((recp = rec_cache_lookup (p, (*recpp)->sysno, recordFlagDelete))) + { + rec_rm (recp); + *recp = *recpp; + } + else + { + rec_cache_insert (p, *recpp, recordFlagDelete); + rec_rm (recpp); + } + *recpp = NULL; +} + +void rec_put (Records p, Record *recpp) +{ + Record *recp; + + if ((recp = rec_cache_lookup (p, (*recpp)->sysno, recordFlagWrite))) + { + rec_rm (recp); + *recp = *recpp; + } + else + { + rec_cache_insert (p, *recpp, recordFlagWrite); + rec_rm (recpp); + } + *recpp = NULL; +} + +void rec_rm (Record *recpp) +{ + int i; + for (i = 0; i < REC_NO_INFO; i++) + free ((*recpp)->info[i]); + free (*recpp); + *recpp = NULL; +} + +Record rec_cp (Record rec) +{ + Record n; + int i; + + if (!(n = malloc (sizeof(*n)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + n->sysno = rec->sysno; + for (i = 0; i < REC_NO_INFO; i++) + if (!rec->info[i]) + { + n->info[i] = NULL; + n->size[i] = 0; + } + else + { + n->size[i] = rec->size[i]; + if (!(n->info[i] = malloc (rec->size[i]))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc. rec_cp"); + exit (1); + } + memcpy (n->info[i], rec->info[i], rec->size[i]); + } + return n; +} + +/* no BF --------------------------------------------------- */ +#else + struct records_info { int rw; int index_fd; @@ -69,26 +643,6 @@ struct record_index_entry { #define REC_HEAD_MAGIC "rechead" -char *rec_strdup (const char *s, size_t *len) -{ - char *p; - - if (!s) - { - *len = 0; - return NULL; - } - *len = strlen(s)+1; - p = malloc (*len); - if (!p) - { - logf (LOG_FATAL|LOG_ERRNO, "malloc"); - exit (1); - } - strcpy (p, s); - return p; -} - static void rec_write_head (Records p) { int r; @@ -497,3 +1051,32 @@ Record rec_cp (Record rec) } return n; } + +void rec_del (Records p, Record *recpp) +{ + assert (0); +} + + +#endif + +char *rec_strdup (const char *s, size_t *len) +{ + char *p; + + if (!s) + { + *len = 0; + return NULL; + } + *len = strlen(s)+1; + p = malloc (*len); + if (!p) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + strcpy (p, s); + return p; +} + diff --git a/index/recindex.h b/index/recindex.h index b12fa8e..ad3d41e 100644 --- a/index/recindex.h +++ b/index/recindex.h @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindex.h,v $ - * Revision 1.3 1995-11-20 16:59:46 adam + * Revision 1.4 1995-11-22 17:19:19 adam + * Record management uses the bfile system. + * + * Revision 1.3 1995/11/20 16:59:46 adam * New update method: the 'old' keys are saved for each records. * * Revision 1.2 1995/11/15 19:13:08 adam @@ -28,6 +31,7 @@ typedef struct record_info { typedef struct records_info *Records; Record rec_cp (Record rec); +void rec_del (Records p, Record *recpp); void rec_rm (Record *recpp); void rec_put (Records p, Record *recpp); Record rec_new (Records p); diff --git a/index/trav.c b/index/trav.c index c929e51..4ea6fb6 100644 --- a/index/trav.c +++ b/index/trav.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: trav.c,v $ - * Revision 1.10 1995-11-21 15:01:16 adam + * Revision 1.11 1995-11-22 17:19:19 adam + * Record management uses the bfile system. + * + * Revision 1.10 1995/11/21 15:01:16 adam * New general match criteria implemented. * New feature: document groups. * @@ -56,7 +59,8 @@ static int repComp (const char *a, const char *b, size_t len) return memcmp (a, b, len); } -static void repositoryExtractR (char *rep, struct recordGroup *rGroup) +static void repositoryExtractR (int deleteFlag, char *rep, + struct recordGroup *rGroup) { struct dir_entry *e; int i; @@ -75,10 +79,10 @@ static void repositoryExtractR (char *rep, struct recordGroup *rGroup) switch (e[i].kind) { case dirs_file: - fileExtract (NULL, rep, rGroup, 0); + fileExtract (NULL, rep, rGroup, deleteFlag); break; case dirs_dir: - repositoryExtractR (rep, rGroup); + repositoryExtractR (deleteFlag, rep, rGroup); break; } } @@ -220,11 +224,21 @@ void repositoryUpdate (struct recordGroup *rGroup) dict_close (dict); } -void repositoryExtract (struct recordGroup *rGroup) +void repositoryDelete (struct recordGroup *rGroup) { char src[256]; assert (rGroup->path); strcpy (src, rGroup->path); - repositoryExtractR (src, rGroup); + repositoryExtractR (1, src, rGroup); } + +void repositoryAdd (struct recordGroup *rGroup) +{ + char src[256]; + + assert (rGroup->path); + strcpy (src, rGroup->path); + repositoryExtractR (0, src, rGroup); +} +