From f3e73ab63fbc960d863d9c14bab3b9e79c400ffa Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Wed, 15 Nov 1995 19:13:07 +0000 Subject: [PATCH] Work on record management. --- index/attribute.c | 12 +- index/extract.c | 41 +++++- index/recindex.c | 393 +++++++++++++++++++++++++++++++++++++++++++++++++---- index/recindex.h | 29 ++-- 4 files changed, 434 insertions(+), 41 deletions(-) diff --git a/index/attribute.c b/index/attribute.c index 64d7035..e96e572 100644 --- a/index/attribute.c +++ b/index/attribute.c @@ -1,4 +1,13 @@ /* + * Copyright (C) 1994-1995, Index Data I/S + * All rights reserved. + * Sebastian Hammer, Adam Dickmeiss + * + * $Log: attribute.c,v $ + * Revision 1.2 1995-11-15 19:13:07 adam + * Work on record management. + * + * * This interface is used by other modules (the Z-server in particular) * to normalize the attributes given in queries. */ @@ -8,8 +17,7 @@ #include #include #include - -#include "d1_attset.h" +#include #include "attribute.h" diff --git a/index/extract.c b/index/extract.c index fffe509..abe3c88 100644 --- a/index/extract.c +++ b/index/extract.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.23 1995-10-27 14:00:10 adam + * Revision 1.24 1995-11-15 19:13:08 adam + * Work on record management. + * + * Revision 1.23 1995/10/27 14:00:10 adam * Implemented detection of database availability. * * Revision 1.22 1995/10/17 18:02:07 adam @@ -88,9 +91,21 @@ #include #include "index.h" +#define RECORD_BASE 1 + +#if RECORD_BASE +#include "recindex.h" +#endif + static Dict file_idx; -static SYSNO sysno_next; + + +#if RECORD_BASE +static Records records = NULL; +#else static int sys_idx_fd = -1; +static SYSNO sysno_next; +#endif static int key_cmd; static int key_sysno; @@ -103,8 +118,9 @@ static int key_file_no; void key_open (int mem) { +#if !RECORD_BASE void *file_key; - +#endif if (mem < 50000) mem = 50000; key_buf = xmalloc (mem); @@ -118,6 +134,10 @@ void key_open (int mem) logf (LOG_FATAL, "dict_open fail of %s", "fileidx"); exit (1); } +#if RECORD_BASE + assert (!records); + records = rec_open (1); +#else file_key = dict_lookup (file_idx, "."); if (file_key) memcpy (&sysno_next, (char*)file_key+1, sizeof(sysno_next)); @@ -128,6 +148,7 @@ void key_open (int mem) logf (LOG_FATAL|LOG_ERRNO, "open %s", FNAME_SYS_IDX); exit (1); } +#endif } struct encode_info { @@ -241,8 +262,12 @@ int key_close (void) { key_flush (); xfree (key_buf); +#if RECORD_BASE + rec_close (&records); +#else close (sys_idx_fd); dict_insert (file_idx, ".", sizeof(sysno_next), &sysno_next); +#endif dict_close (file_idx); return key_file_no; } @@ -399,11 +424,21 @@ void file_extract (int cmd, const char *fname, const char *kname, file_info = dict_lookup (file_idx, kname); if (!file_info) { +#if RECORD_BASE + Record rec = rec_new (records); + + sysno = rec->sysno; + dict_insert (file_idx, kname, sizeof(sysno), &sysno); + rec->info[0] = rec_strdup (file_type); + rec->info[1] = rec_strdup (kname); + rec_put (records, rec); +#else sysno = sysno_next++; dict_insert (file_idx, kname, sizeof(sysno), &sysno); lseek (sys_idx_fd, sysno * SYS_IDX_ENTRY_LEN, SEEK_SET); write (sys_idx_fd, file_type, strlen (file_type)+1); write (sys_idx_fd, kname, strlen(kname)+1); +#endif } else memcpy (&sysno, (char*) file_info+1, sizeof(sysno)); diff --git a/index/recindex.c b/index/recindex.c index f64816c..b8ccae7 100644 --- a/index/recindex.c +++ b/index/recindex.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindex.c,v $ - * Revision 1.1 1995-11-15 14:46:20 adam + * Revision 1.2 1995-11-15 19:13:08 adam + * Work on record management. + * + * Revision 1.1 1995/11/15 14:46:20 adam * Started work on better record management system. * */ @@ -18,29 +21,85 @@ #include "recindex.h" +struct records_info { + int rw; + int index_fd; + char *index_fname; + int data_fd; + char *data_fname; + struct records_head { + char magic[8]; + int no_records; + int index_free; + int index_last; + int data_size; + int data_slack; + int data_used; + } head; + char *tmp_buf; + int tmp_size; + int cache_size; + int cache_cur; + int cache_max; + struct record_cache_entry *record_cache; +}; + +struct record_cache_entry { + Record rec; + int dirty; +}; + +struct record_index_entry { + union { + struct { + int offset; + int size; + } used; + struct { + int next; + } free; + } u; +}; + #define REC_HEAD_MAGIC "rechead" +char *rec_strdup (const char *s) +{ + char *p; + + if (!s) + return NULL; + p = malloc (strlen(s)+1); + if (!p) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + strcpy (p, s); + return p; +} + static void rec_write_head (Records p) { int r; assert (p); - assert (p->fd != -1); - if (lseek (p->fd, (off_t) 0, SEEK_SET) == -1) + assert (p->index_fd != -1); + if (lseek (p->index_fd, (off_t) 0, SEEK_SET) == -1) { - logf (LOG_FATAL|LOG_ERRNO, "lseek to 0 in %s", p->fname); + logf (LOG_FATAL|LOG_ERRNO, "lseek to 0 in %s", p->index_fname); exit (1); } - r = write (p->fd, &p->head, sizeof(p->head)); + r = write (p->index_fd, &p->head, sizeof(p->head)); switch (r) { case -1: - logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->fname); + logf (LOG_FATAL|LOG_ERRNO, "write head of %s", p->index_fname); exit (1); case sizeof(p->head): break; default: - logf (LOG_FATAL, "write head of %s. wrote %d", p->fname, r); + logf (LOG_FATAL, "write head of %s. wrote %d", p->index_fname, r); exit (1); } } @@ -55,60 +114,348 @@ Records rec_open (int rw) logf (LOG_FATAL|LOG_ERRNO, "malloc"); exit (1); } - p->fname = "recindex"; - p->fd = open (p->fname, rw ? (O_RDWR|O_CREAT) : O_RDONLY, 0666); - if (p->fd == -1) + p->rw = rw; + p->tmp_buf = NULL; + p->tmp_size = 0; + p->data_fname = "recdata"; + p->data_fd = -1; + p->index_fname = "recindex"; + p->index_fd = open (p->index_fname, + rw ? (O_RDWR|O_CREAT) : O_RDONLY, 0666); + if (p->index_fd == -1) { - logf (LOG_FATAL|LOG_ERRNO, "open %s", p->fname); + logf (LOG_FATAL|LOG_ERRNO, "open %s", p->index_fname); exit (1); } - r = read (p->fd, &p->head, sizeof(p->head)); + r = read (p->index_fd, &p->head, sizeof(p->head)); switch (r) { case -1: - logf (LOG_FATAL|LOG_ERRNO, "read %s", p->fname); + logf (LOG_FATAL|LOG_ERRNO, "read %s", p->index_fname); exit (1); case 0: + p->head.index_free = 0; + p->head.index_last = 1; p->head.no_records = 0; - p->head.freelist = 0; + p->head.data_size = 0; + p->head.data_slack = 0; + p->head.data_used = 0; if (rw) rec_write_head (p); break; case sizeof(p->head): if (memcmp (p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic))) { - logf (LOG_FATAL, "read %s. bad header", p->fname); + logf (LOG_FATAL, "read %s. bad header", p->index_fname); exit (1); } break; default: logf (LOG_FATAL, "read head of %s. expected %d. got %d", - p->fname, sizeof(p->head), r); + p->index_fname, sizeof(p->head), r); + exit (1); + } + p->data_fd = open (p->data_fname, + rw ? (O_RDWR|O_CREAT) : O_RDONLY, 0666); + if (p->data_fd == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "open %s", p->data_fname); + exit (1); + } + p->cache_max = 100; + p->cache_cur = 0; + if (!(p->record_cache = malloc (sizeof(*p->record_cache)*p->cache_max))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); exit (1); } return p; } -void rec_close (Records p) +static void read_indx (Records p, int sysno, void *buf, int itemsize) +{ + int r; + off_t pos = (sysno-1)*itemsize + sizeof(p->head); + + if (lseek (p->index_fd, pos, SEEK_SET) == (pos) -1) + { + logf (LOG_FATAL|LOG_ERRNO, "seek in %s to pos %ld", + p->index_fname, (long) pos); + exit (1); + } + r = read (p->index_fd, buf, itemsize); + if (r != itemsize) + { + if (r == -1) + logf (LOG_FATAL|LOG_ERRNO, "read in %s at pos %ld", + p->index_fname, (long) pos); + else + logf (LOG_FATAL, "read in %s at pos %ld", + p->index_fname, (long) pos); + exit (1); + } +} + +static void rec_write_single (Records p, Record rec) +{ + struct record_index_entry entry; + int r, i, size = 0, got; + char *cptr; + off_t pos = (rec->sysno-1)*sizeof(entry) + sizeof(p->head); + + for (i = 0; i < REC_NO_INFO; i++) + if (!rec->info[i]) + size++; + else + size += strlen(rec->info[i])+1; + + entry.u.used.offset = p->head.data_size; + entry.u.used.size = size; + p->head.data_size += size; + p->head.data_used += size; + + if (lseek (p->index_fd, pos, SEEK_SET) == (pos) -1) + { + logf (LOG_FATAL|LOG_ERRNO, "seek in %s to pos %ld", + p->index_fname, (long) pos); + exit (1); + } + r = write (p->index_fd, &entry, sizeof(entry)); + if (r != sizeof(entry)) + { + if (r == -1) + logf (LOG_FATAL|LOG_ERRNO, "write of %s at pos %ld", + p->index_fname, (long) pos); + else + logf (LOG_FATAL, "write of %s at pos %ld", + p->index_fname, (long) pos); + exit (1); + } + if (lseek (p->data_fd, entry.u.used.offset, SEEK_SET) == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "lseek in %s to pos %ld", + p->data_fname, entry.u.used.offset); + exit (1); + } + if (p->tmp_size < entry.u.used.size) + { + free (p->tmp_buf); + p->tmp_size = entry.u.used.size + 16384; + if (!(p->tmp_buf = malloc (p->tmp_size))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + } + cptr = p->tmp_buf; + for (i = 0; i < REC_NO_INFO; i++) + if (!rec->info[i]) + *cptr++ = '\0'; + else + { + strcpy (cptr, rec->info[i]); + cptr += strlen(rec->info[i]) + 1; + } + for (got = 0; got < entry.u.used.size; got += r) + { + r = write (p->data_fd, p->tmp_buf + got, entry.u.used.size - got); + if (r <= 0) + { + logf (LOG_FATAL|LOG_ERRNO, "write of %s", p->data_fname); + exit (1); + } + got += r; + } +} + +static void rec_cache_flush (Records p) +{ + int i; + for (i = 0; icache_cur; i++) + { + struct record_cache_entry *e = p->record_cache + i; + if (e->dirty) + rec_write_single (p, e->rec); + rec_rm (e->rec); + } + p->cache_cur = 0; +} + +static Record *rec_cache_lookup (Records p, int sysno, int dirty) +{ + int i; + for (i = 0; icache_cur; i++) + { + struct record_cache_entry *e = p->record_cache + i; + if (e->rec->sysno == sysno) + { + if (dirty) + e->dirty = 1; + return &e->rec; + } + } + return NULL; +} + +static void rec_cache_insert (Records p, Record rec, int dirty) { - if (p->fd != -1) - close (p->fd); - free (p); + struct record_cache_entry *e; + + if (p->cache_cur == p->cache_max) + rec_cache_flush (p); + assert (p->cache_cur < p->cache_max); + + e = p->record_cache + (p->cache_cur)++; + e->dirty = 1; + e->rec = rec_cp (rec); +} + +void rec_close (Records *p) +{ + assert (*p); + + rec_cache_flush (*p); + free ((*p)->record_cache); + + if ((*p)->index_fd != -1) + close ((*p)->index_fd); + + if ((*p)->data_fd != -1) + close ((*p)->data_fd); + + free ((*p)->tmp_buf); + + free (*p); + *p = NULL; } Record rec_get (Records p, int sysno) { + int i; + Record rec, *recp; + struct record_index_entry entry; + int r, got; + char *nptr; + + assert (sysno > 0); assert (p); - return NULL; + + if ((recp = rec_cache_lookup (p, sysno, 0))) + return rec_cp (*recp); + + read_indx (p, sysno, &entry, sizeof(entry)); + + if (!(rec = malloc (sizeof(*rec)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + if (lseek (p->data_fd, entry.u.used.offset, SEEK_SET) == -1) + { + logf (LOG_FATAL|LOG_ERRNO, "lseek in %s to pos %ld", + p->data_fname, entry.u.used.offset); + exit (1); + } + if (p->tmp_size < entry.u.used.size) + { + free (p->tmp_buf); + p->tmp_size = entry.u.used.size + 16384; + if (!(p->tmp_buf = malloc (p->tmp_size))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + } + for (got = 0; got < entry.u.used.size; got += r) + { + r = read (p->data_fd, p->tmp_buf + got, entry.u.used.size - got); + if (r <= 0) + { + logf (LOG_FATAL|LOG_ERRNO, "read of %s", p->data_fname); + exit (1); + } + got += r; + } + rec->sysno = sysno; + + nptr = p->tmp_buf; + for (i = 0; i < REC_NO_INFO; i++) + if (*nptr) + { + rec->info[i] = rec_strdup (nptr); + nptr += strlen(nptr)+1; + } + else + { + nptr++; + rec->info[i] = NULL; + } + rec_cache_insert (p, rec, 0); + return rec; } Record rec_new (Records p) { + int sysno, i; + Record rec; + assert (p); - return NULL; + if (!(rec = malloc (sizeof(*rec)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + if (p->head.index_free == 0) + sysno = (p->head.index_last)++; + else + { + struct record_index_entry entry; + + read_indx (p, p->head.index_free, &entry, sizeof(entry)); + sysno = p->head.index_free; + p->head.index_free = entry.u.free.next; + } + (p->head.no_records)++; + rec->sysno = sysno; + for (i = 0; i < REC_NO_INFO; i++) + rec->info[i] = NULL; + rec_cache_insert (p, rec, 1); + return rec; } void rec_put (Records p, Record rec) { - assert (p); + Record *recp; + + if ((recp = rec_cache_lookup (p, rec->sysno, 1))) + { + rec_rm (*recp); + *recp = rec_cp (rec); + } + else + rec_cache_insert (p, rec, 1); +} + +void rec_rm (Record rec) +{ + int i; + for (i = 0; i < REC_NO_INFO; i++) + free (rec->info[i]); + free (rec); +} + +Record rec_cp (Record rec) +{ + Record n; + int i; + + if (!(n = malloc (sizeof(*n)))) + { + logf (LOG_FATAL|LOG_ERRNO, "malloc"); + exit (1); + } + n->sysno = rec->sysno; + for (i = 0; i < REC_NO_INFO; i++) + n->info[i] = rec_strdup (rec->info[i]); + return n; } diff --git a/index/recindex.h b/index/recindex.h index 6d0b23c..474d485 100644 --- a/index/recindex.h +++ b/index/recindex.h @@ -4,27 +4,30 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindex.h,v $ - * Revision 1.1 1995-11-15 14:46:21 adam + * Revision 1.2 1995-11-15 19:13:08 adam + * Work on record management. + * + * Revision 1.1 1995/11/15 14:46:21 adam * Started work on better record management system. * */ #include -typedef struct records_info { - int fd; - char *fname; - struct records_head { - char magic[8]; - int no_records; - int freelist; - } head; -} *Records; +#define REC_NO_INFO 4 typedef struct record_info { int sysno; - char *type; - char *fname; - char *kinfo; + char *info[REC_NO_INFO]; } *Record; +typedef struct records_info *Records; + +Record rec_cp (Record rec); +void rec_rm (Record rec); +void rec_put (Records p, Record rec); +Record rec_new (Records p); +Record rec_get (Records p, int sysno); +void rec_close (Records *p); +Records rec_open (int rw); +char *rec_strdup (const char *s); -- 1.7.10.4