1 /* $Id: cfile.c,v 1.38 2006-11-08 22:08:27 adam Exp $
2 Copyright (C) 1995-2006
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
27 #include <idzebra/util.h>
28 #include <yaz/yaz-util.h>
32 static int write_head(CFile cf)
34 int left = cf->head.hash_size * sizeof(zint);
37 const char *tab = (char*) cf->array;
41 while (left >= (int) HASH_BSIZE)
43 r = mf_write(cf->hash_mf, bno++, 0, 0, tab);
50 r = mf_write(cf->hash_mf, bno, 0, left, tab);
54 static int read_head(CFile cf)
56 int left = cf->head.hash_size * sizeof(zint);
58 char *tab = (char*) cf->array;
62 while (left >= (int) HASH_BSIZE)
64 if (mf_read(cf->hash_mf, bno++, 0, 0, tab) == -1)
71 if (mf_read(cf->hash_mf, bno, 0, left, tab) == -1)
78 CFile cf_open(MFile mf, MFile_area area, const char *fname,
79 int block_size, int wflag, int *firstp)
83 CFile cf = (CFile) xmalloc(sizeof(*cf));
86 yaz_log(YLOG_DEBUG, "cf: open %s %s", cf->rmf->name,
87 wflag ? "rdwr" : "rd");
95 cf->bucket_lru_front = cf->bucket_lru_back = NULL;
96 cf->bucket_in_memory = 0;
97 cf->max_bucket_in_memory = 100;
99 cf->iobuf = (char *) xmalloc(block_size);
100 memset(cf->iobuf, 0, block_size);
108 zebra_mutex_init(&cf->mutex);
110 sprintf(path, "%s-b", fname);
111 if (!(cf->block_mf = mf_open(area, path, block_size, wflag)))
113 yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
117 sprintf(path, "%s-i", fname);
118 if (!(cf->hash_mf = mf_open(area, path, HASH_BSIZE, wflag)))
120 yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
124 ret = mf_read(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
131 if (ret == 0 || !cf->head.state)
135 cf->head.block_size = block_size;
136 cf->head.hash_size = 199;
137 hash_bytes = cf->head.hash_size * sizeof(zint);
138 cf->head.flat_bucket = cf->head.next_bucket = cf->head.first_bucket =
139 (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2;
140 cf->head.next_block = 1;
141 cf->array = (zint *) xmalloc(hash_bytes);
142 for (i = 0; i<cf->head.hash_size; i++)
146 if (mf_write(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head))
161 assert(cf->head.block_size == block_size);
162 assert(cf->head.hash_size > 2);
163 hash_bytes = cf->head.hash_size * sizeof(zint);
164 assert(cf->head.next_bucket > 0);
165 assert(cf->head.next_block > 0);
166 if (cf->head.state == 1)
167 cf->array = (zint *) xmalloc(hash_bytes);
170 if (read_head(cf) == -1)
176 if (cf->head.state == 1)
178 cf->parray = (struct CFile_hash_bucket **)
179 xmalloc(cf->head.hash_size * sizeof(*cf->parray));
180 for (i = 0; i<cf->head.hash_size; i++)
181 cf->parray[i] = NULL;
186 static int cf_hash(CFile cf, zint no)
188 return (int) (((no >> 3) % cf->head.hash_size));
191 static void release_bucket(CFile cf, struct CFile_hash_bucket *p)
194 p->lru_prev->lru_next = p->lru_next;
196 cf->bucket_lru_back = p->lru_next;
198 p->lru_next->lru_prev = p->lru_prev;
200 cf->bucket_lru_front = p->lru_prev;
202 *p->h_prev = p->h_next;
204 p->h_next->h_prev = p->h_prev;
206 --(cf->bucket_in_memory);
210 static void flush_bucket(CFile cf, int no_to_flush)
213 struct CFile_hash_bucket *p;
215 for (i = 0; i != no_to_flush; i++)
217 p = cf->bucket_lru_back;
222 mf_write(cf->hash_mf, p->ph.this_bucket, 0, 0, &p->ph);
225 release_bucket(cf, p);
229 static struct CFile_hash_bucket *alloc_bucket(CFile cf, zint block_no, int hno)
231 struct CFile_hash_bucket *p, **pp;
233 if (cf->bucket_in_memory == cf->max_bucket_in_memory)
235 assert(cf->bucket_in_memory < cf->max_bucket_in_memory);
236 ++(cf->bucket_in_memory);
237 p = (struct CFile_hash_bucket *) xmalloc(sizeof(*p));
240 p->lru_prev = cf->bucket_lru_front;
241 if (cf->bucket_lru_front)
242 cf->bucket_lru_front->lru_next = p;
244 cf->bucket_lru_back = p;
245 cf->bucket_lru_front = p;
247 pp = cf->parray + hno;
251 (*pp)->h_prev = &p->h_next;
256 static struct CFile_hash_bucket *get_bucket(CFile cf, zint block_no, int hno)
258 struct CFile_hash_bucket *p;
260 p = alloc_bucket(cf, block_no, hno);
262 if (!mf_read(cf->hash_mf, block_no, 0, 0, &p->ph))
264 yaz_log(YLOG_FATAL|YLOG_ERRNO, "read get_bucket");
265 release_bucket(cf, p);
268 assert(p->ph.this_bucket == block_no);
272 static struct CFile_hash_bucket *new_bucket(CFile cf, zint *block_nop, int hno)
274 struct CFile_hash_bucket *p;
278 block_no = *block_nop = cf->head.next_bucket++;
279 p = alloc_bucket(cf, block_no, hno);
282 for (i = 0; i<HASH_BUCKET; i++)
287 p->ph.next_bucket = 0;
288 p->ph.this_bucket = block_no;
292 static int cf_lookup_flat(CFile cf, zint no, zint *vno)
294 zint hno = (no*sizeof(zint))/HASH_BSIZE;
295 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
298 mf_read(cf->hash_mf, hno+cf->head.next_bucket, off, sizeof(zint), vno);
304 static int cf_lookup_hash(CFile cf, zint no, zint *vno)
306 int hno = cf_hash(cf, no);
307 struct CFile_hash_bucket *hb;
311 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
313 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
314 if (hb->ph.no[i] == no)
317 *vno = hb->ph.vno[i];
321 for (block_no = cf->array[hno]; block_no; block_no = hb->ph.next_bucket)
323 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
325 if (hb->ph.this_bucket == block_no)
331 /* extra check ... */
332 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
334 if (hb->ph.this_bucket == block_no)
336 yaz_log(YLOG_FATAL, "Found hash bucket on other chain(1)");
339 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
340 if (hb->ph.no[i] == no)
342 yaz_log(YLOG_FATAL, "Found hash bucket on other chain (2)");
348 hb = get_bucket(cf, block_no, hno);
351 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
352 if (hb->ph.no[i] == no)
354 *vno = hb->ph.vno[i];
361 static int cf_write_flat(CFile cf, zint no, zint vno)
363 zint hno = (no*sizeof(zint))/HASH_BSIZE;
364 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
366 hno += cf->head.next_bucket;
367 if (hno >= cf->head.flat_bucket)
368 cf->head.flat_bucket = hno+1;
370 return mf_write(cf->hash_mf, hno, off, sizeof(zint), &vno);
373 static int cf_moveto_flat(CFile cf)
375 struct CFile_hash_bucket *p;
379 yaz_log(YLOG_LOG, "cf: Moving to flat shadow: %s", cf->rmf->name);
380 yaz_log(YLOG_DEBUG, "cf: Moving to flat shadow: %s", cf->rmf->name);
381 yaz_log(YLOG_DEBUG, "cf: hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT " total="
383 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
384 cf->head.next_bucket - cf->head.first_bucket);
385 assert(cf->head.state == 1);
386 flush_bucket(cf, -1);
387 assert(cf->bucket_in_memory == 0);
388 p = (struct CFile_hash_bucket *) xmalloc(sizeof(*p));
389 for (i = cf->head.first_bucket; i < cf->head.next_bucket; i++)
391 if (mf_read(cf->hash_mf, i, 0, 0, &p->ph) != 1)
393 yaz_log(YLOG_FATAL|YLOG_ERRNO, "read bucket moveto flat");
397 for (j = 0; j < HASH_BUCKET && p->ph.vno[j]; j++)
399 if (cf_write_flat(cf, p->ph.no[j], p->ph.vno[j]))
416 static int cf_lookup(CFile cf, zint no, zint *vno)
418 if (cf->head.state > 1)
419 return cf_lookup_flat(cf, no, vno);
420 return cf_lookup_hash(cf, no, vno);
423 static zint cf_new_flat(CFile cf, zint no)
425 zint vno = (cf->head.next_block)++;
427 cf_write_flat(cf, no, vno);
431 static zint cf_new_hash(CFile cf, zint no)
433 int hno = cf_hash(cf, no);
434 struct CFile_hash_bucket *hbprev = NULL, *hb = cf->parray[hno];
435 zint *bucketpp = &cf->array[hno];
437 zint vno = (cf->head.next_block)++;
439 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
440 if (!hb->ph.vno[HASH_BUCKET-1])
441 for (i = 0; i<HASH_BUCKET; i++)
453 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
454 if (hb->ph.this_bucket == *bucketpp)
456 bucketpp = &hb->ph.next_bucket;
464 /* extra check ... */
465 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
467 if (hb->ph.this_bucket == *bucketpp)
469 yaz_log(YLOG_FATAL, "Found hash bucket on other chain");
475 hb = get_bucket(cf, *bucketpp, hno);
478 for (i = 0; i<HASH_BUCKET; i++)
486 bucketpp = &hb->ph.next_bucket;
491 hb = new_bucket(cf, bucketpp, hno);
497 zint cf_new(CFile cf, zint no)
499 if (cf->head.state > 1)
500 return cf_new_flat(cf, no);
501 if (cf->no_miss*2 > cf->no_hits)
503 if (cf_moveto_flat(cf))
505 assert(cf->head.state > 1);
506 return cf_new_flat(cf, no);
508 return cf_new_hash(cf, no);
512 /** \brief reads block from commit area
513 \param cf commit file
514 \param no block number
515 \param offset offset in block
516 \param nbytes number of bytes to read
517 \param buf buffer for content (if read was succesful)
518 \retval 0 block could not be fully read
519 \retval 1 block could be read
522 int cf_read(CFile cf, zint no, int offset, int nbytes, void *buf)
528 zebra_mutex_lock(&cf->mutex);
529 ret = cf_lookup(cf, no, &block);
530 zebra_mutex_unlock(&cf->mutex);
533 /* block could not be read or error */
536 if (mf_read(cf->block_mf, block, offset, nbytes, buf) != 1)
538 yaz_log(YLOG_FATAL|YLOG_ERRNO, "cf_read no=" ZINT_FORMAT " block=" ZINT_FORMAT, no, block);
544 /** \brief writes block to commit area
545 \param cf commit file
546 \param no block number
547 \param offset offset in block
548 \param nbytes number of bytes to be written
549 \param buf buffer to be written
550 \retval 0 block written
553 int cf_write(CFile cf, zint no, int offset, int nbytes, const void *buf)
559 zebra_mutex_lock(&cf->mutex);
561 ret = cf_lookup(cf, no, &block);
565 zebra_mutex_unlock(&cf->mutex);
570 block = cf_new(cf, no);
573 zebra_mutex_unlock(&cf->mutex);
576 if (offset || nbytes)
578 mf_read(cf->rmf, no, 0, 0, cf->iobuf);
579 memcpy(cf->iobuf + offset, buf, nbytes);
585 zebra_mutex_unlock(&cf->mutex);
586 return mf_write(cf->block_mf, block, offset, nbytes, buf);
589 int cf_close(CFile cf)
591 yaz_log(YLOG_DEBUG, "cf: close hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT
592 " total=" ZINT_FORMAT,
593 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
594 cf->head.next_bucket - cf->head.first_bucket);
595 flush_bucket(cf, -1);
600 mf_write(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
603 mf_close(cf->hash_mf);
606 mf_close(cf->block_mf);
610 zebra_mutex_destroy(&cf->mutex);
618 * indent-tabs-mode: nil
620 * vim: shiftwidth=4 tabstop=8 expandtab