2 * Copyright (C) 1994, Index Data I/S
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.21 1996-03-29 14:11:47 quinn
10 * Revision 1.20 1996/03/19 13:14:57 quinn
13 * Revision 1.19 1996/02/10 12:20:56 quinn
14 * *** empty log message ***
16 * Revision 1.18 1996/02/06 10:19:56 quinn
17 * Attempt at fixing bug. Not all blocks were read before they were unlinked
18 * prior to a remap operation.
20 * Revision 1.17 1995/12/06 15:48:44 quinn
21 * Fixed update-problem.
23 * Revision 1.16 1995/12/06 14:48:26 quinn
24 * Fixed some strange bugs.
26 * Revision 1.15 1995/12/06 09:59:45 quinn
27 * Fixed memory-consumption bug in memory.c
28 * Added more blocksizes to the default ISAM configuration.
30 * Revision 1.14 1995/11/24 17:26:19 quinn
31 * Mostly about making some ISAM stuff in the config file optional.
33 * Revision 1.13 1995/10/17 18:03:15 adam
34 * Commented out qsort in is_merge.
36 * Revision 1.12 1995/09/06 16:11:41 adam
37 * Keysize parameter to is_open (if non-zero).
39 * Revision 1.11 1995/09/04 12:33:46 adam
40 * Various cleanup. YAZ util used instead.
42 * Revision 1.10 1994/09/28 16:58:32 quinn
45 * Revision 1.9 1994/09/28 12:56:15 quinn
46 * Added access functions (ISPT)
48 * Revision 1.8 1994/09/28 12:32:17 quinn
51 * Revision 1.7 1994/09/28 11:56:25 quinn
52 * Added sort of input to is_merge
54 * Revision 1.6 1994/09/28 11:29:33 quinn
55 * Added cmp parameter.
57 * Revision 1.5 1994/09/27 20:03:50 quinn
58 * Seems relatively bug-free.
60 * Revision 1.4 1994/09/26 17:11:29 quinn
63 * Revision 1.3 1994/09/26 17:06:35 quinn
66 * Revision 1.1 1994/09/12 08:02:13 quinn
84 static ispt_struct *ispt_freelist = 0;
88 int total_merge_operations;
90 int dub_items_removed;
94 int delete_insert_noop;
103 static ISPT ispt_alloc()
110 ispt_freelist = ispt_freelist->next;
113 p = xmalloc(sizeof(ispt_struct));
117 static void ispt_free(ISPT pt)
119 pt->next = ispt_freelist;
123 static int splitargs(const char *s, char *bf[], int max)
128 while (*s && isspace(*s))
136 logf (LOG_WARN, "Ignoring extra args to is resource");
140 while (*s && !isspace(*s))
149 ISAM is_open(const char *name, int (*cmp)(const void *p1, const void *p2),
150 int writeflag, int keysize)
153 char *nm, *r, *pp[IS_MAX_BLOCKTYPES+1], m[2];
154 int num, size, rs, tmp, i;
157 logf (LOG_DEBUG, "is_open(%s, %s)", name, writeflag ? "RW" : "RDONLY");
160 statistics.total_merge_operations = 0;
161 statistics.total_items = 0;
162 statistics.dub_items_removed = 0;
163 statistics.new_items = 0;
164 statistics.failed_deletes = 0;
165 statistics.skipped_inserts = 0;
166 statistics.delete_insert_noop = 0;
167 statistics.delete_replace = 0;
168 statistics.delete = 0;
169 statistics.remaps = 0;
170 statistics.new_tables = 0;
171 statistics.block_jumps = 0;
172 statistics.tab_deletes = 0;
175 new = xmalloc(sizeof(*new));
176 new->writeflag = writeflag;
177 for (i = 0; i < IS_MAX_BLOCKTYPES; i++)
178 new->types[i].index = 0; /* dummy */
180 /* determine number and size of blocktypes */
181 if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
182 "blocktypes", 0), "64 512 4K 32K")) ||
183 !(num = splitargs(r, pp, IS_MAX_BLOCKTYPES)))
185 logf (LOG_FATAL, "Failed to locate resource %s", nm);
188 new->num_types = num;
189 for (i = 0; i < num; i++)
191 if ((rs = sscanf(pp[i], "%d%1[bBkKmM]", &size, m)) < 1)
193 logf (LOG_FATAL, "Error in resource %s: %s", r, pp[i]);
201 new->types[i].blocksize = size; break;
203 new->types[i].blocksize = size * 1024; break;
205 new->types[i].blocksize = size * 1048576; break;
207 logf (LOG_FATAL, "Illegal size suffix: %c", *m);
210 new->types[i].dbuf = xmalloc(new->types[i].blocksize);
213 if (!(new->types[i].bf = bf_open(strconcat(name, m, 0),
214 new->types[i].blocksize, writeflag)))
216 logf (LOG_FATAL, "bf_open failed");
219 if ((rs = is_rb_read(&new->types[i], &th)) > 0)
221 if (th.blocksize != new->types[i].blocksize)
223 logf (LOG_FATAL, "File blocksize mismatch in %s", name);
226 new->types[i].freelist = th.freelist;
227 new->types[i].top = th.top;
229 else if (writeflag) /* write dummy superblock to determine top */
231 if ((rs = is_rb_write(&new->types[i], &th)) <=0) /* dummy */
233 logf (LOG_FATAL, "Failed to write initial superblock.");
236 new->types[i].freelist = -1;
237 new->types[i].top = rs;
239 /* ELSE: this is an empty file opened in read-only mode. */
242 new->keysize = keysize;
245 if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
249 logf (LOG_FATAL, "Failed to locate resource %s", nm);
252 if ((new->keysize = atoi(r)) <= 0)
254 logf (LOG_FATAL, "Must specify positive keysize.");
259 /* determine repack percent */
260 if (!(r = res_get_def(common_resource, nm = strconcat(name, ".", "repack",
261 0), IS_DEF_REPACK_PERCENT)))
263 logf (LOG_FATAL, "Failed to locate resource %s", nm);
266 new->repack = atoi(r);
268 /* determine max keys/blocksize */
269 if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
270 "maxkeys", 0), "50 640 10000")) || !(num = splitargs(r, pp,
273 logf (LOG_FATAL, "Failed to locate resource %s", nm);
276 if (num < new->num_types -1)
278 logf (LOG_FATAL, "Not enough elements in %s", nm);
281 for (i = 0; i < num; i++)
283 if ((rs = sscanf(pp[i], "%d", &tmp)) < 1)
285 logf (LOG_FATAL, "Error in resource %s: %s", r, pp[i]);
288 new->types[i].max_keys = tmp;
291 /* determine max keys/block */
292 for (i = 0; i < new->num_types; i++)
294 if (!new->types[i].index)
296 new->types[i].max_keys_block = (new->types[i].blocksize - 2 *
297 sizeof(int)) / new->keysize;
298 new->types[i].max_keys_block0 = (new->types[i].blocksize - 3 *
299 sizeof(int)) / new->keysize;
302 new->types[i].max_keys_block = new->types[i].max_keys_block0 /
304 if (new->types[i].max_keys_block0 < 1)
306 logf (LOG_FATAL, "Blocksize too small in %s", name);
311 /* determine nice fill rates */
312 if (!(r = res_get_def(common_resource, nm = strconcat(name, ".",
313 "nicefill", 0), "90 90 90 95")) || !(num = splitargs(r, pp,
316 logf (LOG_FATAL, "Failed to locate resource %s", nm);
319 if (num < new->num_types)
321 logf (LOG_FATAL, "Not enough elements in %s", nm);
324 for (i = 0; i < num; i++)
326 if ((rs = sscanf(pp[i], "%d", &tmp)) < 1)
328 logf (LOG_FATAL, "Error in resource %s: %s", r, pp[i]);
331 new->types[i].nice_keys_block = (new->types[i].max_keys_block0 * tmp) /
333 if (new->types[i].nice_keys_block < 1)
334 new->types[i].nice_keys_block = 1;
337 new->cmp = cmp ? cmp : is_default_cmp;
344 int is_close(ISAM is)
349 logf (LOG_DEBUG, "is_close()");
350 for (i = 0; i < is->num_types; i++)
356 th.blocksize = is->types[i].blocksize;
357 th.keysize = is->keysize;
358 th.freelist = is->types[i].freelist;
359 th.top = is->types[i].top;
360 if (is_rb_write(&is->types[i], &th) < 0)
362 logf (LOG_FATAL, "Failed to write headerblock");
366 bf_close(is->types[i].bf);
371 logf(LOG_LOG, "ISAM statistics:");
372 logf(LOG_LOG, "total_merge_operations %d",
373 statistics.total_merge_operations);
374 logf(LOG_LOG, "total_items %d", statistics.total_items);
375 logf(LOG_LOG, "dub_items_removed %d",
376 statistics.dub_items_removed);
377 logf(LOG_LOG, "new_items %d", statistics.new_items);
378 logf(LOG_LOG, "failed_deletes %d",
379 statistics.failed_deletes);
380 logf(LOG_LOG, "skipped_inserts %d",
381 statistics.skipped_inserts);
382 logf(LOG_LOG, "delete_insert_noop %d",
383 statistics.delete_insert_noop);
384 logf(LOG_LOG, "delete_replace %d",
385 statistics.delete_replace);
386 logf(LOG_LOG, "delete %d", statistics.delete);
387 logf(LOG_LOG, "remaps %d", statistics.remaps);
388 logf(LOG_LOG, "block_jumps %d", statistics.block_jumps);
389 logf(LOG_LOG, "tab_deletes %d", statistics.tab_deletes);
395 static ISAM_P is_address(int type, int pos)
404 ISAM_P is_merge(ISAM is, ISAM_P pos, int num, char *data)
408 char keybuf[IS_MAX_RECORD];
409 int oldnum, oldtype, i;
410 char operation, *record;
412 statistics.total_merge_operations++;
413 statistics.total_items += num;
415 statistics.new_tables++;
417 is_m_establish_tab(is, &tab, pos);
419 if (is_m_read_full(&tab, tab.data) < 0)
421 logf (LOG_FATAL, "read_full failed");
424 oldnum = tab.num_records;
425 oldtype = tab.pos_type;
428 operation = *(data)++;
429 record = (char*) data;
430 data += is_keysize(is);
432 while (num && !memcmp(record - 1, data, is_keysize(tab.is) + 1))
434 data += 1 + is_keysize(is);
436 statistics.dub_items_removed++;
438 if ((res = is_m_seek_record(&tab, record)) > 0) /* no match */
440 if (operation == KEYOP_INSERT)
442 logf (LOG_DEBUG, "XXInserting new record.");
443 is_m_write_record(&tab, record);
444 statistics.new_items++;
448 logf (LOG_DEBUG, "XXDeletion failed to find match.");
449 statistics.failed_deletes++;
452 else /* match found */
454 if (operation == KEYOP_INSERT)
456 logf (LOG_DEBUG, "XXSkipping insertion - match found.");
457 statistics.skipped_inserts++;
460 else if (operation == KEYOP_DELETE)
462 /* try to avoid needlessly moving data */
463 if (num && *(data) == KEYOP_INSERT)
465 /* next key is identical insert? - NOOP - skip it */
466 if (!memcmp(record, data + 1, is_keysize(is)))
468 logf (LOG_DEBUG, "XXNoop delete. skipping.");
469 data += 1 + is_keysize(is);
471 while (num && !memcmp(data, data + is_keysize(tab.is) +
472 1, is_keysize(tab.is) + 1))
474 data += 1 + is_keysize(is);
476 statistics.dub_items_removed++;
478 statistics.delete_insert_noop++;
481 /* else check if next key can fit in this position */
482 if (is_m_peek_record(&tab, keybuf) &&
483 (*is->cmp)(data + 1, keybuf) < 0)
485 logf (LOG_DEBUG, "XXReplacing record.");
486 is_m_replace_record(&tab, data + 1);
487 data += 1 + is_keysize(is);
489 while (num && !memcmp(data, data + is_keysize(tab.is) +
490 1, is_keysize(tab.is) + 1))
492 data += 1 + is_keysize(is);
494 statistics.dub_items_removed++;
496 statistics.delete_replace++;
500 logf (LOG_DEBUG, "Deleting record.");
501 is_m_delete_record(&tab);
507 while (i < tab.is->num_types - 1 && tab.num_records >
508 tab.is->types[i].max_keys)
510 if (i != tab.pos_type)
512 /* read remaining blocks */
513 for (; tab.cur_mblock; tab.cur_mblock = tab.cur_mblock->next)
514 if (tab.cur_mblock->state < IS_MBSTATE_CLEAN)
515 is_m_read_full(&tab, tab.cur_mblock);
519 statistics.block_jumps++;
521 if (!oldnum || tab.pos_type != oldtype || (abs(oldnum - tab.num_records) *
522 100) / oldnum > tab.is->repack)
532 pos = is_address(tab.pos_type, tab.data->diskpos);
537 statistics.tab_deletes++;
539 is_m_release_tab(&tab);
544 * Locate a table of keys in an isam file. The ISPT is an individual
545 * position marker for that table.
547 ISPT is_position(ISAM is, ISAM_P pos)
552 is_m_establish_tab(is, &p->tab, pos);
559 void is_pt_free(ISPT ip)
561 is_m_release_tab(&ip->tab);
566 * Read a key from a table.
568 int is_readkey(ISPT ip, void *buf)
570 return is_m_read_record(&ip->tab, buf, 0);
573 int is_numkeys(ISPT ip)
575 return is_m_num_records(&ip->tab);
578 void is_rewind(ISPT ip)
580 is_m_rewind(&ip->tab);