+ if (!cache_block (b, pos, p->buf, 0))
+ {
+ yaz_log(b->log_io, "bf_read: open_block");
+ if (!bf_read(b->file[cat].bf, pos/CAT_MAX, 0, 0, p->buf))
+ {
+ yaz_log(YLOG_FATAL, "isamb: read fail for pos=%ld block=%ld",
+ (long) pos, (long) pos/CAT_MAX);
+ abort();
+ }
+ }
+ p->bytes = (char *)p->buf + offset;
+ p->leaf = p->buf[0];
+ p->size = (p->buf[1] + 256 * p->buf[2]) - offset;
+ if (p->size < 0)
+ {
+ yaz_log(YLOG_FATAL, "Bad block size %d in pos=" ZINT_FORMAT "\n",
+ p->size, pos);
+ }
+ assert (p->size >= 0);
+ src = (char*) p->buf + 3;
+ decode_ptr(&src, &p->no_items);
+
+ p->offset = 0;
+ p->dirty = 0;
+ p->deleted = 0;
+ p->decodeClientData = (*b->method->codec.start)();
+ return p;
+}
+
+struct ISAMB_block *new_block (ISAMB b, int leaf, int cat)
+{
+ struct ISAMB_block *p;
+
+ p = xmalloc(sizeof(*p));
+ p->buf = xmalloc(b->file[cat].head.block_size);
+
+ if (!b->file[cat].head.free_list)
+ {
+ zint block_no;
+ block_no = b->file[cat].head.last_block++;
+ p->pos = block_no * CAT_MAX + cat;
+ }
+ else
+ {
+ p->pos = b->file[cat].head.free_list;
+ assert((p->pos & CAT_MASK) == cat);
+ if (!cache_block (b, p->pos, p->buf, 0))
+ {
+ yaz_log(b->log_io, "bf_read: new_block");
+ if (!bf_read(b->file[cat].bf, p->pos/CAT_MAX, 0, 0, p->buf))
+ {
+ yaz_log(YLOG_FATAL, "isamb: read fail for pos=%ld block=%ld",
+ (long) p->pos/CAT_MAX, (long) p->pos/CAT_MAX);
+ abort ();
+ }
+ }
+ yaz_log(b->log_freelist, "got block " ZINT_FORMAT " from freelist %d:" ZINT_FORMAT, p->pos,
+ cat, p->pos/CAT_MAX);
+ memcpy (&b->file[cat].head.free_list, p->buf, sizeof(zint));
+ }
+ p->cat = cat;
+ b->file[cat].head_dirty = 1;
+ memset (p->buf, 0, b->file[cat].head.block_size);
+ p->bytes = (char*)p->buf + b->file[cat].head.block_offset;
+ p->leaf = leaf;
+ p->size = 0;
+ p->dirty = 1;
+ p->deleted = 0;
+ p->offset = 0;
+ p->no_items = 0;
+ p->decodeClientData = (*b->method->codec.start)();
+ return p;
+}
+
+struct ISAMB_block *new_leaf (ISAMB b, int cat)
+{
+ return new_block (b, 1, cat);
+}
+
+
+struct ISAMB_block *new_int (ISAMB b, int cat)
+{
+ return new_block (b, 0, cat);
+}
+
+static void check_block (ISAMB b, struct ISAMB_block *p)
+{
+ assert(b); /* mostly to make the compiler shut up about unused b */
+ if (p->leaf)
+ {
+ ;
+ }
+ else
+ {
+ /* sanity check */
+ char *startp = p->bytes;
+ const char *src = startp;
+ char *endp = p->bytes + p->size;
+ ISAM_P pos;
+ void *c1 = (*b->method->codec.start)();
+
+ decode_ptr(&src, &pos);
+ assert ((pos&CAT_MASK) == p->cat);
+ while (src != endp)
+ {
+#if INT_ENCODE
+ char file_item_buf[DST_ITEM_MAX];
+ char *file_item = file_item_buf;
+ (*b->method->codec.reset)(c1);
+ (*b->method->codec.decode)(c1, &file_item, &src);
+#else
+ zint item_len;
+ decode_item_len(&src, &item_len);
+ assert (item_len > 0 && item_len < 80);
+ src += item_len;
+#endif
+ decode_ptr(&src, &pos);
+ if ((pos&CAT_MASK) != p->cat)
+ {
+ assert ((pos&CAT_MASK) == p->cat);
+ }
+ }
+ (*b->method->codec.stop)(c1);
+ }
+}
+
+void close_block(ISAMB b, struct ISAMB_block *p)
+{
+ if (!p)
+ return;
+ if (p->deleted)
+ {
+ yaz_log(b->log_freelist, "release block " ZINT_FORMAT " from freelist %d:" ZINT_FORMAT,
+ p->pos, p->cat, p->pos/CAT_MAX);
+ memcpy (p->buf, &b->file[p->cat].head.free_list, sizeof(zint));
+ b->file[p->cat].head.free_list = p->pos;
+ if (!cache_block (b, p->pos, p->buf, 1))
+ {
+ yaz_log(b->log_io, "bf_write: close_block (deleted)");
+ bf_write(b->file[p->cat].bf, p->pos/CAT_MAX, 0, 0, p->buf);
+ }
+ }
+ else if (p->dirty)
+ {
+ int offset = b->file[p->cat].head.block_offset;
+ int size = p->size + offset;
+ char *dst = (char*)p->buf + 3;
+ assert (p->size >= 0);
+
+ /* memset becuase encode_ptr usually does not write all bytes */
+ memset(p->buf, 0, b->file[p->cat].head.block_offset);
+ p->buf[0] = p->leaf;
+ p->buf[1] = size & 255;
+ p->buf[2] = size >> 8;
+ encode_ptr(&dst, p->no_items);
+ check_block(b, p);
+ if (!cache_block (b, p->pos, p->buf, 1))
+ {
+ yaz_log(b->log_io, "bf_write: close_block");
+ bf_write(b->file[p->cat].bf, p->pos/CAT_MAX, 0, 0, p->buf);
+ }
+ }
+ (*b->method->codec.stop)(p->decodeClientData);
+ xfree(p->buf);
+ xfree(p);
+}
+
+int insert_sub (ISAMB b, struct ISAMB_block **p,
+ void *new_item, int *mode,
+ ISAMC_I *stream,
+ struct ISAMB_block **sp,
+ void *sub_item, int *sub_size,
+ const void *max_item);
+
+int insert_int (ISAMB b, struct ISAMB_block *p, void *lookahead_item,
+ int *mode,
+ ISAMC_I *stream, struct ISAMB_block **sp,
+ void *split_item, int *split_size, const void *last_max_item)
+{
+ char *startp = p->bytes;
+ const char *src = startp;
+ char *endp = p->bytes + p->size;
+ ISAM_P pos;
+ struct ISAMB_block *sub_p1 = 0, *sub_p2 = 0;
+ char sub_item[DST_ITEM_MAX];
+ int sub_size;
+ int more = 0;
+ zint diff_terms = 0;
+ void *c1 = (*b->method->codec.start)();
+
+ *sp = 0;
+
+ assert(p->size >= 0);
+ decode_ptr(&src, &pos);
+ while (src != endp)
+ {
+ int d;
+ const char *src0 = src;
+#if INT_ENCODE
+ char file_item_buf[DST_ITEM_MAX];
+ char *file_item = file_item_buf;
+ (*b->method->codec.reset)(c1);
+ (*b->method->codec.decode)(c1, &file_item, &src);
+ d = (*b->method->compare_item)(file_item_buf, lookahead_item);
+ if (d > 0)
+ {
+ sub_p1 = open_block(b, pos);
+ assert (sub_p1);
+ diff_terms -= sub_p1->no_items;
+ more = insert_sub (b, &sub_p1, lookahead_item, mode,
+ stream, &sub_p2,
+ sub_item, &sub_size, file_item_buf);
+ diff_terms += sub_p1->no_items;
+ src = src0;
+ break;
+ }
+#else
+ zint item_len;
+ decode_item_len(&src, &item_len);
+ d = (*b->method->compare_item)(src, lookahead_item);
+ if (d > 0)
+ {
+ sub_p1 = open_block(b, pos);
+ assert (sub_p1);
+ diff_terms -= sub_p1->no_items;
+ more = insert_sub (b, &sub_p1, lookahead_item, mode,
+ stream, &sub_p2,
+ sub_item, &sub_size, src);
+ diff_terms += sub_p1->no_items;
+ src = src0;
+ break;
+ }
+ src += item_len;
+#endif
+ decode_ptr(&src, &pos);
+ }
+ if (!sub_p1)
+ {
+ /* we reached the end. So lookahead > last item */
+ sub_p1 = open_block(b, pos);
+ assert (sub_p1);
+ diff_terms -= sub_p1->no_items;
+ more = insert_sub (b, &sub_p1, lookahead_item, mode, stream, &sub_p2,
+ sub_item, &sub_size, last_max_item);
+ diff_terms += sub_p1->no_items;
+ }
+ if (sub_p2)
+ diff_terms += sub_p2->no_items;
+ if (diff_terms)
+ {
+ p->dirty = 1;
+ p->no_items += diff_terms;
+ }
+ if (sub_p2)
+ {
+ /* there was a split - must insert pointer in this one */
+ char dst_buf[DST_BUF_SIZE];
+ char *dst = dst_buf;
+#if INT_ENCODE
+ const char *sub_item_ptr = sub_item;
+#endif
+ assert (sub_size < 80 && sub_size > 1);
+
+ memcpy (dst, startp, src - startp);
+
+ dst += src - startp;
+
+#if INT_ENCODE
+ (*b->method->codec.reset)(c1);
+ (*b->method->codec.encode)(c1, &dst, &sub_item_ptr);
+#else
+ encode_item_len (&dst, sub_size); /* sub length and item */
+ memcpy (dst, sub_item, sub_size);
+ dst += sub_size;
+#endif
+
+ encode_ptr(&dst, sub_p2->pos); /* pos */
+
+ if (endp - src) /* remaining data */
+ {
+ memcpy (dst, src, endp - src);
+ dst += endp - src;
+ }
+ p->size = dst - dst_buf;
+ assert (p->size >= 0);
+
+ if (p->size <= b->file[p->cat].head.block_max)
+ {
+ /* it fits OK in this block */
+ memcpy (startp, dst_buf, dst - dst_buf);
+
+ close_block(b, sub_p2);
+ }
+ else
+ {
+ /* must split _this_ block as well .. */
+ struct ISAMB_block *sub_p3;
+#if INT_ENCODE
+ char file_item_buf[DST_ITEM_MAX];
+ char *file_item = file_item_buf;
+#else
+ zint split_size_tmp;
+#endif
+ zint no_items_first_half = 0;
+ int p_new_size;
+ const char *half;
+ src = dst_buf;
+ endp = dst;
+
+ p->dirty = 1;
+ close_block(b, sub_p2);
+
+ half = src + b->file[p->cat].head.block_size/2;
+ decode_ptr(&src, &pos);
+
+ /* read sub block so we can get no_items for it */
+ sub_p3 = open_block(b, pos);
+ no_items_first_half += sub_p3->no_items;
+ close_block(b, sub_p3);
+
+ while (src <= half)
+ {
+#if INT_ENCODE
+ file_item = file_item_buf;
+ (*b->method->codec.reset)(c1);
+ (*b->method->codec.decode)(c1, &file_item, &src);
+#else
+ decode_item_len(&src, &split_size_tmp);
+ *split_size = (int) split_size_tmp;
+ src += *split_size;
+#endif
+ decode_ptr(&src, &pos);
+
+ /* read sub block so we can get no_items for it */
+ sub_p3 = open_block(b, pos);
+ no_items_first_half += sub_p3->no_items;
+ close_block(b, sub_p3);
+ }
+ /* p is first half */
+ p_new_size = src - dst_buf;
+ memcpy (p->bytes, dst_buf, p_new_size);
+
+#if INT_ENCODE
+ file_item = file_item_buf;
+ (*b->method->codec.reset)(c1);
+ (*b->method->codec.decode)(c1, &file_item, &src);
+ *split_size = file_item - file_item_buf;
+ memcpy(split_item, file_item_buf, *split_size);
+#else
+ decode_item_len(&src, &split_size_tmp);
+ *split_size = (int) split_size_tmp;
+ memcpy (split_item, src, *split_size);
+ src += *split_size;
+#endif
+ /* *sp is second half */
+ *sp = new_int (b, p->cat);
+ (*sp)->size = endp - src;
+ memcpy ((*sp)->bytes, src, (*sp)->size);
+
+ p->size = p_new_size;
+
+ /* adjust no_items in first&second half */
+ (*sp)->no_items = p->no_items - no_items_first_half;
+ p->no_items = no_items_first_half;
+ }
+ p->dirty = 1;
+ }
+ close_block(b, sub_p1);
+ (*b->method->codec.stop)(c1);
+ return more;
+}
+
+int insert_leaf (ISAMB b, struct ISAMB_block **sp1, void *lookahead_item,
+ int *lookahead_mode, ISAMC_I *stream,
+ struct ISAMB_block **sp2,
+ void *sub_item, int *sub_size,
+ const void *max_item)
+{
+ struct ISAMB_block *p = *sp1;
+ char *endp = 0;
+ const char *src = 0;
+ char dst_buf[DST_BUF_SIZE], *dst = dst_buf;
+ int new_size;
+ void *c1 = (*b->method->codec.start)();
+ void *c2 = (*b->method->codec.start)();
+ int more = 1;
+ int quater = b->file[b->no_cat-1].head.block_max / 4;
+ char *mid_cut = dst_buf + quater * 2;
+ char *tail_cut = dst_buf + quater * 3;
+ char *maxp = dst_buf + b->file[b->no_cat-1].head.block_max;
+ char *half1 = 0;
+ char *half2 = 0;
+ char cut_item_buf[DST_ITEM_MAX];
+ int cut_item_size = 0;
+ int no_items = 0; /* number of items (total) */
+ int no_items_1 = 0; /* number of items (first half) */
+ int inserted_dst_bytes = 0;
+
+ if (p && p->size)
+ {
+ char file_item_buf[DST_ITEM_MAX];
+ char *file_item = file_item_buf;
+
+ src = p->bytes;
+ endp = p->bytes + p->size;
+ (*b->method->codec.decode)(c1, &file_item, &src);
+ while (1)
+ {
+ const char *dst_item = 0; /* resulting item to be inserted */
+ char *lookahead_next;
+ char *dst_0 = dst;
+ int d = -1;
+
+ if (lookahead_item)
+ d = (*b->method->compare_item)(file_item_buf, lookahead_item);
+
+ /* d now holds comparison between existing file item and
+ lookahead item
+ d = 0: equal
+ d > 0: lookahead before file
+ d < 0: lookahead after file
+ */
+ if (d > 0)
+ {
+ /* lookahead must be inserted */
+ dst_item = lookahead_item;
+ /* if this is not an insertion, it's really bad .. */
+ if (!*lookahead_mode)
+ {
+ yaz_log(YLOG_WARN, "isamb: Inconsistent register (1)");
+ assert (*lookahead_mode);
+ }
+ }
+ else
+ dst_item = file_item_buf;
+
+ if (!*lookahead_mode && d == 0)
+ {
+ /* it's a deletion and they match so there is nothing to be
+ inserted anyway .. But mark the thing bad (file item
+ was part of input.. The item will not be part of output */
+ p->dirty = 1;
+ }
+ else if (!half1 && dst > mid_cut)
+ {
+ /* we have reached the splitting point for the first time */
+ const char *dst_item_0 = dst_item;
+ half1 = dst; /* candidate for splitting */
+
+ /* encode the resulting item */
+ (*b->method->codec.encode)(c2, &dst, &dst_item);
+
+ cut_item_size = dst_item - dst_item_0;
+ assert(cut_item_size > 0);
+ memcpy (cut_item_buf, dst_item_0, cut_item_size);
+
+ half2 = dst;
+ no_items_1 = no_items;
+ no_items++;
+ }
+ else
+ {
+ /* encode the resulting item */
+ (*b->method->codec.encode)(c2, &dst, &dst_item);
+ no_items++;