1c70d4f7cd10c6ff37d7008759116802b4564428
[idzebra-moved-to-github.git] / trunc.c
1 /* $Id: trunc.c,v 1.63 2005-08-25 10:37:05 adam Exp $
2    Copyright (C) 1995-2005
3    Index Data ApS
4
5 This file is part of the Zebra server.
6
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
10 version.
11
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
20 02111-1307, USA.
21 */
22
23
24 #include <stdio.h>
25 #include <assert.h>
26
27 #include "index.h"
28 #include <rset.h>
29
30 struct trunc_info {
31     int  *ptr;
32     int  *indx;
33     char **heap;
34     int  heapnum;
35     int  (*cmp)(const void *p1, const void *p2);
36     int  keysize;
37     char *swapbuf;
38     char *tmpbuf;
39     char *buf;
40 };
41
42 static void heap_swap(struct trunc_info *ti, int i1, int i2)
43 {
44     int swap;
45
46     swap = ti->ptr[i1];
47     ti->ptr[i1] = ti->ptr[i2];
48     ti->ptr[i2] = swap;
49 }
50
51 static void heap_delete(struct trunc_info *ti)
52 {
53     int cur = 1, child = 2;
54
55     heap_swap(ti, 1, ti->heapnum--);
56     while (child <= ti->heapnum) {
57         if (child < ti->heapnum &&
58             (*ti->cmp)(ti->heap[ti->ptr[child]],
59                        ti->heap[ti->ptr[1+child]]) > 0)
60             child++;
61         if ((*ti->cmp)(ti->heap[ti->ptr[cur]],
62                        ti->heap[ti->ptr[child]]) > 0)
63         {
64             heap_swap(ti, cur, child);
65             cur = child;
66             child = 2*cur;
67         }
68         else
69             break;
70     }
71 }
72
73 static void heap_insert(struct trunc_info *ti, const char *buf, int indx)
74 {
75     int cur, parent;
76
77     cur = ++(ti->heapnum);
78     memcpy(ti->heap[ti->ptr[cur]], buf, ti->keysize);
79     ti->indx[ti->ptr[cur]] = indx;
80     parent = cur/2;
81     while (parent && (*ti->cmp)(ti->heap[ti->ptr[parent]],
82                                 ti->heap[ti->ptr[cur]]) > 0)
83     {
84         heap_swap(ti, cur, parent);
85         cur = parent;
86         parent = cur/2;
87     }
88 }
89
90 static struct trunc_info *heap_init(int size, int key_size,
91                                     int (*cmp)(const void *p1,
92                                                const void *p2))
93 {
94     struct trunc_info *ti = (struct trunc_info *) xmalloc(sizeof(*ti));
95     int i;
96
97     ++size;
98     ti->heapnum = 0;
99     ti->keysize = key_size;
100     ti->cmp = cmp;
101     ti->indx = (int *) xmalloc(size * sizeof(*ti->indx));
102     ti->heap = (char **) xmalloc(size * sizeof(*ti->heap));
103     ti->ptr = (int *) xmalloc(size * sizeof(*ti->ptr));
104     ti->swapbuf = (char *) xmalloc(ti->keysize);
105     ti->tmpbuf = (char *) xmalloc(ti->keysize);
106     ti->buf = (char *) xmalloc(size * ti->keysize);
107     for (i = size; --i >= 0; )
108     {
109         ti->ptr[i] = i;
110         ti->heap[i] = ti->buf + ti->keysize * i;
111     }
112     return ti;
113 }
114
115 static void heap_close(struct trunc_info *ti)
116 {
117     xfree(ti->ptr);
118     xfree(ti->indx);
119     xfree(ti->heap);
120     xfree(ti->swapbuf);
121     xfree(ti->tmpbuf);
122     xfree(ti->buf);
123     xfree(ti);
124 }
125
126 static RSET rset_trunc_r(ZebraHandle zi, const char *term, int length,
127                          const char *flags, ISAM_P *isam_p, int from, int to,
128                          int merge_chunk, int preserve_position,
129                          int term_type, NMEM rset_nmem,
130                          struct rset_key_control *kctrl, int scope,
131                          TERMID termid)
132 {
133     RSET result;
134     RSFD result_rsfd;
135     int nn = 0;
136
137     result = rstemp_create(rset_nmem, kctrl, scope,
138                            res_get(zi->res, "setTmpDir"), termid);
139     result_rsfd = rset_open(result, RSETF_WRITE);
140
141     if (to - from > merge_chunk)
142     {
143         RSFD *rsfd;
144         RSET *rset;
145         int i, i_add = (to-from)/merge_chunk + 1;
146         struct trunc_info *ti;
147         int rscur = 0;
148         int rsmax = (to-from)/i_add + 1;
149         NMEM rset_nmem_sub = nmem_create(); /* all sub rsets not needed
150                                                after this */
151         
152         rset = (RSET *) xmalloc(sizeof(*rset) * rsmax);
153         rsfd = (RSFD *) xmalloc(sizeof(*rsfd) * rsmax);
154         
155         for (i = from; i < to; i += i_add)
156         {
157             if (i_add <= to - i)
158                 rset[rscur] = rset_trunc_r(zi, term, length, flags,
159                                            isam_p, i, i+i_add,
160                                            merge_chunk, preserve_position,
161                                            term_type, rset_nmem_sub, 
162                                            kctrl, scope, 0);
163             else
164                 rset[rscur] = rset_trunc_r(zi, term, length, flags,
165                                            isam_p, i, to,
166                                            merge_chunk, preserve_position,
167                                            term_type, rset_nmem_sub, 
168                                            kctrl, scope, 0);
169             rscur++;
170         }
171         ti = heap_init (rscur, sizeof(struct it_key), key_compare_it);
172         for (i = rscur; --i >= 0; )
173         {
174             rsfd[i] = rset_open(rset[i], RSETF_READ);
175             if (rset_read(rsfd[i], ti->tmpbuf, 0))
176                 heap_insert(ti, ti->tmpbuf, i);
177             else
178             {
179                 rset_close(rsfd[i]);
180                 rset_delete(rset[i]);
181             }
182         }
183         while (ti->heapnum)
184         {
185             int n = ti->indx[ti->ptr[1]];
186
187             rset_write(result_rsfd, ti->heap[ti->ptr[1]]);
188             nn++;
189
190             while (1)
191             {
192                 if(!rset_read (rsfd[n], ti->tmpbuf,0))
193                 {
194                     heap_delete(ti);
195                     rset_close(rsfd[n]);
196                     rset_delete(rset[n]);
197                     break;
198                 }
199                 if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1)
200                 {
201                     heap_delete(ti);
202                     heap_insert(ti, ti->tmpbuf, n);
203                     break;
204                 }
205             }
206         }
207         xfree(rset);
208         xfree(rsfd);
209         heap_close(ti);
210         nmem_destroy(rset_nmem_sub);
211     }
212     else if (zi->reg->isamc)
213     {
214         ISAMC_PP *ispt;
215         int i;
216         struct trunc_info *ti;
217
218         ispt = (ISAMC_PP *) xmalloc(sizeof(*ispt) * (to-from));
219
220         ti = heap_init(to-from, sizeof(struct it_key),
221                        key_compare_it);
222         for (i = to-from; --i >= 0; )
223         {
224             ispt[i] = isamc_pp_open(zi->reg->isamc, isam_p[from+i]);
225             if (isamc_pp_read(ispt[i], ti->tmpbuf))
226                 heap_insert(ti, ti->tmpbuf, i);
227             else
228                 isamc_pp_close(ispt[i]);
229         }
230         while (ti->heapnum)
231         {
232             int n = ti->indx[ti->ptr[1]];
233
234             rset_write(result_rsfd, ti->heap[ti->ptr[1]]);
235             nn++;
236             if (preserve_position)
237             {
238                 heap_delete(ti);
239                 if (isamc_pp_read(ispt[n], ti->tmpbuf))
240                     heap_insert(ti, ti->tmpbuf, n);
241                 else
242                     isamc_pp_close(ispt[n]);
243             }
244             else
245             {
246                 while (1)
247                 {
248                     if (!isamc_pp_read(ispt[n], ti->tmpbuf))
249                     {
250                         heap_delete(ti);
251                         isamc_pp_close(ispt[n]);
252                         break;
253                     }
254                     if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1)
255                     {
256                         heap_delete(ti);
257                         heap_insert(ti, ti->tmpbuf, n);
258                         break;
259                     }
260                 }
261             }
262         }
263         heap_close(ti);
264         xfree(ispt);
265     }
266     else if (zi->reg->isams)
267     {
268         ISAMS_PP *ispt;
269         int i;
270         struct trunc_info *ti;
271         int nn = 0;
272
273         ispt = (ISAMS_PP *) xmalloc(sizeof(*ispt) * (to-from));
274
275         ti = heap_init(to-from, sizeof(struct it_key),
276                        key_compare_it);
277         for (i = to-from; --i >= 0; )
278         {
279             ispt[i] = isams_pp_open(zi->reg->isams, isam_p[from+i]);
280             if (isams_pp_read(ispt[i], ti->tmpbuf))
281                 heap_insert(ti, ti->tmpbuf, i);
282             else
283                 isams_pp_close(ispt[i]);
284         }
285         while (ti->heapnum)
286         {
287             int n = ti->indx[ti->ptr[1]];
288
289             rset_write(result_rsfd, ti->heap[ti->ptr[1]]);
290             nn++;
291             while (1)
292             {
293                 if (!isams_pp_read(ispt[n], ti->tmpbuf))
294                 {
295                     heap_delete(ti);
296                     isams_pp_close(ispt[n]);
297                     break;
298                 }
299                 if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1)
300                 {
301                     heap_delete(ti);
302                     heap_insert(ti, ti->tmpbuf, n);
303                     break;
304                 }
305             }
306         }
307         heap_close(ti);
308         xfree(ispt);
309     }
310     else if (zi->reg->isamb)
311     {
312         ISAMB_PP *ispt;
313         int i;
314         struct trunc_info *ti;
315
316         ispt = (ISAMB_PP *) xmalloc(sizeof(*ispt) * (to-from));
317
318         ti = heap_init(to-from, sizeof(struct it_key),
319                        key_compare_it);
320         for (i = to-from; --i >= 0; )
321         {
322             if (isam_p[from+i]) {
323                 ispt[i] = isamb_pp_open(zi->reg->isamb, isam_p[from+i], scope);
324                 if (isamb_pp_read(ispt[i], ti->tmpbuf))
325                     heap_insert(ti, ti->tmpbuf, i);
326                 else
327                     isamb_pp_close(ispt[i]);
328             }
329         }
330         while (ti->heapnum)
331         {
332             int n = ti->indx[ti->ptr[1]];
333
334             rset_write(result_rsfd, ti->heap[ti->ptr[1]]);
335             nn++;
336
337             if (preserve_position)
338             {
339                 heap_delete(ti);
340                 if (isamb_pp_read(ispt[n], ti->tmpbuf))
341                     heap_insert(ti, ti->tmpbuf, n);
342                 else
343                     isamb_pp_close(ispt[n]);
344             }
345             else
346             {
347                 while (1)
348                 {
349                     if (!isamb_pp_read(ispt[n], ti->tmpbuf))
350                     {
351                         heap_delete(ti);
352                         isamb_pp_close(ispt[n]);
353                         break;
354                     }
355                     if ((*ti->cmp)(ti->tmpbuf, ti->heap[ti->ptr[1]]) > 1)
356                     {
357                         heap_delete(ti);
358                         heap_insert(ti, ti->tmpbuf, n);
359                         break;
360                     }
361                 }
362             }
363         }
364         heap_close(ti);
365         xfree(ispt);
366     }
367     else
368         yaz_log(YLOG_WARN, "Unknown isam set in rset_trunc_r");
369
370     rset_close(result_rsfd);
371     return result;
372 }
373
374 static int isams_trunc_cmp(const void *p1, const void *p2)
375 {
376     ISAM_P i1 = *(ISAM_P*) p1;
377     ISAM_P i2 = *(ISAM_P*) p2;
378
379     if (i1 > i2)
380         return 1;
381     else if (i1 < i2)
382         return -1;
383     return 0;
384 }
385
386 static int isamc_trunc_cmp(const void *p1, const void *p2)
387 {
388     ISAM_P i1 = *(ISAM_P*) p1;
389     ISAM_P i2 = *(ISAM_P*) p2;
390     zint d;
391
392     d = (isamc_type(i1) - isamc_type(i2));
393     if (d == 0)
394         d = isamc_block(i1) - isamc_block(i2);
395     if (d > 0)
396         return 1;
397     else if (d < 0)
398         return -1;
399     return 0;
400 }
401
402 RSET rset_trunc(ZebraHandle zi, ISAM_P *isam_p, int no,
403                 const char *term, int length, const char *flags,
404                 int preserve_position, int term_type, NMEM rset_nmem,
405                 struct rset_key_control *kctrl, int scope,
406                 struct ord_list *ol, int reg_type,
407                 zint hits_limit, const char *term_ref_id)
408 {
409     TERMID termid;
410     RSET result;
411     int trunc_chunk;
412     
413     termid = rset_term_create(term, length, flags, term_type, rset_nmem, ol,
414                               reg_type, hits_limit, term_ref_id);
415     if (no < 1)
416         return rsnull_create(rset_nmem, kctrl, termid);
417     
418     if (zi->reg->isams)
419     {
420         if (no == 1)
421             return rsisams_create(rset_nmem, kctrl, scope,
422                                   zi->reg->isams, *isam_p, termid);
423         qsort(isam_p, no, sizeof(*isam_p), isams_trunc_cmp);
424     }
425     else if (zi->reg->isamc)
426     {
427         if (no == 1)
428             return rsisamc_create(rset_nmem, kctrl, scope,
429                                   zi->reg->isamc, *isam_p, termid);
430         qsort(isam_p, no, sizeof(*isam_p), isamc_trunc_cmp);
431     }
432     else if (zi->reg->isamb)
433     {
434         int trunc_limit = atoi(res_get_def(zi->res, "trunclimit", "10000"));
435         if (no == 1)
436             return rsisamb_create(rset_nmem, kctrl, scope,
437                                   zi->reg->isamb, *isam_p, termid);
438         else if (no < trunc_limit) 
439         {
440             RSET r;
441             RSET *rsets = xmalloc(no*sizeof(RSET)); /* use nmem! */
442             int i;
443             for (i = 0; i<no; i++)
444                 rsets[i] = rsisamb_create(rset_nmem, kctrl, scope,
445                                           zi->reg->isamb, isam_p[i],
446                                           0 /* termid */);
447             r = rsmulti_or_create(rset_nmem, kctrl, scope,
448                                   termid /* termid */,
449                                   no, rsets);
450             xfree(rsets);
451             return r;
452         } 
453         qsort(isam_p, no, sizeof(*isam_p), isamc_trunc_cmp);
454     }
455     else
456     {
457         yaz_log(YLOG_WARN, "Unknown isam set in rset_trunc");
458         return rsnull_create(rset_nmem, kctrl, 0);
459     }
460     trunc_chunk = atoi(res_get_def(zi->res, "truncchunk", "20"));
461     result = rset_trunc_r(zi, term, length, flags, isam_p, 0, no, trunc_chunk,
462                           preserve_position, term_type, rset_nmem, kctrl,
463                           scope, termid);
464     return result;
465 }
466