Better check of marc leader info YAZ-691
[yaz-moved-to-github.git] / src / rpn2solr.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2013 Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file
7  * \brief Implements RPN to SOLR conversion
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #include <yaz/rpn2solr.h>
17 #include <yaz/xmalloc.h>
18 #include <yaz/diagbib1.h>
19 #include <yaz/z-core.h>
20 #include <yaz/wrbuf.h>
21
22 static void wrbuf_vputs(const char *buf, void *client_data)
23 {
24     wrbuf_write((WRBUF) client_data, buf, strlen(buf));
25 }
26
27 static const char *lookup_index_from_string_attr(Z_AttributeList *attributes)
28 {
29     int j;
30     int server_choice = 1;
31     for (j = 0; j < attributes->num_attributes; j++)
32     {
33         Z_AttributeElement *ae = attributes->attributes[j];
34         if (*ae->attributeType == 1) /* use attribute */
35         {
36             if (ae->which == Z_AttributeValue_complex)
37             {
38                 Z_ComplexAttribute *ca = ae->value.complex;
39                 int i;
40                 for (i = 0; i < ca->num_list; i++)
41                 {
42                     Z_StringOrNumeric *son = ca->list[i];
43                     if (son->which == Z_StringOrNumeric_string)
44                         return son->u.string;
45                 }
46             }
47             server_choice = 0; /* not serverChoice because we have use attr */
48         }
49     }
50     if (server_choice)
51         return "cql.serverChoice";
52     return 0;
53 }
54
55 static const char *lookup_relation_index_from_attr(Z_AttributeList *attributes)
56 {
57     int j;
58     for (j = 0; j < attributes->num_attributes; j++)
59     {
60         Z_AttributeElement *ae = attributes->attributes[j];
61         if (*ae->attributeType == 2) /* relation attribute */
62         {
63             if (ae->which == Z_AttributeValue_numeric)
64             {
65                 /* Only support for numeric relation */
66                 Odr_int *relation = ae->value.numeric;
67                 /* map this numeric to representation in SOLR */
68                 switch (*relation)
69                 {
70                     /* Unsure on whether this is the relation attribute constants? */
71                 case Z_ProximityOperator_Prox_lessThan:
72                     return "<";
73                 case Z_ProximityOperator_Prox_lessThanOrEqual:
74                     return "le";
75                 case Z_ProximityOperator_Prox_equal:
76                     return ":";
77                 case Z_ProximityOperator_Prox_greaterThanOrEqual:
78                     return "ge";
79                 case Z_ProximityOperator_Prox_greaterThan:
80                     return ">";
81                 case Z_ProximityOperator_Prox_notEqual:
82                     return 0;
83                 case 100:
84                     /* phonetic is not implemented */
85                     return 0;
86                 case 101:
87                     /* stem is not not implemented */
88                     return 0;
89                 case 102:
90                     /* relevance is supported in SOLR, but not implemented yet */
91                     return 0;
92                 default:
93                     /* Invalid relation */
94                     return 0;
95                 }
96             }
97             else {
98                 /*  Can we have a complex relation value?
99                     Should we implement something?
100                 */
101             }
102         }
103     }
104     return ":";
105 }
106
107 static int check_range(solr_transform_t ct, Z_Complex *q,
108                        Z_AttributesPlusTerm **p_apt1,
109                        Z_AttributesPlusTerm **p_apt2)
110 {
111     Z_Operator *op = q->roperator;
112     if (op->which == Z_Operator_and &&
113         q->s1->which == Z_RPNStructure_simple &&
114         q->s2->which == Z_RPNStructure_simple &&
115         q->s1->u.simple->which == Z_Operand_APT &&
116         q->s2->u.simple->which == Z_Operand_APT)
117     {
118         Z_AttributesPlusTerm *apt1 = q->s1->u.simple->u.attributesPlusTerm;
119         Z_AttributesPlusTerm *apt2 = q->s2->u.simple->u.attributesPlusTerm;
120         const char *i1 = solr_lookup_reverse(ct, "index.", apt1->attributes);
121         const char *i2 = solr_lookup_reverse(ct, "index.", apt2->attributes);
122         const char *rel1 = solr_lookup_reverse(ct, "relation.",
123                                                apt1->attributes);
124         const char *rel2 = solr_lookup_reverse(ct, "relation.",
125                                                apt2->attributes);
126         if (!rel1)
127             rel1 = lookup_relation_index_from_attr(apt1->attributes);
128         if (!rel2)
129             rel2 = lookup_relation_index_from_attr(apt2->attributes);
130         if (!i1)
131             i1 = lookup_index_from_string_attr(apt1->attributes);
132         if (!i2)
133             i2 = lookup_index_from_string_attr(apt2->attributes);
134         if (i1 && i2 && !strcmp(i1, i2) && rel1 && rel2)
135         {
136             if ((rel1[0] == '>' || rel1[0] == 'g') &&
137                 (rel2[0] == '<' || rel2[0] == 'l'))
138             {
139                 *p_apt1 = apt1;
140                 *p_apt2 = apt2;
141                 return 1;
142             }
143             if ((rel2[0] == '>' || rel2[0] == 'g') &&
144                 (rel1[0] == '<' || rel1[0] == 'l'))
145             {
146                 *p_apt1 = apt2;
147                 *p_apt2 = apt1;
148                 return 1;
149             }
150         }
151     }
152     return 0;
153 }
154
155 static int rpn2solr_attr(solr_transform_t ct,
156                          Z_AttributeList *attributes, WRBUF w)
157 {
158     const char *index = solr_lookup_reverse(ct, "index.", attributes);
159     const char *structure = solr_lookup_reverse(ct, "structure.", attributes);
160
161     /* if transform (properties) do not match, we'll just use a USE string attribute (bug #2978) */
162     if (!index)
163         index = lookup_index_from_string_attr(attributes);
164     if (!index)
165     {
166         solr_transform_set_error(ct,
167                                  YAZ_BIB1_UNSUPP_USE_ATTRIBUTE, 0);
168         return -1;
169     }
170     /* for serverChoice we omit index+relation+structure */
171     if (strcmp(index, "cql.serverChoice"))
172     {
173         wrbuf_puts(w, index);
174         wrbuf_puts(w, ":");
175         if (structure)
176         {
177             if (strcmp(structure, "*"))
178             {
179                 wrbuf_puts(w, "/");
180                 wrbuf_puts(w, structure);
181                 wrbuf_puts(w, " ");
182             }
183         }
184     }
185     return 0;
186 }
187
188 static Odr_int get_truncation(Z_AttributesPlusTerm *apt)
189 {
190     int j;
191     Z_AttributeList *attributes = apt->attributes;
192     for (j = 0; j < attributes->num_attributes; j++)
193     {
194         Z_AttributeElement *ae = attributes->attributes[j];
195         if (*ae->attributeType == 5) /* truncation attribute */
196         {
197             if (ae->which == Z_AttributeValue_numeric)
198             {
199                 return *(ae->value.numeric);
200             }
201             else if (ae->which == Z_AttributeValue_complex) {
202                 ;
203                 //yaz_log(YLOG_DEBUG, "Z_Attribute_complex");
204                 /* Complex: Shouldn't happen */
205             }
206         }
207     }
208     /* No truncation given */
209     return 0;
210 }
211
212 #define SOLR_SPECIAL "+-&|!(){}[]^\"~*?:\\"
213
214 static int emit_term(solr_transform_t ct, WRBUF w, Z_Term *term, Odr_int trunc)
215 {
216     size_t lterm = 0;
217     const char *sterm = 0;
218     switch (term->which)
219     {
220     case Z_Term_general:
221         lterm = term->u.general->len;
222         sterm = (const char *) term->u.general->buf;
223         break;
224     case Z_Term_numeric:
225         wrbuf_printf(w, ODR_INT_PRINTF, *term->u.numeric);
226         break;
227     case Z_Term_characterString:
228         sterm = term->u.characterString;
229         lterm = strlen(sterm);
230         break;
231     default:
232         solr_transform_set_error(ct, YAZ_BIB1_TERM_TYPE_UNSUPP, 0);
233         return -1;
234     }
235
236     if (sterm)
237     {
238         size_t i;
239         int must_quote = 0;
240
241         for (i = 0 ; i < lterm; i++)
242             if (sterm[i] == ' ')
243                 must_quote = 1;
244         if (must_quote)
245             wrbuf_puts(w, "\"");
246         for (i = 0 ; i < lterm; i++)
247         {
248             if (sterm[i] == '\\' && i < lterm - 1)
249             {
250                 i++;
251                 if (strchr(SOLR_SPECIAL, sterm[i]))
252                     wrbuf_putc(w, '\\');
253                 wrbuf_putc(w, sterm[i]);
254             }
255             else if (sterm[i] == '?' && trunc == 104)
256             {
257                 wrbuf_putc(w, '*');
258             }
259             else if (sterm[i] == '#' && trunc == 104)
260             {
261                 wrbuf_putc(w, '?');
262             }
263             else if (strchr(SOLR_SPECIAL, sterm[i]))
264             {
265                 wrbuf_putc(w, '\\');
266                 wrbuf_putc(w, sterm[i]);
267             }
268             else
269                 wrbuf_putc(w, sterm[i]);
270         }
271         if (trunc == 1)
272             wrbuf_puts(w, "*");
273         if (must_quote)
274             wrbuf_puts(w, "\"");
275     }
276     return 0;
277 }
278
279 static int rpn2solr_simple(solr_transform_t ct,
280                            void (*pr)(const char *buf, void *client_data),
281                            void *client_data,
282                            Z_AttributesPlusTerm *apt, WRBUF w,
283                            Z_AttributesPlusTerm *apt2)
284  {
285      int ret = 0;
286      Z_Term *term = apt->term;
287      Odr_int trunc = get_truncation(apt);
288      const char *relation2 = 0;
289      const char *relation1 = solr_lookup_reverse(ct, "relation.",
290                                                  apt->attributes);
291      /* Attempt to fix bug #2978: Look for a relation attribute */
292      if (!relation1)
293          relation1 = lookup_relation_index_from_attr(apt->attributes);
294      if (!relation1)
295      {
296          solr_transform_set_error(ct, YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE, 0);
297          return -1;
298      }
299      if (apt2)
300      {
301          relation2 = solr_lookup_reverse(ct, "relation.",
302                                          apt2->attributes);
303          if (!relation2)
304              relation2 = lookup_relation_index_from_attr(apt2->attributes);
305      }
306      wrbuf_rewind(w);
307      ret = rpn2solr_attr(ct, apt->attributes, w);
308      if (ret)
309          return ret;
310      if (trunc == 0 || trunc == 1 || trunc == 100 || trunc == 104)
311              ;
312      else
313      {
314          solr_transform_set_error(ct, YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE, 0);
315          return -1;
316      }
317
318      if (!relation1)
319          ret = emit_term(ct, w, term, trunc);
320      else if (relation1[0] == '<' || relation1[0] == 'l')
321      {
322          wrbuf_puts(w, "[* TO ");
323          ret = emit_term(ct, w, term, trunc);
324          if (!strcmp(relation1, "le") || !strcmp(relation1, "<="))
325              wrbuf_puts(w, "]");
326          else
327              wrbuf_puts(w, "}");
328      }
329      else if (relation1[0] == '>' || relation1[0] == 'g')
330      {
331          if (!strcmp(relation1, ">=") || !strcmp(relation1, "ge"))
332              wrbuf_puts(w, "[");
333          else
334              wrbuf_puts(w, "{");
335          ret = emit_term(ct, w, term, trunc);
336          wrbuf_puts(w, " TO ");
337          if (apt2)
338          {
339              emit_term(ct, w, apt2->term, 0);
340              if (!relation2 || !strcmp(relation2, "<=") ||
341                  !strcmp(relation2, "le"))
342                  wrbuf_puts(w, "]");
343              else
344                  wrbuf_puts(w, "}");
345          }
346          else
347              wrbuf_puts(w, "*]");
348      }
349      else
350          ret = emit_term(ct, w, term, trunc);
351      if (ret == 0)
352          pr(wrbuf_cstr(w), client_data);
353      return ret;
354  }
355
356
357 static int rpn2solr_structure(solr_transform_t ct,
358                               void (*pr)(const char *buf, void *client_data),
359                                void *client_data,
360                               Z_RPNStructure *q, int nested,
361                               WRBUF w)
362 {
363     if (q->which == Z_RPNStructure_simple)
364     {
365         if (q->u.simple->which != Z_Operand_APT)
366         {
367             solr_transform_set_error(
368                 ct, YAZ_BIB1_RESULT_SET_UNSUPP_AS_A_SEARCH_TERM, 0);
369             return -1;
370         }
371         else
372             return rpn2solr_simple(ct, pr, client_data,
373                                    q->u.simple->u.attributesPlusTerm, w, 0);
374     }
375     else
376     {
377         Z_Operator *op = q->u.complex->roperator;
378         Z_AttributesPlusTerm *apt1, *apt2;
379         int r;
380
381         if (check_range(ct, q->u.complex, &apt1, &apt2))
382             return rpn2solr_simple(ct, pr, client_data, apt1, w, apt2);
383         if (nested)
384             pr("(", client_data);
385
386         r = rpn2solr_structure(ct, pr, client_data, q->u.complex->s1, 1, w);
387         if (r)
388             return r;
389         switch(op->which)
390         {
391         case  Z_Operator_and:
392             pr(" AND ", client_data);
393             break;
394         case  Z_Operator_or:
395             pr(" OR ", client_data);
396             break;
397         case  Z_Operator_and_not:
398             pr(" AND NOT ", client_data);
399             break;
400         case  Z_Operator_prox:
401             solr_transform_set_error(ct, YAZ_BIB1_UNSUPP_SEARCH, 0);
402             return -1;
403         }
404         r = rpn2solr_structure(ct, pr, client_data, q->u.complex->s2, 1, w);
405         if (nested)
406             pr(")", client_data);
407         return r;
408     }
409 }
410
411 int solr_transform_rpn2solr_stream(solr_transform_t ct,
412                                    void (*pr)(const char *buf, void *client_data),
413                                    void *client_data,
414                                    Z_RPNQuery *q)
415 {
416     int r;
417     WRBUF w = wrbuf_alloc();
418     solr_transform_set_error(ct, 0, 0);
419     r = rpn2solr_structure(ct, pr, client_data, q->RPNStructure, 0, w);
420     wrbuf_destroy(w);
421     return r;
422 }
423
424
425 int solr_transform_rpn2solr_wrbuf(solr_transform_t ct,
426                                   WRBUF w,
427                                   Z_RPNQuery *q)
428 {
429     return solr_transform_rpn2solr_stream(ct, wrbuf_vputs, w, q);
430 }
431
432 /*
433  * Local variables:
434  * c-basic-offset: 4
435  * c-file-style: "Stroustrup"
436  * indent-tabs-mode: nil
437  * End:
438  * vim: shiftwidth=4 tabstop=8 expandtab
439  */
440