Comment
[yaz-moved-to-github.git] / client / client.c
index 6157a37..a02c99c 100644 (file)
@@ -2,7 +2,7 @@
  * Copyright (c) 1995-2003, Index Data
  * See the file LICENSE for details.
  *
- * $Id: client.c,v 1.191 2003-04-30 14:04:45 adam Exp $
+ * $Id: client.c,v 1.215 2003-12-18 17:02:24 mike Exp $
  */
 
 #include <stdio.h>
@@ -41,6 +41,7 @@
 #include <yaz/ill.h>
 #include <yaz/srw.h>
 #include <yaz/yaz-ccl.h>
+#include <yaz/cql.h>
 
 #if HAVE_READLINE_READLINE_H
 #include <readline/readline.h>
 #define C_PROMPT "Z> "
 
 static char *codeset = 0;               /* character set for output */
-
+static int hex_dump = 0;
+static char *dump_file_prefix = 0;
 static ODR out, in, print;              /* encoding and decoding streams */
+#if HAVE_XML2
 static ODR srw_sr_odr_out = 0;
 static Z_SRW_PDU *srw_sr = 0;
+#endif
 static FILE *apdu_file = 0;
 static FILE *ber_file = 0;
 static COMSTACK conn = 0;               /* our z-association */
@@ -77,7 +81,6 @@ static Z_ElementSetNames *elementSetNames = 0;
 static int setno = 1;                   /* current set offset */
 static enum oid_proto protocol = PROTO_Z3950;      /* current app protocol */
 static enum oid_value recordsyntax = VAL_USMARC;
-//static enum oid_value schema = VAL_NONE;
 static char *schema = 0;
 static int sent_close = 0;
 static NMEM session_mem = NULL;         /* memory handle for init-response */
@@ -85,10 +88,14 @@ static Z_InitResponse *session = 0;     /* session parameters */
 static char last_scan_line[512] = "0";
 static char last_scan_query[512] = "0";
 static char ccl_fields[512] = "default.bib";
-static char* esPackageName = 0;
-static char* yazProxy = 0;
+/* ### How can I set this path to use wherever YAZ is installed? */
+static char cql_fields[512] = "/usr/local/share/yaz/etc/pqf.properties";
+static char *esPackageName = 0;
+static char *yazProxy = 0;
 static int kilobytes = 1024;
-static char* yazCharset = 0;
+static char *negotiationCharset = 0;
+static char *outputCharset = 0;
+static char *marcCharset = 0;
 static char* yazLang = 0;
 
 static char last_cmd[32] = "?";
@@ -97,16 +104,20 @@ static char *refid = NULL;
 static char *last_open_command = NULL;
 static int auto_reconnect = 0;
 
+static char cur_host[200];
+
 typedef enum {
     QueryType_Prefix,
     QueryType_CCL,
     QueryType_CCL2RPN,
-    QueryType_CQL
+    QueryType_CQL,
+    QueryType_CQL2RPN
 } QueryType;
 
 static QueryType queryType = QueryType_Prefix;
 
 static CCL_bibset bibset;               /* CCL bibset handle */
+static cql_transform_t cqltrans;       /* CQL context-set handle */
 
 #if HAVE_READLINE_COMPLETION_OVER
 
@@ -145,26 +156,42 @@ const char* query_type_as_string(QueryType q)
     case QueryType_CCL: return "CCL (CCL sent to server) ";
     case QueryType_CCL2RPN: return "CCL -> RPN (RPN sent to server)";
     case QueryType_CQL: return "CQL (CQL sent to server)";
+    case QueryType_CQL2RPN: return "CQL -> RPN (RPN sent to server)";
     default: 
         return "unknown Query type internal yaz-client error";
     }
 }
 
-
-void do_hex_dump(char* buf,int len) 
+static void do_hex_dump(const char* buf, int len) 
 {
-#if 0 
-    int i,x;
-    for( i=0; i<len ; i=i+16 ) 
-    {                  
-        printf(" %4.4d ",i);
-        for(x=0 ; i+x<len && x<16; ++x) 
-        {
-            printf("%2.2X ",(unsigned int)((unsigned char)buf[i+x]));
-        }
-        printf("\n");
+    if (hex_dump)
+    {
+       int i,x;
+       for( i=0; i<len ; i=i+16 ) 
+       {                       
+           printf(" %4.4d ",i);
+           for(x=0 ; i+x<len && x<16; ++x) 
+           {
+               printf("%2.2X ",(unsigned int)((unsigned char)buf[i+x]));
+           }
+           printf("\n");
+       }
+    }
+    if (dump_file_prefix)
+    {
+       static int no = 0;
+       if (++no < 1000 && strlen(dump_file_prefix) < 500)
+       {
+           char fname[1024];
+           FILE *of;
+           sprintf (fname, "%s.%03d.raw", dump_file_prefix, no);
+           of = fopen(fname, "wb");
+           
+           fwrite (buf, 1, len, of);
+           
+           fclose(of);
+       }
     }
-#endif
 }
 
 void add_otherInfos(Z_APDU *a) 
@@ -181,7 +208,7 @@ void add_otherInfos(Z_APDU *a)
     }   
 }
 
-void send_apdu(Z_APDU *a)
+int send_apdu(Z_APDU *a)
 {
     char *buf;
     int len;
@@ -197,20 +224,21 @@ void send_apdu(Z_APDU *a)
     {
         odr_perror(out, "Encoding APDU");
         close_session();
-        return;
+        return 0;
     }
     buf = odr_getbuf(out, &len, 0);
     if (ber_file)
         odr_dumpBER(ber_file, buf, len);
     /* printf ("sending APDU of size %d\n", len); */
+    do_hex_dump(buf, len);
     if (cs_put(conn, buf, len) < 0)
     {
         fprintf(stderr, "cs_put: %s", cs_errmsg(cs_errno(conn)));
         close_session();
-        return;
+        return 0;
     }
-    do_hex_dump(buf,len);
     odr_reset(out); /* release the APDU structure  */
+    return 1;
 }
 
 static void print_stringn(const unsigned char *buf, size_t len)
@@ -276,7 +304,7 @@ static void send_initRequest(const char* type_and_host)
         yaz_oi_set_string_oidval(&req->otherInfo, out, VAL_PROXY,
         1, type_and_host);
     
-    if (yazCharset || yazLang) {
+    if (negotiationCharset || yazLang) {
        Z_OtherInformation **p;
        Z_OtherInformationUnit *p0;
        
@@ -287,16 +315,23 @@ static void send_initRequest(const char* type_and_host)
                
                p0->which = Z_OtherInfo_externallyDefinedInfo;
                p0->information.externallyDefinedInfo =
-                       yaz_set_proposal_charneg(out,
-                               (const char**)&yazCharset, (yazCharset)?1:0,
-                               (const char**)&yazLang, (yazLang)?1:0, 1);
+                       yaz_set_proposal_charneg(
+                            out,
+                            (const char**)&negotiationCharset, 
+                            negotiationCharset ? 1 : 0,
+                            (const char**)&yazLang, yazLang ? 1 : 0, 1);
        }
     }
     
-    send_apdu(apdu);
-    printf("Sent initrequest.\n");
+    if (send_apdu(apdu))
+        printf("Sent initrequest.\n");
 }
 
+
+/* These two are used only from process_initResponse() */
+static void render_initUserInfo(Z_OtherInformation *ui1);
+static void render_diag(Z_DiagnosticFormat *diag);
+
 static int process_initResponse(Z_InitResponse *res)
 {
     int ver = 0;
@@ -320,19 +355,25 @@ static int process_initResponse(Z_InitResponse *res)
         printf("Version: %s\n", res->implementationVersion);
     if (res->userInformationField)
     {
-        printf("UserInformationfield:\n");
-        if (!z_External(print, (Z_External**)&res-> userInformationField,
-            0, 0))
-        {
-            odr_perror(print, "Printing userinfo\n");
-            odr_reset(print);
-        }
-        if (res->userInformationField->which == Z_External_octet)
-        {
-            printf("Guessing visiblestring:\n");
-            printf("'%s'\n", res->userInformationField->u. octet_aligned->buf);
-        }
-        odr_reset (print);
+       Z_External *uif = res->userInformationField;
+       if (uif->which == Z_External_userInfo1) {
+           render_initUserInfo(uif->u.userInfo1);
+       } else {
+           printf("UserInformationfield:\n");
+           if (!z_External(print, (Z_External**)&uif, 0, 0)) {
+               odr_perror(print, "Printing userinfo\n");
+               odr_reset(print);
+           }
+           if (uif->which == Z_External_octet) {
+               printf("Guessing visiblestring:\n");
+               printf("'%s'\n", uif->u. octet_aligned->buf);
+           } else if (uif->which == Z_External_single) {
+               /* Peek at any private Init-diagnostic APDUs */
+               Odr_any *sat = uif->u.single_ASN1_type;
+               printf("### NAUGHTY: External is '%s'\n", sat->buf);
+           }
+           odr_reset (print);
+       }
     }
     printf ("Options:");
     if (ODR_MASK_GET(res->options, Z_Options_search))
@@ -388,6 +429,15 @@ static int process_initResponse(Z_InitResponse *res)
             
             yaz_get_response_charneg(session_mem, p, &charset, &lang,
                                      &selected);
+
+           if (outputCharset && negotiationCharset) {
+                   odr_set_charset (out, charset, outputCharset);
+                   odr_set_charset (in, outputCharset, charset);
+           }
+           else {
+                   odr_set_charset (out, 0, 0);
+                   odr_set_charset (in, 0, 0);
+           }
             
             printf("Accepted character set : %s\n", charset);
             printf("Accepted code language : %s\n", lang ? lang : "none");
@@ -398,6 +448,60 @@ static int process_initResponse(Z_InitResponse *res)
     return 0;
 }
 
+
+static void render_initUserInfo(Z_OtherInformation *ui1) {
+    int i;
+    printf("Init response contains %d otherInfo unit%s:\n",
+          ui1->num_elements, ui1->num_elements == 1 ? "" : "s");
+
+    for (i = 0; i < ui1->num_elements; i++) {
+       Z_OtherInformationUnit *unit = ui1->list[i];
+       printf("  %d: otherInfo unit contains ", i+1);
+       if (unit->which == Z_OtherInfo_externallyDefinedInfo &&
+           unit->information.externallyDefinedInfo->which ==
+           Z_External_diag1) {
+           render_diag(unit->information.externallyDefinedInfo->u.diag1);
+       } else {
+           printf("unsupported otherInfo unit type %d\n", unit->which);
+       }
+    }
+}
+
+
+/* ### should this share code with display_diagrecs()? */
+static void render_diag(Z_DiagnosticFormat *diag) {
+    int i;
+
+    printf("%d diagnostic%s:\n", diag->num, diag->num == 1 ? "" : "s");
+    for (i = 0; i < diag->num; i++) {
+       Z_DiagnosticFormat_s *ds = diag->elements[i];
+       printf("    %d: ", i+1);
+       switch (ds->which) {
+       case Z_DiagnosticFormat_s_defaultDiagRec: {
+           Z_DefaultDiagFormat *dd = ds->u.defaultDiagRec;
+           /* ### should check `dd->diagnosticSetId' */
+           printf("code=%d (%s)", *dd->condition,
+                  diagbib1_str(*dd->condition));
+           /* Both types of addinfo are the same, so use type-pun */
+           if (dd->u.v2Addinfo != 0)
+               printf(",\n\taddinfo='%s'", dd->u.v2Addinfo);
+           break;
+       }
+       case Z_DiagnosticFormat_s_explicitDiagnostic:
+           printf("Explicit diagnostic (not supported)");
+           break;
+       default:
+           printf("Unrecognised diagnostic type %d", ds->which);
+           break;
+       }
+
+       if (ds->message != 0)
+           printf(", message='%s'", ds->message);
+       printf("\n");
+    }
+}
+
+
 static int set_base(const char *arg)
 {
     int i;
@@ -527,7 +631,6 @@ int session_connect(const char *arg)
 
 int cmd_open(const char *arg)
 {
-    static char cur_host[200];
     if (arg)
     {
         strncpy (cur_host, arg, sizeof(cur_host)-1);
@@ -568,14 +671,20 @@ int cmd_authentication(const char *arg)
     r = sscanf (arg, "%39s %39s %39s", user, group, pass);
     if (r == 0)
     {
-        printf("Auth field set to null\n");
+        printf("Authentication set to null\n");
         auth = 0;
     }
     if (r == 1)
     {
         auth = &au;
-        au.which = Z_IdAuthentication_open;
-        au.u.open = user;
+       if (!strcmp(user, "-")) {
+           au.which = Z_IdAuthentication_anonymous;
+           printf("Authentication set to Anonymous\n");
+       } else {
+           au.which = Z_IdAuthentication_open;
+           au.u.open = user;
+           printf("Authentication set to Open (%s)\n", user);
+       }
     }
     if (r == 2)
     {
@@ -585,6 +694,7 @@ int cmd_authentication(const char *arg)
         idPass.groupId = NULL;
         idPass.userId = user;
         idPass.password = group;
+       printf("Authentication set to User (%s), Pass (%s)\n", user, group);
     }
     if (r == 3)
     {
@@ -594,6 +704,8 @@ int cmd_authentication(const char *arg)
         idPass.groupId = group;
         idPass.userId = user;
         idPass.password = pass;
+       printf("Authentication set to User (%s), Group (%s), Pass (%s)\n",
+              user, group, pass);
     }
     return 1;
 }
@@ -704,26 +816,35 @@ static void display_record(Z_External *r)
                                         &result, &rlen)> 0)
                 {
                     char *from = 0;
-                    if (ent->value == VAL_USMARC)
+                    if (marcCharset && !strcmp(marcCharset, "auto"))
                     {
-                        if (octet_buf[9] == 'a')
-                            from = "UTF-8";
+                        if (ent->value == VAL_USMARC)
+                        {
+                            if (octet_buf[9] == 'a')
+                                from = "UTF-8";
+                            else
+                                from = "MARC-8";
+                        }
                         else
-                            from = "MARC8";
+                            from = "ISO-8859-1";
                     }
-                    else
-                        from = "ISO-8859-1";
-
-                    if (codeset && from)
+                   else if (marcCharset)
+                       from = marcCharset;
+                    if (outputCharset && from)
                     {   
-                        printf ("convert from %s to %s\n", from, codeset);
-                        cd = yaz_iconv_open(codeset, from);
+                        cd = yaz_iconv_open(outputCharset, from);
+                        printf ("convert from %s to %s", from, 
+                                outputCharset);
+                       if (!cd)
+                           printf (" unsupported\n");
+                       else
+                           printf ("\n");
                     }
                     if (!cd)
                         fwrite (result, 1, rlen, stdout);
                     else
                     {
-                        char outbuf[12];
+                        char outbuf[6];
                         size_t inbytesleft = rlen;
                         const char *inp = result;
                         
@@ -731,9 +852,11 @@ static void display_record(Z_External *r)
                         {
                             size_t outbytesleft = sizeof(outbuf);
                             char *outp = outbuf;
-                            size_t r = yaz_iconv (cd, (char**) &inp,
-                                                  &inbytesleft, 
-                                                  &outp, &outbytesleft);
+                           size_t r;
+
+                            r = yaz_iconv (cd, (char**) &inp,
+                                          &inbytesleft, 
+                                          &outp, &outbytesleft);
                             if (r == (size_t) (-1))
                             {
                                 int e = yaz_iconv_error(cd);
@@ -785,6 +908,107 @@ static void display_record(Z_External *r)
         puts (wrbuf_buf(w));
         wrbuf_free(w, 1);
     }
+    else if ( /* OPAC display not complete yet .. */
+            ent && ent->value == VAL_OPAC)
+    {
+       int i;
+       if (r->u.opac->bibliographicRecord)
+           display_record(r->u.opac->bibliographicRecord);
+       for (i = 0; i<r->u.opac->num_holdingsData; i++)
+       {
+           Z_HoldingsRecord *h = r->u.opac->holdingsData[i];
+           if (h->which == Z_HoldingsRecord_marcHoldingsRecord)
+           {
+               printf ("MARC holdings %d\n", i);
+               display_record(h->u.marcHoldingsRecord);
+           }
+           else if (h->which == Z_HoldingsRecord_holdingsAndCirc)
+           {
+               int j;
+
+               Z_HoldingsAndCircData *data = h->u.holdingsAndCirc;
+
+               printf ("Data holdings %d\n", i);
+               if (data->typeOfRecord)
+                   printf ("typeOfRecord: %s\n", data->typeOfRecord);
+               if (data->encodingLevel)
+                   printf ("encodingLevel: %s\n", data->encodingLevel);
+               if (data->receiptAcqStatus)
+                   printf ("receiptAcqStatus: %s\n", data->receiptAcqStatus);
+               if (data->generalRetention)
+                   printf ("generalRetention: %s\n", data->generalRetention);
+               if (data->completeness)
+                   printf ("completeness: %s\n", data->completeness);
+               if (data->dateOfReport)
+                   printf ("dateOfReport: %s\n", data->dateOfReport);
+               if (data->nucCode)
+                   printf ("nucCode: %s\n", data->nucCode);
+               if (data->localLocation)
+                   printf ("localLocation: %s\n", data->localLocation);
+               if (data->shelvingLocation)
+                   printf ("shelvingLocation: %s\n", data->shelvingLocation);
+               if (data->callNumber)
+                   printf ("callNumber: %s\n", data->callNumber);
+               if (data->copyNumber)
+                   printf ("copyNumber: %s\n", data->copyNumber);
+               if (data->publicNote)
+                   printf ("publicNote: %s\n", data->publicNote);
+               if (data->reproductionNote)
+                   printf ("reproductionNote: %s\n", data->reproductionNote);
+               if (data->termsUseRepro)
+                   printf ("termsUseRepro: %s\n", data->termsUseRepro);
+               if (data->enumAndChron)
+                   printf ("enumAndChron: %s\n", data->enumAndChron);
+               for (j = 0; j<data->num_volumes; j++)
+               {
+                   printf ("volume %d\n", j);
+                   if (data->volumes[j]->enumeration)
+                       printf (" enumeration: %s\n",
+                               data->volumes[j]->enumeration);
+                   if (data->volumes[j]->chronology)
+                       printf (" chronology: %s\n",
+                               data->volumes[j]->chronology);
+                   if (data->volumes[j]->enumAndChron)
+                       printf (" enumAndChron: %s\n",
+                               data->volumes[j]->enumAndChron);
+               }
+               for (j = 0; j<data->num_circulationData; j++)
+               {
+                   printf ("circulation %d\n", j);
+                   if (data->circulationData[j]->availableNow)
+                       printf (" availableNow: %d\n",
+                               *data->circulationData[j]->availableNow);
+                   if (data->circulationData[j]->availablityDate)
+                       printf (" availabiltyDate: %s\n",
+                               data->circulationData[j]->availablityDate);
+                   if (data->circulationData[j]->availableThru)
+                       printf (" availableThru: %s\n",
+                               data->circulationData[j]->availableThru);
+                   if (data->circulationData[j]->restrictions)
+                       printf (" restrictions: %s\n",
+                               data->circulationData[j]->restrictions);
+                   if (data->circulationData[j]->itemId)
+                       printf (" itemId: %s\n",
+                               data->circulationData[j]->itemId);
+                   if (data->circulationData[j]->renewable)
+                       printf (" renewable: %d\n",
+                               *data->circulationData[j]->renewable);
+                   if (data->circulationData[j]->onHold)
+                       printf (" onHold: %d\n",
+                               *data->circulationData[j]->onHold);
+                   if (data->circulationData[j]->enumAndChron)
+                       printf (" enumAndChron: %s\n",
+                               data->circulationData[j]->enumAndChron);
+                   if (data->circulationData[j]->midspine)
+                       printf (" midspine: %s\n",
+                               data->circulationData[j]->midspine);
+                   if (data->circulationData[j]->temporaryLocation)
+                       printf (" temporaryLocation: %s\n",
+                               data->circulationData[j]->temporaryLocation);
+               }
+           }
+       }
+    }
     else 
     {
         printf("Unknown record representation.\n");
@@ -902,8 +1126,8 @@ static int send_deleteResultSetRequest(const char *arg)
 #if HAVE_XML2
 static int send_srw(Z_SRW_PDU *sr)
 {
-    const char *charset = 0;
-    const char *host_port = 0;
+    const char *charset = negotiationCharset;
+    const char *host_port = cur_host;
     char *path = 0;
     char ctype[50];
     Z_SOAP_Handler h[2] = {
@@ -941,7 +1165,7 @@ static int send_srw(Z_SRW_PDU *sr)
             memcpy (h, cp0, cp1 - cp0);
             h[cp1-cp0] = '\0';
             z_HTTP_header_add(out, &gdu->u.HTTP_Request->headers,
-                              "host", h);
+                              "Host", h);
         }
     }
 
@@ -973,10 +1197,14 @@ static int send_srw(Z_SRW_PDU *sr)
         char *buf_out;
         int len_out;
         int r;
+       if (apdu_file && !z_GDU(print, &gdu, 0, 0))
+            printf ("Failed to print outgoing APDU\n");
         buf_out = odr_getbuf(out, &len_out, 0);
         
         /* we don't odr_reset(out), since we may need the buffer again */
 
+       do_hex_dump(buf_out, len_out);
+
         r = cs_put(conn, buf_out, len_out);
 
         odr_destroy(o);
@@ -1029,8 +1257,10 @@ static int send_searchRequest(const char *arg)
     Odr_oct ccl_query;
     YAZ_PQF_Parser pqf_parser;
     Z_External *ext;
+    QueryType myQueryType = queryType;
+    char pqfbuf[512];
 
-    if (queryType == QueryType_CCL2RPN)
+    if (myQueryType == QueryType_CCL2RPN)
     {
         rpn = ccl_find_str(bibset, arg, &error, &pos);
         if (error)
@@ -1038,7 +1268,32 @@ static int send_searchRequest(const char *arg)
             printf("CCL ERROR: %s\n", ccl_err_msg(error));
             return 0;
         }
+    } else if (myQueryType == QueryType_CQL2RPN) {
+       /* ### All this code should be wrapped in a utility function */
+       CQL_parser parser;
+       struct cql_node *node;
+       const char *addinfo;
+       if (cqltrans == 0) {
+            printf("Can't use CQL: no translation file.  Try set_cqlfile\n");
+           return 0;
+       }
+       parser = cql_parser_create();
+       if ((error = cql_parser_string(parser, arg)) != 0) {
+            printf("Can't parse CQL: must be a syntax error\n");
+            return 0;
+       }
+       node = cql_parser_result(parser);
+       if ((error = cql_transform_buf(cqltrans, node, pqfbuf,
+                                      sizeof pqfbuf)) != 0) {
+           error = cql_transform_error(cqltrans, &addinfo);
+           printf ("Can't convert CQL to PQF: %s (addinfo=%s)\n",
+                   cql_strerror(error), addinfo);
+            return 0;
+        }
+       arg = pqfbuf;
+       myQueryType = QueryType_Prefix;
     }
+
     req->referenceId = set_refid (out);
     if (!strcmp(arg, "@big")) /* strictly for troublemaking */
     {
@@ -1078,7 +1333,7 @@ static int send_searchRequest(const char *arg)
 
     req->query = &query;
 
-    switch (queryType)
+    switch (myQueryType)
     {
     case QueryType_Prefix:
         query.which = Z_Query_type_1;
@@ -1129,9 +1384,9 @@ static int send_searchRequest(const char *arg)
         printf ("Unsupported query type\n");
         return 0;
     }
-    send_apdu(apdu);
+    if (send_apdu(apdu))
+        printf("Sent searchRequest.\n");
     setno = 1;
-    printf("Sent searchRequest.\n");
     return 2;
 }
 
@@ -1676,14 +1931,24 @@ static int only_z3950()
     return 0;
 }
 
+static int cmd_update_common(const char *arg, int version);
+
 static int cmd_update(const char *arg)
 {
+    return cmd_update_common(arg, 1);
+}
+
+static int cmd_update0(const char *arg)
+{
+    return cmd_update_common(arg, 0);
+}
+
+static int cmd_update_common(const char *arg, int version)
+{
     Z_APDU *apdu = zget_APDU(out, Z_APDU_extendedServicesRequest );
     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
     Z_External *r;
     int oid[OID_SIZE];
-    Z_IUOriginPartToKeep *toKeep;
-    Z_IUSuppliedRecords *notToKeep;
     oident update_oid;
     char action[20], recid[20], fname[80];
     int action_no;
@@ -1747,7 +2012,10 @@ static int cmd_update(const char *arg)
 
     update_oid.proto = PROTO_Z3950;
     update_oid.oclass = CLASS_EXTSERV;
-    update_oid.value = VAL_DBUPDATE;
+    if (version == 0)
+       update_oid.value = VAL_DBUPDATE0;
+    else
+       update_oid.value = VAL_DBUPDATE;
     oid_ent_to_oid (&update_oid, oid);
     req->packageType = odr_oiddup(out,oid);
     req->packageName = esPackageName;
@@ -1759,41 +2027,90 @@ static int cmd_update(const char *arg)
     r->direct_reference = odr_oiddup(out,oid);
     r->indirect_reference = 0;
     r->descriptor = 0;
-    r->which = Z_External_update;
-    r->u.update = (Z_IUUpdate *) odr_malloc(out, sizeof(*r->u.update));
-    r->u.update->which = Z_IUUpdate_esRequest;
-    r->u.update->u.esRequest = (Z_IUUpdateEsRequest *)
-        odr_malloc(out, sizeof(*r->u.update->u.esRequest));
-    toKeep = r->u.update->u.esRequest->toKeep = (Z_IUOriginPartToKeep *)
-        odr_malloc(out, sizeof(*r->u.update->u.esRequest->toKeep));
-    toKeep->databaseName = databaseNames[0];
-    toKeep->schema = 0;
-    toKeep->elementSetName = 0;
-    toKeep->actionQualifier = 0;
-    toKeep->action = (int *) odr_malloc(out, sizeof(*toKeep->action));
-    *toKeep->action = action_no;
-
-    notToKeep = r->u.update->u.esRequest->notToKeep = (Z_IUSuppliedRecords *)
-        odr_malloc(out, sizeof(*r->u.update->u.esRequest->notToKeep));
-    notToKeep->num = 1;
-    notToKeep->elements = (Z_IUSuppliedRecords_elem **)
-        odr_malloc(out, sizeof(*notToKeep->elements));
-    notToKeep->elements[0] = (Z_IUSuppliedRecords_elem *)
-        odr_malloc(out, sizeof(**notToKeep->elements));
-    notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
-    if (*recid)
-    {
-        notToKeep->elements[0]->u.opaque = (Odr_oct *)
-            odr_malloc (out, sizeof(Odr_oct));
-        notToKeep->elements[0]->u.opaque->buf = (unsigned char *) recid;
-        notToKeep->elements[0]->u.opaque->size = strlen(recid);
-        notToKeep->elements[0]->u.opaque->len = strlen(recid);
+    if (version == 0)
+    {
+       Z_IU0OriginPartToKeep *toKeep;
+       Z_IU0SuppliedRecords *notToKeep;
+
+       r->which = Z_External_update0;
+       r->u.update0 = (Z_IU0Update *) odr_malloc(out, sizeof(*r->u.update0));
+       r->u.update0->which = Z_IUUpdate_esRequest;
+       r->u.update0->u.esRequest = (Z_IU0UpdateEsRequest *)
+           odr_malloc(out, sizeof(*r->u.update0->u.esRequest));
+       toKeep = r->u.update0->u.esRequest->toKeep = (Z_IU0OriginPartToKeep *)
+           odr_malloc(out, sizeof(*r->u.update0->u.esRequest->toKeep));
+       
+       toKeep->databaseName = databaseNames[0];
+       toKeep->schema = 0;
+       toKeep->elementSetName = 0;
+
+       toKeep->action = (int *) odr_malloc(out, sizeof(*toKeep->action));
+       *toKeep->action = action_no;
+       
+       notToKeep = r->u.update0->u.esRequest->notToKeep = (Z_IU0SuppliedRecords *)
+           odr_malloc(out, sizeof(*r->u.update0->u.esRequest->notToKeep));
+       notToKeep->num = 1;
+       notToKeep->elements = (Z_IU0SuppliedRecords_elem **)
+           odr_malloc(out, sizeof(*notToKeep->elements));
+       notToKeep->elements[0] = (Z_IU0SuppliedRecords_elem *)
+           odr_malloc(out, sizeof(**notToKeep->elements));
+       notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
+       if (*recid)
+       {
+           notToKeep->elements[0]->u.opaque = (Odr_oct *)
+               odr_malloc (out, sizeof(Odr_oct));
+           notToKeep->elements[0]->u.opaque->buf = (unsigned char *) recid;
+           notToKeep->elements[0]->u.opaque->size = strlen(recid);
+           notToKeep->elements[0]->u.opaque->len = strlen(recid);
+       }
+       else
+           notToKeep->elements[0]->u.opaque = 0;
+       notToKeep->elements[0]->supplementalId = 0;
+       notToKeep->elements[0]->correlationInfo = 0;
+       notToKeep->elements[0]->record = record_this;
     }
     else
-        notToKeep->elements[0]->u.opaque = 0;
-    notToKeep->elements[0]->supplementalId = 0;
-    notToKeep->elements[0]->correlationInfo = 0;
-    notToKeep->elements[0]->record = record_this;
+    {
+       Z_IUOriginPartToKeep *toKeep;
+       Z_IUSuppliedRecords *notToKeep;
+
+       r->which = Z_External_update;
+       r->u.update = (Z_IUUpdate *) odr_malloc(out, sizeof(*r->u.update));
+       r->u.update->which = Z_IUUpdate_esRequest;
+       r->u.update->u.esRequest = (Z_IUUpdateEsRequest *)
+           odr_malloc(out, sizeof(*r->u.update->u.esRequest));
+       toKeep = r->u.update->u.esRequest->toKeep = (Z_IUOriginPartToKeep *)
+           odr_malloc(out, sizeof(*r->u.update->u.esRequest->toKeep));
+       
+       toKeep->databaseName = databaseNames[0];
+       toKeep->schema = 0;
+       toKeep->elementSetName = 0;
+       toKeep->actionQualifier = 0;
+       toKeep->action = (int *) odr_malloc(out, sizeof(*toKeep->action));
+       *toKeep->action = action_no;
+
+       notToKeep = r->u.update->u.esRequest->notToKeep = (Z_IUSuppliedRecords *)
+           odr_malloc(out, sizeof(*r->u.update->u.esRequest->notToKeep));
+       notToKeep->num = 1;
+       notToKeep->elements = (Z_IUSuppliedRecords_elem **)
+           odr_malloc(out, sizeof(*notToKeep->elements));
+       notToKeep->elements[0] = (Z_IUSuppliedRecords_elem *)
+           odr_malloc(out, sizeof(**notToKeep->elements));
+       notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
+       if (*recid)
+       {
+           notToKeep->elements[0]->u.opaque = (Odr_oct *)
+               odr_malloc (out, sizeof(Odr_oct));
+           notToKeep->elements[0]->u.opaque->buf = (unsigned char *) recid;
+           notToKeep->elements[0]->u.opaque->size = strlen(recid);
+           notToKeep->elements[0]->u.opaque->len = strlen(recid);
+       }
+       else
+           notToKeep->elements[0]->u.opaque = 0;
+       notToKeep->elements[0]->supplementalId = 0;
+       notToKeep->elements[0]->correlationInfo = 0;
+       notToKeep->elements[0]->record = record_this;
+    }
     
     send_apdu(apdu);
 
@@ -2041,7 +2358,8 @@ static int send_SRW_presentRequest(const char *arg)
 
 static void close_session (void)
 {
-    cs_close (conn);
+    if (conn)
+        cs_close (conn);
     conn = 0;
     if (session_mem)
     {
@@ -2256,7 +2574,9 @@ int send_sortrequest(const char *arg, int newset)
 
 void display_term(Z_TermInfo *t)
 {
-    if (t->term->which == Z_Term_general)
+    if (t->displayTerm)
+        printf("%s", t->displayTerm);
+    else if (t->term->which == Z_Term_general)
     {
         printf("%.*s", t->term->u.general->len, t->term->u.general->buf);
         sprintf(last_scan_line, "%.*s", t->term->u.general->len,
@@ -2477,6 +2797,8 @@ int cmd_querytype (const char *arg)
         queryType = QueryType_CCL2RPN;
     else if (!strcmp(arg, "cql"))
         queryType = QueryType_CQL;        
+    else if (!strcmp (arg, "cql2rpn") || !strcmp (arg, "cqlrpn"))
+        queryType = QueryType_CQL2RPN;
     else
     {
         printf ("Querytype must be one of:\n");
@@ -2484,6 +2806,7 @@ int cmd_querytype (const char *arg)
         printf (" ccl            - CCL query\n");
         printf (" ccl2rpn        - CCL query converted to RPN\n");
         printf (" cql            - CQL\n");
+        printf (" cql2rpn        - CQL query converted to RPN\n");
         return 0;
     }
     return 1;
@@ -2524,27 +2847,30 @@ int cmd_packagename(const char* arg)
     xfree (esPackageName);
     esPackageName = NULL;
     if (*arg)
-    {
-        esPackageName = (char *) xmalloc (strlen(arg)+1);
-        strcpy (esPackageName, arg);
-    }
+        esPackageName = xstrdup(arg);
     return 1;
 }
 
 int cmd_proxy(const char* arg)
 {
-    if (*arg == '\0') {
-               xfree (yazProxy);
-               yazProxy = NULL;
-       
-    }
     xfree (yazProxy);
     yazProxy = NULL;
     if (*arg)
-    {
-        yazProxy = (char *) xmalloc (strlen(arg)+1);
-        strcpy (yazProxy, arg);
-    } 
+        yazProxy = xstrdup (arg);
+    return 1;
+}
+
+int cmd_marccharset(const char *arg)
+{
+    char l1[30];
+
+    *l1 = 0;
+    if (sscanf(arg, "%29s", l1) < 1)
+        return 1;
+    xfree (marcCharset);
+    marcCharset = 0;
+    if (strcmp(l1, "-"))
+        marcCharset = xstrdup(l1);
     return 1;
 }
 
@@ -2555,18 +2881,43 @@ int cmd_charset(const char* arg)
     *l1 = *l2 = 0;
     if (sscanf(arg, "%29s %29s", l1, l2) < 1)
     {
-       printf("Current character set is `%s'\n", (yazCharset) ? yazCharset:NULL);
+       printf("Current negotiation character set is `%s'\n", 
+               negotiationCharset ? negotiationCharset: "none");
+       printf("Current output character set is `%s'\n", 
+               outputCharset ? outputCharset: "none");
        return 1;
     }
-    xfree (yazCharset);
-    yazCharset = NULL;
-    if (*l1)
-        yazCharset = xstrdup(l1);
-    if (*l2)
+    xfree (negotiationCharset);
+    negotiationCharset = NULL;
+    if (*l1 && strcmp(l1, "-"))
     {
-        odr_set_charset (out, l1, l2);
-        odr_set_charset (in, l2, l1);
+        negotiationCharset = xstrdup(l1);
+        printf ("Character set negotiation : %s\n", negotiationCharset);
     }
+    else
+        printf ("Character set negotiation disabled\n");
+    if (*l2)
+    {
+        xfree (outputCharset);
+        outputCharset = 0;
+        if (!strcmp(l2, "auto") && codeset)
+        {
+            if (codeset)
+            {
+                printf ("output charset: %s\n", codeset);
+                outputCharset = xstrdup(codeset);
+
+
+            }
+            else
+                printf ("No codeset found on this system\n");
+        }
+        else if (strcmp(l2, "-"))
+            outputCharset = xstrdup(l2);
+        else
+            printf ("Output charset conversion disabled\n");
+    } 
+
     return 1;
 }
 
@@ -2684,6 +3035,22 @@ int cmd_set_cclfile(const char* arg)
     return 0;
 }
 
+int cmd_set_cqlfile(const char* arg)
+{
+    cql_transform_t newcqltrans;
+
+    if ((newcqltrans = cql_transform_open_fname(arg)) == 0) {
+        perror("unable to open CQL file");
+       return 0;
+    }
+    if (cqltrans != 0)
+       cql_transform_close(cqltrans);
+
+    cqltrans = newcqltrans;
+    strcpy(cql_fields, arg);
+    return 0;
+}
+
 int cmd_set_auto_reconnect(const char* arg)
 {  
     if(strlen(arg)==0) {
@@ -2724,17 +3091,6 @@ int cmd_set_marcdump(const char* arg)
     return 1;
 }
 
-int cmd_set_proxy(const char* arg)
-{
-    if(yazProxy) free(yazProxy);
-    yazProxy=NULL;
-    
-    if(strlen(arg) > 1) {
-        yazProxy=strdup(arg);
-    }
-    return 1;
-}
-
 /* 
    this command takes 3 arge {name class oid} 
 */
@@ -2870,6 +3226,10 @@ static void initialize(void)
         ccl_qual_file (bibset, inf);
         fclose (inf);
     }
+
+    cqltrans = cql_transform_open_fname(cql_fields);
+    /* If this fails, no problem: we detect cqltrans == 0 later */
+
 #if HAVE_READLINE_READLINE_H
     rl_attempted_completion_function = (CPPFunction*)readline_completer;
 #endif
@@ -2892,14 +3252,16 @@ static void handle_srw_response(Z_SRW_searchRetrieveResponse *res)
 {
     int i;
 
-    printf ("Received SRW SearchRetrieveResponse\n");
+    printf ("Received SRW SearchRetrieve Response\n");
     
     for (i = 0; i<res->num_diagnostics; i++)
     {
-        printf ("SRW diagnostic %d %s\nDetails: %s\n",
+        printf ("SRW diagnostic %d %s\n",
                 *res->diagnostics[i].code,
-                yaz_diag_srw_str(*res->diagnostics[i].code),
-                res->diagnostics[i].details);
+                yaz_diag_srw_str(*res->diagnostics[i].code));
+
+       if (res->diagnostics[i].details)
+            printf ("Details: %s\n", res->diagnostics[i].details);
     }
     if (res->numberOfRecords)
         printf ("Number of hits: %d\n", *res->numberOfRecords);
@@ -2976,17 +3338,20 @@ static void http_response(Z_HTTP_Response *hres)
         }
         close_session();
     }
-    if (!strcmp(hres->version, "1.0"))
-    {
-        /* HTTP 1.0: only if Keep-Alive we stay alive.. */
-        if (!connection_head || strcmp(connection_head, "Keep-Alive"))
-            close_session();
-    }
-    else 
+    else
     {
-        /* HTTP 1.1: only if no close we stay alive .. */
-        if (connection_head && !strcmp(connection_head, "close"))
-            close_session();
+       if (!strcmp(hres->version, "1.0"))
+       {
+           /* HTTP 1.0: only if Keep-Alive we stay alive.. */
+           if (!connection_head || strcmp(connection_head, "Keep-Alive"))
+               close_session();
+       }
+       else 
+       {
+           /* HTTP 1.1: only if no close we stay alive .. */
+           if (connection_head && !strcmp(connection_head, "close"))
+               close_session();
+       }
     }
 }
 #endif
@@ -3015,6 +3380,8 @@ void wait_and_handle_response()
                 
                 buf_out = odr_getbuf(out, &len_out, 0);
                 
+               do_hex_dump(buf_out, len_out);
+
                 cs_put(conn, buf_out, len_out);
                 
                 odr_reset(out);
@@ -3030,6 +3397,7 @@ void wait_and_handle_response()
         odr_reset(out);
         odr_reset(in); /* release APDU from last round */
         record_last = 0;
+        do_hex_dump(netbuffer, res);
         odr_setbuf(in, netbuffer, res, 0);
         
         if (!z_GDU(in, &gdu, 0, 0))
@@ -3042,8 +3410,9 @@ void wait_and_handle_response()
             fprintf(f, "---------\n");
             if (apdu_file)
                 z_GDU(print, &gdu, 0, 0);
-            close_session ();
-            break;
+            if (conn && cs_more(conn))
+                continue;
+           break;
         }
         if (ber_file)
             odr_dumpBER(ber_file, netbuffer, res);
@@ -3269,7 +3638,8 @@ int cmd_list_all(const char* args) {
             printf("Authentication       : Unknown\n");
         }
     }
-    if ( yazCharset ) printf("Character set        : `%s'\n", (yazCharset) ? yazCharset:NULL);
+    if (negotiationCharset)
+        printf("Neg. Character set   : `%s'\n", negotiationCharset);
     
     /* bases */
     printf("Bases                : ");
@@ -3278,6 +3648,7 @@ int cmd_list_all(const char* args) {
     
     /* Query options */
     printf("CCL file             : %s\n",ccl_fields);
+    printf("CQL file             : %s\n",cql_fields);
     printf("Query type           : %s\n",query_type_as_string(queryType));
     
     printf("Named Result Sets    : %s\n",setnumber==-1?"off":"on");
@@ -3366,9 +3737,11 @@ static struct {
     {"refid", cmd_refid, "<id>",NULL,0,NULL},
     {"itemorder", cmd_itemorder, "ill|item <itemno>",NULL,0,NULL},
     {"update", cmd_update, "<action> <recid> [<file>]",NULL,0,NULL},
+    {"update0", cmd_update0, "<action> <recid> [<file>]",NULL,0,NULL},
     {"packagename", cmd_packagename, "<packagename>",NULL,0,NULL},
     {"proxy", cmd_proxy, "[('tcp'|'ssl')]<host>[':'<port>]",NULL,0,NULL},
-    {"charset", cmd_charset, "<charset_name>",NULL,0,NULL},
+    {"charset", cmd_charset, "<nego_charset> <output_charset>",NULL,0,NULL},
+    {"marccharset", cmd_marccharset, "<charset_name>",NULL,0,NULL},
     {"lang", cmd_lang, "<language_code>",NULL,0,NULL},
     {".", cmd_source, "<filename>",NULL,1,NULL},
     {"!", cmd_subshell, "Subshell command",NULL,1,NULL},
@@ -3376,6 +3749,7 @@ static struct {
     {"set_berfile", cmd_set_berfile, "<filename>",NULL,1,NULL},
     {"set_marcdump", cmd_set_marcdump," <filename>",NULL,1,NULL},
     {"set_cclfile", cmd_set_cclfile," <filename>",NULL,1,NULL},
+    {"set_cqlfile", cmd_set_cqlfile," <filename>",NULL,1,NULL},
     {"set_auto_reconnect", cmd_set_auto_reconnect," on|off",complete_auto_reconnect,1,NULL},
        {"set_otherinfo", cmd_set_otherinfo,"<otherinfoinddex> <oid> <string>",NULL,0,NULL},
     {"register_oid", cmd_register_oid,"<name> <class> <oid>",NULL,0,NULL},
@@ -3663,6 +4037,17 @@ static void client(void)
     }
 }
 
+static void show_version(void)
+{
+    char vstr[20];
+
+    yaz_version(vstr, 0);
+    printf ("YAZ version: %s\n", YAZ_VERSION);
+    if (strcmp(vstr, YAZ_VERSION))
+       printf ("YAZ DLL/SO: %s\n", vstr);
+    exit(0);
+}
+
 int main(int argc, char **argv)
 {
     char *prog = *argv;
@@ -3680,8 +4065,10 @@ int main(int argc, char **argv)
     codeset = nl_langinfo(CODESET);
 #endif
 #endif
+    if (codeset)
+       outputCharset = xstrdup(codeset);
 
-    while ((ret = options("k:c:a:b:m:v:p:u:t:", argv, argc, &arg)) != -2)
+    while ((ret = options("k:c:q:a:b:m:v:p:u:t:Vxd:", argv, argc, &arg)) != -2)
     {
         switch (ret)
         {
@@ -3693,6 +4080,9 @@ int main(int argc, char **argv)
                 strcat (open_command, arg);
             }
             break;
+       case 'd':
+           dump_file_prefix = arg;
+           break;
         case 'k':
             kilobytes = atoi(arg);
             break;
@@ -3704,12 +4094,16 @@ int main(int argc, char **argv)
             }
             break;
         case 't':
-            codeset = arg;
+            outputCharset = xstrdup(arg);
             break;
         case 'c':
             strncpy (ccl_fields, arg, sizeof(ccl_fields)-1);
             ccl_fields[sizeof(ccl_fields)-1] = '\0';
             break;
+        case 'q':
+            strncpy (cql_fields, arg, sizeof(cql_fields)-1);
+            cql_fields[sizeof(cql_fields)-1] = '\0';
+            break;
         case 'b':
             if (!strcmp(arg, "-"))
                 ber_file=stderr;
@@ -3722,6 +4116,9 @@ int main(int argc, char **argv)
             else
                 apdu_file=fopen(arg, "a");
             break;
+       case 'x':
+           hex_dump = 1;
+           break;
         case 'p':
             yazProxy=strdup(arg);
             break;
@@ -3734,12 +4131,16 @@ int main(int argc, char **argv)
             }
             break;
         case 'v':
-            yaz_log_init (yaz_log_mask_str(arg), "", NULL);
+            yaz_log_init(yaz_log_mask_str(arg), "", 0);
+            break;
+        case 'V':
+           show_version();
             break;
         default:
             fprintf (stderr, "Usage: %s [-m <marclog>] [ -a <apdulog>] "
-                     "[-b berdump] [-c cclfields]\n      [-p <proxy-addr>] [-u <auth>] "
-                     "[-k size] [<server-addr>]\n",
+                     "[-b berdump] [-c cclfields] \n"
+                    "[-q cqlfields] [-p <proxy-addr>] [-u <auth>] "
+                     "[-k size] [-d dump] [-V] [<server-addr>]\n",
                      prog);
             exit (1);
         }