Merge branch 'master' into fetch_more

diff --git a/src/client.c b/src/client.c
index ee6ead2..5f2df7b 100644 (file)
--- a/src/client.c
+++ b/src/client.c
@@ -115,6 +115,7 @@ struct client {
     char *addinfo; // diagnostic info for most resent error
     Odr_int hits;
     int record_offset;
+    int show_stat_no;
     int filtered; // When using local:, this will count the number of filtered records.
     int maxrecs;
     int startrecs;
@@ -763,6 +764,58 @@ static const char *get_strategy_plus_sort(struct client *l, const char *field)
     return strategy_plus_sort;
 }
 
+void client_update_show_stat(struct client *cl, int cmd)
+{
+    if (cmd == 0)
+        cl->show_stat_no = 0;
+    else if (cmd == 1)
+        cl->show_stat_no++;
+}
+
+int client_fetch_more(struct client *cl)
+{
+    struct session_database *sdb = client_get_database(cl);
+    const char *str;
+    int extend_recs = 0;
+    int number;
+
+    yaz_log(YLOG_LOG, "cl=%s show_stat_no=%d got=%d",
+            client_get_id(cl), cl->show_stat_no, cl->record_offset);
+    if (cl->show_stat_no < cl->record_offset)
+        return 0;
+    yaz_log(YLOG_LOG, "cl=%s Trying to get more", client_get_id(cl));
+
+    str = session_setting_oneval(sdb, PZ_EXTENDRECS);
+    if (str && *str)
+        extend_recs = atoi(str);
+
+    if (extend_recs > cl->hits)
+        extend_recs = cl->hits;
+
+    number = extend_recs - cl->record_offset;
+    if (number > 0)
+    {
+        ZOOM_resultset set = cl->resultset;
+        struct connection *co = client_get_connection(cl);
+
+        str = session_setting_oneval(sdb, PZ_REQUESTSYNTAX);
+        ZOOM_resultset_option_set(set, "preferredRecordSyntax", str);
+        str = session_setting_oneval(sdb, PZ_ELEMENTS);
+        if (str && *str)
+            ZOOM_resultset_option_set(set, "elementSetName", str);
+
+        ZOOM_resultset_records(set, 0, cl->record_offset, number);
+        client_set_state(cl, Client_Working);
+        connection_continue(co);
+        return 1;
+    }
+    else
+    {
+        yaz_log(YLOG_LOG, "cl=%s. OK no more in total set", client_get_id(cl));
+    }
+    return 0;
+}
+
 int client_parse_init(struct client *cl, int same_search)
 {
     cl->same_search = same_search;

diff --git a/src/client.h b/src/client.h
index bb66d60..d50fbdf 100644 (file)
--- a/src/client.h
+++ b/src/client.h
@@ -80,6 +80,7 @@ int client_prep_connection(struct client *cl,
                            iochan_man_t iochan,
                            const struct timeval *abstime);
 int client_start_search(struct client *cl);
+int client_fetch_more(struct client *cl);
 int client_parse_init(struct client *cl, int same_search);
 int client_parse_range(struct client *cl, const char *startrecs, const char *maxrecs);
 int client_parse_sort(struct client *cl, struct reclist_sortparms *sp);
@@ -116,6 +117,7 @@ const char *client_get_facet_limit_local(struct client *cl,
                                          NMEM nmem, int *num, char ***values);
 
 int client_test_sort_order(struct client *cl, struct reclist_sortparms *sp);
+void client_update_show_stat(struct client *cl, int cmd);
 
 #endif
 

diff --git a/src/http_command.c b/src/http_command.c
index f2f9b20..7620e92 100644 (file)
--- a/src/http_command.c
+++ b/src/http_command.c
@@ -93,6 +93,8 @@ struct http_sessions {
 static YAZ_MUTEX g_http_session_mutex = 0;
 static int g_http_sessions = 0;
 
+static void show_records_ready(void *data);
+
 int get_version(struct http_request *rq) {
     const char *version = http_argbyname(rq, "version");
     int version_no = 0;
@@ -1118,7 +1120,10 @@ static void show_records(struct http_channel *c, struct http_session *s, int act
 
     }
 
-    rl = show_range_start(s->psession, sp, startn, &numn, &total, &total_hits, &approx_hits);
+    rl = show_range_start(s->psession, sp, startn, &numn, &total,
+                          &total_hits, &approx_hits, show_records_ready, c);
+    if (!rl)
+        return;
 
     response_open(c, "show");
     wrbuf_printf(c->wrbuf, "\n<activeclients>%d</activeclients>\n", active);

diff --git a/src/session.c b/src/session.c
index 16f7794..e81c221 100644 (file)
--- a/src/session.c
+++ b/src/session.c
@@ -1229,10 +1229,44 @@ void show_single_stop(struct session *se, struct record_cluster *rec)
 }
 
 
+int session_fetch_more(struct session *se)
+{
+    struct client_list *l;
+    int ret = 0;
+
+    for (l = se->clients_active; l; l = l->next)
+    {
+        struct client *cl = l->client;
+        if (client_get_state(cl) == Client_Idle)
+        {
+            if (client_fetch_more(cl))
+            {
+                session_log(se, YLOG_LOG, "%s: more to fetch",
+                            client_get_id(cl));
+                ret = 1;
+            }
+            else
+            {
+                session_log(se, YLOG_LOG, "%s: no more to fetch",
+                            client_get_id(cl));
+            }
+        }
+        else
+        {
+            session_log(se, YLOG_LOG, "%s: no fetch due to state=%s",
+                        client_get_id(cl), client_get_state_str(cl));
+        }
+
+    }
+    return ret;
+}
+
 struct record_cluster **show_range_start(struct session *se,
                                          struct reclist_sortparms *sp,
                                          int start, int *num, int *total,
-                                         Odr_int *sumhits, Odr_int *approx_hits)
+                                         Odr_int *sumhits, Odr_int *approx_hits,
+                                         void (*show_records_ready)(void *data),
+                                         struct http_channel *chan)
 {
     struct record_cluster **recs = 0;
     struct reclist_sortparms *spp;
@@ -1264,13 +1298,24 @@ struct record_cluster **show_range_start(struct session *se,
     reclist_enter(se->reclist);
     *total = reclist_get_num_records(se->reclist);
 
+    for (l = se->clients_active; l; l = l->next)
+        client_update_show_stat(l->client, 0);
+
     for (i = 0; i < start; i++)
-        if (!reclist_read_record(se->reclist))
+    {
+        struct record_cluster *r = reclist_read_record(se->reclist);
+        if (!r)
         {
             *num = 0;
             break;
         }
-
+        else
+        {
+            struct record *rec = r->records;
+            for (;rec; rec = rec->next)
+                client_update_show_stat(rec->client, 1);
+        }
+    }
     if (*num > 0)
         recs =
             nmem_malloc(se->nmem, *num * sizeof(struct record_cluster *));
@@ -1282,7 +1327,13 @@ struct record_cluster **show_range_start(struct session *se,
             *num = i;
             break;
         }
-        recs[i] = r;
+        else
+        {
+            struct record *rec = r->records;
+            for (;rec; rec = rec->next)
+                client_update_show_stat(rec->client, 1);
+            recs[i] = r;
+        }
     }
     reclist_leave(se->reclist);
 #if USE_TIMING
@@ -1292,6 +1343,25 @@ struct record_cluster **show_range_start(struct session *se,
             yaz_timing_get_sys(t));
     yaz_timing_destroy(&t);
 #endif
+
+    if (!session_fetch_more(se))
+        session_log(se, YLOG_LOG, "can not fetch more");
+    else
+    {
+        show_range_stop(se, recs);
+        session_log(se, YLOG_LOG, "fetching more in progress");
+        if (session_set_watch(se, SESSION_WATCH_SHOW,
+                              show_records_ready, chan, chan))
+        {
+            session_log(se, YLOG_WARN, "Ignoring show block");
+            session_enter(se, "show_range_start");
+        }
+        else
+        {
+            session_log(se, YLOG_LOG, "session watch OK");
+            return 0;
+        }
+    }
     return recs;
 }
 

diff --git a/src/session.h b/src/session.h
index 66c41ba..0b05afc 100644 (file)
--- a/src/session.h
+++ b/src/session.h
@@ -170,7 +170,12 @@ enum pazpar2_error_code session_search(struct session *s, const char *query,
 struct record_cluster **show_range_start(struct session *s,
                                          struct reclist_sortparms *sp,
                                          int start,
-                                         int *num, int *total, Odr_int *sumhits, Odr_int *approximation);
+                                         int *num, int *total,
+                                         Odr_int *sumhits,
+                                         Odr_int *approximation,
+                                         void (*ready)(void *data),
+                                         struct http_channel *chan);
+int session_fetch_more(struct session *s);
 void show_range_stop(struct session *s, struct record_cluster **recs);
 
 struct record_cluster *show_single_start(struct session *s, const char *id,

diff --git a/src/settings.c b/src/settings.c
index 233125e..d2e15ec 100644 (file)
--- a/src/settings.c
+++ b/src/settings.c
@@ -81,6 +81,7 @@ static char *hard_settings[] = {
     "pz:sortmap:",
     "pz:present_chunk",
     "pz:block_timeout",
+    "pz:extendrecs",
     0
 };
 

diff --git a/src/settings.h b/src/settings.h
index f6a33fd..b0da583 100644 (file)
--- a/src/settings.h
+++ b/src/settings.h
@@ -54,7 +54,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #define PZ_SORTMAP              31
 #define PZ_PRESENT_CHUNK        32
 #define PZ_BLOCK_TIMEOUT        33
-#define PZ_MAX_EOF              34
+#define PZ_EXTENDRECS           34
+#define PZ_MAX_EOF              35
 
 struct setting
 {

diff --git a/test/gils_service.xml b/test/gils_service.xml
index daecf4d..984bab6 100644 (file)
--- a/test/gils_service.xml
+++ b/test/gils_service.xml
@@ -23,6 +23,7 @@
        <set name="pz:apdulog" value="1"/>
 
        <set name="pz:maxrecs" value="3" />
+       <set name="pz:extendrecs" value="6" />
       </settings>
 
       <metadata name="url" merge="unique"/>

diff --git a/test/test_http_28.res b/test/test_http_28.res
index d32263c..f14ba5c 100644 (file)
--- a/test/test_http_28.res
+++ b/test/test_http_28.res
@@ -1,10 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <show><status>OK</status>
 <activeclients>0</activeclients>
-<merged>3</merged>
+<merged>6</merged>
 <total>17</total>
 <start>0</start>
-<num>3</num>
+<num>6</num>
 <hit>
  <md-title>UTAH GEOCHROMOMETRY</md-title>
  <location id="z3950.indexdata.com/gils"
@@ -12,54 +12,108 @@
   <md-title>UTAH GEOCHROMOMETRY</md-title>
  </location>
  <count>1</count>
- <relevance>86304</relevance>
+ <relevance>100941</relevance>
  <relevance_info>
 field=title content=UTAH GEOCHROMOMETRY;
 utah: w[1] += w(6) / (1+log2(1+lead_decay(0.000000) * length(0)));
 utah: tf[1] += w[1](6) / length(2) (3.000000);
 relevance = 0;
-idf[1] = log(((1 + total(3))/termoccur(3));
-utah: relevance += 100000 * tf[1](3.000000) * idf[1](0.287682) (86304);
-score = relevance(86304);
+idf[1] = log(((1 + total(6))/termoccur(5));
+utah: relevance += 100000 * tf[1](3.000000) * idf[1](0.336472) (100941);
+score = relevance(100941);
  </relevance_info>
  <recid>content: title utah geochromometry author medium book</recid>
 </hit>
 <hit>
+ <md-title>UTAH CRIB FILE</md-title>
+ <location id="z3950.indexdata.com/gils"
+    name="Index Data GILS test server" checksum="2596737976">
+  <md-title>UTAH CRIB FILE</md-title>
+ </location>
+ <count>1</count>
+ <relevance>67294</relevance>
+ <relevance_info>
+field=title content=UTAH CRIB FILE;
+utah: w[1] += w(6) / (1+log2(1+lead_decay(0.000000) * length(0)));
+utah: tf[1] += w[1](6) / length(3) (2.000000);
+relevance = 0;
+idf[1] = log(((1 + total(6))/termoccur(5));
+utah: relevance += 100000 * tf[1](2.000000) * idf[1](0.336472) (67294);
+score = relevance(67294);
+ </relevance_info>
+ <recid>content: title utah crib file author medium book</recid>
+</hit>
+<hit>
  <md-title>UTAH EARTHQUAKE EPICENTERS</md-title>
  <location id="z3950.indexdata.com/gils"
     name="Index Data GILS test server" checksum="1725776531">
   <md-title>UTAH EARTHQUAKE EPICENTERS</md-title>
  </location>
  <count>1</count>
- <relevance>57536</relevance>
+ <relevance>67294</relevance>
  <relevance_info>
 field=title content=UTAH EARTHQUAKE EPICENTERS;
 utah: w[1] += w(6) / (1+log2(1+lead_decay(0.000000) * length(0)));
 utah: tf[1] += w[1](6) / length(3) (2.000000);
 relevance = 0;
-idf[1] = log(((1 + total(3))/termoccur(3));
-utah: relevance += 100000 * tf[1](2.000000) * idf[1](0.287682) (57536);
-score = relevance(57536);
+idf[1] = log(((1 + total(6))/termoccur(5));
+utah: relevance += 100000 * tf[1](2.000000) * idf[1](0.336472) (67294);
+score = relevance(67294);
  </relevance_info>
  <recid>content: title utah earthquake epicenters author medium book</recid>
 </hit>
 <hit>
+ <md-title>UTAH OIL FIELD FILE</md-title>
+ <location id="z3950.indexdata.com/gils"
+    name="Index Data GILS test server" checksum="2248353398">
+  <md-title>UTAH OIL FIELD FILE</md-title>
+ </location>
+ <count>1</count>
+ <relevance>50470</relevance>
+ <relevance_info>
+field=title content=UTAH OIL FIELD FILE;
+utah: w[1] += w(6) / (1+log2(1+lead_decay(0.000000) * length(0)));
+utah: tf[1] += w[1](6) / length(4) (1.500000);
+relevance = 0;
+idf[1] = log(((1 + total(6))/termoccur(5));
+utah: relevance += 100000 * tf[1](1.500000) * idf[1](0.336472) (50470);
+score = relevance(50470);
+ </relevance_info>
+ <recid>content: title utah oil field file author medium book</recid>
+</hit>
+<hit>
  <md-title>UTAH GEOLOGICAL AND MINERAL SURVEY PUBLICATIONS</md-title>
  <location id="z3950.indexdata.com/gils"
     name="Index Data GILS test server" checksum="1899968820">
   <md-title>UTAH GEOLOGICAL AND MINERAL SURVEY PUBLICATIONS</md-title>
  </location>
  <count>1</count>
- <relevance>28768</relevance>
+ <relevance>33647</relevance>
  <relevance_info>
 field=title content=UTAH GEOLOGICAL AND MINERAL SURVEY PUBLICATIONS;
 utah: w[1] += w(6) / (1+log2(1+lead_decay(0.000000) * length(0)));
 utah: tf[1] += w[1](6) / length(6) (1.000000);
 relevance = 0;
-idf[1] = log(((1 + total(3))/termoccur(3));
-utah: relevance += 100000 * tf[1](1.000000) * idf[1](0.287682) (28768);
-score = relevance(28768);
+idf[1] = log(((1 + total(6))/termoccur(5));
+utah: relevance += 100000 * tf[1](1.000000) * idf[1](0.336472) (33647);
+score = relevance(33647);
  </relevance_info>
  <recid>content: title utah geological and mineral survey publications author medium book</recid>
 </hit>
+<hit>
+ <md-title>APPLIED GEOLOGY FILE</md-title>
+ <location id="z3950.indexdata.com/gils"
+    name="Index Data GILS test server" checksum="2422545687">
+  <md-title>APPLIED GEOLOGY FILE</md-title>
+ </location>
+ <count>1</count>
+ <relevance>0</relevance>
+ <relevance_info>
+relevance = 0;
+idf[1] = log(((1 + total(6))/termoccur(5));
+utah: relevance += 100000 * tf[1](0.000000) * idf[1](0.336472) (0);
+score = relevance(0);
+ </relevance_info>
+ <recid>content: title applied geology file author medium book</recid>
+</hit>
 </show>
\ No newline at end of file