complete error timeout handling for all tests

diff --git a/lib/ZOOM/IRSpy.pm b/lib/ZOOM/IRSpy.pm
index bfd593c..4e2040c 100644 (file)
--- a/lib/ZOOM/IRSpy.pm
+++ b/lib/ZOOM/IRSpy.pm
@@ -448,6 +448,7 @@ sub check {
            my $conn = $copy_conn[$i0];
            #print "connection $i0 of $nconn/", scalar(@conn), " is $conn\n";
            next if !defined $conn;
+
            if (!$conn->current_task()) {
                if (!$conn->next_task()) {
                    # Out of tasks: we need a new test
@@ -460,11 +461,13 @@ sub check {
                        $conn->log("irspy_test",
                                   "checking for next test after '$address'");
                        $nextaddr = $this->_next_test($address);
-                       if ($nextaddr && $conn->record->zoom_error->{TIMEOUT} >= $max_timeout_errors) {
-                           $conn->log("irspy", "Got to many timeouts, stop testing: " . $conn->record->zoom_error->{TIMEOUT});
-                           $nextaddr = "";
-                        }
                    }
+
+                    if (ZOOM::IRSpy::Test::zoom_error_timeout_check($conn)) {
+                       $conn->log("irspy", "Got to many timeouts, stop testing");
+                       undef $nextaddr;
+                    }
+
                    if (!defined $nextaddr) {
                        $conn->log("irspy", "has no more tests: removing");
                        $this->_rewrite_irspy_record($conn);
@@ -506,6 +509,13 @@ sub check {
 
                my $task = $conn->next_task();
                die "no next task queued for $conn" if !defined $task;
+
+               # do not run the next task if we got too many timeouts
+                if (ZOOM::IRSpy::Test::zoom_error_timeout_check($conn)) {
+                    $conn->log("irspy_task", "Got to many timeouts for this target, do not start a new task");
+                    next;
+                }
+
                $conn->log("irspy_task", "preparing task $task");
                $conn->next_task(0);
                $conn->current_task($task);

diff --git a/lib/ZOOM/IRSpy/Test.pm b/lib/ZOOM/IRSpy/Test.pm
index 097b9c5..0bf89bd 100644 (file)
--- a/lib/ZOOM/IRSpy/Test.pm
+++ b/lib/ZOOM/IRSpy/Test.pm
@@ -7,6 +7,9 @@ use warnings;
 
 use Scalar::Util;
 
+use Exporter 'import';
+our @EXPORT = qw(zoom_error_timeout_update zoom_error_timeout_check); 
+
 =head1 NAME
 
 ZOOM::IRSpy::Test - base class for tests in IRSpy
@@ -21,6 +24,7 @@ I<## To follow>
 
 =cut
 
+
 sub subtests { () }
 
 sub timeout { undef }
@@ -33,6 +37,29 @@ sub start {
 }
 
 
+our $max_timeout_errors = $ZOOM::IRSpy::max_timeout_errors;
+
+sub zoom_error_timeout_update {
+    my ($conn, $exception) = @_;
+
+    if ($exception =~ /Timeout/i) {
+        $conn->record->zoom_error->{TIMEOUT}++;
+        $conn->log("irspy_test", "Increase timeout error counter to: " .
+                $conn->record->zoom_error->{TIMEOUT});
+    }
+}
+
+sub zoom_error_timeout_check {
+    my $conn = shift;
+
+    if ($conn->record->zoom_error->{TIMEOUT} >= $max_timeout_errors) {
+        $conn->log("irspy_test", "Got $max_timeout_errors or more timeouts, give up...");
+        return  1;
+    }
+
+    return 0;
+}
+
 =head1 SEE ALSO
 
 ZOOM::IRSpy

diff --git a/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm b/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm
index 7da2596..9acc097 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm
+++ b/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm
@@ -103,6 +103,7 @@ sub error {
     my($conn, $task, $test_args, $exception) = @_;
 
     $conn->log("irspy_test", "Named resultset check failed:", $exception);
+    zoom_error_timeout_update($conn, $exception);
     return ZOOM::IRSpy::Status::TASK_DONE;
 }
 

diff --git a/lib/ZOOM/IRSpy/Test/Search/Bib1.pm b/lib/ZOOM/IRSpy/Test/Search/Bib1.pm
index 64c0d0e..fd4d1d6 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/Search/Bib1.pm
+++ b/lib/ZOOM/IRSpy/Test/Search/Bib1.pm
@@ -51,6 +51,7 @@ sub error {
     $conn->log("irspy_test", "search on access-point $attr had error: ",
               $exception);
     update($conn, $attr, 0);
+    zoom_error_timeout_update($conn, $exception);
 
     return ZOOM::IRSpy::Status::TEST_BAD
        if ($exception->code() == 1 || # permanent system error
@@ -68,5 +69,4 @@ sub update {
                                             'ok'        => $ok);
 }
 
-
 1;

diff --git a/lib/ZOOM/IRSpy/Test/Search/Boolean.pm b/lib/ZOOM/IRSpy/Test/Search/Boolean.pm
index 29bc211..ea8233a 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/Search/Boolean.pm
+++ b/lib/ZOOM/IRSpy/Test/Search/Boolean.pm
@@ -52,6 +52,7 @@ sub error {
     $conn->log("irspy_test", "search using boolean operator ", $operator,
                              " had error: ", $exception);
     update($conn, $operator, 0);
+    zoom_error_timeout_update($conn, $exception);
     return ZOOM::IRSpy::Status::TASK_DONE;
 }
 

diff --git a/lib/ZOOM/IRSpy/Test/Search/CQL.pm b/lib/ZOOM/IRSpy/Test/Search/CQL.pm
index c4f049b..5c21281 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/Search/CQL.pm
+++ b/lib/ZOOM/IRSpy/Test/Search/CQL.pm
@@ -52,6 +52,7 @@ sub error {
     $task->{rs}->destroy();
     $conn->log("irspy_test", "CQL search on '$index' had error: $exception");
     $conn->record()->store_result("search_cql", index => $index, ok => 0);
+    zoom_error_timeout_update($conn, $exception);
     return ZOOM::IRSpy::Status::TEST_BAD
        if $exception->code() == 11; # Unsupported query type
 

diff --git a/lib/ZOOM/IRSpy/Test/Search/DBDate.pm b/lib/ZOOM/IRSpy/Test/Search/DBDate.pm
index 981e92e..b349f1b 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/Search/DBDate.pm
+++ b/lib/ZOOM/IRSpy/Test/Search/DBDate.pm
@@ -50,6 +50,7 @@ sub error {
     my $rec = $conn->record();
     $rec->append_entry("irspy:status", "<irspy:search_dbdate ok='0'>" .
                       isodate(time()) . "</irspy:search_dbdate>");
+    zoom_error_timeout_update($conn, $exception);
     return ZOOM::IRSpy::Status::TEST_BAD;
 }
 

diff --git a/lib/ZOOM/IRSpy/Test/Search/Dan1.pm b/lib/ZOOM/IRSpy/Test/Search/Dan1.pm
index cbee4b4..415c5aa 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/Search/Dan1.pm
+++ b/lib/ZOOM/IRSpy/Test/Search/Dan1.pm
@@ -48,6 +48,7 @@ sub error {
     $conn->log("irspy_test", "search on access-point $attr had error: ",
               $exception);
     update($conn, $attr, 0);
+    zoom_error_timeout_update($conn, $exception);
 
     ### How about TEST_BAD if $exception->code() == 121?
     return ZOOM::IRSpy::Status::TASK_DONE;

diff --git a/lib/ZOOM/IRSpy/Test/Search/Explain.pm b/lib/ZOOM/IRSpy/Test/Search/Explain.pm
index 7bdad09..087b846 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/Search/Explain.pm
+++ b/lib/ZOOM/IRSpy/Test/Search/Explain.pm
@@ -53,6 +53,7 @@ sub error {
     $task->{rs}->destroy();
     $conn->log("irspy_test", "Explain category lookup failed: ", $exception);
     update($conn, $category, 0);
+    zoom_error_timeout_update($conn, $exception);
 
     return ZOOM::IRSpy::Status::TEST_BAD
        if ($exception->code() == 109 || # Database unavailable

diff --git a/lib/ZOOM/IRSpy/Test/Search/Title.pm b/lib/ZOOM/IRSpy/Test/Search/Title.pm
index 2e123f9..16d3a06 100644 (file)
--- a/lib/ZOOM/IRSpy/Test/Search/Title.pm
+++ b/lib/ZOOM/IRSpy/Test/Search/Title.pm
@@ -44,6 +44,7 @@ sub error {
     my $rec = $conn->record();
     $rec->append_entry("irspy:status", "<irspy:search_title ok='0'>" .
                       isodate(time()) . "</irspy:search_title>");
+    zoom_error_timeout_update($conn, $exception);
     return ZOOM::IRSpy::Status::TEST_BAD;
 }