X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=robot.tcl;h=2e175d623cb41f9c62a06a7b0d318a29dca11b45;hb=833faf12d797d629cae34abc8e84e88a6044eb7f;hp=7c3dbe6c63ec7dcb741e3fd0fc56b0f0ecff6466;hpb=ca0419e13f1efe7c3d56d59766f48a826cd6c080;p=tclrobot.git

diff --git a/robot.tcl b/robot.tcl
index 7c3dbe6..2e175d6 100755
--- a/robot.tcl
+++ b/robot.tcl
@@ -1,5 +1,5 @@
 #!/usr/bin/tclsh 
-# $Id: robot.tcl,v 1.21 2001/10/26 13:26:11 adam Exp $
+# $Id: robot.tcl,v 1.25 2001/11/07 11:50:07 adam Exp $
 #
 proc RobotFileNext1 {area lead} {
     # puts "RobotFileNext1 area=$area lead=$lead"
@@ -441,8 +441,9 @@ proc RobotTextHtml {url out} {
             }
 	    puts $out {></meta>}
 	} body {
-	    regsub -all -nocase {<script([^<]|(<!.*>))*</script>} $body {} abody
-	    regsub -all {<[^\>]+>} $abody {} nbody
+	    regsub -all {<!--[^-]*->} $body { } abody
+	    regsub -all -nocase {<script[^<]*</script>} $abody {} bbody
+	    regsub -all {<[^\>]+>} $bbody {} nbody
 	    puts $out "<documentcontent>"
             puts $out $nbody
             puts $out "</documentcontent>"
@@ -597,19 +598,9 @@ proc RobotTextPlain {url out} {
     }
 }
 
-proc Robot200 {url} {
+proc RobotWriteMetadata {url out} {
     global URL domains
-    
-    set out [RobotFileOpen raw $URL($url,hostport) $URL($url,path)]
-    puts -nonewline $out $URL($url,buf)
-    RobotFileClose $out
-
-    if {![checkrule mime $URL($url,head,content-type)]} {
-        RobotError $url mimedeny
-        return
-    }
 
-    set out [RobotFileOpen visited $URL($url,hostport) $URL($url,path)]
     puts $out "<zmbot>"
 
     set distance 1000
@@ -640,8 +631,23 @@ proc Robot200 {url} {
         }
     }
     puts $out "</zmbot>"
+}
+
+proc Robot200 {url} {
+    global URL domains
+    
+    set out [RobotFileOpen raw $URL($url,hostport) $URL($url,path)]
+    puts -nonewline $out $URL($url,buf)
     RobotFileClose $out
-    # puts "Parsing done"
+
+    if {![checkrule mime $URL($url,head,content-type)]} {
+        RobotError $url mimedeny
+        return
+    }
+    set out [RobotFileOpen visited $URL($url,hostport) $URL($url,path)]
+    RobotWriteMetadata $url $out
+    RobotFileClose $out
+
     RobotFileUnlink unvisited $URL($url,hostport) $URL($url,path)
 }
 
@@ -861,12 +867,29 @@ proc checkrule {type this} {
             }
             # consider type
             if {[lindex $l 1] != $type} continue
-            # consider mask
-            if {![string match [lindex $l 2] $this]} continue
+            # consider mask (! negates)
+            set masks [lindex $l 2]
+	    set ok 0
+	    foreach mask $masks {	
+                if {$debuglevel > 4} {
+                    puts "consider single mask $mask"
+                }
+                if {[string index $mask 0] == "!"} {
+                    set mask [string range $mask 1 end]
+                    if {[string match $mask $this]}  continue
+                } else {
+                    if {![string match $mask $this]} continue
+                }
+                set ok 1
+            }
+            if {$debuglevel > 4} {
+                puts "ok = $ok"
+            }
+            if {!$ok} continue
             # OK, we have a match
             if {[lindex $l 0] == "allow"} {
                 if {$debuglevel > 3} {
-                    puts "CHECKRULE MATH OK"
+                    puts "CHECKRULE MATCH OK"
                 }
                 return 1
             } else {
@@ -878,7 +901,7 @@ proc checkrule {type this} {
         }
     }
     if {$debuglevel > 3} {
-        puts "CHECKRULE MATH OK"
+        puts "CHECKRULE MATCH OK"
     }
     return 1
 }