Updated version to 0.2.1.

[tclrobot.git] / robot.tcl

diff --git a/robot.tcl b/robot.tcl
index a90d6e8..608f562 100755 (executable)
--- a/robot.tcl
+++ b/robot.tcl
@@ -1,5 +1,5 @@
 #!/usr/bin/tclsh 
-# $Id: robot.tcl,v 1.43 2003/06/11 09:40:22 adam Exp $
+# $Id: robot.tcl,v 1.47 2003/12/10 09:58:22 adam Exp $
 #
 proc RobotFileNext1 {area lead} {
     # puts "RobotFileNext1 area=$area lead=$lead"
@@ -32,6 +32,7 @@ proc RobotFileNext1 {area lead} {
 }
 
 proc RobotWriteRecord {outf fromurl distance} {
+    puts $outf {<?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?>}
     puts $outf "<zmbot>"
     puts $outf "<distance>"
     puts $outf $distance
@@ -47,6 +48,7 @@ proc RobotReadRecord {inf fromurlx distancex} {
     upvar $distancex distance
     gets $inf
     gets $inf
+    gets $inf
     set distance [string trim [gets $inf]]
     # puts "got distance = $distance"
     gets $inf
@@ -608,6 +610,15 @@ proc RobotRedirect {task url tourl code} {
     }
 }
 
+proc wellform {body} {
+    regsub -all {<!--[^-]*-->} $body { } abody
+    regsub -all -nocase {<script[^<]*</script>} $abody {} body
+    regsub -all {<[^\>]+>} $body {} abody
+    regsub -all {&nbsp;} $abody { } body
+    regsub -all {&} $body {&amp;} abody
+    return $abody
+}
+
 proc link {task url out href body distance} {
     global URL control
     if {[expr $distance > $control($task,distance)]} return
@@ -616,7 +627,8 @@ proc link {task url out href body distance} {
     
     puts $out "<cr>"
     puts $out "<identifier>$href</identifier>"
-    puts $out "<description>$body</description>"
+    set abody [wellform $body]
+    puts $out "<description>$abody</description>"
     puts $out "</cr>"
     
     if {![RobotFileExist $task visited $host $path]} {
@@ -695,7 +707,7 @@ proc RobotTextHtml {task url out} {
                         set metacontent $parm($a)
                     }
                 }
-               unset parm($al)
+               unset parm($a)
             }
            puts $out "></meta>"
             # go through robots directives (af any)
@@ -712,11 +724,9 @@ proc RobotTextHtml {task url out} {
             # don't print title of document content if noindex is used
             if {!$noindex} {
                 puts $out "<title>$title</title>"
-                regsub -all {<!--[^-]*-->} $body { } abody
-                regsub -all -nocase {<script[^<]*</script>} $abody {} bbody
-                regsub -all {<[^\>]+>} $bbody {} nbody
+               set bbody [wellform $body]
                 puts $out "<documentcontent>"
-                puts $out $nbody
+                puts $out $bbody
                 puts $out "</documentcontent>"
             }
         } -nonest base {
@@ -727,7 +737,7 @@ proc RobotTextHtml {task url out} {
             set href [string trim $parm(href)]
             if {![RobotHref $task $url href host path]} continue
             set URL($task,$url,bpath) $path
-        } a {
+        } -nonest a {
             # <a href="...."> .. </a> 
             # we're not using nonest - otherwise body isn't set
             if {$nofollow} continue
@@ -801,6 +811,8 @@ proc RobotTextPlain {task url out} {
 proc RobotWriteMetadata {task url out} {
     global URL
 
+    set charset $URL($task,$url,charset)
+    puts $out "<?xml version=\"1.0\" encoding=\"$charset\" standalone=\"yes\"?>"
     puts $out "<zmbot>"
 
     set distance 1000
@@ -886,6 +898,7 @@ proc RobotReadHeader {task url sock} {
            set version {}
            set headbuf [string range $URL($task,$url,buf) 0 $n]
            incr n 4
+           set URL($task,$url,charset) ISO-8859-1
            set URL($task,$url,buf) [string range $URL($task,$url,buf) $n end]
            
            regexp {^HTTP/([0-9.]+)[ ]+([0-9]+)} $headbuf x version code
@@ -894,6 +907,7 @@ proc RobotReadHeader {task url sock} {
                if {[regexp {^([^:]+):[ ]+([^;]*)} $line x name value]} {
                    set URL($task,$url,head,[string tolower $name]) [string trim $value]
                }
+               regexp {^Content-Type:.*charset=([A-Za-z0-9_-]*)} $line x URL($task,$url,charset)
            }
            puts "HTTP CODE $code"
            set URL($task,$url,state) skip