From 9d3f82cd1140362487d8fa6372cac1b24a49d21e Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Wed, 14 Nov 2001 09:15:23 +0000 Subject: [PATCH] File status written with counts of areas: unvisited, bad, visited. Tag area src=.. used for relative links. --- robot.tcl | 245 ++++++++++++++++++++++++------------------------------------- 1 file changed, 94 insertions(+), 151 deletions(-) diff --git a/robot.tcl b/robot.tcl index ee70b9a..b6466c3 100755 --- a/robot.tcl +++ b/robot.tcl @@ -1,5 +1,5 @@ #!/usr/bin/tclsh -# $Id: robot.tcl,v 1.28 2001/11/13 11:17:26 adam Exp $ +# $Id: robot.tcl,v 1.29 2001/11/14 09:15:23 adam Exp $ # proc RobotFileNext1 {area lead} { # puts "RobotFileNext1 area=$area lead=$lead" @@ -69,7 +69,9 @@ proc RobotFileNext {area} { if {![string length $n]} { set robotSeq -1 flush stdout - puts "Round robin un,ba,vi=$status(unvisited),$status(bad),$status(visited)" + set statusfile [open status w] + puts $statusfile "$status(unvisited) $status(bad) $status(visited)" + close $statusfile return wait } incr robotSeq @@ -89,14 +91,20 @@ proc RobotFileNext {area} { proc RobotFileExist {area host path} { - # puts "RobotFileExist begin area=$area host=$host path=$path" + global debuglevel + + if {$debuglevel > 3} { + puts "RobotFileExist begin area=$area host=$host path=$path" + } set lpath [split $path /] set l [llength $lpath] incr l -1 set t [lindex $lpath $l] incr l -1 set npath $area/$host[join [lrange $lpath 0 $l] /d]/f$t - # puts "RobotFileExist end npath=$npath" + if {$debuglevel > 3} { + puts "RobotFileExist end npath=$npath" + } return [file exists $npath] } @@ -136,11 +144,14 @@ proc RobotFileOpen {area host path {mode w}} { set orgPwd [pwd] global workdir global status + global debuglevel if {![info exists workdir]} { return stdout } - #puts "RobotFileOpen orgPwd=$orgPwd area=$area host=$host path=$path mode=$mode" + if {$debuglevel > 3} { + puts "RobotFileOpen orgPwd=$orgPwd area=$area host=$host path=$path mode=$mode" + } if {[string compare $orgPwd $workdir]} { puts "ooops. RobotFileOpen failed" puts "workdir = $workdir" @@ -169,11 +180,14 @@ proc RobotFileOpen {area host path {mode w}} { } set d [lindex $comp $len] if {[string length $d]} { - if {[file isdirectory $d]} { - set out [open $d/f $mode] - } else { - set out [open f$d $mode] - } + set out [open f$d $mode] + if {0} { + if {[file isfile $d/f]} { + set out [open $d/f $mode] + } else { + set out [open f$d $mode] + } + } } else { set out [open f $mode] } @@ -320,7 +334,11 @@ proc RobotHref {url hrefx hostx pathx} { } if {[string first / $surl]} { # relative path - regexp {^([^\#?]*)} $URL($url,path) x dpart + set curpath $URL($url,path) + if {[info exists URL($url,bpath)]} { + set curpath $URL($url,bpath) + } + regexp {^([^\#?]*)} $curpath x dpart set l [string last / $dpart] if {[expr $l >= 0]} { set surl [string range $dpart 0 $l]$surl @@ -433,12 +451,62 @@ proc RobotRedirect {url tourl code} { } } +proc link {url out href body distance} { + global URL maxdistance + if {[expr $distance > $maxdistance]} return + + if {![RobotHref $url href host path]} return + + puts $out "" + puts $out "$href" + puts $out "$body" + puts $out "" + + if {![RobotFileExist visited $host $path]} { + set olddistance 1000 + if {![RobotFileExist bad $host $path]} { + if {[RobotFileExist unvisited $host $path]} { + set inf [RobotFileOpen unvisited $host $path r] + RobotReadRecord $inf oldurl olddistance + RobotFileClose $inf + } + } else { + set olddistance 0 + } + if {[string length $olddistance] == 0} { + set olddistance 1000 + } + if {[expr $distance < $olddistance]} { + set outf [RobotFileOpen unvisited $host $path] + RobotWriteRecord $outf $url $distance + RobotFileClose $outf + } + } elseif {[string compare $href $url]} { + set inf [RobotFileOpen visited $host $path r] + RobotReadRecord $inf xurl olddistance + close $inf + if {[string length $olddistance] == 0} { + set olddistance 1000 + } + if {[expr $distance < $olddistance]} { + puts "OK remarking url=$url href=$href" + puts "olddistance = $olddistance" + puts "newdistance = $distance" + set outf [RobotFileOpen unvisited $host $path] + RobotWriteRecord $outf $url $distance + RobotFileClose $outf + } + } +} + proc RobotTextHtml {url out} { global URL maxdistance set distance 0 + set fdistance 0 if {$maxdistance < 1000 && [info exists URL($url,dist)]} { - set distance [expr $URL($url,dist) + 1] + set fdistance $URL($url,dist) + set distance [expr $fdistance + 1] } htmlSwitch $URL($url,buf) \ title { @@ -459,156 +527,28 @@ proc RobotTextHtml {url out} { puts $out "" puts $out $nbody puts $out "" + } -nonest base { + if {![info exists parm(href)]} { + continue + } + set href [string trim $parm(href)] + if {![RobotHref $url href host path]} continue + set URL($url,bpath) $path } -nonest a { if {![info exists parm(href)]} { - puts "no href" continue } - if {[expr $distance <= $maxdistance]} { - set href [string trim $parm(href)] - if {![RobotHref $url href host path]} continue - - puts $out "" - puts $out "$href" - puts $out "$body" - puts $out "" - - if {![RobotFileExist visited $host $path]} { - set olddistance 1000 - if {![RobotFileExist bad $host $path]} { - if {[RobotFileExist unvisited $host $path]} { - set inf [RobotFileOpen unvisited $host $path r] - RobotReadRecord $inf oldurl olddistance - RobotFileClose $inf - } - } else { - set olddistance 0 - } - if {[string length $olddistance] == 0} { - set olddistance 1000 - } - if {[expr $distance < $olddistance]} { - set outf [RobotFileOpen unvisited $host $path] - RobotWriteRecord $outf $url $distance - RobotFileClose $outf - } - } elseif {[string compare $href $url]} { - set inf [RobotFileOpen visited $host $path r] - RobotReadRecord $inf xurl olddistance - close $inf - if {[string length $olddistance] == 0} { - set olddistance 1000 - } - if {[expr $distance < $olddistance]} { - puts "OK remarking url=$url href=$href" - puts "olddistance = $olddistance" - puts "newdistance = $distance" - set outf [RobotFileOpen unvisited $host $path] - RobotWriteRecord $outf $url $distance - RobotFileClose $outf - } - } - } + link $url $out [string trim $parm(href)] $body $distance } -nonest area { if {![info exists parm(href)]} { - puts "no href" continue } - if {[expr $distance <= $maxdistance]} { - set href [string trim $parm(href)] - if {![RobotHref $url href host path]} continue - - puts $out "" - puts $out "$href" - puts $out "" - puts $out "" - - if {![RobotFileExist visited $host $path]} { - set olddistance 1000 - if {![RobotFileExist bad $host $path]} { - if {[RobotFileExist unvisited $host $path]} { - set inf [RobotFileOpen unvisited $host $path r] - RobotReadRecord $inf oldurl olddistance - RobotFileClose $inf - } - } else { - set olddistance 0 - } - if {[string length $olddistance] == 0} { - set olddistance 1000 - } - if {[expr $distance < $olddistance]} { - set outf [RobotFileOpen unvisited $host $path] - RobotWriteRecord $outf $url $distance - RobotFileClose $outf - } - } elseif {[string compare $href $url]} { - set inf [RobotFileOpen visited $host $path r] - RobotReadRecord $inf xurl olddistance - close $inf - if {[string length $olddistance] == 0} { - set olddistance 1000 - } - if {[expr $distance < $olddistance]} { - puts "OK remarking url=$url href=$href" - puts "olddistance = $olddistance" - puts "newdistance = $distance" - set outf [RobotFileOpen unvisited $host $path] - RobotWriteRecord $outf $url $distance - RobotFileClose $outf - } - } - } + link $url $out [string trim $parm(href)] $body $distance } -nonest frame { if {![info exists parm(src)]} { - puts "no src" continue } - if {[expr $distance <= $maxdistance]} { - set href [string trim $parm(src)] - if {![RobotHref $url href host path]} continue - - puts $out "" - puts $out "$href" - puts $out "" - puts $out "" - - if {![RobotFileExist visited $host $path]} { - set olddistance 1000 - if {![RobotFileExist bad $host $path]} { - if {[RobotFileExist unvisited $host $path]} { - set inf [RobotFileOpen unvisited $host $path r] - RobotReadRecord $inf oldurl olddistance - RobotFileClose $inf - } - } else { - set olddistance 0 - } - if {[string length $olddistance] == 0} { - set olddistance 1000 - } - if {[expr $distance < $olddistance]} { - set outf [RobotFileOpen unvisited $host $path] - RobotWriteRecord $outf $url $distance - RobotFileClose $outf - } - } elseif {[string compare $href $url]} { - set inf [RobotFileOpen visited $host $path r] - RobotReadRecord $inf xurl olddistance - close $inf - if {[string length $olddistance] == 0} { - set olddistance 1000 - } - if {[expr $distance < $olddistance]} { - puts "OK remarking url=$url href=$href" - puts "olddistance = $olddistance" - puts "newdistance = $distance" - set outf [RobotFileOpen unvisited $host $path] - RobotWriteRecord $outf $url $distance - RobotFileClose $outf - } - } - } + link $url $out [string trim $parm(src)] $body $fdistance } } @@ -1082,4 +1022,7 @@ while {$robotsRunning} { vwait robotsRunning } -puts "End un,ba,vi=$status(unvisited),$status(bad),$status(visited)" +set statusfile [open status w] +puts $statusfile "$status(unvisited) $status(bad) $status(visited)" +close $statusfile + -- 1.7.10.4