+ set buffer [read $sock 16384]
+ set readCount [string length $buffer]
+
+ if {$readCount <= 0} {
+ Robot200 $url
+ RobotRestart $url $sock
+ } elseif {!$binary && [string first \0 $buffer] >= 0} {
+ Robot200 $url
+ RobotRestart $url $sock
+ } else {
+ # puts "Got $readCount bytes"
+ set URL($url,buf) $URL($url,buf)$buffer
+ }
+}
+
+proc RobotReadHeader {url sock} {
+ global URL debuglevel
+
+ if {$debuglevel > 1} {
+ puts "HTTP head $url"
+ }
+ if {[catch {set buffer [read $sock 2148]}]} {
+ RobotError $url 404
+ RobotRestart $url $sock
+ return
+ }
+ set readCount [string length $buffer]
+
+ if {$readCount <= 0} {
+ RobotError $url 404
+ RobotRestart $url $sock
+ } else {
+ # puts "Got $readCount bytes"
+ set URL($url,buf) $URL($url,buf)$buffer
+
+ set n [string first \r\n\r\n $URL($url,buf)]
+ if {$n > 1} {
+ set code 0
+ set version {}
+ set headbuf [string range $URL($url,buf) 0 $n]
+ incr n 4
+ set URL($url,buf) [string range $URL($url,buf) $n end]
+
+ regexp {^HTTP/([0-9.]+)[ ]+([0-9]+)} $headbuf x version code
+ set lines [split $headbuf \n]
+ foreach line $lines {
+ if {[regexp {^([^:]+):[ ]+([^;]*)} $line x name value]} {
+ set URL($url,head,[string tolower $name]) [string trim $value]
+ }
+ }
+ puts "HTTP CODE $code"
+ set URL($url,state) skip
+ switch $code {
+ 301 {
+ RobotRedirect $url $URL($url,head,location) 301
+ RobotRestart $url $sock
+ }
+ 302 {
+ RobotRedirect $url $URL($url,head,location) 302
+ RobotRestart $url $sock
+ }
+ 200 {
+ if {![info exists URL($url,head,content-type)]} {
+ set URL($url,head,content-type) {}
+ }
+ set binary 1
+ switch -glob -- $URL($url,head,content-type) {
+ text/* {
+ set binary 0
+ }
+ }
+ if {![regexp {/robots.txt$} $url]} {
+ if {![checkrule mime $URL($url,head,content-type)]} {
+ RobotError $url mimedeny
+ RobotRestart $url $sock
+ return
+ }
+ }
+ fileevent $sock readable [list RobotReadContent $url $sock $binary]
+ }
+ default {
+ RobotError $url $code
+ RobotRestart $url $sock
+ }
+ }
+ }
+ }
+}
+
+proc RobotSockCancel {url sock} {
+
+ puts "RobotSockCancel sock=$sock url=$url"
+ RobotError $url 401
+ RobotRestart $url $sock
+}
+
+proc RobotConnect {url sock} {
+ global URL agent acceptLanguage
+
+ fconfigure $sock -translation {lf crlf} -blocking 0
+ fileevent $sock readable [list RobotReadHeader $url $sock]