From 6939cb252548edb02672e3bec5b009aa815e4c25 Mon Sep 17 00:00:00 2001 From: Wolfram Schneider Date: Wed, 21 Apr 2010 14:20:22 +0200 Subject: [PATCH] add missing pages and reorder entries Google has crawled more than 10.900 pages from irspy.indexdata.com - how stupid! --- web/htdocs/robots.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/web/htdocs/robots.txt b/web/htdocs/robots.txt index beef7ee..d19bcff 100644 --- a/web/htdocs/robots.txt +++ b/web/htdocs/robots.txt @@ -1,8 +1,11 @@ User-agent: * -Disallow: /full.html -Disallow: /all.html +Disallow: /admin/ Disallow: /add.html -Disallow: /upload.html +Disallow: /all.html +Disallow: /doc.html +Disallow: /find.html +Disallow: /full.html Disallow: /raw.html -Disallow: /admin/ +Disallow: /stats.html +Disallow: /upload.html -- 1.7.10.4