User-Agent: * 
 Allow: / 
 Disallow: /cgi-bin

User-agent: MFHttpScan
Disallow: /

User-agent: Mozilla/3.0 (compatible; Indy Library)
Disallow: /