# Tell search engines not to index these folders
#disallow whole site
#User-agent: *
#Disallow: /

User-agent: * 
Disallow: /cgi-bin/ 
Disallow: /catalog/includes/
Disallow: /catalog/admin/ 
Disallow: /catalog/pub/ 
Disallow: /catalog/download/


#disallow google images bot
#User-Agent: Googlebot-Image
#Disallow: /images/

#Stop bad robots

User-agent: aipbot 
Disallow: / 

User-agent: BecomeBot 
Disallow: / 


User-agent: psbot 
Disallow: /