User-agent: BUbiNG User-agent: AlphaSeoBot User-agent: ltx71 User-agent: Companybook-Crawler User-agent: BDCbot User-agent: spbot User-agent: SemrushBot User-agent: SiteAuditBot User-agent: AhrefsBot User-agent: MJ12bot User-agent: DotBot User-agent: omgili User-agent: BLEXBot User-agent: magpie-crawler User-agent: ExtLinksBot User-agent: netseer User-agent: weborama-fetcher User-agent: Linkfluence User-agent: sentibot User-agent: SEOkicks User-agent: Barkrowler User-agent: CCBot User-agent: trendictionbot User-agent: Amazonbot User-agent: serpstatbot User-agent: PetalBot User-agent: DataForSeoBot User-agent: CensysInspect User-agent: AwarioRssBot User-agent: AwarioSmartBot User-agent: AwarioBot User-agent: PhxBot User-agent: Bytespider User-agent: bl.uk_lddc_bot User-agent: ecoresearchCrawler User-agent: turnitinbot User-agent: Zoominfobot User-agent: dataprovider User-agent: VelenPublicWebCrawler User-agent: DomainStatsBot User-agent: HypeStat User-agent: panscient.com User-agent: YaK User-agent: LCC User-agent: MakeMerryBot User-agent: IonCrawl User-agent: AndersPinkBot User-agent: CheckMarkNetwork/1.0 (+https://www.checkmarknetwork.com/spider.html) Disallow: / # Most desirable images are hosted on S3. These images will just be icons and stuff, so block them. User-agent: Googlebot-Image Disallow: / Allow: /site/images/support/ # allowing access to image scripts and topic pages for proper sharing User-agent: facebookexternalhit Disallow: /images/ Disallow: /pm Disallow: /album Disallow: /mbactions Disallow: /register Disallow: /email Disallow: /search Disallow: /upload Disallow: /printthread Disallow: /post/show_single_post Disallow: /post/printadd Disallow: /post/upost Disallow: /post/hpt Disallow: /subscribe Disallow: /calendar Disallow: /calendar/newevent Disallow: /calendar/daydetail?*&nav= Disallow: /calendar/display*view=weekly Disallow: /calendar/display*view=monthly Disallow: /calendar/showbirthday Disallow: /external Disallow: /tool/view/gb/private Disallow: /tool/view/gb/email Disallow: /tool/pm Disallow: /tool/members/ Disallow: /tool/ticket/ Disallow: /cgi/view/poll.cgi Disallow: /cgi/view/topsites.cgi Disallow: /cgi/view/member.cgi Disallow: /cgi/view/out.cgi Disallow: /?authtoken= Disallow: *?*&sort= Disallow: *?sort= Allow: /site/images/support/ Crawl-Delay: 15 # allowing access to topic pages for proper context ads User-agent: Mediapartners-Google Disallow: /images/ Disallow: /pm Disallow: /album Disallow: /mbactions Disallow: /register Disallow: /email Disallow: /search Disallow: /profile Disallow: /file Disallow: /thumb Disallow: /upload Disallow: /printthread Disallow: /post/show_single_post Disallow: /post/printadd Disallow: /post/upost Disallow: /post/hpt Disallow: /subscribe Disallow: /calendar Disallow: /calendar/newevent Disallow: /calendar/daydetail?*&nav= Disallow: /calendar/display*view=weekly Disallow: /calendar/display*view=monthly Disallow: /calendar/showbirthday Disallow: /external Disallow: /tool/view/gb/private Disallow: /tool/view/gb/email Disallow: /tool/pm Disallow: /tool/members/ Disallow: /tool/ticket/ Disallow: /cgi/view/poll.cgi Disallow: /cgi/view/topsites.cgi Disallow: /cgi/view/member.cgi Disallow: /cgi/view/out.cgi Disallow: /?authtoken= Crawl-Delay: 15 # Disallow pages with very little content, duplicate content, or different links pointing to the same content User-agent: * Disallow: /images/ Disallow: /pm Disallow: /album Disallow: /mbactions Disallow: /register Disallow: /email Disallow: /search Disallow: /profile Disallow: /file Disallow: /thumb Disallow: /upload Disallow: /printthread Disallow: /post/show_single_post Disallow: /post/printadd Disallow: /post/upost Disallow: /post/hpt Disallow: /subscribe Disallow: /calendar Disallow: /calendar/newevent Disallow: /calendar/daydetail?*&nav= Disallow: /calendar/display*view=weekly Disallow: /calendar/display*view=monthly Disallow: /calendar/showbirthday Disallow: /external Disallow: /tags Disallow: /tool/view/gb/private Disallow: /tool/view/gb/email Disallow: /tool/pm Disallow: /tool/members/ Disallow: /tool/ticket/ Disallow: /cgi/view/poll.cgi Disallow: /cgi/view/topsites.cgi Disallow: /cgi/view/member.cgi Disallow: /cgi/view/out.cgi Disallow: /?authtoken= Disallow: /contact?*subject= Disallow: /post*?goto= Disallow: /post*&goto= Disallow: /post*?id= Disallow: *?*&sort= Disallow: *?sort= Disallow: *?full_version= Disallow: *?*full_version= Disallow: /?s= Disallow: /?t= Disallow: /?p= Disallow: /?id= Disallow: /?profile= Disallow: /*.jpg$ Disallow: /*.jpeg$ Disallow: /*.gif$ Disallow: /*.png$ Disallow: /tool/members/login?action=logout Disallow: /new$ Disallow: /top$ Disallow: /top$ Disallow: /top?period= Allow: /site/images/support/ Allow: /tool/members/signup Allow: /tool/members/login Allow: /images/favicon/ Crawl-Delay: 15 # End