User-agent: * # these generally need logins, javascript or at least humans to do # something. It's pointless for the crawlers to crawl them... Disallow: /api/ Disallow: /messages Disallow: /invite Disallow: /signout Disallow: /signin Disallow: /pictures/upload Disallow: /owner/ Disallow: /map/ Disallow: /submit/ Disallow: /call/ Disallow: /nfredirect/ Disallow: /directions Disallow: /merchant-verification Disallow: /brands/ Disallow: /css/ Disallow: /user/ # http://www.google.com/support/webmasters/bin/answer.py?answer=35303 # Googlebot supports wildcards - the have a tool in their "webmaster tools" # to check the rules (these work) Disallow: /*?inline= Disallow: /*?map=inline Disallow: /*?pictures=inline Disallow: /*?pictures=1 Disallow: /*?videos=inline Disallow: /*?videos=1 # http://www.edochan.com/programming/pf.htm User-agent: Fasterfox Disallow: / # http://sites.google.com/site/bendercrawler User-agent: Bender Disallow: / Sitemap: https://www.weblocal.ca/sitemap-urls-index.xml.gz