# robots.txt # Default Rule # Allow only public entry points # User-agent: * Disallow: /cgi-bin/ # non public area Disallow: /css/ # Disallow indexing css Disallow: /images/ # Disallow indexing images Disallow: /img/ # Disallow indexing img Disallow: /js/ # Disallow indexing js