User-agent: * Disallow: /admin # Disallow: /assets/ # Block the /assets directory. Disallow: *.pdf$ # Block pdf files. Non-standard but should for major search engines.