User-agent: *
# user agent with an asterix tells all web crawlers to follow the rules shown beneath it
Disallow: /cgi-bin/
Disallow: /wp-admin/
# disallow the wp admin and includes directories as these contain WordPress software application data
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /category/
# categories are duplicate content, the crawlers will still find your posts
Disallow: /trackback/
Disallow: /author/
Disallow: /page/
Disallow: /tag/
Disallow: /search/
Disallow: /feed/
# feeds are duplicate content but don’t worry, crawlers will find your feeds elsewhere anyway
Disallow: /comments/
Disallow: /2010/
Disallow: /wp-
Disallow: /forum
Disallow: /forum/