User-agent: *
Allow: /
Disallow: /subscription/complete
Disallow: /admin

# We don't want google to crawl our landing pages under /join, because we're using them for really
# specific things
User-agent: *
Disallow: /join/

# Marketing uses this url to direct new conversions to for tracking
User-agent: *
Disallow: /welcome-home

# Alleviate duplicate content penalties from signin links
User-agent: *
Disallow: /signin

#Prevent crawling of profile pages
User-agent: *
Disallow: /profiles

# Prevent crawling of answer pages
User-agent: *
Disallow: /library/answer

# Prevent crawling of new relic junk pages
User-agent: *
Disallow: /*/aggregate$

Sitemap: https://th-sitemap.s3.amazonaws.com/sitemap.xml.gz