User-agent: *
Crawl-delay: 20
Disallow: /home/_investment-news-articles
Disallow: /home/_investment-ideas
Disallow: /home/_asx-announcements
Disallow: /home/_smh-guide
Disallow: /home/_my-investsmart-widget
Disallow: /search/_shares
Disallow: /search/_Shares
Disallow: /search/_announcements
Disallow: /search/_Announcements
Disallow: /search/_dividends
Disallow: /search/_Dividends
Disallow: /search/_funds
Disallow: /search/_Funds
Disallow: /search/_news
Disallow: /search/_News
Disallow: /investment-news/newsletter/
Disallow: /*.asp$
Disallow: /*_price-chart
Disallow: /portfolio-manager/diversified-portfolios/_getdetails
Disallow: /portfolio-manager/diversified-portfolios/_getholdings
Disallow: /portfolio-manager/diversified-portfolios/_getdepositswithdrawls
Disallow: /portfolio-manager/diversified-portfolios/_gettransactions
Disallow: /portfolio-manager/diversified-portfolios/_getdividendsinterest
Disallow: /portfolio-manager/diversified-portfolios/_getfees
Disallow: /test/throwerror
Disallow: /test/version
Disallow: /shares/announcement-redirect/*
Disallow: /*&size=*
Disallow: /resources/pdf/*
Disallow: /identity/*
Disallow: /account/*
Disallow: /events/registermodel
Disallow: /widgets/*
Disallow: /*_related-articles*

User-agent: Yandex
User-agent: Baiduspider
User-agent: Baidu
User-agent: Mail.RU_Bot/2.0
User-agent: Mail.RU_Bot
User-agent: Mail.RU
User-agent: MegaIndex.ru/2.0
User-agent: MegaIndex.ru
User-agent: MegaIndex
User-agent: Vagabondo
User-agent: Exabot
User-agent: BSpider
User-agent: Screaming Frog SEO Spider
Disallow: /

# Certain social media sites are whitelisted to allow crawlers to access page markup
User-agent: Twitterbot
 Disallow:

Sitemap: https://www.investsmart.com.au/sitemap-is.xml
Sitemap: https://www.investsmart.com.au/sitemap-events.xml
Sitemap: https://www.investsmart.com.au/sitemap-myportfolio.xml
Sitemap: https://www.investsmart.com.au/sitemap-ourstory.xml
Sitemap: https://www.investsmart.com.au/sitemap-news.xml
Sitemap: https://www.investsmart.com.au/sitemap-model-portfolios.xml
Sitemap: https://www.investsmart.com.au/sitemap-insights.xml
Sitemap: https://www.investsmart.com.au/sitemap-videos.xml