User-Agent: *
Disallow: /UNUser/Documents/*
Disallow: /unuser/documents/*
Disallow: /UNUser/Documents/
Disallow: /unuser/documents/
Disallow: /Styles/
Disallow: /Scripts/
Disallow: /Images/
Disallow: /Content/
Disallow: /Web References/
Disallow: /Views/
Disallow: /Service References/
Disallow: /apple-touch-icon.png
Disallow: /AccessDeniedError.htm
Disallow: /favicon.ico
Disallow: /FileNotFoundError.htm
Disallow: /GenericError.htm
Disallow: /InternalError.htm
#Following line might need to be updated in live environment
Disallow: /PollPageUNGMWWW.htm	
#The below line is to block the bots like MJ12bot that are constantly polling invalid UNGM urls that has a matching pattern of /SolveUrl?noticeID etc - ref: UNGMVRR#6185
Allow: /SolveUrl?url
Disallow: /SolveUrl?

#The below lines are to block URL that were frequently accessed by search engine bots
User-Agent: Bingbot
User-Agent: Googlebot
User-Agent: facebot
User-Agent: YandexBot
User-Agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
User-Agent: Baiduspider
User-Agent: Slurp
User-Agent: ia_archiver
User-Agent: DuckDuckBot
Disallow: /Account/Registration/Individual
Disallow: /Areas/Public/Downloads/
Disallow: /HarmonizationDoc/
Disallow: /Public/UNSPSC/Excel
Disallow: /Publications/
Disallow: /Styles/images/
Disallow: /SustainableProcurement/toolsUN/
Disallow: /UNUser/Documents/DownloadPublicDocument
Disallow: /HarmonizationDoc/