From a4a5ab0f0bf8ac0c00c4f9d3642bed8705de0876 Mon Sep 17 00:00:00 2001 From: Harsha Deepa <44192755+harshadeepa@users.noreply.github.com> Date: Thu, 3 Oct 2019 15:17:47 +0530 Subject: [PATCH] Disallowed crawlers/spiders from crawling updates --- robots.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robots.txt b/robots.txt index 214e4119..1eba72b6 100644 --- a/robots.txt +++ b/robots.txt @@ -1,4 +1,4 @@ # www.robotstxt.org/ # www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 - +Disallow: /updates/* User-agent: *