diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
index d734f4555..2f846f8b2 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
@@ -180,7 +180,7 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
                                 redir);
                     }
 
-                    response = http.getProtocolOutput(redir.toString(), Metadata.empty);
+                    response = http.getProtocolOutput(redir.toString(), fetchRobotsMd);
                     code = response.getStatusCode();
                     bytesFetched.add(
                             response.getContent() != null ? response.getContent().length : 0);