From de1196f24556122510f48338e0b8e6d21c88b28e Mon Sep 17 00:00:00 2001
From: Julien Nioche <julien@digitalpebble.com>
Date: Sat, 18 Jul 2026 10:24:29 +0100
Subject: [PATCH] bugfix - max content not applied for robots.txt when
 redirecting

Signed-off-by: Julien Nioche <julien@digitalpebble.com>
---
 .../org/apache/stormcrawler/protocol/HttpRobotRulesParser.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
index d734f4555..2f846f8b2 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
@@ -180,7 +180,7 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
                                 redir);
                     }
 
-                    response = http.getProtocolOutput(redir.toString(), Metadata.empty);
+                    response = http.getProtocolOutput(redir.toString(), fetchRobotsMd);
                     code = response.getStatusCode();
                     bytesFetched.add(
                             response.getContent() != null ? response.getContent().length : 0);