Fix #304: Use HTML test document instead of sitemap XML in SubDocumentsFilterTest

core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsFilterTest.java

@@ -18,8 +18,6 @@
 package org.apache.stormcrawler.parse.filter;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.bolt.JSoupParserBolt;
 import org.apache.stormcrawler.parse.ParsingTester;
@@ -36,12 +34,10 @@ void setupParserBolt() {
     }
 
     @Test
-    void testSitemapSubdocuments() throws IOException {
-        Map<String, Object> config = new HashMap<>();
-        config.put("detect.mimetype", false);
-        prepareParserBolt("test.subdocfilter.json", config);
+    void testSubdocuments() throws IOException {
+        prepareParserBolt("test.subdocfilter.json");
         Metadata metadata = new Metadata();
-        parse("https://stormcrawler.apache.org/sitemap.xml", "stormcrawler.sitemap.xml", metadata);
+        parse("https://stormcrawler.apache.org/", "subdocuments.html", metadata);
         Assertions.assertEquals(7, output.getEmitted().size());
     }
 }

core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java

@@ -17,64 +17,61 @@
 
 package org.apache.stormcrawler.parse.filter;
 
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathFactory;
-import org.apache.stormcrawler.parse.ParseData;
 import org.apache.stormcrawler.parse.ParseFilter;
 import org.apache.stormcrawler.parse.ParseResult;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
 import org.w3c.dom.DocumentFragment;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
 public class SubDocumentsParseFilter extends ParseFilter {
-    private static final org.slf4j.Logger LOG =
-            LoggerFactory.getLogger(SubDocumentsParseFilter.class);
 
     @Override
     public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
+        if (doc == null) {
+            return;
+        }
 
-        InputStream stream = new ByteArrayInputStream(content);
-
-        try {
-            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-            Document document = factory.newDocumentBuilder().parse(stream);
-            Element root = document.getDocumentElement();
-
-            XPath xPath = XPathFactory.newInstance().newXPath();
-            XPathExpression expression = xPath.compile("//url");
-
-            NodeList nodes = (NodeList) expression.evaluate(root, XPathConstants.NODESET);
-
-            for (int i = 0; i < nodes.getLength(); i++) {
-                Node node = nodes.item(i);
-
-                expression = xPath.compile("loc");
-                Node child = (Node) expression.evaluate(node, XPathConstants.NODE);
-
-                // create a subdocument for each url found in the sitemap
-                ParseData parseData = parse.get(child.getTextContent());
-
-                NodeList children = node.getChildNodes();
-                for (int j = 0; j < children.getLength(); j++) {
-                    Node n = children.item(j);
-                    parseData.put(n.getNodeName(), n.getTextContent());
+        NodeList links = findElements(doc, "a");
+        for (int i = 0; i < links.getLength(); i++) {
+            Node node = links.item(i);
+            if (node instanceof Element) {
+                String href = ((Element) node).getAttribute("href");
+                if (href != null && !href.isEmpty()) {
+                    parse.get(href);
                 }
             }
-        } catch (Exception e) {
-            LOG.error("Error processing sitemap from {}: {}", url, e);
         }
     }
 
+    private NodeList findElements(Node node, String tagName) {
+        if (node instanceof org.w3c.dom.Document) {
+            return ((org.w3c.dom.Document) node).getElementsByTagName(tagName);
+        }
+        Node child = node.getFirstChild();
+        while (child != null) {
+            if (child instanceof Element) {
+                return ((Element) child).getElementsByTagName(tagName);
+            }
+            child = child.getNextSibling();
+        }
+        return new EmptyNodeList();
+    }
+
     @Override
     public boolean needsDOM() {
         return true;
     }
+
+    private static class EmptyNodeList implements NodeList {
+        @Override
+        public Node item(int index) {
+            return null;
+        }
+
+        @Override
+        public int getLength() {
+            return 0;
+        }
+    }
 }

core/src/test/resources/subdocuments.html

@@ -0,0 +1,31 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<html>
+<head><title>Test page for SubDocumentsParseFilter</title></head>
+<body>
+<div>
+  <a href="https://stormcrawler.apache.org/about/">About</a>
+  <a href="https://stormcrawler.apache.org/index.html">Index</a>
+  <a href="https://stormcrawler.apache.org/download/index.html">Download</a>
+  <a href="https://stormcrawler.apache.org/getting-started/">Getting Started</a>
+  <a href="https://stormcrawler.apache.org/faq/">FAQ</a>
+  <a href="https://stormcrawler.apache.org/support/">Support</a>
+</div>
+</body>
+</html>