fix for yasserg#346

13401095975 · Oct 17, 2018 · 6a8d22b · 6a8d22b
1 parent 4683cc7
commit 6a8d22b
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 5 deletions.
diff --git a/crawler4j/pom.xml b/crawler4j/pom.xml
@@ -189,10 +189,6 @@
 		<artifactId>tika-parsers</artifactId>
 		<version>${apache.tika.version}</version>
 		<exclusions>
-			<exclusion>
-				<groupId>org.apache.poi</groupId>
-				<artifactId>poi</artifactId>
-			</exclusion>
 			<exclusion>
 				<groupId>org.apache.poi</groupId>
 				<artifactId>poi-ooxml</artifactId>

diff --git a/crawler4j/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java b/crawler4j/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java
@@ -47,6 +47,10 @@ public HtmlParseData parse(Page page, String contextURL) throws ParseException {
         HtmlContentHandler contentHandler = new HtmlContentHandler();
         Metadata metadata = new Metadata();
 
+        if (page.getContentType() != null) {
+            metadata.add(Metadata.CONTENT_TYPE, page.getContentType());
+        }
+
         try (InputStream inputStream = new ByteArrayInputStream(page.getContentData())) {
             htmlParser.parse(inputStream, contentHandler, metadata, parseContext);
         } catch (Exception e) {

diff --git a/crawler4j/src/test/groovy/edu/uci/ics/crawler4j/parser/HtmlParserTest.groovy b/crawler4j/src/test/groovy/edu/uci/ics/crawler4j/parser/HtmlParserTest.groovy
@@ -9,7 +9,7 @@ import org.apache.http.entity.*
 class HtmlParserTest extends Specification {
 
     def "can parse html page"() {
-        def parser = new TikaHtmlParser(new CrawlConfig())
+        def parser = new TikaHtmlParser(new CrawlConfig(), null)
         def url = new WebURL(url: "http://wiki.c2.com/")
         def file = new File("src/test/resources/html/wiki.c2.com.html")
         def contentType = new ContentType("text/html", Charset.forName("UTF-8"))