Merge branch 'master' into PR/halt-on-error

crawler4j-examples/crawler4j-examples-base/src/main/resources/logback.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE XML>
+<configuration>
+    <property name="LOG_HOME" value="target/test-logs" />
+    <property name="LOG_FILE_NAME" value="/crawler4j.log" />
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>INFO</level>
+        </filter>
+        <encoder>
+            <pattern>%date{HH:mm:ss} %-5level [%thread] - [%logger{0}]- %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+        <file>${LOG_HOME}/${LOG_FILE_NAME}</file>
+        <append>false</append>
+        <encoder>
+            <pattern>%date %-5level [%thread] - [%logger] - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="com.github.tomakehurst" level="ERROR"/>
+    <logger name="org.apache.http" level="WARN" />
+    <logger name="org.eclipse.jetty" level="ERROR" />
+
+    <root level="DEBUG">
+        <appender-ref ref="STDOUT" />
+        <appender-ref ref="FILE" />
+    </root>
+</configuration>

crawler4j/pom.xml

@@ -19,6 +19,7 @@
 		<apache.tika.version>1.17</apache.tika.version>
         <guava.version>26.0-jre</guava.version>
         <public.suffix.list.version>2.2.0</public.suffix.list.version>
+        <url-detector.version>0.1.20</url-detector.version>
 		<!--test dependency versions -->
 		<junit.version>4.12</junit.version>
 		<wiremock.version>2.14.0</wiremock.version>
@@ -153,31 +154,83 @@
 					</execution>
 				</executions>
 			</plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>enforce-banned-dependencies</id>
+                        <goals>
+                            <goal>enforce</goal>
+                        </goals>
+                        <configuration>
+                            <rules>
+                                <bannedDependencies>
+                                    <excludes>
+                                        <!-- 
+                                            Banning JCL because we use SLF4J (and jcl-over-slf4j).
+                                            If this occurs in your build you should find what 
+                                            dependency is bringing in JCL and add an exclusion 
+                                            rule to prevent it.
+                                        -->
+                                        <exclude>commons-logging</exclude>
+                                        <!--
+                                            Banning Log4j because we use SLF4J/Logback.  If you
+                                            come across this, exclude the Log4j dependency and 
+                                            add the log4j-over-slf4j library to replace it.
+                                        -->
+                                        <exclude>log4j</exclude>
+                                        <exclude>org.apache.logging.log4j</exclude>
+                                    </excludes>
+                                </bannedDependencies>
+                            </rules>
+                            <fail>true</fail>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
 		</plugins>
 	</build>
 
 	<dependencies>
 
-    <!-- Compile time Dependencies -->
-
+    <!-- Logging API -->
     <dependency>
-        <!-- Logging framework -->
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-api</artifactId>
         <version>${slf4j.version}</version>
     </dependency>
     <dependency>
-        <!-- Implementation of slf4j -->
+        <groupId>org.slf4j</groupId>
+        <artifactId>jcl-over-slf4j</artifactId>
+        <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>jul-to-slf4j</artifactId>
+        <version>${slf4j.version}</version>
+    </dependency>
+
+    <!-- Logging Provider -->
+    <dependency>
         <groupId>ch.qos.logback</groupId>
         <artifactId>logback-classic</artifactId>
         <version>${logback.version}</version>
         <scope>runtime</scope>
     </dependency>
+
+    <!-- Compile time Dependencies -->
 	<dependency>
 		<groupId>org.apache.httpcomponents</groupId>
 		<artifactId>httpclient</artifactId>
 		<version>${apache.http.components.version}</version>
 		<scope>compile</scope>
+		<exclusions>
+		    <exclusion>
+		        <groupId>commons-logging</groupId>
+		        <artifactId>commons-logging</artifactId>
+		    </exclusion>
+		</exclusions>
 	</dependency>
 	<dependency>
 		<groupId>com.sleepycat</groupId>
@@ -367,6 +420,11 @@
 			</exclusion>
 		</exclusions>
 	</dependency>
+    <dependency>
+        <groupId>io.github.pgalbraith</groupId>
+        <artifactId>url-detector</artifactId>
+        <version>${url-detector.version}</version>
+    </dependency>
     <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>