Upgrade to log4j2 dependency and configuration (apache#680)

* Add log4j2 dependency and configuration * add yamal configuration * Add more roll-over-file configuration + log4j2-web module
teaey · Feb 23, 2018 · 21d3307 · 21d3307
1 parent e581d95
commit 21d3307
Show file tree

Hide file tree

Showing 16 changed files with 213 additions and 148 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,7 +6,7 @@ dependency-reduced-pom.xml
 logs
 /data
 pulsar-broker/tmp.*
-pulsar-broker/src/test/resources/log4j.properties
+pulsar-broker/src/test/resources/log4j2.yaml
 *.log
 
 *.versionsBackup

diff --git a/all/pom.xml b/all/pom.xml
@@ -65,13 +65,33 @@
     </dependency>
 
     <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
+      <artifactId>log4j</artifactId>
+      <groupId>log4j</groupId>
     </dependency>
 
     <dependency>
-      <artifactId>log4j</artifactId>
-      <groupId>log4j</groupId>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-web</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
     </dependency>
 
     <dependency>

diff --git a/bin/bookkeeper b/bin/bookkeeper
@@ -51,7 +51,7 @@ BINDIR=`dirname "$0"`
 BK_HOME=`cd $BINDIR/..;pwd`
 
 DEFAULT_CONF=$BK_HOME/conf/bookkeper.conf
-DEFAULT_LOG_CONF=$BK_HOME/conf/log4j.properties
+DEFAULT_LOG_CONF=$BK_HOME/conf/log4j2.yaml
 
 source $BK_HOME/conf/bkenv.sh
 
@@ -146,7 +146,7 @@ COMMAND=$1
 shift
 
 if [ $COMMAND == "shell" ]; then
-    DEFAULT_LOG_CONF=$BK_HOME/conf/log4j.shell.properties
+    DEFAULT_LOG_CONF=$BK_HOME/conf/log4j2.yaml
 fi
 
 if [ -z "$BOOKIE_CONF" ]; then
@@ -159,7 +159,7 @@ fi
 
 BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH:$BOOKIE_EXTRA_CLASSPATH"
 BOOKIE_CLASSPATH="`dirname $BOOKIE_LOG_CONF`:$BOOKIE_CLASSPATH"
-OPTS="$OPTS -Dlog4j.configuration=`basename $BOOKIE_LOG_CONF`"
+OPTS="$OPTS -Dlog4j.configurationFile=`basename $BOOKIE_LOG_CONF`"
 
 OPTS="-cp $BOOKIE_CLASSPATH $OPTS"
 
@@ -169,12 +169,12 @@ OPTS="$OPTS $BOOKIE_EXTRA_OPTS"
 OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
 
 # log directory & file
-BOOKIE_ROOT_LOGGER=${BOOKIE_ROOT_LOGGER:-"INFO,CONSOLE"}
+BOOKIE_LOG_APPENDER=${BOOKIE_LOG_APPENDER:-"Console"}
 BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"$BK_HOME/logs"}
 BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"bookkeeper-server.log"}
 
 #Configure log configuration system properties
-OPTS="$OPTS -Dbookkeeper.root.logger=$BOOKIE_ROOT_LOGGER"
+OPTS="$OPTS -Dpulsar.log.appender=$BOOKIE_LOG_APPENDER"
 OPTS="$OPTS -Dbookkeeper.log.dir=$BOOKIE_LOG_DIR"
 OPTS="$OPTS -Dbookkeeper.log.file=$BOOKIE_LOG_FILE"
 

diff --git a/bin/pulsar b/bin/pulsar
@@ -29,7 +29,7 @@ DEFAULT_DISCOVERY_CONF=$PULSAR_HOME/conf/discovery.conf
 DEFAULT_PROXY_CONF=$PULSAR_HOME/conf/proxy.conf
 DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf
 DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf
-DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j.properties
+DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
 
 if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ]
 then
@@ -174,7 +174,7 @@ fi
 
 PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
-OPTS="$OPTS -Dlog4j.configuration=`basename $PULSAR_LOG_CONF`"
+OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
 
 # Ensure we can read bigger content from ZK. (It might be
 # rarely needed when trying to list many z-nodes under a
@@ -186,11 +186,11 @@ OPTS="-cp $PULSAR_CLASSPATH $OPTS"
 OPTS="$OPTS $PULSAR_EXTRA_OPTS"
 
 # log directory & file
-PULSAR_ROOT_LOGGER=${PULSAR_ROOT_LOGGER:-"INFO,CONSOLE"}
+PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
 PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
 
 #Configure log configuration system properties
-OPTS="$OPTS -Dpulsar.root.logger=$PULSAR_ROOT_LOGGER"
+OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
 OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
 
 #Change to PULSAR_HOME to support relative paths

diff --git a/bin/pulsar-admin b/bin/pulsar-admin
@@ -22,7 +22,7 @@ BINDIR=$(dirname "$0")
 PULSAR_HOME=`cd $BINDIR/..;pwd`
 
 DEFAULT_CLIENT_CONF=$PULSAR_HOME/conf/client.conf
-DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j.properties
+DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
 
 if [ -f "$PULSAR_HOME/conf/pulsar_tools_env.sh" ]
 then
@@ -88,20 +88,20 @@ fi
 
 PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
-OPTS="$OPTS -Dlog4j.configuration=`basename $PULSAR_LOG_CONF`"
+OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
 OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
 
 OPTS="-cp $PULSAR_CLASSPATH $OPTS"
 
 OPTS="$OPTS $PULSAR_EXTRA_OPTS"
 
 # log directory & file
-PULSAR_ROOT_LOGGER=${PULSAR_ROOT_LOGGER:-"INFO,CONSOLE"}
 PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
+PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
 
 #Configure log configuration system properties
-OPTS="$OPTS -Dpulsar.root.logger=$PULSAR_ROOT_LOGGER"
 OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
+OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
 
 #Change to PULSAR_HOME to support relative paths
 cd "$PULSAR_HOME"

diff --git a/bin/pulsar-client b/bin/pulsar-client
@@ -22,7 +22,7 @@ BINDIR=$(dirname "$0")
 PULSAR_HOME=`cd $BINDIR/..;pwd`
 
 DEFAULT_CLIENT_CONF=$PULSAR_HOME/conf/client.conf
-DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j.shell.properties
+DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
 
 if [ -f "$PULSAR_HOME/conf/pulsar_tools_env.sh" ]
 then
@@ -88,20 +88,20 @@ fi
 
 PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
-OPTS="$OPTS -Dlog4j.configuration=`basename $PULSAR_LOG_CONF`"
+OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
 OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
 
 OPTS="-cp $PULSAR_CLASSPATH $OPTS"
 
 OPTS="$OPTS $PULSAR_EXTRA_OPTS"
 
 # log directory & file
-PULSAR_ROOT_LOGGER=${PULSAR_ROOT_LOGGER:-"INFO,CONSOLE"}
 PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
+PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
 
 #Configure log configuration system properties
-OPTS="$OPTS -Dpulsar.root.logger=$PULSAR_ROOT_LOGGER"
 OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
+OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
 
 #Change to PULSAR_HOME to support relative paths
 cd "$PULSAR_HOME"

diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon
@@ -44,7 +44,7 @@ then
 fi
 
 PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
-PULSAR_ROOT_LOGGER=${PULSAR_ROOT_LOGGER:-'INFO,ROLLINGFILE'}
+PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RollingFile"}
 PULSAR_STOP_TIMEOUT=${PULSAR_STOP_TIMEOUT:-30}
 PULSAR_PID_DIR=${PULSAR_PID_DIR:-$PULSAR_HOME/bin}
 
@@ -90,7 +90,7 @@ case $command in
 esac
 
 export PULSAR_LOG_DIR=$PULSAR_LOG_DIR
-export PULSAR_ROOT_LOGGER=$PULSAR_ROOT_LOGGER
+export PULSAR_LOG_APPENDER=$PULSAR_LOG_APPENDER
 export PULSAR_LOG_FILE=pulsar-$command-$HOSTNAME.log
 
 pid=$PULSAR_PID_DIR/pulsar-$command.pid

diff --git a/bin/pulsar-perf b/bin/pulsar-perf
@@ -22,7 +22,7 @@ BINDIR=$(dirname "$0")
 PULSAR_HOME=`cd $BINDIR/..;pwd`
 
 DEFAULT_CLIENT_CONF=$PULSAR_HOME/conf/client.conf
-DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j.properties
+DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
 
 if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ]
 then
@@ -117,18 +117,18 @@ fi
 
 PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
-OPTS="$OPTS -Dlog4j.configuration=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true"
+OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true"
 
 OPTS="-cp $PULSAR_CLASSPATH $OPTS"
 OPTS="$OPTS $PULSAR_EXTRA_OPTS"
 
 # log directory & file
-PULSAR_ROOT_LOGGER=${PULSAR_ROOT_LOGGER:-"INFO,CONSOLE"}
+PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
 PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
 PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-perftest.log"}
 
 #Configure log configuration system properties
-OPTS="$OPTS -Dpulsar.root.logger=$PULSAR_ROOT_LOGGER"
+OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
 OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
 OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE"
 

diff --git a/conf/log4j.properties b/conf/log4j.properties
diff --git a/conf/log4j.shell.properties b/conf/log4j.shell.properties
diff --git a/conf/log4j2-scripts/filter.js b/conf/log4j2-scripts/filter.js
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// This is a sample file which can be used by log4j2 to: only log debug-statement which has text: pulsar-topic-name
+var result = false;
+var topicName = "pulsar-topic-name";
+/*
+ * Find more logEvent attributes at : 
+ * https://github.com/apache/logging-log4j2/blob/dbd2d252a1b4139a9bd9eb213c89f28498db6dcf/log4j-core/src/main/java/org/apache/logging/log4j/core/LogEvent.java
+ */
+if (logEvent.getLevel() == "DEBUG"){
+        if(logEvent.getMessage().getFormattedMessage().indexOf(topicName)!=-1) {
+                result = true;  
+        }
+} else {
+        result = true;
+}
+result;