Skip to content

Commit

Permalink
[Functions] Completed work for PR apache#7555 to allow customizable f…
Browse files Browse the repository at this point in the history
…unction logging (apache#10389)

* Completed work for PR apache#7555 to allow customizable function logging

* Fixed license header

* PR apache#10389 Added null check to prevent test failures.

* PR apache#10389 Added debug logging statement

* PR apache#10389 Made changes more robust, added default value, and fixed NullPointerException

* PR apache#10389 Improved backwards compatibility

Co-authored-by: Devin Bost <[email protected]>
  • Loading branch information
devinbost and Devin Bost authored Apr 29, 2021
1 parent 61639a2 commit d757db1
Show file tree
Hide file tree
Showing 3 changed files with 148 additions and 2 deletions.
8 changes: 7 additions & 1 deletion bin/pulsar
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf
DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf
DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
DEFAULT_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
DEFAULT_FUNCTIONS_LOG_CONF=$PULSAR_HOME/conf/functions_log4j2.xml

# functions related variables
FUNCTIONS_HOME=$PULSAR_HOME/pulsar-functions
Expand Down Expand Up @@ -157,6 +158,7 @@ where command is one of:
or command is the full name of a class with a defined main() method.
Environment variables:
FUNCTIONS_LOG_CONF Function Log4j configuration file (default $DEFAULT_FUNCTIONS_LOG_CONF)
PULSAR_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF)
PULSAR_BROKER_CONF Configuration file for broker (default: $DEFAULT_BROKER_CONF)
PULSAR_BOOKKEEPER_CONF Configuration file for bookie (default: $DEFAULT_BOOKKEEPER_CONF)
Expand Down Expand Up @@ -264,6 +266,10 @@ if [ -z "$PULSAR_PRESTO_CONF" ]; then
PULSAR_PRESTO_CONF=$DEFAULT_PULSAR_PRESTO_CONF
fi

if [ -z "$FUNCTIONS_LOG_CONF"]; then
FUNCTIONS_LOG_CONF=$DEFAULT_FUNCTIONS_LOG_CONF
fi

PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
Expand Down Expand Up @@ -305,7 +311,7 @@ OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}"
OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}"
OPTS="$OPTS -Dpulsar.functions.extra.dependencies.dir=${FUNCTIONS_EXTRA_DEPS_DIR}"
OPTS="$OPTS -Dpulsar.functions.instance.classpath=${PULSAR_CLASSPATH}"

OPTS="$OPTS -Dpulsar.functions.log.conf=${FUNCTIONS_LOG_CONF}"

ZK_OPTS=" -Dzookeeper.4lw.commands.whitelist=* -Dzookeeper.snapshot.trust.empty=true"

Expand Down
130 changes: 130 additions & 0 deletions conf/functions_log4j2.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<Configuration>
<name>pulsar-functions-instance</name>
<monitorInterval>30</monitorInterval>
<Properties>
<Property>
<name>pulsar.log.appender</name>
<value>RollingFile</value>
</Property>
<Property>
<name>pulsar.log.level</name>
<value>info</value>
</Property>
<Property>
<name>bk.log.level</name>
<value>info</value>
</Property>
</Properties>
<Appenders>
<Console>
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
<RollingFile>
<name>RollingFile</name>
<fileName>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log</fileName>
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
<interval>1</interval>
<modulate>true</modulate>
</TimeBasedTriggeringPolicy>
<SizeBasedTriggeringPolicy>
<size>1 GB</size>
</SizeBasedTriggeringPolicy>
<CronTriggeringPolicy>
<schedule>0 0 0 * * ?</schedule>
</CronTriggeringPolicy>
</Policies>
<DefaultRolloverStrategy>
<Delete>
<basePath>${sys:pulsar.function.log.dir}</basePath>
<maxDepth>2</maxDepth>
<IfFileName>
<glob>*/${sys:pulsar.function.log.file}*log.gz</glob>
</IfFileName>
<IfLastModified>
<age>30d</age>
</IfLastModified>
</Delete>
</DefaultRolloverStrategy>
</RollingFile>
<RollingRandomAccessFile>
<name>BkRollingFile</name>
<fileName>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk</fileName>
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
<interval>1</interval>
<modulate>true</modulate>
</TimeBasedTriggeringPolicy>
<SizeBasedTriggeringPolicy>
<size>1 GB</size>
</SizeBasedTriggeringPolicy>
<CronTriggeringPolicy>
<schedule>0 0 0 * * ?</schedule>
</CronTriggeringPolicy>
</Policies>
<DefaultRolloverStrategy>
<Delete>
<basePath>${sys:pulsar.function.log.dir}</basePath>
<maxDepth>2</maxDepth>
<IfFileName>
<glob>*/${sys:pulsar.function.log.file}.bk*log.gz</glob>
</IfFileName>
<IfLastModified>
<age>30d</age>
</IfLastModified>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
</Appenders>
<Loggers>
<Logger>
<name>org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper</name>
<level>${sys:bk.log.level}</level>
<additivity>false</additivity>
<AppenderRef>
<ref>BkRollingFile</ref>
</AppenderRef>
</Logger>
<Root>
<level>info</level>
<AppenderRef>
<ref>${sys:pulsar.log.appender}</ref>
<level>${sys:pulsar.log.level}</level>
</AppenderRef>
</Root>
</Loggers>
</Configuration>
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator;
import org.apache.pulsar.functions.utils.FunctionCommon;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
Expand Down Expand Up @@ -103,7 +104,16 @@ class ProcessRuntime implements Runtime {
}
switch (instanceConfig.getFunctionDetails().getRuntime()) {
case JAVA:
logConfigFile = "java_instance_log4j2.xml";
String logConfigPath = System.getProperty("pulsar.functions.log.conf");
if(log.isDebugEnabled()){
log.debug("The loaded value of pulsar.functions.log.conf is {}", logConfigPath);
}
// Added null check to prevent test failures
if(logConfigPath != null && Files.exists(Paths.get(logConfigPath))){
logConfigFile = logConfigPath;
} else { // Keeping existing file for backwards compatibility
logConfigFile = "java_instance_log4j2.xml";
}
break;
case PYTHON:
logConfigFile = System.getenv("PULSAR_HOME") + "/conf/functions-logging/logging_config.ini";
Expand Down

0 comments on commit d757db1

Please sign in to comment.