Skip to content

Commit

Permalink
Update the default log pattern (apache#11935)
Browse files Browse the repository at this point in the history
Master Issue: apache#8298

### Motivation
To make it easy to track problems and resolve them, we should add date for the logs by default.

### Modifications
Updated the default logs pattern, added date.


* Update the default log pattern

* fix clang format
  • Loading branch information
tomscut authored Sep 9, 2021
1 parent 3cd5b9e commit 767d2a4
Show file tree
Hide file tree
Showing 24 changed files with 70 additions and 46 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
*/
package org.apache.pulsar.tests;

import org.apache.logging.log4j.core.util.datetime.FixedDateFormat;

import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.management.LockInfo;
Expand Down Expand Up @@ -68,7 +70,8 @@ static String buildThreadDump() {
// fallback to using JMX for creating the thread dump
StringBuilder dump = new StringBuilder();

DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS");
DateFormat dateFormat = new SimpleDateFormat(
FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern());
dump.append(String.format("Timestamp: %s", dateFormat.format(new Date())));
dump.append("\n\n");

Expand Down
2 changes: 1 addition & 1 deletion buildtools/src/main/resources/log4j2.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<Configuration status="INFO">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level [%t{12}] %c{1.}@%L - %msg %X%n" />
<PatternLayout pattern="%d{ISO8601_OFFSET_DATE_TIME_HHMM} %-5level [%t{12}] %c{1.}@%L - %msg %X%n" />
</Console>
</Appenders>
<Loggers>
Expand Down
6 changes: 3 additions & 3 deletions conf/functions_log4j2.xml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
<RollingFile>
Expand All @@ -49,7 +49,7 @@
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -82,7 +82,7 @@
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down
4 changes: 2 additions & 2 deletions conf/log4j2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ Configuration:
name: Console
target: SYSTEM_OUT
PatternLayout:
Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n"

# Rolling file appender configuration
RollingFile:
Expand All @@ -63,7 +63,7 @@ Configuration:
filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz"
immediateFlush: false
PatternLayout:
Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n"
Policies:
TimeBasedTriggeringPolicy:
interval: 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.bookkeeper.util.DirectMemoryUtils;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.util.datetime.FixedDateFormat;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
Expand Down Expand Up @@ -316,7 +317,8 @@ public void shutdown() {


public static void main(String[] args) throws Exception {
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS");
DateFormat dateFormat = new SimpleDateFormat(
FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern());
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> {
System.out.println(String.format("%s [%s] error Uncaught exception in thread %s: %s",
dateFormat.format(new Date()), thread.getContextClassLoader(),
Expand Down
4 changes: 2 additions & 2 deletions pulsar-client-cpp/include/pulsar/ConsoleLoggerFactory.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ class ConsoleLoggerFactoryImpl;
* The default LoggerFactory of Client if `USE_LOG4CXX` macro was not defined during compilation.
*
*
* The log format is "yyyy-mm-dd hh:MM:ss.xxx <level> <thread-id> <file>:<line> | <msg>", like
* The log format is "yyyy-MM-dd HH:mm:ss,SSS Z <level> <thread-id> <file>:<line> | <msg>", like
*
* ```
* 2021-03-24 17:35:46.571 INFO [0x10a951e00] ConnectionPool:85 | Created connection for ...
* 2021-03-24 17:35:46,571 +0800 INFO [0x10a951e00] ConnectionPool:85 | Created connection for ...
* ```
*
* It uses `std::cout` to prints logs to standard output. You can use this factory class to change your log
Expand Down
4 changes: 2 additions & 2 deletions pulsar-client-cpp/include/pulsar/FileLoggerFactory.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ class FileLoggerFactoryImpl;
/**
* A logger factory that is appending logs to a single file.
*
* The log format is "yyyy-mm-dd hh:MM:ss.xxx <level> <thread-id> <file>:<line> | <msg>", like
* The log format is "yyyy-MM-dd HH:mm:ss,SSS Z <level> <thread-id> <file>:<line> | <msg>", like
*
* ```
* 2021-03-24 17:35:46.571 INFO [0x10a951e00] ConnectionPool:85 | Created connection for ...
* 2021-03-24 17:35:46,571 +0800 INFO [0x10a951e00] ConnectionPool:85 | Created connection for ...
* ```
*
* Example:
Expand Down
3 changes: 2 additions & 1 deletion pulsar-client-cpp/lib/Log4cxxLogger.cc
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ std::unique_ptr<LoggerFactory> Log4CxxLoggerFactory::create() {
if (!LogManager::getLoggerRepository()->isConfigured()) {
LogManager::getLoggerRepository()->setConfigured(true);
LoggerPtr root = log4cxx::Logger::getRootLogger();
static const LogString TTCC_CONVERSION_PATTERN(LOG4CXX_STR("%d{HH:mm:ss.SSS} [%t] %-5p %l - %m%n"));
static const LogString TTCC_CONVERSION_PATTERN(
LOG4CXX_STR("%d{yyyy-MM-dd HH:mm:ss,SSS Z} [%t] %-5p %l - %m%n"));
LayoutPtr layout(new PatternLayout(TTCC_CONVERSION_PATTERN));
AppenderPtr appender(new ConsoleAppender(layout));
root->setLevel(log4cxx::Level::getInfo());
Expand Down
4 changes: 4 additions & 0 deletions pulsar-discovery-service/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>

</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.Map;
import java.util.TreeMap;

import org.apache.logging.log4j.core.util.datetime.FixedDateFormat;
import org.apache.pulsar.common.configuration.PulsarConfigurationLoader;
import org.apache.pulsar.common.util.CmdGenerateDocs;
import org.apache.pulsar.discovery.service.DiscoveryService;
Expand Down Expand Up @@ -65,7 +66,8 @@ public static void init(String configFile) throws Exception {
removeHandlersForRootLogger();
install();

DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS");
DateFormat dateFormat = new SimpleDateFormat(
FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern());
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> {
System.out.println(String.format("%s [%s] error Uncaught exception in thread %s: %s", dateFormat.format(new Date()), thread.getContextClassLoader(), thread.getName(), exception.getMessage()));
});
Expand Down
2 changes: 1 addition & 1 deletion pulsar-functions/localrun/src/main/resources/log4j2.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
<PatternLayout pattern="%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
<RollingFile>
Expand All @@ -49,7 +49,7 @@
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -82,7 +82,7 @@
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
</Appenders>
Expand Down
4 changes: 4 additions & 0 deletions pulsar-io/canal/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@
<artifactId>canal.client</artifactId>
<version>1.1.4</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.Date;
import java.util.List;
import com.alibaba.fastjson.JSON;
import org.apache.logging.log4j.core.util.datetime.FixedDateFormat;
import org.apache.pulsar.io.core.annotations.Connector;
import org.apache.pulsar.io.core.annotations.IOType;

Expand All @@ -55,7 +56,8 @@ public CanalMessage extractValue(List<FlatMessage> flatMessages) {
String messages = JSON.toJSONString(flatMessages, SerializerFeature.WriteMapNullValue);
CanalMessage canalMessage = new CanalMessage();
Date date = new Date();
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat dateFormat = new SimpleDateFormat(
FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern());
canalMessage.setTimestamp(dateFormat.format(date));
canalMessage.setId(this.messageId);
canalMessage.setMessage(messages);
Expand Down
4 changes: 4 additions & 0 deletions pulsar-proxy/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,10 @@
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
</dependencies>
<profiles>
<profile>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import static org.slf4j.bridge.SLF4JBridgeHandler.removeHandlersForRootLogger;

import com.google.common.annotations.VisibleForTesting;
import org.apache.logging.log4j.core.util.datetime.FixedDateFormat;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.authentication.AuthenticationService;
Expand Down Expand Up @@ -103,7 +104,8 @@ public ProxyServiceStarter(String[] args) throws Exception {
removeHandlersForRootLogger();
install();

DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS");
DateFormat dateFormat = new SimpleDateFormat(
FixedDateFormat.FixedFormat.ISO8601_OFFSET_DATE_TIME_HHMM.getPattern());
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> {
System.out.println(String.format("%s [%s] error Uncaught exception in thread %s: %s", dateFormat.format(new Date()), thread.getContextClassLoader(), thread.getName(), exception.getMessage()));
});
Expand Down
8 changes: 4 additions & 4 deletions site2/docs/functions-develop.md
Original file line number Diff line number Diff line change
Expand Up @@ -683,7 +683,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
<RollingFile>
Expand All @@ -692,7 +692,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -725,7 +725,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -830,7 +830,7 @@ The `<AppenderRef>` is defined in the `<Appenders>` section, such as:
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
```
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -684,7 +684,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
<RollingFile>
Expand All @@ -693,7 +693,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -726,7 +726,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -831,7 +831,7 @@ The `<AppenderRef>` is defined in the `<Appenders>` section, such as:
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
```
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -684,7 +684,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
<RollingFile>
Expand All @@ -693,7 +693,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -726,7 +726,7 @@ To customize the function log level, create or update `functions_log4j2.xml` in
<filePattern>${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz</filePattern>
<immediateFlush>true</immediateFlush>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy>
Expand Down Expand Up @@ -831,7 +831,7 @@ The `<AppenderRef>` is defined in the `<Appenders>` section, such as:
<name>Console</name>
<target>SYSTEM_OUT</target>
<PatternLayout>
<Pattern>%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n</Pattern>
<Pattern>%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n</Pattern>
</PatternLayout>
</Console>
```
Expand Down
Loading

0 comments on commit 767d2a4

Please sign in to comment.