[Issue apache#11799][logging] Interface for structured event logging (a…

…pache#11800)

This change has just the interface for structured event logging.
Implementation will follow.

Master issue: apache#11799

pom.xml

@@ -1973,6 +1973,8 @@ flexible messaging model and an intuitive client API.</description>
         <module>pulsar-client-auth-sasl</module>
         <module>pulsar-config-validation</module>
 
+        <module>structured-event-log</module>
+
         <!-- transaction related modules -->
         <module>pulsar-transaction</module>
 

structured-event-log/pom.xml

@@ -0,0 +1,56 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.pulsar</groupId>
+    <artifactId>pulsar</artifactId>
+    <version>2.9.0-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <artifactId>structured-event-log</artifactId>
+  <name>Structured event logger</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.testng</groupId>
+      <artifactId>testng</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/Event.java

@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.structuredeventlog;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+/**
+ * Structured Logged Event interface.
+ *
+ * This interface is used to add context information to the log event and eventually log.
+ */
+public interface Event {
+    /**
+     * Create a new child event. The child event will inherit the trace ID of the event
+     * from which it was created. The child event parentId will be the ID of the event
+     * from which it was created.
+     * The child will inherit resources from its parent event, but not attributes.
+     *
+     * @return a new child event
+     */
+    Event newChildEvent();
+
+    /**
+     * Set the trace ID of the event. Normally this will be inherited from the parent
+     * event. In the case of a root event, the trace ID will be automatically generated.
+     * This method only needs to be used if a traceID has been received from elsewhere,
+     * such as from a user request.
+     * @param traceId the traceId
+     * @return this
+     */
+    Event traceId(String traceId);
+
+    /**
+     * Set the parent ID of the event. Normally this is set automatically for child events.
+     * For root events, it is normally empty unless provided by some other means, such
+     * as via a user request.
+     * @param parentId the parentId
+     * @return this
+     */
+    Event parentId(String parentId);
+
+    /**
+     * Mark this event as timed.
+     * Timed events will measure the duration between #timed() being called
+     * and logged being called. The duration will be in milliseconds.
+     *
+     * <pre>
+     * Event e = logger.newRootEvent().timed();
+     * // do something that takes time.
+     * e.log(Events.SOME_EVENT);
+     * </pre>
+     *
+     * @return this
+     */
+    Event timed();
+
+    /**
+     * Mark this event as sampled.
+     * Sampled events will only log once per specified duration.
+     * The sampling key is used to scope the rate limit. All events using the
+     * same sampling key will observe the same rate limit.
+     * Sampled events are most useful in the data plane, where, if there is an
+     * error for one event, there's likely to be a lot of other events which are
+     * almost identical.
+     *
+     * @param samplingKey a key by which to scope the rate limiting
+     * @param duration the duration for which one event will be logged
+     * @param unit the duration unit
+     * @return this
+     */
+    Event sampled(Object samplingKey, int duration, TimeUnit unit);
+
+    /**
+     * Add resources for the event from an EventResources object.
+     * @see #resource(java.lang.String,java.lang.Object)
+     * @return this
+     */
+    Event resources(EventResources attrs);
+
+    /**
+     * Add a resource for the event. Resources are inherited by
+     * child events.
+     * @param key the key to identify the resource
+     * @param value the value which will be logged for the resource.
+     *        This is converted to a string before logging.
+     * @return this
+     */
+    Event resource(String key, Object value);
+
+    /**
+     * Add a resource for the event using a supplier. The supplier is
+     * used in the case that generating the string from the object is
+     * expensive or we want to generate a custom string.
+     * @param key the key to identify the resource
+     * @param value a supplier which returns the value to be logged for
+     *              this resource
+     * @see #resource(java.lang.String,java.lang.Object)
+     */
+    Event resource(String key, Supplier<String> value);
+
+    /**
+     * Add an attribute for the event. Attributes are not inherited
+     * by child events.
+     * @param key the key to identify the attribute
+     * @param value the value which will be logged for the attribute.
+     *        This is converted to a string, using Object#toString() before logging.
+     * @return this
+     */
+    Event attr(String key, Object value);
+
+    /**
+     * Add an attribute for the event using a supplier.
+     * @param key the key to identify the attribute
+     * @param value a supplier which returns the value to be logged for
+     *              this attribute
+     * @return this
+     */
+    Event attr(String key, Supplier<String> value);
+
+    /**
+     * Attach an exception to the event.
+     * @param t the exception
+     * @return this
+     */
+    Event exception(Throwable t);
+
+    /**
+     * Log this event at the error level.
+     * @return this
+     */
+    Event atError();
+
+    /**
+     * Log this event at the info level (the default).
+     * @return this
+     */
+    Event atInfo();
+
+    /**
+     * Log this event at the warn level.
+     * @return this
+     */
+    Event atWarn();
+
+    /**
+     * Log the event, using an enum as the message.
+     * Logging with a enum allows documentation and annotations, such as subcomponent
+     * to be attached to the message.
+     * @param event the event message, in enum form
+     */
+    void log(Enum<?> event);
+
+    /**
+     * Log the event, using a string.
+     * @param event the event message.
+     */
+    void log(String event);
+
+    /**
+     * Stash this log event to bridge across an unmodifiable API call.
+     * @see StructuredEventLog#unstash()
+     */
+    void stash();
+}

structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/EventResources.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.structuredeventlog;
+
+import java.util.function.Supplier;
+
+/**
+ * A container for resources
+ * The container allows resources to be shared across multiple
+ * events with different root events. An example usage of this would be used
+ * for a server connection. We want all events initiated on the connection to
+ * share resource values, such as remote and local socket. However, each root
+ * event on the connection may be for a different trace.
+ */
+public interface EventResources {
+    /**
+     * Add a resource for the event. Resources are inherited by
+     * child events.
+     * @param key the key to identify the resource
+     * @param value the value which will be logged for the resource.
+     *        This is converted to a string before logging.
+     * @return this
+     */
+    EventResources resource(String key, Object value);
+
+    /**
+     * Add a resource for the event using a supplier. The supplier is
+     * used in the case that generating the string from the object is
+     * expensive or we want to generate a custom string.
+     * @param key the key to identify the resource
+     * @param value a supplier which returns the value to be logged for
+     *              this resource
+     * @see #resource(java.lang.String,java.lang.Object)
+     */
+    EventResources resource(String key, Supplier<String> value);
+}

structured-event-log/src/main/java/org/apache/pulsar/structuredeventlog/StructuredEventLog.java

@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.structuredeventlog;
+
+/**
+ * Structured event logging interface
+ *
+ * Allows resources and attribute key value pairs to be attached to a logged event.
+ *
+ * Basic usage:
+ * <pre>
+ * StructuredEventLog logger = StructuredEventLog.newLogger();
+ * logger.newRootEvent()
+ *     .resource("remote", remoteAddr)
+ *     .resource("local", localAddr)
+ *     .attr("path", request.getPath())
+ *     .log(Events.READ_REQUEST);
+ * </pre>
+ */
+public interface StructuredEventLog {
+    /**
+     * Create a new root event. Root events occur in response to some external stimulus, such as
+     * a user request, a timer being triggered or a threshold being crossed.
+     *
+     * The level of the event is INFO by default.
+     *
+     * The root event will generate a new traceId, and will have a empty parent Id. If this
+     * information is provided, as can be the case with a user request, they can be set
+     * with Event#traceId(String) and Event#parentId(String).
+     */
+    Event newRootEvent();
+
+    /**
+     * Create an new event resources object, which can be used across multiple
+     * root events.
+     */
+    EventResources newEventResources();
+
+    /**
+     * Retrieves an event from the call stack. This can be used, along with Event#stash(),
+     * to bridge an event across an API without having to modify the API to pass the event
+     * object.
+     *
+     * For example, the child event, METHOD2, in the following example, will share the traceId
+     * with METHOD1, and METHOD1's id will be match the parentId of METHOD2.
+     *
+     * <pre>
+     * void method1() {
+     *    Event e = logger.newRootEvent()
+     *        .timed()
+     *        .resource("foo", bar);
+     *
+     *    e.stash();
+     *    unmodifiableMethod();
+     *    e.log(Events.METHOD1);
+     * }
+     *
+     * void unmodifiableMethod() {
+     *    logger.unstash().newChildEvent().log(Events.METHOD2);
+     * }
+     * </pre>
+     *
+     * This should be used sparingly.
+     */
+    Event unstash();
+
+    /**
+     * Create a new logger object, from which root events can be created.
+     */
+    public static StructuredEventLog newLogger() {
+        return null;
+    }
+}