[patch][pulsar-sql] Add Java version trim agent for presto 332 (apach…

…e#15236) Fix apache#14951 ### Motivation The presto 332 couldn't parse Java version like this `11.0.14.1`, so add a Java version trim agent to walk around the problem. This is a temporary patch, after the presto upgrade to 332+, we could remove this. ### Modifications Add a Java version trim agent to format the system property `java.version` for the Pulsar SQL plugin.
AlphaWang · Apr 24, 2022 · a1aa18f · a1aa18f
1 parent 3823ddf
commit a1aa18f
Show file tree

Hide file tree

Showing 12 changed files with 155 additions and 4 deletions.
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
@@ -604,9 +604,8 @@ jobs:
           - name: Pulsar IO - Oracle
             group: PULSAR_IO_ORA
 
-          # disable Sql integration tests until https://github.com/apache/pulsar/issues/14951 has been resolved
-          #- name: Sql
-          #  group: SQL
+          - name: Sql
+            group: SQL
 
     steps:
       - name: checkout

diff --git a/conf/presto/jvm.config b/conf/presto/jvm.config
@@ -27,3 +27,4 @@
 -XX:+ExitOnOutOfMemoryError
 -Dpresto-temporarily-allow-java8=true
 -Djdk.attach.allowAttachSelf=true
+-javaagent:java-version-trim-agent.jar
diff --git a/pulsar-sql/java-version-trim-agent/pom.xml b/pulsar-sql/java-version-trim-agent/pom.xml
@@ -0,0 +1,58 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>pulsar-sql</artifactId>
+        <groupId>org.apache.pulsar</groupId>
+        <version>2.11.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>java-version-trim-agent</artifactId>
+    <name>Pulsar SQL :: Java Version Trim Agent</name>
+
+    <build>
+        <finalName>java-version-trim-agent</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>3.1.0</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                        </manifest>
+                        <manifestEntries>
+                            <Premain-Class>org.apache.pulsar.sql.agent.TrimJavaVersionAgent</Premain-Class>
+                            <Agent-Class>org.apache.pulsar.sql.agent.TrimJavaVersionAgent</Agent-Class>
+                            <Can-Redefine-Classes>true</Can-Redefine-Classes>
+                            <Can-Retransform-Classes>true</Can-Retransform-Classes>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/...va-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/TrimJavaVersionAgent.java b/...va-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/TrimJavaVersionAgent.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.sql.agent;
+
+import java.lang.instrument.Instrumentation;
+import java.util.logging.Logger;
+
+/**
+ * The presto 332 couldn't parse Java version like this `11.0.14.1`,
+ * so add java version trim agent to walk around the problem.
+ *
+ * After the presto upgrade to 332+, we could remove this.
+ */
+public class TrimJavaVersionAgent {
+
+    private static final Logger logger = Logger.getLogger(TrimJavaVersionAgent.class.getName());
+
+    private static final String JAVA_VERSION = "java.version";
+
+    public static String trimJavaVersion(String javaVersion) {
+        String[] arr = javaVersion.split("\\.");
+        if (arr.length <= 3) {
+            return javaVersion;
+        }
+        return arr[0] + "." + arr[1] + "." + arr[2];
+    }
+
+    public static void premain(String agentArgs, Instrumentation inst) {
+        String javaVersion = System.getProperty(JAVA_VERSION);
+        String trimVersion = trimJavaVersion(javaVersion);
+        logger.info("original java version " + javaVersion + " => trim java version " + trimVersion);
+        System.setProperty(JAVA_VERSION, trimVersion);
+    }
+
+}
diff --git a/...r-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/package-info.java b/...r-sql/java-version-trim-agent/src/main/java/org/apache/pulsar/sql/agent/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Implementation of the connector to the Presto engine.
+ */
+package org.apache.pulsar.sql.agent;
diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml
@@ -34,6 +34,7 @@
     <modules>
         <module>presto-pulsar</module>
         <module>presto-pulsar-plugin</module>
+        <module>java-version-trim-agent</module>
         <module>presto-distribution</module>
     </modules>
 

diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml
@@ -194,6 +194,13 @@
       <version>${jackson.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>java-version-trim-agent</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+
   </dependencies>
 
   <dependencyManagement>

diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml
@@ -40,6 +40,11 @@
             <outputDirectory>bin/</outputDirectory>
             <fileMode>644</fileMode>
         </file>
+        <file>
+            <source>${basedir}/../java-version-trim-agent/target/java-version-trim-agent.jar</source>
+            <destName>java-version-trim-agent.jar</destName>
+            <outputDirectory>/</outputDirectory>
+        </file>
     </files>
     <fileSets>
         <fileSet>

diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml
@@ -104,6 +104,12 @@
             <version>${presto.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>${javax.annotation-api.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>io.prestosql</groupId>
             <artifactId>presto-main</artifactId>

diff --git a/tests/docker-images/latest-version-image/conf/presto/jvm.config b/tests/docker-images/latest-version-image/conf/presto/jvm.config
@@ -28,3 +28,4 @@
 -XX:+ExitOnOutOfMemoryError
 -Dpresto-temporarily-allow-java8=true
 -Djdk.attach.allowAttachSelf=true
+-javaagent:java-version-trim-agent.jar
diff --git a/...rc/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java b/...rc/test/java/org/apache/pulsar/tests/integration/presto/TestPrestoQueryTieredStorage.java
@@ -41,7 +41,6 @@
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
-
 /**
  * Test presto query from tiered storage, the Pulsar SQL is cluster mode.
  */

diff --git a/...egration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java b/...egration/src/test/java/org/apache/pulsar/tests/integration/suites/PulsarSQLTestSuite.java
@@ -23,6 +23,7 @@
 import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
+
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.PulsarClientException;
-Original file line number
+Diff line change
@@ Expand Up / @@ -41,7 +41,6 @@ @@
     import org.testng.Assert;
     import org.testng.annotations.Test;
     /**
      * Test presto query from tiered storage, the Pulsar SQL is cluster mode.
      */
@@ Expand Down @@