[common] Load plugin classes directly from jar without decompression (a…

…pache#1996)
liming30 · Sep 15, 2023 · b3f1031 · b3f1031
1 parent d9326c6
commit b3f1031
Show file tree

Hide file tree

Showing 8 changed files with 28 additions and 93 deletions.
diff --git a/paimon-codegen-loader/pom.xml b/paimon-codegen-loader/pom.xml
@@ -65,10 +65,10 @@ under the License.
                 <artifactId>maven-dependency-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>copy-table-planner-jars</id>
+                        <id>copy-codegen-classes</id>
                         <phase>prepare-package</phase>
                         <goals>
-                            <goal>copy</goal>
+                            <goal>unpack</goal>
                         </goals>
                         <configuration>
                             <artifactItems>
@@ -78,10 +78,9 @@ under the License.
                                     <version>${project.version}</version>
                                     <type>jar</type>
                                     <overWrite>true</overWrite>
-                                    <destFileName>paimon-codegen.jar</destFileName>
                                 </artifactItem>
                             </artifactItems>
-                            <outputDirectory>${project.build.directory}/classes</outputDirectory>
+                            <outputDirectory>${project.build.directory}/classes/paimon-codegen</outputDirectory>
                         </configuration>
                     </execution>
                 </executions>

diff --git a/paimon-codegen-loader/src/main/java/org/apache/paimon/codegen/CodeGenLoader.java b/paimon-codegen-loader/src/main/java/org/apache/paimon/codegen/CodeGenLoader.java
@@ -23,7 +23,7 @@
 /** Loader to load codegen classes. */
 public class CodeGenLoader {
 
-    private static final String CODEGEN_FAT_JAR = "paimon-codegen.jar";
+    private static final String CODEGEN_CLASSES_DIR = "paimon-codegen";
 
     // Singleton lazy initialization
 
@@ -32,7 +32,7 @@ public class CodeGenLoader {
     private static synchronized PluginLoader getLoader() {
         if (loader == null) {
             // Avoid NoClassDefFoundError without cause by exception
-            loader = new PluginLoader(CODEGEN_FAT_JAR);
+            loader = new PluginLoader(CODEGEN_CLASSES_DIR);
         }
         return loader;
     }

diff --git a/paimon-common/src/main/java/org/apache/paimon/plugin/PluginLoader.java b/paimon-common/src/main/java/org/apache/paimon/plugin/PluginLoader.java
@@ -18,24 +18,11 @@
 
 package org.apache.paimon.plugin;
 
-import org.apache.paimon.utils.FileIOUtils;
-import org.apache.paimon.utils.IOUtils;
-import org.apache.paimon.utils.LocalFileUtils;
-import org.apache.paimon.utils.StringUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
 import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Optional;
 import java.util.ServiceLoader;
-import java.util.UUID;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -62,54 +49,22 @@ public class PluginLoader {
 
     private static final String[] COMPONENT_CLASSPATH = new String[] {"org.apache.paimon"};
 
-    private final Path delegateJar;
-
     private final ComponentClassLoader submoduleClassLoader;
 
-    public PluginLoader(String jarName) {
-        try {
-            ClassLoader ownerClassLoader = PluginLoader.class.getClassLoader();
-            Path tmpDirectory =
-                    tmpDirectoryFromYarn()
-                            .orElseGet(() -> Paths.get(System.getProperty("java.io.tmpdir")));
-            Files.createDirectories(
-                    LocalFileUtils.getTargetPathIfContainsSymbolicPath(tmpDirectory));
-            delegateJar =
-                    extractResource(
-                            jarName,
-                            ownerClassLoader,
-                            tmpDirectory,
-                            String.format(
-                                    "%s could not be found.\n"
-                                            + "If you're running a test, please make sure you've built the modules by running\n"
-                                            + "mvn clean install -DskipTests",
-                                    jarName));
-            this.submoduleClassLoader =
-                    new ComponentClassLoader(
-                            new URL[] {delegateJar.toUri().toURL()},
-                            ownerClassLoader,
-                            OWNER_CLASSPATH,
-                            COMPONENT_CLASSPATH);
-        } catch (IOException e) {
-            throw new RuntimeException("Could not initialize the paimon-codegen loader.", e);
+    public PluginLoader(String dirName) {
+        // URL must end with slash,
+        // otherwise URLClassLoader cannot recognize this path as a directory
+        if (!dirName.endsWith("/")) {
+            dirName += "/";
         }
-    }
 
-    private Path extractResource(
-            String resourceName,
-            ClassLoader flinkClassLoader,
-            Path tmpDirectory,
-            String errorMessage)
-            throws IOException {
-        String[] splitName = resourceName.split("\\.");
-        splitName[0] += "_" + UUID.randomUUID();
-        final Path tempFile = Files.createFile(tmpDirectory.resolve(String.join(".", splitName)));
-        final InputStream resourceStream = flinkClassLoader.getResourceAsStream(resourceName);
-        if (resourceStream == null) {
-            throw new RuntimeException(errorMessage);
-        }
-        IOUtils.copyBytes(resourceStream, Files.newOutputStream(tempFile));
-        return tempFile;
+        ClassLoader ownerClassLoader = PluginLoader.class.getClassLoader();
+        this.submoduleClassLoader =
+                new ComponentClassLoader(
+                        new URL[] {ownerClassLoader.getResource(dirName)},
+                        ownerClassLoader,
+                        OWNER_CLASSPATH,
+                        COMPONENT_CLASSPATH);
     }
 
     public <T> T discover(Class<T> clazz) {
@@ -141,21 +96,4 @@ public <T> T newInstance(String name) {
     public ClassLoader submoduleClassLoader() {
         return submoduleClassLoader;
     }
-
-    @Override
-    protected void finalize() throws IOException {
-        submoduleClassLoader.close();
-        FileIOUtils.deleteFileOrDirectory(delegateJar.toFile());
-    }
-
-    private static Optional<Path> tmpDirectoryFromYarn() {
-        String localDirs = System.getenv("LOCAL_DIRS");
-        if (!StringUtils.isBlank(localDirs)) {
-            String[] paths = localDirs.split(",|" + File.pathSeparator);
-            if (paths.length > 0) {
-                return Optional.of(Paths.get(paths[0]));
-            }
-        }
-        return Optional.empty();
-    }
 }
diff --git a/paimon-filesystems/paimon-oss/pom.xml b/paimon-filesystems/paimon-oss/pom.xml
@@ -61,10 +61,10 @@
                 <artifactId>maven-dependency-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>copy-oss-jar</id>
+                        <id>copy-oss-classes</id>
                         <phase>prepare-package</phase>
                         <goals>
-                            <goal>copy</goal>
+                            <goal>unpack</goal>
                         </goals>
                         <configuration>
                             <artifactItems>
@@ -74,10 +74,9 @@
                                     <version>${project.version}</version>
                                     <type>jar</type>
                                     <overWrite>true</overWrite>
-                                    <destFileName>paimon-plugin-oss.jar</destFileName>
+                                    <outputDirectory>${project.build.directory}/classes/paimon-plugin-oss</outputDirectory>
                                 </artifactItem>
                             </artifactItems>
-                            <outputDirectory>${project.build.directory}/classes</outputDirectory>
                         </configuration>
                     </execution>
                 </executions>

diff --git a/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java b/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
@@ -31,7 +31,7 @@
 /** A {@link PluginLoader} to load oss. */
 public class OSSLoader implements FileIOLoader {
 
-    private static final String OSS_JAR = "paimon-plugin-oss.jar";
+    private static final String OSS_CLASSES_DIR = "paimon-plugin-oss";
 
     private static final String OSS_CLASS = "org.apache.paimon.oss.OSSFileIO";
 
@@ -42,7 +42,7 @@ public class OSSLoader implements FileIOLoader {
     private static synchronized PluginLoader getLoader() {
         if (loader == null) {
             // Avoid NoClassDefFoundError without cause by exception
-            loader = new PluginLoader(OSS_JAR);
+            loader = new PluginLoader(OSS_CLASSES_DIR);
         }
         return loader;
     }

diff --git a/paimon-filesystems/paimon-s3/pom.xml b/paimon-filesystems/paimon-s3/pom.xml
@@ -99,10 +99,10 @@
                 <artifactId>maven-dependency-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>copy-s3-jar</id>
+                        <id>copy-s3-classes</id>
                         <phase>prepare-package</phase>
                         <goals>
-                            <goal>copy</goal>
+                            <goal>unpack</goal>
                         </goals>
                         <configuration>
                             <artifactItems>
@@ -112,10 +112,9 @@
                                     <version>${project.version}</version>
                                     <type>jar</type>
                                     <overWrite>true</overWrite>
-                                    <destFileName>paimon-plugin-s3.jar</destFileName>
+                                    <outputDirectory>${project.build.directory}/classes/paimon-plugin-s3</outputDirectory>
                                 </artifactItem>
                             </artifactItems>
-                            <outputDirectory>${project.build.directory}/classes</outputDirectory>
                         </configuration>
                     </execution>
                 </executions>

diff --git a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
@@ -31,7 +31,7 @@
 /** A {@link PluginLoader} to load oss. */
 public class S3Loader implements FileIOLoader {
 
-    private static final String S3_JAR = "paimon-plugin-s3.jar";
+    private static final String S3_CLASSES_DIR = "paimon-plugin-s3";
 
     private static final String S3_CLASS = "org.apache.paimon.s3.S3FileIO";
 
@@ -42,7 +42,7 @@ public class S3Loader implements FileIOLoader {
     private static synchronized PluginLoader getLoader() {
         if (loader == null) {
             // Avoid NoClassDefFoundError without cause by exception
-            loader = new PluginLoader(S3_JAR);
+            loader = new PluginLoader(S3_CLASSES_DIR);
         }
         return loader;
     }

diff --git a/pom.xml b/pom.xml
@@ -70,7 +70,7 @@ under the License.
 
     <properties>
         <project.build.outputTimestamp>2023-09-06T02:16:30Z</project.build.outputTimestamp>
-        <paimon.shade.version>0.4.0-incubating</paimon.shade.version>
+        <paimon.shade.version>0.6-SNAPSHOT</paimon.shade.version>
         <paimon.shade.jackson.version>2.14.2</paimon.shade.jackson.version>
         <paimon.shade.guava.version>30.1.1-jre</paimon.shade.guava.version>
         <paimon.shade.caffeine.version>2.9.3</paimon.shade.caffeine.version>