From 0a21309423cc6dfe1b14ad176bc363d64a4b7947 Mon Sep 17 00:00:00 2001 From: albertshau Date: Thu, 8 Nov 2018 17:46:10 -0800 Subject: [PATCH] CDAP-14562 fixed a classloading issue with output formats Instead of just using the output format's classloader as the context classloading when creating the RecordWriter, use a CombineClassLoader of the output format's classload and the current context classloader. This prevents issues where classes in the Hadoop classpath are not visible when creating the RecordWriter. More specifically, this fixes issues with XML parsers required by the S3 filesystem. If the program/plugin does not have xml dependencies packaged (they shouldn't need to), it should not result in ClassNotFound exceptions. --- .../runtime/batch/dataset/output/MultipleOutputs.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cdap-app-fabric/src/main/java/co/cask/cdap/internal/app/runtime/batch/dataset/output/MultipleOutputs.java b/cdap-app-fabric/src/main/java/co/cask/cdap/internal/app/runtime/batch/dataset/output/MultipleOutputs.java index 99d5d1842dc5..0df40b3ebe12 100644 --- a/cdap-app-fabric/src/main/java/co/cask/cdap/internal/app/runtime/batch/dataset/output/MultipleOutputs.java +++ b/cdap-app-fabric/src/main/java/co/cask/cdap/internal/app/runtime/batch/dataset/output/MultipleOutputs.java @@ -19,6 +19,7 @@ import co.cask.cdap.app.metrics.MapReduceMetrics; import co.cask.cdap.common.conf.ConfigurationUtil; import co.cask.cdap.common.lang.ClassLoaders; +import co.cask.cdap.common.lang.CombineClassLoader; import co.cask.cdap.proto.id.EntityId; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; @@ -193,9 +194,16 @@ private synchronized RecordWriter getRecordWriter(String namedOutput) throws IOE } ClassLoader outputFormatClassLoader = outputFormatClass.getClassLoader(); + // Use a CombineClassLoader of the output format's classloader and the context classloader + // This is to prevent class not found issues for classes that are visible to the system, but not to the + // program/plugin. More specifically, this happens for XML parsers that are in the Hadoop classpath but usually + // not packaged in programs and plugins. + // see CDAP-14562 for more info + ClassLoader outputClassLoader = new CombineClassLoader(outputFormatClassLoader, + Thread.currentThread().getContextClassLoader()); // This is needed in case the OutputFormat's classloader conflicts with the program classloader (for example, // TableOutputFormat). - ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(outputFormatClassLoader); + ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(outputClassLoader); try { // We use ReflectionUtils to instantiate the OutputFormat, because it also calls setConf on the object, if it