diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractContext.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractContext.java index de28480620d7..b5a7d887703e 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractContext.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractContext.java @@ -73,6 +73,8 @@ import io.cdap.cdap.common.internal.remote.RemoteClientFactory; import io.cdap.cdap.common.lang.ClassLoaders; import io.cdap.cdap.common.lang.CombineClassLoader; +import io.cdap.cdap.common.lang.FilterClassLoader; +import io.cdap.cdap.common.lang.FilterClassLoader.Filter; import io.cdap.cdap.common.logging.LoggingContext; import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; @@ -732,7 +734,23 @@ protected ClassLoader createProgramInvocationClassLoader() { ClassLoader pluginsClassLoader = PluginClassLoaders.createFilteredPluginsClassLoader( program.getApplicationSpecification().getPlugins(), pluginInstantiator); - return new CombineClassLoader(null, program.getClassLoader(), pluginsClassLoader, + FilterClassLoader filteredProgramClassLoader = new FilterClassLoader( + program.getClassLoader(), + new Filter() { + @Override + public boolean acceptResource(String resource) { + return !(resource.contains("org/apache/hadoop/fs/s3a") + || resource.contains("org/apache/hadoop/fs/s3native")); + } + + @Override + public boolean acceptPackage(String packageName) { + return !(packageName.contains("org.apache.hadoop.fs.s3a") + || packageName.contains("org.apache.hadoop.fs.s3native")); + } + }); + + return new CombineClassLoader(null, filteredProgramClassLoader, pluginsClassLoader, getClass().getClassLoader()); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginClassLoader.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginClassLoader.java index ffb06321fb42..472c68480bdf 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginClassLoader.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginClassLoader.java @@ -22,6 +22,7 @@ import io.cdap.cdap.app.program.ManifestFields; import io.cdap.cdap.common.lang.CombineClassLoader; import io.cdap.cdap.common.lang.DirectoryClassLoader; +import io.cdap.cdap.common.lang.FilterClassLoader.Filter; import io.cdap.cdap.common.lang.PackageFilterClassLoader; import io.cdap.cdap.internal.app.runtime.ProgramClassLoader; import java.io.File; @@ -79,8 +80,12 @@ static ClassLoader createParent(ClassLoader templateClassLoader) { // from the parent of the template program class loader (which is a filtered // CDAP classloader), followed by template export-packages, then by a plugin // lib jars. + PackageFilterClassLoader parent = + new PackageFilterClassLoader(programClassLoader.getParent(), + (packageName) -> !(packageName.contains("org.apache.hadoop.fs.s3a") + || packageName.contains("org.apache.hadoop.fs.s3native"))); CombineClassLoader classLoader = new CombineClassLoader( - programClassLoader.getParent(), filteredTemplateClassLoader); + parent, filteredTemplateClassLoader); // Creating a CloseableClassLoader ensures that the caller can close the // PackageFilterClassLoader that is held by the CombineClassLoader. return new CloseableClassLoader(classLoader, filteredTemplateClassLoader); @@ -93,6 +98,20 @@ static ClassLoader createParent(ClassLoader templateClassLoader) { this.exportPackages = ManifestFields.getExportPackages(getManifest()); } + // @Override + // protected synchronized Class loadClass(String name, boolean resolve) + // throws ClassNotFoundException { + // if (exportPackages.contains(getPackage(name))) { + // try { + // return findClass(name); + // } catch (ClassNotFoundException e) { + // return super.loadClass(name, resolve); + // } + // } else { + // return super.loadClass(name, resolve); + // } + // } + /** * @return the plugin's artifact id */ diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkClassLoader.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkClassLoader.java index 73cfe50f97e7..e2c7a6ec48df 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkClassLoader.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkClassLoader.java @@ -21,6 +21,8 @@ import io.cdap.cdap.api.spark.SparkExecutionContext; import io.cdap.cdap.common.lang.ClassLoaders; import io.cdap.cdap.common.lang.CombineClassLoader; +import io.cdap.cdap.common.lang.FilterClassLoader; +import io.cdap.cdap.common.lang.FilterClassLoader.Filter; import io.cdap.cdap.internal.app.runtime.ProgramClassLoader; import io.cdap.cdap.internal.app.runtime.plugin.PluginClassLoaders; import org.apache.spark.SparkConf; @@ -122,11 +124,27 @@ public JavaSparkExecutionContext createJavaExecutionContext(SparkExecutionContex * Creates the delegating list of ClassLoader. Used by constructor only. */ private static List createDelegateClassLoaders(SparkRuntimeContext context) { + FilterClassLoader filteredProgramClassLoader = new FilterClassLoader( + context.getProgram().getClassLoader(), + new Filter() { + @Override + public boolean acceptResource(String resource) { + return !(resource.contains("org/apache/hadoop/fs/s3a") + || resource.contains("org/apache/hadoop/fs/s3native")); + } + + @Override + public boolean acceptPackage(String packageName) { + return !(packageName.contains("org.apache.hadoop.fs.s3a") + || packageName.contains("org.apache.hadoop.fs.s3native")); + } + }); return Arrays.asList( - context.getProgram().getClassLoader(), - PluginClassLoaders.createFilteredPluginsClassLoader(context.getApplicationSpecification().getPlugins(), - context.getPluginInstantiator()), - SparkClassLoader.class.getClassLoader() + filteredProgramClassLoader, + PluginClassLoaders.createFilteredPluginsClassLoader( + context.getApplicationSpecification().getPlugins(), + context.getPluginInstantiator()), + SparkClassLoader.class.getClassLoader() ); } } diff --git a/cdap-standalone/pom.xml b/cdap-standalone/pom.xml index 347be68a2e29..9cc11152909d 100644 --- a/cdap-standalone/pom.xml +++ b/cdap-standalone/pom.xml @@ -267,71 +267,6 @@ - - - org.apache.hadoop - hadoop-aws - ${sdk.hadoop.version} - - - com.amazonaws - aws-java-sdk-s3 - 1.11.133 - - - com.amazonaws - aws-java-sdk-kms - 1.11.133 - - - com.amazonaws - aws-java-sdk-core - 1.11.133 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - - - com.amazonaws - jmespath-java - 1.11.133 - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - joda-time - joda-time - 2.9.9 - - - org.apache.httpcomponents - httpclient - 4.5.2 - - - commons-logging - commons-logging - 1.2 - - - org.apache.httpcomponents - httpcore - 4.4.4 - compile - - - software.amazon.ion - ion-java - 1.0.2 - - io.cdap.cdap cdap-ui