Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove s3 connector from platform #15881

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

remove s3 connector from platform #15881

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/AbstractContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@
import io.cdap.cdap.common.internal.remote.RemoteClientFactory;
import io.cdap.cdap.common.lang.ClassLoaders;
import io.cdap.cdap.common.lang.CombineClassLoader;
import io.cdap.cdap.common.lang.FilterClassLoader;
import io.cdap.cdap.common.lang.FilterClassLoader.Filter;
import io.cdap.cdap.common.logging.LoggingContext;
import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService;
import io.cdap.cdap.common.namespace.NamespaceQueryAdmin;
Expand Down Expand Up @@ -732,7 +734,23 @@ protected ClassLoader createProgramInvocationClassLoader() {
ClassLoader pluginsClassLoader = PluginClassLoaders.createFilteredPluginsClassLoader(
program.getApplicationSpecification().getPlugins(), pluginInstantiator);

return new CombineClassLoader(null, program.getClassLoader(), pluginsClassLoader,
FilterClassLoader filteredProgramClassLoader = new FilterClassLoader(
program.getClassLoader(),
new Filter() {
@Override
public boolean acceptResource(String resource) {
return !(resource.contains("org/apache/hadoop/fs/s3a")
|| resource.contains("org/apache/hadoop/fs/s3native"));
}

@Override
public boolean acceptPackage(String packageName) {
return !(packageName.contains("org.apache.hadoop.fs.s3a")
|| packageName.contains("org.apache.hadoop.fs.s3native"));
}
});

return new CombineClassLoader(null, filteredProgramClassLoader, pluginsClassLoader,
getClass().getClassLoader());
}

Expand Down
21 changes: 20 additions & 1 deletion ...-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/plugin/PluginClassLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.cdap.cdap.app.program.ManifestFields;
import io.cdap.cdap.common.lang.CombineClassLoader;
import io.cdap.cdap.common.lang.DirectoryClassLoader;
import io.cdap.cdap.common.lang.FilterClassLoader.Filter;
import io.cdap.cdap.common.lang.PackageFilterClassLoader;
import io.cdap.cdap.internal.app.runtime.ProgramClassLoader;
import java.io.File;
Expand Down Expand Up @@ -79,8 +80,12 @@ static ClassLoader createParent(ClassLoader templateClassLoader) {
// from the parent of the template program class loader (which is a filtered
// CDAP classloader), followed by template export-packages, then by a plugin
// lib jars.
PackageFilterClassLoader parent =
new PackageFilterClassLoader(programClassLoader.getParent(),
(packageName) -> !(packageName.contains("org.apache.hadoop.fs.s3a")
|| packageName.contains("org.apache.hadoop.fs.s3native")));
CombineClassLoader classLoader = new CombineClassLoader(
programClassLoader.getParent(), filteredTemplateClassLoader);
parent, filteredTemplateClassLoader);
// Creating a CloseableClassLoader ensures that the caller can close the
// PackageFilterClassLoader that is held by the CombineClassLoader.
return new CloseableClassLoader(classLoader, filteredTemplateClassLoader);
Expand All @@ -93,6 +98,20 @@ static ClassLoader createParent(ClassLoader templateClassLoader) {
this.exportPackages = ManifestFields.getExportPackages(getManifest());
}

// @Override
// protected synchronized Class<?> loadClass(String name, boolean resolve)
// throws ClassNotFoundException {
// if (exportPackages.contains(getPackage(name))) {
// try {
// return findClass(name);
// } catch (ClassNotFoundException e) {
// return super.loadClass(name, resolve);
// }
// } else {
// return super.loadClass(name, resolve);
// }
// }

/**
* @return the plugin's artifact id
*/
Expand Down
26 changes: 22 additions & 4 deletions cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/SparkClassLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import io.cdap.cdap.api.spark.SparkExecutionContext;
import io.cdap.cdap.common.lang.ClassLoaders;
import io.cdap.cdap.common.lang.CombineClassLoader;
import io.cdap.cdap.common.lang.FilterClassLoader;
import io.cdap.cdap.common.lang.FilterClassLoader.Filter;
import io.cdap.cdap.internal.app.runtime.ProgramClassLoader;
import io.cdap.cdap.internal.app.runtime.plugin.PluginClassLoaders;
import org.apache.spark.SparkConf;
Expand Down Expand Up @@ -122,11 +124,27 @@ public JavaSparkExecutionContext createJavaExecutionContext(SparkExecutionContex
* Creates the delegating list of ClassLoader. Used by constructor only.
*/
private static List<ClassLoader> createDelegateClassLoaders(SparkRuntimeContext context) {
FilterClassLoader filteredProgramClassLoader = new FilterClassLoader(
context.getProgram().getClassLoader(),
new Filter() {
@Override
public boolean acceptResource(String resource) {
return !(resource.contains("org/apache/hadoop/fs/s3a")
|| resource.contains("org/apache/hadoop/fs/s3native"));
}

@Override
public boolean acceptPackage(String packageName) {
return !(packageName.contains("org.apache.hadoop.fs.s3a")
|| packageName.contains("org.apache.hadoop.fs.s3native"));
}
});
return Arrays.asList(
context.getProgram().getClassLoader(),
PluginClassLoaders.createFilteredPluginsClassLoader(context.getApplicationSpecification().getPlugins(),
context.getPluginInstantiator()),
SparkClassLoader.class.getClassLoader()
filteredProgramClassLoader,
PluginClassLoaders.createFilteredPluginsClassLoader(
context.getApplicationSpecification().getPlugins(),
context.getPluginInstantiator()),
SparkClassLoader.class.getClassLoader()
);
}
}
Expand Down
65 changes: 0 additions & 65 deletions cdap-standalone/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,71 +267,6 @@
</exclusion>
</exclusions>
</dependency>
<!--Amazon S3 dependencies-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${sdk.hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.133</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-kms</artifactId>
<version>1.11.133</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
<version>1.11.133</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>jmespath-java</artifactId>
<version>1.11.133</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.9</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.4</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>software.amazon.ion</groupId>
<artifactId>ion-java</artifactId>
<version>1.0.2</version>
</dependency>
<!--End of Amazon S3 dependencies-->
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-ui</artifactId>
Expand Down