Skip to content

Commit

Permalink
Extract reusable classes
Browse files Browse the repository at this point in the history 
Signed-off-by: Hongxin Liang <[email protected]>
  • Loading branch information
@honnix
honnix committed Sep 25, 2023
1 parent 04e6fdc commit db89324
Show file tree
Hide file tree
Showing 56 changed files with 476 additions and 211 deletions.
182 changes: 182 additions & 0 deletions jflyte-utils/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2021 Flyte Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.flyte</groupId>
<artifactId>flytekit-parent</artifactId>
<version>0.4.23-SNAPSHOT</version>
</parent>

<artifactId>jflyte-utils</artifactId>

<name>JFlyte Utils - Utilities for jflyte</name>
<description>Primarily used by jflyte, but can also be used to extend or build a jflyte alternative</description>

<dependencies>
<!-- compile -->
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flyteidl-protos</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-api</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-local-engine</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>jflyte-api</artifactId>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>jflyte-aws</artifactId>
<!-- We want to have the dependency, so it is already built before building the docker image,
but we want to avoid the artifacts from the module to pollute jflyte -->
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>jflyte-google-cloud</artifactId>
<!-- We want to have the dependency, so it is already built before building the docker image,
but we want to avoid the artifacts from the module to pollute jflyte -->
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
</dependency>
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
</dependency>

<!-- runtime -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>runtime</scope>
</dependency>

<!-- provided -->
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>provided</scope>
</dependency>

<!-- test -->
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-jackson</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.flyte</groupId>
<artifactId>flytekit-java</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-testing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.marschall</groupId>
<artifactId>memoryfilesystem</artifactId>
<scope>test</scope>
</dependency>
<!-- ensure there is at least one slf4j implementation in test classpath so that we can test
org/slf4j/impl/StaticLoggerBinder.class is only discovered in child class loader -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
2 changes: 1 addition & 1 deletion .../main/java/org/flyte/jflyte/Artifact.java → ...java/org/flyte/jflyte/utils/Artifact.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import com.google.auto.value.AutoValue;
import org.flyte.jflyte.api.FileSystem;
Expand Down
6 changes: 3 additions & 3 deletions ...java/org/flyte/jflyte/ArtifactStager.java → ...rg/flyte/jflyte/utils/ArtifactStager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import static com.google.common.base.Verify.verify;

Expand Down Expand Up @@ -56,7 +56,7 @@
* for directories as well, where directory would be unique and contain hash code of all of it's
* contents.
*/
class ArtifactStager {
public class ArtifactStager {
private static final Logger LOG = LoggerFactory.getLogger(ArtifactStager.class);

private final String stagingLocation;
Expand All @@ -69,7 +69,7 @@ class ArtifactStager {
this.executorService = executorService;
}

static ArtifactStager create(
public static ArtifactStager create(
Config config, Collection<ClassLoader> modules, ExecutorService executorService) {
try {
String stagingLocation = config.stagingLocation();
Expand Down
2 changes: 1 addition & 1 deletion ...flyte/AuthorizationHeaderInterceptor.java → ...utils/AuthorizationHeaderInterceptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import io.grpc.CallOptions;
import io.grpc.Channel;
Expand Down
6 changes: 3 additions & 3 deletions ...g/flyte/jflyte/ChildFirstClassLoader.java → ...e/jflyte/utils/ChildFirstClassLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import com.google.errorprone.annotations.Var;
import java.io.IOException;
Expand All @@ -36,7 +36,7 @@
* between the code in parent class loader, and the code loaded in child class loaders, and we pass
* instances of these classes around.
*/
class ChildFirstClassLoader extends URLClassLoader {
public class ChildFirstClassLoader extends URLClassLoader {

// we have to load these classes in parent class loader
// it's base shared between all plugins and user code
Expand Down Expand Up @@ -66,7 +66,7 @@ public URL nextElement() {
}
}

ChildFirstClassLoader(URL[] urls) {
public ChildFirstClassLoader(URL[] urls) {
super(urls, ChildFirstClassLoader.class.getClassLoader());
}

Expand Down
16 changes: 8 additions & 8 deletions ...n/java/org/flyte/jflyte/ClassLoaders.java → .../org/flyte/jflyte/utils/ClassLoaders.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap;
import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
Expand All @@ -33,29 +33,29 @@
import org.slf4j.LoggerFactory;

/** Utility to work with class loaders. */
class ClassLoaders {
public class ClassLoaders {

private static final Logger LOG = LoggerFactory.getLogger(ClassLoaders.class);

private ClassLoaders() {
throw new UnsupportedOperationException();
}

static Map<String, ClassLoader> forModuleDir(String dir) {
public static Map<String, ClassLoader> forModuleDir(String dir) {
return listDirectory(new File(dir)).stream()
.filter(File::isDirectory)
.map(subDir -> Maps.immutableEntry(subDir.getAbsolutePath(), forDirectory(subDir)))
.collect(toUnmodifiableMap());
}

static ClassLoader forDirectory(File dir) {
public static ClassLoader forDirectory(File dir) {
LOG.debug("Loading jars from [{}]", dir.getAbsolutePath());

return AccessController.doPrivileged(
(PrivilegedAction<ClassLoader>) () -> new ChildFirstClassLoader(getClassLoaderUrls(dir)));
}

static URL[] getClassLoaderUrls(File dir) {
private static URL[] getClassLoaderUrls(File dir) {
Preconditions.checkNotNull(dir, "dir is null");

return listDirectory(dir).stream()
Expand All @@ -73,7 +73,7 @@ static URL[] getClassLoaderUrls(File dir) {
.toArray(URL[]::new);
}

static List<File> listDirectory(File file) {
private static List<File> listDirectory(File file) {
if (!file.exists()) {
throw new RuntimeException(
String.format("Directory doesn't exist [%s]", file.getAbsolutePath()));
Expand All @@ -88,7 +88,7 @@ static List<File> listDirectory(File file) {
return ImmutableList.copyOf(files);
}

static <V> V withClassLoader(ClassLoader classLoader, Callable<V> callable) {
public static <V> V withClassLoader(ClassLoader classLoader, Callable<V> callable) {
ClassLoader originalContextClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(classLoader);

Expand Down
2 changes: 1 addition & 1 deletion .../org/flyte/jflyte/CompletableFutures.java → ...lyte/jflyte/utils/CompletableFutures.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import java.util.ArrayList;
import java.util.List;
Expand Down
16 changes: 8 additions & 8 deletions ...rc/main/java/org/flyte/jflyte/Config.java → ...n/java/org/flyte/jflyte/utils/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,52 +14,52 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.flyte.jflyte;
package org.flyte.jflyte.utils;

import com.google.auto.value.AutoValue;
import javax.annotation.Nullable;

/** Configuration file for jflyte. */
@AutoValue
abstract class Config {
public abstract class Config {

/**
* Get flyte platform URL.
*
* @return flyte platform URL.
*/
abstract String platformUrl();
public abstract String platformUrl();

/**
* Get flyte docker image.
*
* @return flyte docker image.
*/
abstract String image();
public abstract String image();

/**
* Get flyte staging location path.
*
* @return flyte staging location path.
*/
@Nullable
abstract String stagingLocation();
public abstract String stagingLocation();

/**
* Get flyte module dir path.
*
* @return flyte module dir path.
*/
abstract String moduleDir();
public abstract String moduleDir();

/**
* Get if the auth with the platform is secure or not.
*
* @return boolean indicating if platform is insecure or not.
*/
abstract boolean platformInsecure();
public abstract boolean platformInsecure();

static Config load() {
public static Config load() {
return Config.builder()
.platformUrl(getenv("FLYTE_PLATFORM_URL"))
.moduleDir(getenv("FLYTE_INTERNAL_MODULE_DIR"))
Expand Down
Loading

0 comments on commit db89324

Please sign in to comment.