diff --git a/jflyte-utils/pom.xml b/jflyte-utils/pom.xml new file mode 100644 index 000000000..e9c8cb7f8 --- /dev/null +++ b/jflyte-utils/pom.xml @@ -0,0 +1,182 @@ + + + + 4.0.0 + + + org.flyte + flytekit-parent + 0.4.23-SNAPSHOT + + + jflyte-utils + + JFlyte Utils - Utilities for jflyte + Primarily used by jflyte, but can also be used to extend or build a jflyte alternative + + + + + org.flyte + flyteidl-protos + + + org.flyte + flytekit-api + + + org.flyte + flytekit-local-engine + + + org.flyte + jflyte-api + + + org.flyte + jflyte-aws + + provided + + + org.flyte + jflyte-google-cloud + + provided + + + org.slf4j + slf4j-api + + + org.slf4j + jul-to-slf4j + + + info.picocli + picocli + + + io.grpc + grpc-netty + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + io.netty + netty-tcnative-boringssl-static + + + + + org.slf4j + slf4j-simple + runtime + + + + + com.google.auto.value + auto-value-annotations + provided + + + com.google.auto.service + auto-service-annotations + provided + + + com.google.errorprone + error_prone_annotations + provided + + + com.google.code.findbugs + jsr305 + provided + + + + + org.flyte + flytekit-jackson + test + + + org.flyte + flytekit-java + test + + + org.junit.jupiter + junit-jupiter + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.hamcrest + hamcrest + test + + + io.grpc + grpc-testing + test + + + org.mockito + mockito-core + test + + + com.github.marschall + memoryfilesystem + test + + + + ch.qos.logback + logback-classic + test + + + + + + + maven-jar-plugin + + + maven-dependency-plugin + + + + diff --git a/jflyte/src/main/java/org/flyte/jflyte/Artifact.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/Artifact.java similarity index 97% rename from jflyte/src/main/java/org/flyte/jflyte/Artifact.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/Artifact.java index a0427a02c..e641b9369 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/Artifact.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/Artifact.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.auto.value.AutoValue; import org.flyte.jflyte.api.FileSystem; diff --git a/jflyte/src/main/java/org/flyte/jflyte/ArtifactStager.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ArtifactStager.java similarity index 98% rename from jflyte/src/main/java/org/flyte/jflyte/ArtifactStager.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ArtifactStager.java index 73434ee4b..2ba960f12 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ArtifactStager.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ArtifactStager.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static com.google.common.base.Verify.verify; @@ -56,7 +56,7 @@ * for directories as well, where directory would be unique and contain hash code of all of it's * contents. */ -class ArtifactStager { +public class ArtifactStager { private static final Logger LOG = LoggerFactory.getLogger(ArtifactStager.class); private final String stagingLocation; @@ -69,7 +69,7 @@ class ArtifactStager { this.executorService = executorService; } - static ArtifactStager create( + public static ArtifactStager create( Config config, Collection modules, ExecutorService executorService) { try { String stagingLocation = config.stagingLocation(); diff --git a/jflyte/src/main/java/org/flyte/jflyte/AuthorizationHeaderInterceptor.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/AuthorizationHeaderInterceptor.java similarity index 98% rename from jflyte/src/main/java/org/flyte/jflyte/AuthorizationHeaderInterceptor.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/AuthorizationHeaderInterceptor.java index e1cee67b9..18be42e80 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/AuthorizationHeaderInterceptor.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/AuthorizationHeaderInterceptor.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import io.grpc.CallOptions; import io.grpc.Channel; diff --git a/jflyte/src/main/java/org/flyte/jflyte/ChildFirstClassLoader.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ChildFirstClassLoader.java similarity index 96% rename from jflyte/src/main/java/org/flyte/jflyte/ChildFirstClassLoader.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ChildFirstClassLoader.java index db337fe0f..fc59b7e18 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ChildFirstClassLoader.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ChildFirstClassLoader.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.errorprone.annotations.Var; import java.io.IOException; @@ -36,7 +36,7 @@ * between the code in parent class loader, and the code loaded in child class loaders, and we pass * instances of these classes around. */ -class ChildFirstClassLoader extends URLClassLoader { +public class ChildFirstClassLoader extends URLClassLoader { // we have to load these classes in parent class loader // it's base shared between all plugins and user code @@ -66,7 +66,7 @@ public URL nextElement() { } } - ChildFirstClassLoader(URL[] urls) { + public ChildFirstClassLoader(URL[] urls) { super(urls, ChildFirstClassLoader.class.getClassLoader()); } diff --git a/jflyte/src/main/java/org/flyte/jflyte/ClassLoaders.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ClassLoaders.java similarity index 87% rename from jflyte/src/main/java/org/flyte/jflyte/ClassLoaders.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ClassLoaders.java index 01abbfe56..6c3e01d9b 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ClassLoaders.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ClassLoaders.java @@ -14,9 +14,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; /** Utility to work with class loaders. */ -class ClassLoaders { +public class ClassLoaders { private static final Logger LOG = LoggerFactory.getLogger(ClassLoaders.class); @@ -41,21 +41,21 @@ private ClassLoaders() { throw new UnsupportedOperationException(); } - static Map forModuleDir(String dir) { + public static Map forModuleDir(String dir) { return listDirectory(new File(dir)).stream() .filter(File::isDirectory) .map(subDir -> Maps.immutableEntry(subDir.getAbsolutePath(), forDirectory(subDir))) .collect(toUnmodifiableMap()); } - static ClassLoader forDirectory(File dir) { + public static ClassLoader forDirectory(File dir) { LOG.debug("Loading jars from [{}]", dir.getAbsolutePath()); return AccessController.doPrivileged( (PrivilegedAction) () -> new ChildFirstClassLoader(getClassLoaderUrls(dir))); } - static URL[] getClassLoaderUrls(File dir) { + private static URL[] getClassLoaderUrls(File dir) { Preconditions.checkNotNull(dir, "dir is null"); return listDirectory(dir).stream() @@ -73,7 +73,7 @@ static URL[] getClassLoaderUrls(File dir) { .toArray(URL[]::new); } - static List listDirectory(File file) { + private static List listDirectory(File file) { if (!file.exists()) { throw new RuntimeException( String.format("Directory doesn't exist [%s]", file.getAbsolutePath())); @@ -88,7 +88,7 @@ static List listDirectory(File file) { return ImmutableList.copyOf(files); } - static V withClassLoader(ClassLoader classLoader, Callable callable) { + public static V withClassLoader(ClassLoader classLoader, Callable callable) { ClassLoader originalContextClassLoader = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(classLoader); diff --git a/jflyte/src/main/java/org/flyte/jflyte/CompletableFutures.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/CompletableFutures.java similarity index 97% rename from jflyte/src/main/java/org/flyte/jflyte/CompletableFutures.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/CompletableFutures.java index ea3986924..3bd782eba 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/CompletableFutures.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/CompletableFutures.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import java.util.ArrayList; import java.util.List; diff --git a/jflyte/src/main/java/org/flyte/jflyte/Config.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/Config.java similarity index 88% rename from jflyte/src/main/java/org/flyte/jflyte/Config.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/Config.java index f4abbac10..ffc16dd1f 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/Config.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/Config.java @@ -14,28 +14,28 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.auto.value.AutoValue; import javax.annotation.Nullable; /** Configuration file for jflyte. */ @AutoValue -abstract class Config { +public abstract class Config { /** * Get flyte platform URL. * * @return flyte platform URL. */ - abstract String platformUrl(); + public abstract String platformUrl(); /** * Get flyte docker image. * * @return flyte docker image. */ - abstract String image(); + public abstract String image(); /** * Get flyte staging location path. @@ -43,23 +43,23 @@ abstract class Config { * @return flyte staging location path. */ @Nullable - abstract String stagingLocation(); + public abstract String stagingLocation(); /** * Get flyte module dir path. * * @return flyte module dir path. */ - abstract String moduleDir(); + public abstract String moduleDir(); /** * Get if the auth with the platform is secure or not. * * @return boolean indicating if platform is insecure or not. */ - abstract boolean platformInsecure(); + public abstract boolean platformInsecure(); - static Config load() { + public static Config load() { return Config.builder() .platformUrl(getenv("FLYTE_PLATFORM_URL")) .moduleDir(getenv("FLYTE_INTERNAL_MODULE_DIR")) diff --git a/jflyte/src/main/java/org/flyte/jflyte/ExecutionConfig.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ExecutionConfig.java similarity index 68% rename from jflyte/src/main/java/org/flyte/jflyte/ExecutionConfig.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ExecutionConfig.java index 1f94bd84f..aeacb9efc 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ExecutionConfig.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ExecutionConfig.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.lang.System.getenv; @@ -22,16 +22,16 @@ /** Configuration values available only during task execution. */ @AutoValue -abstract class ExecutionConfig { +public abstract class ExecutionConfig { abstract String image(); - abstract String project(); + public abstract String project(); - abstract String domain(); + public abstract String domain(); - abstract String version(); + public abstract String version(); - static ExecutionConfig load() { + public static ExecutionConfig load() { return ExecutionConfig.builder() .project(getenv("FLYTE_INTERNAL_PROJECT")) .domain(getenv("FLYTE_INTERNAL_DOMAIN")) @@ -41,20 +41,20 @@ static ExecutionConfig load() { } /** Builder for {@link ExecutionConfig}. */ - static Builder builder() { + public static Builder builder() { return new AutoValue_ExecutionConfig.Builder(); } @AutoValue.Builder - abstract static class Builder { - abstract Builder project(String project); + public abstract static class Builder { + public abstract Builder project(String project); - abstract Builder domain(String domain); + public abstract Builder domain(String domain); - abstract Builder version(String version); + public abstract Builder version(String version); - abstract Builder image(String image); + public abstract Builder image(String image); - abstract ExecutionConfig build(); + public abstract ExecutionConfig build(); } } diff --git a/jflyte/src/main/java/org/flyte/jflyte/FileSystemLoader.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/FileSystemLoader.java similarity index 88% rename from jflyte/src/main/java/org/flyte/jflyte/FileSystemLoader.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/FileSystemLoader.java index cf24a3fa3..8a77c67e7 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/FileSystemLoader.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/FileSystemLoader.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.common.base.Verify; import com.google.common.collect.ImmutableMap; @@ -31,16 +31,16 @@ import org.slf4j.LoggerFactory; /** Load the available FileSystems. */ -class FileSystemLoader { +public class FileSystemLoader { private static final Logger LOG = LoggerFactory.getLogger(FileSystemLoader.class); - static Map loadFileSystems(Collection modules) { + public static Map loadFileSystems(Collection modules) { return modules.stream() .flatMap(module -> ClassLoaders.withClassLoader(module, () -> loadFileSystems().stream())) .collect(Collectors.toMap(FileSystem::getScheme, x -> x)); } - static FileSystem getFileSystem(Map fileSystems, String uri) { + public static FileSystem getFileSystem(Map fileSystems, String uri) { return getFileSystem(fileSystems, URI.create(uri)); } @@ -51,7 +51,7 @@ static FileSystem getFileSystem(Map fileSystems, URI uri) { return Verify.verifyNotNull(fileSystem, "Can't find FileSystem for [%s]", scheme); } - static List loadFileSystems() { + private static List loadFileSystems() { ServiceLoader loader = ServiceLoader.load(FileSystemRegistrar.class); LOG.debug("Discovering FileSystemRegistrar"); diff --git a/jflyte/src/main/java/org/flyte/jflyte/FlyteAdminClient.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/FlyteAdminClient.java similarity index 93% rename from jflyte/src/main/java/org/flyte/jflyte/FlyteAdminClient.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/FlyteAdminClient.java index 9e5ca0f8d..70e446787 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/FlyteAdminClient.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/FlyteAdminClient.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static com.google.common.base.Verify.verifyNotNull; @@ -53,7 +53,7 @@ * This is a thin synchronous wrapper around the auto-generated gRPC stubs for communicating with * the admin service. */ -class FlyteAdminClient implements AutoCloseable { +public class FlyteAdminClient implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(FlyteAdminClient.class); static final String TRIGGERING_PRINCIPAL = "sdk"; @@ -71,7 +71,7 @@ class FlyteAdminClient implements AutoCloseable { this.retries = retries; } - static FlyteAdminClient create( + public static FlyteAdminClient create( String target, boolean insecure, @Nullable TokenSource tokenSource) { ManagedChannelBuilder builder = ManagedChannelBuilder.forTarget(target); @@ -93,7 +93,7 @@ static FlyteAdminClient create( } } - void createTask(TaskIdentifier id, TaskTemplate template) { + public void createTask(TaskIdentifier id, TaskTemplate template) { LOG.debug("createTask {}", id); TaskOuterClass.TaskCreateRequest request = @@ -108,7 +108,7 @@ void createTask(TaskIdentifier id, TaskTemplate template) { idempotentCreate("createTask", id, () -> stub.createTask(request)); } - void createWorkflow( + public void createWorkflow( WorkflowIdentifier id, WorkflowTemplate template, Map subWorkflows) { @@ -130,7 +130,7 @@ void createWorkflow( idempotentCreate("createWorkflow", id, () -> stub.createWorkflow(request)); } - void createLaunchPlan(LaunchPlanIdentifier id, LaunchPlan launchPlan) { + public void createLaunchPlan(LaunchPlanIdentifier id, LaunchPlan launchPlan) { LOG.debug("createLaunchPlan {}", id); LaunchPlanOuterClass.LaunchPlanCreateRequest request = @@ -181,7 +181,7 @@ void createExecution(String domain, String project, LaunchPlanIdentifier launchP } @Nullable - TaskIdentifier fetchLatestTaskId(NamedEntityIdentifier taskId) { + public TaskIdentifier fetchLatestTaskId(NamedEntityIdentifier taskId) { return fetchLatestResource( taskId, request -> stub.listTasks(request).getTasksList(), @@ -190,7 +190,7 @@ TaskIdentifier fetchLatestTaskId(NamedEntityIdentifier taskId) { } @Nullable - WorkflowIdentifier fetchLatestWorkflowId(NamedEntityIdentifier workflowId) { + public WorkflowIdentifier fetchLatestWorkflowId(NamedEntityIdentifier workflowId) { return fetchLatestResource( workflowId, request -> stub.listWorkflows(request).getWorkflowsList(), @@ -199,7 +199,7 @@ WorkflowIdentifier fetchLatestWorkflowId(NamedEntityIdentifier workflowId) { } @Nullable - LaunchPlanIdentifier fetchLatestLaunchPlanId(NamedEntityIdentifier launchPlanId) { + public LaunchPlanIdentifier fetchLatestLaunchPlanId(NamedEntityIdentifier launchPlanId) { return fetchLatestResource( launchPlanId, request -> stub.listLaunchPlans(request).getLaunchPlansList(), diff --git a/jflyte/src/main/java/org/flyte/jflyte/GrpcRetries.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/GrpcRetries.java similarity index 99% rename from jflyte/src/main/java/org/flyte/jflyte/GrpcRetries.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/GrpcRetries.java index 95168d087..94c964569 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/GrpcRetries.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/GrpcRetries.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.auto.value.AutoValue; import com.google.errorprone.annotations.Var; diff --git a/jflyte/src/main/java/org/flyte/jflyte/IdentifierRewrite.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/IdentifierRewrite.java similarity index 95% rename from jflyte/src/main/java/org/flyte/jflyte/IdentifierRewrite.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/IdentifierRewrite.java index 023410d8c..f9591bae2 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/IdentifierRewrite.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/IdentifierRewrite.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; @@ -54,17 +54,18 @@ Visitor visitor() { class Visitor extends WorkflowNodeVisitor { @Override - PartialTaskIdentifier visitTaskIdentifier(PartialTaskIdentifier value) { + protected PartialTaskIdentifier visitTaskIdentifier(PartialTaskIdentifier value) { return apply(value); } @Override - PartialWorkflowIdentifier visitWorkflowIdentifier(PartialWorkflowIdentifier value) { + protected PartialWorkflowIdentifier visitWorkflowIdentifier(PartialWorkflowIdentifier value) { return apply(value); } @Override - PartialLaunchPlanIdentifier visitLaunchPlanIdentifier(PartialLaunchPlanIdentifier value) { + protected PartialLaunchPlanIdentifier visitLaunchPlanIdentifier( + PartialLaunchPlanIdentifier value) { return apply(value); } } diff --git a/jflyte/src/main/java/org/flyte/jflyte/JFlyteCustom.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/JFlyteCustom.java similarity index 93% rename from jflyte/src/main/java/org/flyte/jflyte/JFlyteCustom.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/JFlyteCustom.java index 91edebb0e..198240156 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/JFlyteCustom.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/JFlyteCustom.java @@ -14,9 +14,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableList; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableList; import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; @@ -25,11 +25,11 @@ import org.flyte.api.v1.Struct; @AutoValue -abstract class JFlyteCustom { +public abstract class JFlyteCustom { abstract List artifacts(); - Struct serializeToStruct() { + public Struct serializeToStruct() { Struct jflyte = Struct.of( ImmutableMap.of( @@ -43,7 +43,7 @@ Struct serializeToStruct() { return Struct.of(ImmutableMap.of("jflyte", Struct.Value.ofStructValue(jflyte))); } - static JFlyteCustom deserializeFromStruct(Struct struct) { + public static JFlyteCustom deserializeFromStruct(Struct struct) { Struct.Value jflyteValue = struct.fields().get("jflyte"); Preconditions.checkArgument( diff --git a/jflyte/src/main/java/org/flyte/jflyte/MoreCollectors.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/MoreCollectors.java similarity index 82% rename from jflyte/src/main/java/org/flyte/jflyte/MoreCollectors.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/MoreCollectors.java index ddb0c9c89..cd5a19aac 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/MoreCollectors.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/MoreCollectors.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.util.Collections.unmodifiableMap; import static java.util.stream.Collectors.collectingAndThen; @@ -29,27 +29,27 @@ import java.util.stream.Collector; /** Custom stream java collectors. */ -class MoreCollectors { - static Collector> toUnmodifiableList() { +public class MoreCollectors { + public static Collector> toUnmodifiableList() { return collectingAndThen(toList(), Collections::unmodifiableList); } - static Collector, ?, Map> toUnmodifiableMap() { + public static Collector, ?, Map> toUnmodifiableMap() { return collectingAndThen( toMap(Map.Entry::getKey, Map.Entry::getValue), Collections::unmodifiableMap); } - static Collector> toUnmodifiableMap( + public static Collector> toUnmodifiableMap( Function keyMapper, Function valueMapper) { return collectingAndThen(toMap(keyMapper, valueMapper), Collections::unmodifiableMap); } - static Map mapValues(Map map, Function fn) { + public static Map mapValues(Map map, Function fn) { return map.entrySet().stream() .collect(toUnmodifiableMap(Map.Entry::getKey, x -> fn.apply(x.getValue()))); } - static Map mapValues( + public static Map mapValues( Map map, Function fn, String keyTemplate, Function templateVars) { Map newValues = new LinkedHashMap<>(map.size()); map.forEach( diff --git a/jflyte/src/main/java/org/flyte/jflyte/PackageLoader.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/PackageLoader.java similarity index 97% rename from jflyte/src/main/java/org/flyte/jflyte/PackageLoader.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/PackageLoader.java index 0a015fca9..f3bb3ca1f 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/PackageLoader.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/PackageLoader.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import java.io.IOException; import java.io.UncheckedIOException; @@ -35,10 +35,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class PackageLoader { +public class PackageLoader { private static final Logger LOG = LoggerFactory.getLogger(PackageLoader.class); - static ClassLoader load( + public static ClassLoader load( Map fileSystems, TaskTemplate taskTemplate, ExecutorService executorService) { diff --git a/jflyte/src/main/java/org/flyte/jflyte/ProjectClosure.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProjectClosure.java similarity index 95% rename from jflyte/src/main/java/org/flyte/jflyte/ProjectClosure.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProjectClosure.java index cbe7e16f7..97cbf6199 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ProjectClosure.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProjectClosure.java @@ -14,14 +14,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; -import static org.flyte.jflyte.MoreCollectors.mapValues; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableList; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap; -import static org.flyte.jflyte.QuantityUtil.asJavaQuantity; +import static org.flyte.jflyte.utils.MoreCollectors.mapValues; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableList; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap; +import static org.flyte.jflyte.utils.QuantityUtil.asJavaQuantity; import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; @@ -79,15 +79,15 @@ import org.slf4j.LoggerFactory; @AutoValue -abstract class ProjectClosure { +public abstract class ProjectClosure { private static final Logger LOG = LoggerFactory.getLogger(ProjectClosure.class); - abstract Map taskSpecs(); + public abstract Map taskSpecs(); - abstract Map workflowSpecs(); + public abstract Map workflowSpecs(); - abstract Map launchPlans(); + public abstract Map launchPlans(); ProjectClosure applyCustom(JFlyteCustom custom) { Map rewrittenTaskSpecs = @@ -100,7 +100,7 @@ ProjectClosure applyCustom(JFlyteCustom custom) { .build(); } - void serialize(BiConsumer output) { + public void serialize(BiConsumer output) { int size = taskSpecs().size() + launchPlans().size() + workflowSpecs().size(); int sizeDigits = (int) (Math.log10(size) + 1); AtomicInteger counter = new AtomicInteger(); @@ -141,7 +141,7 @@ private static TaskSpec applyCustom(TaskSpec taskSpec, JFlyteCustom custom) { return TaskSpec.create(rewrittenTaskTemplate); } - static ProjectClosure loadAndStage( + public static ProjectClosure loadAndStage( String packageDir, ExecutionConfig config, Supplier stagerSupplier, @@ -342,7 +342,7 @@ && checkCycles(subWorkflowId, allWorkflows, beingVisited, visited))) { } @VisibleForTesting - static Map collectSubWorkflows( + public static Map collectSubWorkflows( List rewrittenNodes, Map allWorkflows) { return collectSubWorkflowIds(rewrittenNodes).stream() // all identifiers should be rewritten at this point @@ -374,7 +374,7 @@ static Map collectSubWorkflows( .collect(toUnmodifiableMap()); } - static Map collectTasks( + public static Map collectTasks( List rewrittenNodes, Map allTasks) { return collectTaskIds(rewrittenNodes).stream() // all identifiers should be rewritten at this point @@ -407,7 +407,7 @@ private static List collectTaskIds(List rewrittenNo .collect(toUnmodifiableList()); } - static Map createTaskTemplates( + public static Map createTaskTemplates( ExecutionConfig config, Map runnableTasks, Map dynamicWorkflowTasks, @@ -550,7 +550,7 @@ private static TaskTemplate createTaskTemplateForDynamicWorkflow( } @VisibleForTesting - static Struct merge(Struct source, Struct target) { + public static Struct merge(Struct source, Struct target) { Map fields = new HashMap<>(target.fields()); fields.putAll(source.fields()); diff --git a/jflyte/src/main/java/org/flyte/jflyte/ProtoReader.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoReader.java similarity index 88% rename from jflyte/src/main/java/org/flyte/jflyte/ProtoReader.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoReader.java index f14d11d2e..e2fd58b8c 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ProtoReader.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoReader.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import flyteidl.core.Literals; import flyteidl.core.Tasks; @@ -28,14 +28,14 @@ import org.flyte.jflyte.api.FileSystem; /** ProtocolBuffer file reader helper. */ -class ProtoReader { +public class ProtoReader { private final FileSystem inputFs; - ProtoReader(FileSystem inputFs) { + public ProtoReader(FileSystem inputFs) { this.inputFs = inputFs; } - TaskTemplate getTaskTemplate(String uri) { + public TaskTemplate getTaskTemplate(String uri) { try (ReadableByteChannel channel = inputFs.reader(uri)) { Tasks.TaskTemplate proto = Tasks.TaskTemplate.parseFrom(Channels.newInputStream(channel)); @@ -45,7 +45,7 @@ TaskTemplate getTaskTemplate(String uri) { } } - Map getInput(String uri) { + public Map getInput(String uri) { try (ReadableByteChannel channel = inputFs.reader(uri)) { Literals.LiteralMap proto = Literals.LiteralMap.parseFrom(Channels.newInputStream(channel)); diff --git a/jflyte/src/main/java/org/flyte/jflyte/ProtoUtil.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoUtil.java similarity index 96% rename from jflyte/src/main/java/org/flyte/jflyte/ProtoUtil.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoUtil.java index 93c76e173..063cbbffb 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ProtoUtil.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoUtil.java @@ -14,16 +14,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.nullToEmpty; import static java.time.format.DateTimeFormatter.ISO_DATE_TIME; import static java.util.Objects.requireNonNull; -import static org.flyte.jflyte.MoreCollectors.mapValues; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableList; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap; -import static org.flyte.jflyte.QuantityUtil.isValidQuantity; +import static org.flyte.jflyte.utils.MoreCollectors.mapValues; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableList; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap; +import static org.flyte.jflyte.utils.QuantityUtil.isValidQuantity; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -109,7 +109,10 @@ /** Utility to serialize between flytekit-api and flyteidl proto. */ @SuppressWarnings("PreferJavaTimeOverload") -class ProtoUtil { +public class ProtoUtil { + public static final String RUNTIME_FLAVOR = "java"; + public static final String RUNTIME_VERSION = "0.0.1"; + // Datetime is a proto Timestamp and therefore must conform to the Timestamp range. See: // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp private static final Instant DATETIME_MIN = @@ -118,9 +121,6 @@ class ProtoUtil { ISO_DATE_TIME.parse("9999-12-31T23:59:59.999999999Z", Instant::from); private static final Pattern DNS_1123_REGEXP = Pattern.compile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"); - static final String RUNTIME_FLAVOR = "java"; - static final String RUNTIME_VERSION = "0.0.1"; - private ProtoUtil() { throw new UnsupportedOperationException(); } @@ -135,7 +135,7 @@ static Map deserialize(Literals.LiteralMap literalMap) { return inputs; } - static Literal deserialize(Literals.Literal literal) { + public static Literal deserialize(Literals.Literal literal) { switch (literal.getValueCase()) { case SCALAR: return Literal.ofScalar(deserialize(literal.getScalar())); @@ -150,7 +150,7 @@ static Literal deserialize(Literals.Literal literal) { throw new UnsupportedOperationException(String.format("Unsupported Literal [%s]", literal)); } - static Scalar deserialize(Literals.Scalar scalar) { + private static Scalar deserialize(Literals.Scalar scalar) { switch (scalar.getValueCase()) { case PRIMITIVE: return Scalar.ofPrimitive(deserialize(scalar.getPrimitive())); @@ -176,6 +176,7 @@ static Scalar deserialize(Literals.Scalar scalar) { throw new UnsupportedOperationException(String.format("Unsupported Scalar [%s]", scalar)); } + @VisibleForTesting static Primitive deserialize(Literals.Primitive primitive) { switch (primitive.getValueCase()) { case INTEGER: @@ -201,6 +202,7 @@ static Primitive deserialize(Literals.Primitive primitive) { throw new UnsupportedOperationException(String.format("Unsupported Primitive [%s]", primitive)); } + @VisibleForTesting static Blob deserialize(Literals.Blob blob) { BlobType type = BlobType.builder() @@ -213,7 +215,8 @@ static Blob deserialize(Literals.Blob blob) { return Blob.builder().uri(blob.getUri()).metadata(metadata).build(); } - static BlobType.BlobDimensionality deserialize(Types.BlobType.BlobDimensionality dimensionality) { + private static BlobType.BlobDimensionality deserialize( + Types.BlobType.BlobDimensionality dimensionality) { switch (dimensionality) { case SINGLE: return BlobType.BlobDimensionality.SINGLE; @@ -227,7 +230,7 @@ static BlobType.BlobDimensionality deserialize(Types.BlobType.BlobDimensionality String.format("Unsupported BlobDimensionality [%s]", dimensionality)); } - static List deserialize(Literals.LiteralCollection literalCollection) { + private static List deserialize(Literals.LiteralCollection literalCollection) { return literalCollection.getLiteralsList().stream() .map(ProtoUtil::deserialize) .collect(toUnmodifiableList()); @@ -285,7 +288,7 @@ static IdentifierOuterClass.Identifier serialize(PartialIdentifier id) { .build(); } - static IdentifierOuterClass.ResourceType getResourceType(PartialIdentifier id) { + private static IdentifierOuterClass.ResourceType getResourceType(PartialIdentifier id) { if (id instanceof LaunchPlanIdentifier || id instanceof PartialLaunchPlanIdentifier) { // if only Java 14 :( return IdentifierOuterClass.ResourceType.LAUNCH_PLAN; @@ -298,7 +301,7 @@ static IdentifierOuterClass.ResourceType getResourceType(PartialIdentifier id) { throw new IllegalArgumentException("Unknown Identifier type: " + id.getClass()); } - static Tasks.TaskTemplate serialize(TaskIdentifier id, TaskTemplate taskTemplate) { + private static Tasks.TaskTemplate serialize(TaskIdentifier id, TaskTemplate taskTemplate) { return serialize(taskTemplate).toBuilder().setId(serialize(id)).build(); } @@ -434,6 +437,7 @@ static Types.LiteralType serialize(LiteralType literalType) { return builder.build(); } + @VisibleForTesting static LiteralType deserialize(Types.LiteralType proto) { switch (proto.getTypeCase()) { case SIMPLE: @@ -834,7 +838,7 @@ static Workflow.BranchNode serialize(BranchNode branchNode) { return Workflow.BranchNode.newBuilder().setIfElse(serialize(branchNode.ifElse())).build(); } - static Workflow.IfElseBlock serialize(IfElseBlock ifElse) { + private static Workflow.IfElseBlock serialize(IfElseBlock ifElse) { Workflow.IfElseBlock.Builder builder = Workflow.IfElseBlock.newBuilder(); builder.setCase(serialize(ifElse.case_())); @@ -941,6 +945,7 @@ private static Literals.Binding serialize(Binding binding) { .build(); } + @VisibleForTesting static Literals.BindingData serialize(BindingData binding) { Literals.BindingData.Builder builder = Literals.BindingData.newBuilder(); @@ -958,6 +963,7 @@ static Literals.BindingData serialize(BindingData binding) { throw new AssertionError("unexpected BindingData.Kind: " + binding.kind()); } + @VisibleForTesting static Types.OutputReference serialize(OutputReference promise) { return Types.OutputReference.newBuilder() .setNodeId(promise.nodeId()) @@ -1106,6 +1112,7 @@ private static void dateTimeRangeCheck(Instant datetime) { } } + @VisibleForTesting static Literals.Blob serialize(Blob blob) { Literals.BlobMetadata metadata = Literals.BlobMetadata.newBuilder().setType(serialize(blob.metadata().type())).build(); @@ -1113,7 +1120,7 @@ static Literals.Blob serialize(Blob blob) { return Literals.Blob.newBuilder().setUri(blob.uri()).setMetadata(metadata).build(); } - static Literals.Literal serialize(Literal value) { + public static Literals.Literal serialize(Literal value) { Literals.Literal.Builder builder = Literals.Literal.newBuilder(); switch (value.kind()) { @@ -1181,6 +1188,7 @@ static LaunchPlanIdentifier deserializeLaunchPlanId(IdentifierOuterClass.Identif .build(); } + @VisibleForTesting static Types.Error serialize(NodeError e) { return Types.Error.newBuilder() .setMessage(e.message()) @@ -1188,7 +1196,7 @@ static Types.Error serialize(NodeError e) { .build(); } - static Errors.ContainerError serializeThrowable(Throwable e) { + public static Errors.ContainerError serializeThrowable(Throwable e) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); @@ -1199,7 +1207,7 @@ static Errors.ContainerError serializeThrowable(Throwable e) { .build(); } - static Errors.ContainerError.Kind serialize(ContainerError.Kind kind) { + private static Errors.ContainerError.Kind serialize(ContainerError.Kind kind) { switch (kind) { case RECOVERABLE: return Errors.ContainerError.Kind.RECOVERABLE; @@ -1210,7 +1218,7 @@ static Errors.ContainerError.Kind serialize(ContainerError.Kind kind) { throw new AssertionError("unexpected ContainerError.Kind: " + kind); } - static Errors.ContainerError serializeContainerError(ContainerError error) { + public static Errors.ContainerError serializeContainerError(ContainerError error) { return Errors.ContainerError.newBuilder() .setCode(error.getCode()) .setKind(serialize(error.getKind())) @@ -1218,6 +1226,7 @@ static Errors.ContainerError serializeContainerError(ContainerError error) { .build(); } + @VisibleForTesting static ScheduleOuterClass.Schedule serialize(CronSchedule cronSchedule) { String schedule = cronSchedule.schedule(); String offset = cronSchedule.offset(); @@ -1231,7 +1240,7 @@ static ScheduleOuterClass.Schedule serialize(CronSchedule cronSchedule) { return ScheduleOuterClass.Schedule.newBuilder().setCronSchedule(builder.build()).build(); } - static Interface.ParameterMap serializeParameters(Map defaultInputs) { + private static Interface.ParameterMap serializeParameters(Map defaultInputs) { return Interface.ParameterMap.newBuilder() .putAllParameters( defaultInputs.entrySet().stream() diff --git a/jflyte/src/main/java/org/flyte/jflyte/ProtoWriter.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoWriter.java similarity index 90% rename from jflyte/src/main/java/org/flyte/jflyte/ProtoWriter.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoWriter.java index d9d2ed1e5..339e4d0b6 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ProtoWriter.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/ProtoWriter.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import flyteidl.core.DynamicJob; import flyteidl.core.Errors; @@ -32,7 +32,7 @@ import org.slf4j.LoggerFactory; /** ProtocolBuffer file writer helper. */ -class ProtoWriter { +public class ProtoWriter { private static final String OUTPUTS_PB = "outputs.pb"; private static final String FUTURES_PB = "futures.pb"; private static final String ERROR_PB = "error.pb"; @@ -41,7 +41,7 @@ class ProtoWriter { private final String outputPrefix; private final FileSystem outputFs; - ProtoWriter(String outputPrefix, FileSystem outputFs) { + public ProtoWriter(String outputPrefix, FileSystem outputFs) { this.outputPrefix = outputPrefix; this.outputFs = outputFs; } @@ -50,37 +50,37 @@ interface Writer { void write(OutputStream os) throws IOException; } - void writeOutputs(Map outputs) { - String outputUri = normalizeUri(outputPrefix, OUTPUTS_PB); + public void writeFutures(DynamicJobSpec dynamicJobSpec) { + String outputUri = normalizeUri(outputPrefix, FUTURES_PB); writeTo( outputFs, outputUri, outputStream -> { - Literals.LiteralMap proto = ProtoUtil.serialize(outputs); + DynamicJob.DynamicJobSpec proto = ProtoUtil.serialize(dynamicJobSpec); - LOG.debug("writeOutputs {}", proto); + LOG.debug("writeFutures {}", proto); proto.writeTo(outputStream); }); } - void writeFutures(DynamicJobSpec dynamicJobSpec) { - String outputUri = normalizeUri(outputPrefix, FUTURES_PB); + public void writeOutputs(Map outputs) { + String outputUri = normalizeUri(outputPrefix, OUTPUTS_PB); writeTo( outputFs, outputUri, outputStream -> { - DynamicJob.DynamicJobSpec proto = ProtoUtil.serialize(dynamicJobSpec); + Literals.LiteralMap proto = ProtoUtil.serialize(outputs); - LOG.debug("writeFutures {}", proto); + LOG.debug("writeOutputs {}", proto); proto.writeTo(outputStream); }); } - void writeError(Errors.ContainerError containerError) { + public void writeError(Errors.ContainerError containerError) { String outputUri = normalizeUri(outputPrefix, ERROR_PB); writeTo( diff --git a/jflyte/src/main/java/org/flyte/jflyte/QuantityUtil.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/QuantityUtil.java similarity index 98% rename from jflyte/src/main/java/org/flyte/jflyte/QuantityUtil.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/QuantityUtil.java index 64fed6b27..3e1fa9cac 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/QuantityUtil.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/QuantityUtil.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/jflyte/src/main/java/org/flyte/jflyte/Registrars.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/Registrars.java similarity index 93% rename from jflyte/src/main/java/org/flyte/jflyte/Registrars.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/Registrars.java index 1ed0eeef4..342a7b601 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/Registrars.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/Registrars.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import java.util.HashMap; import java.util.Map; @@ -24,14 +24,14 @@ import org.slf4j.LoggerFactory; /** This class consists exclusively of static methods that operate on {@link Registrar}. */ -class Registrars { +public class Registrars { private static final Logger LOG = LoggerFactory.getLogger(Registrars.class); private Registrars() { throw new UnsupportedOperationException(); } - static > Map loadAll( + public static > Map loadAll( Class registrarClass, Map env) { ServiceLoader loader = ServiceLoader.load(registrarClass); diff --git a/jflyte/src/main/java/org/flyte/jflyte/TaskSpec.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/TaskSpec.java similarity index 88% rename from jflyte/src/main/java/org/flyte/jflyte/TaskSpec.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/TaskSpec.java index 319a84096..b390d5f7b 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/TaskSpec.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/TaskSpec.java @@ -14,16 +14,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.auto.value.AutoValue; import org.flyte.api.v1.TaskTemplate; /** Class for Task specification. */ @AutoValue -abstract class TaskSpec { +public abstract class TaskSpec { - abstract TaskTemplate taskTemplate(); + public abstract TaskTemplate taskTemplate(); static TaskSpec create(TaskTemplate taskTemplate) { return new AutoValue_TaskSpec(taskTemplate); diff --git a/jflyte/src/main/java/org/flyte/jflyte/TokenSourceFactoryLoader.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/TokenSourceFactoryLoader.java similarity index 93% rename from jflyte/src/main/java/org/flyte/jflyte/TokenSourceFactoryLoader.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/TokenSourceFactoryLoader.java index 143d30ecd..9e589ea9c 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/TokenSourceFactoryLoader.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/TokenSourceFactoryLoader.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.common.collect.ImmutableMap; import java.util.Collection; @@ -28,10 +28,10 @@ import org.slf4j.LoggerFactory; /** TokenSource loader. */ -class TokenSourceFactoryLoader { +public class TokenSourceFactoryLoader { private static final Logger LOG = LoggerFactory.getLogger(TokenSourceFactoryLoader.class); - static TokenSource getTokenSource(Collection modules, String name) { + public static TokenSource getTokenSource(Collection modules, String name) { return modules.stream() .flatMap( module -> ClassLoaders.withClassLoader(module, () -> loadTokenFactorySources(name))) diff --git a/jflyte/src/main/java/org/flyte/jflyte/WorkflowNodeVisitor.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/WorkflowNodeVisitor.java similarity index 85% rename from jflyte/src/main/java/org/flyte/jflyte/WorkflowNodeVisitor.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/WorkflowNodeVisitor.java index 3d847428e..4ba76b238 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/WorkflowNodeVisitor.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/WorkflowNodeVisitor.java @@ -14,9 +14,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableList; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableList; import java.util.List; import org.flyte.api.v1.BranchNode; @@ -31,16 +31,16 @@ import org.flyte.api.v1.WorkflowTemplate; /** Utility to navigate through the workflow nodes. */ -class WorkflowNodeVisitor { +public class WorkflowNodeVisitor { - WorkflowTemplate visitWorkflowTemplate(WorkflowTemplate template) { + public WorkflowTemplate visitWorkflowTemplate(WorkflowTemplate template) { List newNodes = template.nodes().stream().map(this::visitNode).collect(toUnmodifiableList()); return template.toBuilder().nodes(newNodes).build(); } - Node visitNode(Node node) { + public Node visitNode(Node node) { return node.toBuilder() .branchNode(node.branchNode() != null ? visitBranchNode(node.branchNode()) : null) .taskNode(node.taskNode() != null ? visitTaskNode(node.taskNode()) : null) @@ -87,15 +87,16 @@ WorkflowNode.Reference visitWorkflowNodeReference(WorkflowNode.Reference referen throw new AssertionError("Unexpected WorkflowNode.Reference.Kind: " + reference.kind()); } - PartialTaskIdentifier visitTaskIdentifier(PartialTaskIdentifier value) { + protected PartialTaskIdentifier visitTaskIdentifier(PartialTaskIdentifier value) { return value; } - PartialWorkflowIdentifier visitWorkflowIdentifier(PartialWorkflowIdentifier value) { + protected PartialWorkflowIdentifier visitWorkflowIdentifier(PartialWorkflowIdentifier value) { return value; } - PartialLaunchPlanIdentifier visitLaunchPlanIdentifier(PartialLaunchPlanIdentifier value) { + protected PartialLaunchPlanIdentifier visitLaunchPlanIdentifier( + PartialLaunchPlanIdentifier value) { return value; } } diff --git a/jflyte/src/main/java/org/flyte/jflyte/WorkflowSpec.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/WorkflowSpec.java similarity index 85% rename from jflyte/src/main/java/org/flyte/jflyte/WorkflowSpec.java rename to jflyte-utils/src/main/java/org/flyte/jflyte/utils/WorkflowSpec.java index 286e4f165..f8523759f 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/WorkflowSpec.java +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/WorkflowSpec.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.google.auto.value.AutoValue; import java.util.Map; @@ -23,11 +23,11 @@ /** Class for Workflow specification. */ @AutoValue -abstract class WorkflowSpec { +public abstract class WorkflowSpec { - abstract WorkflowTemplate workflowTemplate(); + public abstract WorkflowTemplate workflowTemplate(); - abstract Map subWorkflows(); + public abstract Map subWorkflows(); static Builder builder() { return new AutoValue_WorkflowSpec.Builder(); diff --git a/jflyte-utils/src/main/java/org/flyte/jflyte/utils/package-info.java b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/package-info.java new file mode 100644 index 000000000..46fc4ac01 --- /dev/null +++ b/jflyte-utils/src/main/java/org/flyte/jflyte/utils/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Flyte Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** Primarily used by jflyte, but can also be used to extend or build a jflyte alternative. */ +package org.flyte.jflyte.utils; diff --git a/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ApiUtils.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ApiUtils.java new file mode 100644 index 000000000..d7f82f19d --- /dev/null +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ApiUtils.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Flyte Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.flyte.jflyte.utils; + +import org.flyte.api.v1.LiteralType; +import org.flyte.api.v1.SimpleType; +import org.flyte.api.v1.Variable; + +/** Contains helper methods to create api objects in a more concise way */ +class ApiUtils { + + private ApiUtils() { + throw new UnsupportedOperationException("Not meant to be created"); + } + + static Variable createVar(SimpleType simpleType) { + return createVar(LiteralType.ofSimpleType(simpleType)); + } + + static Variable createVar(LiteralType type) { + return Variable.builder().literalType(type).description("").build(); + } +} diff --git a/jflyte/src/test/java/org/flyte/jflyte/AuthorizationHeaderInterceptorTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/AuthorizationHeaderInterceptorTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/AuthorizationHeaderInterceptorTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/AuthorizationHeaderInterceptorTest.java index 72e72c436..7aab8f1e9 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/AuthorizationHeaderInterceptorTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/AuthorizationHeaderInterceptorTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; diff --git a/jflyte/src/test/java/org/flyte/jflyte/ChildFirstClassLoaderTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ChildFirstClassLoaderTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/ChildFirstClassLoaderTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/ChildFirstClassLoaderTest.java index e4b182cbd..799f95878 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/ChildFirstClassLoaderTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ChildFirstClassLoaderTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/jflyte/src/test/java/org/flyte/jflyte/CompletableFuturesTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/CompletableFuturesTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/CompletableFuturesTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/CompletableFuturesTest.java index bb660cf7f..bf14e7988 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/CompletableFuturesTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/CompletableFuturesTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; diff --git a/jflyte/src/test/java/org/flyte/jflyte/FileSystemExtension.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/FileSystemExtension.java similarity index 97% rename from jflyte/src/test/java/org/flyte/jflyte/FileSystemExtension.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/FileSystemExtension.java index 2da2bf151..c7c504143 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/FileSystemExtension.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/FileSystemExtension.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import com.github.marschall.memoryfilesystem.MemoryFileSystemBuilder; import java.nio.file.FileSystem; diff --git a/jflyte/src/test/java/org/flyte/jflyte/FlyteAdminClientTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/FlyteAdminClientTest.java similarity index 98% rename from jflyte/src/test/java/org/flyte/jflyte/FlyteAdminClientTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/FlyteAdminClientTest.java index 50ebe0d16..9516e1e74 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/FlyteAdminClientTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/FlyteAdminClientTest.java @@ -14,13 +14,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; -import static org.flyte.jflyte.ApiUtils.createVar; -import static org.flyte.jflyte.FlyteAdminClient.TRIGGERING_PRINCIPAL; -import static org.flyte.jflyte.FlyteAdminClient.USER_TRIGGERED_EXECUTION_NESTING; +import static org.flyte.jflyte.utils.FlyteAdminClient.TRIGGERING_PRINCIPAL; +import static org.flyte.jflyte.utils.FlyteAdminClient.USER_TRIGGERED_EXECUTION_NESTING; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -141,8 +140,8 @@ public void shouldPropagateCreateTaskToStub() { TypedInterface interface_ = TypedInterface.builder() - .inputs(ImmutableMap.of("x", createVar(SimpleType.STRING))) - .outputs(ImmutableMap.of("y", createVar(SimpleType.INTEGER))) + .inputs(ImmutableMap.of("x", ApiUtils.createVar(SimpleType.STRING))) + .outputs(ImmutableMap.of("y", ApiUtils.createVar(SimpleType.INTEGER))) .build(); Container container = diff --git a/jflyte/src/test/java/org/flyte/jflyte/GrpcRetriesTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/GrpcRetriesTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/GrpcRetriesTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/GrpcRetriesTest.java index f28c099be..29af85df2 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/GrpcRetriesTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/GrpcRetriesTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/jflyte/src/test/java/org/flyte/jflyte/GrpcUtils.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/GrpcUtils.java similarity index 97% rename from jflyte/src/test/java/org/flyte/jflyte/GrpcUtils.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/GrpcUtils.java index dee14fb3e..c48f5b395 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/GrpcUtils.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/GrpcUtils.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import io.grpc.BindableService; import io.grpc.ManagedChannel; diff --git a/jflyte/src/test/java/org/flyte/jflyte/IdentifierRewriteTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/IdentifierRewriteTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/IdentifierRewriteTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/IdentifierRewriteTest.java index fd815c525..11a84c40f 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/IdentifierRewriteTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/IdentifierRewriteTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; diff --git a/jflyte/src/test/java/org/flyte/jflyte/InMemoryFileSystem.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/InMemoryFileSystem.java similarity index 98% rename from jflyte/src/test/java/org/flyte/jflyte/InMemoryFileSystem.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/InMemoryFileSystem.java index 44e2cb56e..8b17b5f08 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/InMemoryFileSystem.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/InMemoryFileSystem.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.nio.file.StandardOpenOption.CREATE_NEW; import static java.nio.file.StandardOpenOption.WRITE; diff --git a/jflyte/src/test/java/org/flyte/jflyte/JFlyteCustomTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/JFlyteCustomTest.java similarity index 98% rename from jflyte/src/test/java/org/flyte/jflyte/JFlyteCustomTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/JFlyteCustomTest.java index 31cfa86b3..6e0d88af6 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/JFlyteCustomTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/JFlyteCustomTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; diff --git a/jflyte/src/test/java/org/flyte/jflyte/ProjectClosureTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProjectClosureTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/ProjectClosureTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProjectClosureTest.java index a927b89c5..2d9cfe6a1 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/ProjectClosureTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProjectClosureTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; diff --git a/jflyte/src/test/java/org/flyte/jflyte/ProtoReaderTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoReaderTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/ProtoReaderTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoReaderTest.java index 930647c2d..3a4910df0 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/ProtoReaderTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoReaderTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; diff --git a/jflyte/src/test/java/org/flyte/jflyte/ProtoUtilTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoUtilTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/ProtoUtilTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoUtilTest.java index d1ab8481b..77052e286 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/ProtoUtilTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoUtilTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static flyteidl.core.IdentifierOuterClass.ResourceType.LAUNCH_PLAN; import static flyteidl.core.IdentifierOuterClass.ResourceType.TASK; diff --git a/jflyte/src/test/java/org/flyte/jflyte/ProtoWriterTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoWriterTest.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/ProtoWriterTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoWriterTest.java index fc6fff89d..e3e3c0355 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/ProtoWriterTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/ProtoWriterTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; diff --git a/jflyte/src/test/java/org/flyte/jflyte/QuantityUtilTest.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/QuantityUtilTest.java similarity index 98% rename from jflyte/src/test/java/org/flyte/jflyte/QuantityUtilTest.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/QuantityUtilTest.java index ee89a2635..e03889b72 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/QuantityUtilTest.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/QuantityUtilTest.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/jflyte/src/test/java/org/flyte/jflyte/TestAdminService.java b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/TestAdminService.java similarity index 99% rename from jflyte/src/test/java/org/flyte/jflyte/TestAdminService.java rename to jflyte-utils/src/test/java/org/flyte/jflyte/utils/TestAdminService.java index 91c395fe5..ecca14a90 100644 --- a/jflyte/src/test/java/org/flyte/jflyte/TestAdminService.java +++ b/jflyte-utils/src/test/java/org/flyte/jflyte/utils/TestAdminService.java @@ -14,7 +14,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.flyte.jflyte; +package org.flyte.jflyte.utils; import flyteidl.admin.Common; import flyteidl.admin.ExecutionOuterClass; diff --git a/jflyte/src/test/resources/META-INF/services/org.flyte.jflyte.Foo b/jflyte-utils/src/test/resources/META-INF/services/org.flyte.jflyte.Foo similarity index 100% rename from jflyte/src/test/resources/META-INF/services/org.flyte.jflyte.Foo rename to jflyte-utils/src/test/resources/META-INF/services/org.flyte.jflyte.Foo diff --git a/jflyte/src/test/resources/org/flyte/jflyte/org/slf4j/impl/StaticLoggerBinder.class b/jflyte-utils/src/test/resources/org/flyte/jflyte/utils/org/slf4j/impl/StaticLoggerBinder.class similarity index 100% rename from jflyte/src/test/resources/org/flyte/jflyte/org/slf4j/impl/StaticLoggerBinder.class rename to jflyte-utils/src/test/resources/org/flyte/jflyte/utils/org/slf4j/impl/StaticLoggerBinder.class diff --git a/jflyte/pom.xml b/jflyte/pom.xml index 0bb0d623e..430be0334 100644 --- a/jflyte/pom.xml +++ b/jflyte/pom.xml @@ -55,6 +55,10 @@ org.flyte jflyte-api + + org.flyte + jflyte-utils + org.flyte jflyte-aws diff --git a/jflyte/src/main/java/org/flyte/jflyte/Execute.java b/jflyte/src/main/java/org/flyte/jflyte/Execute.java index c93b76a5b..9cbd5d468 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/Execute.java +++ b/jflyte/src/main/java/org/flyte/jflyte/Execute.java @@ -16,8 +16,8 @@ */ package org.flyte.jflyte; -import static org.flyte.jflyte.ClassLoaders.withClassLoader; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap; +import static org.flyte.jflyte.utils.ClassLoaders.withClassLoader; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap; import java.util.Collection; import java.util.Map; @@ -31,6 +31,14 @@ import org.flyte.api.v1.TaskIdentifier; import org.flyte.api.v1.TaskTemplate; import org.flyte.jflyte.api.FileSystem; +import org.flyte.jflyte.utils.ClassLoaders; +import org.flyte.jflyte.utils.Config; +import org.flyte.jflyte.utils.FileSystemLoader; +import org.flyte.jflyte.utils.PackageLoader; +import org.flyte.jflyte.utils.ProtoReader; +import org.flyte.jflyte.utils.ProtoUtil; +import org.flyte.jflyte.utils.ProtoWriter; +import org.flyte.jflyte.utils.Registrars; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; diff --git a/jflyte/src/main/java/org/flyte/jflyte/ExecuteDynamicWorkflow.java b/jflyte/src/main/java/org/flyte/jflyte/ExecuteDynamicWorkflow.java index 5f1b6ce78..03af3a40f 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ExecuteDynamicWorkflow.java +++ b/jflyte/src/main/java/org/flyte/jflyte/ExecuteDynamicWorkflow.java @@ -18,9 +18,9 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; -import static org.flyte.jflyte.ClassLoaders.withClassLoader; -import static org.flyte.jflyte.MoreCollectors.mapValues; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableList; +import static org.flyte.jflyte.utils.ClassLoaders.withClassLoader; +import static org.flyte.jflyte.utils.MoreCollectors.mapValues; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -53,6 +53,18 @@ import org.flyte.api.v1.WorkflowTemplate; import org.flyte.api.v1.WorkflowTemplateRegistrar; import org.flyte.jflyte.api.FileSystem; +import org.flyte.jflyte.utils.ClassLoaders; +import org.flyte.jflyte.utils.Config; +import org.flyte.jflyte.utils.ExecutionConfig; +import org.flyte.jflyte.utils.FileSystemLoader; +import org.flyte.jflyte.utils.JFlyteCustom; +import org.flyte.jflyte.utils.PackageLoader; +import org.flyte.jflyte.utils.ProjectClosure; +import org.flyte.jflyte.utils.ProtoReader; +import org.flyte.jflyte.utils.ProtoUtil; +import org.flyte.jflyte.utils.ProtoWriter; +import org.flyte.jflyte.utils.Registrars; +import org.flyte.jflyte.utils.WorkflowNodeVisitor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -231,7 +243,7 @@ static class DynamicWorkflowIdentifierRewrite extends WorkflowNodeVisitor { } @Override - PartialTaskIdentifier visitTaskIdentifier(PartialTaskIdentifier value) { + protected PartialTaskIdentifier visitTaskIdentifier(PartialTaskIdentifier value) { if (value.project() == null && value.domain() == null && value.version() == null) { return PartialTaskIdentifier.builder() .name(value.name()) @@ -246,7 +258,7 @@ PartialTaskIdentifier visitTaskIdentifier(PartialTaskIdentifier value) { } @Override - PartialWorkflowIdentifier visitWorkflowIdentifier(PartialWorkflowIdentifier value) { + protected PartialWorkflowIdentifier visitWorkflowIdentifier(PartialWorkflowIdentifier value) { if (value.project() == null && value.domain() == null && value.version() == null) { return PartialWorkflowIdentifier.builder() .name(value.name()) @@ -264,7 +276,8 @@ PartialWorkflowIdentifier visitWorkflowIdentifier(PartialWorkflowIdentifier valu } @Override - PartialLaunchPlanIdentifier visitLaunchPlanIdentifier(PartialLaunchPlanIdentifier value) { + protected PartialLaunchPlanIdentifier visitLaunchPlanIdentifier( + PartialLaunchPlanIdentifier value) { if (value.project() == null && value.domain() == null && value.version() == null) { return PartialLaunchPlanIdentifier.builder() .name(value.name()) diff --git a/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalArgsParser.java b/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalArgsParser.java index 821a31f38..6efd6fcb6 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalArgsParser.java +++ b/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalArgsParser.java @@ -16,7 +16,7 @@ */ package org.flyte.jflyte; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap; import com.google.common.collect.Maps; import java.util.List; diff --git a/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalLoader.java b/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalLoader.java index 6a136d1a8..e599da048 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalLoader.java +++ b/jflyte/src/main/java/org/flyte/jflyte/ExecuteLocalLoader.java @@ -18,9 +18,9 @@ import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toMap; -import static org.flyte.jflyte.ClassLoaders.withClassLoader; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableList; -import static org.flyte.jflyte.MoreCollectors.toUnmodifiableMap; +import static org.flyte.jflyte.utils.ClassLoaders.withClassLoader; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableList; +import static org.flyte.jflyte.utils.MoreCollectors.toUnmodifiableMap; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -37,6 +37,8 @@ import org.flyte.api.v1.Struct; import org.flyte.api.v1.TypedInterface; import org.flyte.api.v1.WorkflowTemplate; +import org.flyte.jflyte.utils.ClassLoaders; +import org.flyte.jflyte.utils.Config; /** Utility class to load modules for execute-local mode. */ public class ExecuteLocalLoader { diff --git a/jflyte/src/main/java/org/flyte/jflyte/Modules.java b/jflyte/src/main/java/org/flyte/jflyte/Modules.java index 194c128f1..705363041 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/Modules.java +++ b/jflyte/src/main/java/org/flyte/jflyte/Modules.java @@ -24,6 +24,7 @@ import org.flyte.api.v1.WorkflowIdentifier; import org.flyte.api.v1.WorkflowTemplate; import org.flyte.api.v1.WorkflowTemplateRegistrar; +import org.flyte.jflyte.utils.Registrars; /** Utility class to load {@link WorkflowTemplate} and {@link RunnableTask}. */ public class Modules { diff --git a/jflyte/src/main/java/org/flyte/jflyte/RegisterWorkflows.java b/jflyte/src/main/java/org/flyte/jflyte/RegisterWorkflows.java index 3ea884b8f..72cf52195 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/RegisterWorkflows.java +++ b/jflyte/src/main/java/org/flyte/jflyte/RegisterWorkflows.java @@ -16,7 +16,7 @@ */ package org.flyte.jflyte; -import static org.flyte.jflyte.TokenSourceFactoryLoader.getTokenSource; +import static org.flyte.jflyte.utils.TokenSourceFactoryLoader.getTokenSource; import java.util.Collection; import java.util.concurrent.Callable; @@ -25,6 +25,12 @@ import java.util.function.Supplier; import javax.annotation.Nullable; import org.flyte.jflyte.api.TokenSource; +import org.flyte.jflyte.utils.ArtifactStager; +import org.flyte.jflyte.utils.ClassLoaders; +import org.flyte.jflyte.utils.Config; +import org.flyte.jflyte.utils.ExecutionConfig; +import org.flyte.jflyte.utils.FlyteAdminClient; +import org.flyte.jflyte.utils.ProjectClosure; import picocli.CommandLine.Command; import picocli.CommandLine.Option; diff --git a/jflyte/src/main/java/org/flyte/jflyte/SerializeWorkflows.java b/jflyte/src/main/java/org/flyte/jflyte/SerializeWorkflows.java index 7d4c2a882..94f20e298 100644 --- a/jflyte/src/main/java/org/flyte/jflyte/SerializeWorkflows.java +++ b/jflyte/src/main/java/org/flyte/jflyte/SerializeWorkflows.java @@ -16,7 +16,7 @@ */ package org.flyte.jflyte; -import static org.flyte.jflyte.TokenSourceFactoryLoader.getTokenSource; +import static org.flyte.jflyte.utils.TokenSourceFactoryLoader.getTokenSource; import com.google.errorprone.annotations.Var; import com.google.protobuf.ByteString; @@ -32,6 +32,12 @@ import java.util.function.Supplier; import javax.annotation.Nullable; import org.flyte.jflyte.api.TokenSource; +import org.flyte.jflyte.utils.ArtifactStager; +import org.flyte.jflyte.utils.ClassLoaders; +import org.flyte.jflyte.utils.Config; +import org.flyte.jflyte.utils.ExecutionConfig; +import org.flyte.jflyte.utils.FlyteAdminClient; +import org.flyte.jflyte.utils.ProjectClosure; import picocli.CommandLine.Command; import picocli.CommandLine.Option; diff --git a/jflyte/update_protos.sh b/jflyte/update_protos.sh deleted file mode 100755 index 074a7fac4..000000000 --- a/jflyte/update_protos.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -FLYTEIDL_VERSION="0.21.3" - -out=src/main/proto - -mkdir -p src/main/proto - -curl -L "https://github.com/lyft/flyteidl/archive/v${FLYTEIDL_VERSION}.tar.gz" | \ - tar xvzf - \ - --strip-components=2 \ - -C "$out/" \ - "flyteidl-${FLYTEIDL_VERSION}/protos/flyteidl/admin/*.proto" \ - "flyteidl-${FLYTEIDL_VERSION}/protos/flyteidl/core/*.proto" \ - "flyteidl-${FLYTEIDL_VERSION}/protos/flyteidl/event/*.proto" \ - "flyteidl-${FLYTEIDL_VERSION}/protos/flyteidl/service/admin.proto" - -# remove trailing whitespace for better OCR -# remove google.api.http options -# remove grpc gateway options -# remove unused import -find "$out" -type f \ - -exec sed -i '' -e "s/ *$//" {} \; \ - -exec sed -i '' -e "/option (google.api.http)/,/};/d" {} \; \ - -exec sed -i '' -e "/option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation)/,/};/d" {} \; \ - -exec sed -i '' -e "/protoc-gen-swagger\/options\/annotations.proto/d" {} \; \ - -exec sed -i '' -e "/google\/api\/annotations.proto/d" {} \; diff --git a/pom.xml b/pom.xml index 2dbdb1c87..83a8aad8a 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,7 @@ jflyte-api jflyte-aws jflyte-google-cloud + jflyte-utils jflyte integration-tests @@ -173,6 +174,11 @@ jflyte ${project.version} + + org.flyte + jflyte-utils + ${project.version} + org.flyte jflyte-aws