[3.10, 3.11, 3.12] Backport data loader (#2363)

Co-authored-by: Peckstadt Yves <[email protected]>

core/src/main/java/com/scalar/db/common/error/CoreError.java

@@ -579,6 +579,32 @@ public enum CoreError implements ScalarDbError {
           + "Primary-key columns must not contain any of the following characters in Cosmos DB: ':', '/', '\\', '#', '?'. Value: %s",
       "",
       ""),
+  DATA_LOADER_DIRECTORY_WRITE_ACCESS_NOT_ALLOWED(
+      Category.USER_ERROR,
+      "0131",
+      "The directory '%s' does not have write permissions. Please ensure that the current user has write access to the directory.",
+      "",
+      ""),
+  DATA_LOADER_DIRECTORY_CREATE_FAILED(
+      Category.USER_ERROR,
+      "0132",
+      "Failed to create the directory '%s'. Please check if you have sufficient permissions and if there are any file system restrictions. Details: %s",
+      "",
+      ""),
+  DATA_LOADER_MISSING_DIRECTORY_NOT_ALLOWED(
+      Category.USER_ERROR, "0133", "Directory path cannot be null or empty.", "", ""),
+  DATA_LOADER_MISSING_FILE_EXTENSION(
+      Category.USER_ERROR,
+      "0134",
+      "No file extension was found on the provided file name %s.",
+      "",
+      ""),
+  DATA_LOADER_INVALID_FILE_EXTENSION(
+      Category.USER_ERROR,
+      "0135",
+      "Invalid file extension: %s. Allowed extensions are: %s",
+      "",
+      ""),
 
   //
   // Errors for the concurrency error category

data-loader/build.gradle

@@ -0,0 +1,27 @@
+subprojects {
+    group = "scalardb.dataloader"
+
+    ext {
+        apacheCommonsLangVersion = '3.14.0'
+        apacheCommonsIoVersion = '2.16.1'
+    }
+    dependencies {
+        // AssertJ
+        testImplementation("org.assertj:assertj-core:${assertjVersion}")
+
+        // JUnit 5
+        testImplementation(platform("org.junit:junit-bom:$junitVersion"))
+        testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion")
+        testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}")
+        testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
+
+        // Apache Commons
+        implementation("org.apache.commons:commons-lang3:${apacheCommonsLangVersion}")
+        implementation("commons-io:commons-io:${apacheCommonsIoVersion}")
+
+        // Mockito
+        testImplementation "org.mockito:mockito-core:${mockitoVersion}"
+        testImplementation "org.mockito:mockito-inline:${mockitoVersion}"
+        testImplementation "org.mockito:mockito-junit-jupiter:${mockitoVersion}"
+    }
+}

data-loader/cli/build.gradle

@@ -0,0 +1,59 @@
+plugins {
+    id 'net.ltgt.errorprone' version "${errorpronePluginVersion}"
+    id 'com.github.johnrengelman.shadow' version "${shadowPluginVersion}"
+    id 'com.github.spotbugs' version "${spotbugsPluginVersion}"
+    id 'application'
+}
+
+application {
+    mainClass = 'com.scalar.db.dataloader.cli.DataLoaderCli'
+}
+
+archivesBaseName = "scalardb-data-loader-cli"
+
+dependencies {
+    implementation project(':core')
+    implementation project(':data-loader:core')
+    implementation "org.slf4j:slf4j-simple:${slf4jVersion}"
+    implementation "info.picocli:picocli:${picocliVersion}"
+
+    // for SpotBugs
+    compileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}"
+    testCompileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}"
+
+    // for Error Prone
+    errorprone "com.google.errorprone:error_prone_core:${errorproneVersion}"
+    errorproneJavac "com.google.errorprone:javac:${errorproneJavacVersion}"
+}
+
+javadoc {
+    title = "ScalarDB Data Loader CLI"
+}
+
+// Build a fat jar
+shadowJar {
+    archiveClassifier.set("")
+    manifest {
+        attributes 'Main-Class': 'com.scalar.db.dataloader.DataLoaderCli'
+    }
+    mergeServiceFiles()
+}
+
+spotless {
+    java {
+        target 'src/*/java/**/*.java'
+        importOrder()
+        removeUnusedImports()
+        googleJavaFormat(googleJavaFormatVersion)
+    }
+}
+
+spotbugsMain.reports {
+    html.enabled = true
+}
+spotbugsMain.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml")
+
+spotbugsTest.reports {
+    html.enabled = true
+}
+spotbugsTest.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml")

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/DataLoaderCli.java

@@ -0,0 +1,27 @@
+package com.scalar.db.dataloader.cli;
+
+import com.scalar.db.dataloader.cli.command.dataexport.ExportCommand;
+import com.scalar.db.dataloader.cli.command.dataimport.ImportCommand;
+import picocli.CommandLine;
+
+/** The main class to start the ScalarDB Data loader CLI tool */
+@CommandLine.Command(
+    description = "ScalarDB Data Loader CLI",
+    mixinStandardHelpOptions = true,
+    version = "1.0",
+    subcommands = {ImportCommand.class, ExportCommand.class})
+public class DataLoaderCli {
+
+  /**
+   * Main method to start the ScalarDB Data Loader CLI tool
+   *
+   * @param args the command line arguments
+   */
+  public static void main(String[] args) {
+    int exitCode =
+        new CommandLine(new DataLoaderCli())
+            .setCaseInsensitiveEnumValuesAllowed(true)
+            .execute(args);
+    System.exit(exitCode);
+  }
+}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverter.java

@@ -0,0 +1,32 @@
+package com.scalar.db.dataloader.cli.command;
+
+import com.scalar.db.dataloader.core.ColumnKeyValue;
+import picocli.CommandLine;
+
+/**
+ * Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object. The
+ * string format should be "key=value".
+ */
+public class ColumnKeyValueConverter implements CommandLine.ITypeConverter<ColumnKeyValue> {
+
+  /**
+   * Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object.
+   *
+   * @param keyValue the string representation of the key-value pair in the format "key=value"
+   * @return a {@link ColumnKeyValue} object representing the key-value pair
+   * @throws IllegalArgumentException if the input string is not in the expected format
+   */
+  @Override
+  public ColumnKeyValue convert(String keyValue) {
+    if (keyValue == null) {
+      throw new IllegalArgumentException("Key-value cannot be null");
+    }
+    String[] parts = keyValue.split("=", 2);
+    if (parts.length != 2 || parts[0].trim().isEmpty() || parts[1].trim().isEmpty()) {
+      throw new IllegalArgumentException("Invalid key-value format: " + keyValue);
+    }
+    String columnName = parts[0].trim();
+    String value = parts[1].trim();
+    return new ColumnKeyValue(columnName, value);
+  }
+}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java

@@ -0,0 +1,70 @@
+package com.scalar.db.dataloader.cli.command.dataexport;
+
+import com.scalar.db.common.error.CoreError;
+import com.scalar.db.dataloader.cli.exception.DirectoryValidationException;
+import com.scalar.db.dataloader.cli.exception.InvalidFileExtensionException;
+import com.scalar.db.dataloader.cli.util.DirectoryUtils;
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import javax.annotation.Nullable;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
+import picocli.CommandLine;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Spec;
+
+@CommandLine.Command(name = "export", description = "Export data from a ScalarDB table")
+public class ExportCommand extends ExportCommandOptions implements Callable<Integer> {
+
+  private static final List<String> ALLOWED_EXTENSIONS = Arrays.asList("csv", "json", "jsonl");
+
+  @Spec CommandSpec spec;
+
+  @Override
+  public Integer call() throws Exception {
+    validateOutputDirectory(outputFilePath);
+    return 0;
+  }
+
+  private void validateOutputDirectory(@Nullable String path)
+      throws DirectoryValidationException, InvalidFileExtensionException {
+    if (path == null || path.isEmpty()) {
+      // It is ok for the output file path to be null or empty as a default file name will be used
+      // if not provided
+      return;
+    }
+
+    File file = new File(path);
+
+    if (file.isDirectory()) {
+      validateDirectory(path);
+    } else {
+      validateFileExtension(file.getName());
+      validateDirectory(file.getParent());
+    }
+  }
+
+  private void validateDirectory(String directoryPath) throws DirectoryValidationException {
+    // If the directory path is null or empty, use the current working directory
+    if (directoryPath == null || directoryPath.isEmpty()) {
+      DirectoryUtils.validateTargetDirectory(DirectoryUtils.getCurrentWorkingDirectory());
+    } else {
+      DirectoryUtils.validateTargetDirectory(directoryPath);
+    }
+  }
+
+  private void validateFileExtension(String filename) throws InvalidFileExtensionException {
+    String extension = FilenameUtils.getExtension(filename);
+    if (StringUtils.isBlank(extension)) {
+      throw new InvalidFileExtensionException(
+          CoreError.DATA_LOADER_MISSING_FILE_EXTENSION.buildMessage(filename));
+    }
+    if (!ALLOWED_EXTENSIONS.contains(extension.toLowerCase())) {
+      throw new InvalidFileExtensionException(
+          CoreError.DATA_LOADER_INVALID_FILE_EXTENSION.buildMessage(
+              extension, String.join(", ", ALLOWED_EXTENSIONS)));
+    }
+  }
+}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java

@@ -0,0 +1,14 @@
+package com.scalar.db.dataloader.cli.command.dataexport;
+
+import picocli.CommandLine;
+
+/** A class to represent the command options for the export command. */
+public class ExportCommandOptions {
+
+  @CommandLine.Option(
+      names = {"--output-file", "-o"},
+      paramLabel = "<OUTPUT_FILE>",
+      description =
+          "Path and name of the output file for the exported data (default: <table_name>.<format>)")
+  protected String outputFilePath;
+}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java

@@ -0,0 +1,14 @@
+package com.scalar.db.dataloader.cli.command.dataimport;
+
+import java.util.concurrent.Callable;
+import picocli.CommandLine;
+
+@CommandLine.Command(name = "import", description = "Import data into a ScalarDB table")
+public class ImportCommand extends ImportCommandOptions implements Callable<Integer> {
+  @CommandLine.Spec CommandLine.Model.CommandSpec spec;
+
+  @Override
+  public Integer call() throws Exception {
+    return 0;
+  }
+}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommandOptions.java

@@ -0,0 +1,3 @@
+package com.scalar.db.dataloader.cli.command.dataimport;
+
+public class ImportCommandOptions {}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/DirectoryValidationException.java

@@ -0,0 +1,9 @@
+package com.scalar.db.dataloader.cli.exception;
+
+/** Exception thrown when there is an error validating a directory. */
+public class DirectoryValidationException extends Exception {
+
+  public DirectoryValidationException(String message) {
+    super(message);
+  }
+}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/InvalidFileExtensionException.java

@@ -0,0 +1,9 @@
+package com.scalar.db.dataloader.cli.exception;
+
+/** An exception thrown when the file extension is invalid. */
+public class InvalidFileExtensionException extends Exception {
+
+  public InvalidFileExtensionException(String message) {
+    super(message);
+  }
+}

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/DirectoryUtils.java

@@ -0,0 +1,62 @@
+package com.scalar.db.dataloader.cli.util;
+
+import com.scalar.db.common.error.CoreError;
+import com.scalar.db.dataloader.cli.exception.DirectoryValidationException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.apache.commons.lang3.StringUtils;
+
+/** Utility class for validating and handling directories. */
+public final class DirectoryUtils {
+
+  private DirectoryUtils() {
+    // restrict instantiation
+  }
+
+  /**
+   * Validates the provided directory path. Ensures that the directory exists and is writable. If
+   * the directory doesn't exist, a creation attempt is made.
+   *
+   * @param directoryPath the directory path to validate
+   * @throws DirectoryValidationException if the directory is not writable or cannot be created
+   */
+  public static void validateTargetDirectory(String directoryPath)
+      throws DirectoryValidationException {
+    if (StringUtils.isBlank(directoryPath)) {
+      throw new IllegalArgumentException(
+          CoreError.DATA_LOADER_MISSING_DIRECTORY_NOT_ALLOWED.buildMessage());
+    }
+
+    Path path = Paths.get(directoryPath);
+
+    if (Files.exists(path)) {
+      // Check if the provided directory is writable
+      if (!Files.isWritable(path)) {
+        throw new DirectoryValidationException(
+            CoreError.DATA_LOADER_DIRECTORY_WRITE_ACCESS_NOT_ALLOWED.buildMessage(
+                path.toAbsolutePath()));
+      }
+
+    } else {
+      // Create the directory if it doesn't exist
+      try {
+        Files.createDirectories(path);
+      } catch (IOException e) {
+        throw new DirectoryValidationException(
+            CoreError.DATA_LOADER_DIRECTORY_CREATE_FAILED.buildMessage(
+                path.toAbsolutePath(), e.getMessage()));
+      }
+    }
+  }
+
+  /**
+   * Returns the current working directory.
+   *
+   * @return the current working directory
+   */
+  public static String getCurrentWorkingDirectory() {
+    return Paths.get(System.getProperty("user.dir")).toAbsolutePath().toString();
+  }
+}