From 350c31a6dc8328b5cdebe909c8d64520c382ecbf Mon Sep 17 00:00:00 2001 From: Hiroyuki Yamada Date: Wed, 27 Nov 2024 13:40:26 +0900 Subject: [PATCH] Backport to branch(3.10) : [3.10, 3.11, 3.12] Backport data loader (#2365) Co-authored-by: Toshihiro Suzuki Co-authored-by: Peckstadt Yves --- data-loader/build.gradle | 27 +++++ data-loader/cli/build.gradle | 59 +++++++++ .../db/dataloader/cli/DataLoaderCli.java | 27 +++++ .../cli/command/ColumnKeyValueConverter.java | 32 +++++ .../cli/command/dataexport/ExportCommand.java | 70 +++++++++++ .../dataexport/ExportCommandOptions.java | 14 +++ .../cli/command/dataimport/ImportCommand.java | 14 +++ .../dataimport/ImportCommandOptions.java | 3 + .../DirectoryValidationException.java | 9 ++ .../InvalidFileExtensionException.java | 9 ++ .../dataloader/cli/util/DirectoryUtils.java | 62 ++++++++++ .../command/ColumnKeyValueConverterTest.java | 62 ++++++++++ .../command/dataexport/ExportCommandTest.java | 112 ++++++++++++++++++ .../cli/util/DirectoryUtilsTest.java | 87 ++++++++++++++ data-loader/core/build.gradle | 43 +++++++ .../db/dataloader/core/ColumnKeyValue.java | 36 ++++++ .../scalar/db/dataloader/core/FileFormat.java | 8 ++ settings.gradle | 2 + 18 files changed, 676 insertions(+) create mode 100644 data-loader/build.gradle create mode 100644 data-loader/cli/build.gradle create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/DataLoaderCli.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverter.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommandOptions.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/DirectoryValidationException.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/InvalidFileExtensionException.java create mode 100644 data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/DirectoryUtils.java create mode 100644 data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverterTest.java create mode 100644 data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java create mode 100644 data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java create mode 100644 data-loader/core/build.gradle create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/ColumnKeyValue.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/FileFormat.java diff --git a/data-loader/build.gradle b/data-loader/build.gradle new file mode 100644 index 0000000000..dc73d45023 --- /dev/null +++ b/data-loader/build.gradle @@ -0,0 +1,27 @@ +subprojects { + group = "scalardb.dataloader" + + ext { + apacheCommonsLangVersion = '3.14.0' + apacheCommonsIoVersion = '2.16.1' + } + dependencies { + // AssertJ + testImplementation("org.assertj:assertj-core:${assertjVersion}") + + // JUnit 5 + testImplementation(platform("org.junit:junit-bom:$junitVersion")) + testImplementation("org.junit.jupiter:junit-jupiter:$junitVersion") + testImplementation("org.junit.jupiter:junit-jupiter-api:${junitVersion}") + testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}") + + // Apache Commons + implementation("org.apache.commons:commons-lang3:${apacheCommonsLangVersion}") + implementation("commons-io:commons-io:${apacheCommonsIoVersion}") + + // Mockito + testImplementation "org.mockito:mockito-core:${mockitoVersion}" + testImplementation "org.mockito:mockito-inline:${mockitoVersion}" + testImplementation "org.mockito:mockito-junit-jupiter:${mockitoVersion}" + } +} diff --git a/data-loader/cli/build.gradle b/data-loader/cli/build.gradle new file mode 100644 index 0000000000..5c495467d2 --- /dev/null +++ b/data-loader/cli/build.gradle @@ -0,0 +1,59 @@ +plugins { + id 'net.ltgt.errorprone' version "${errorpronePluginVersion}" + id 'com.github.johnrengelman.shadow' version "${shadowPluginVersion}" + id 'com.github.spotbugs' version "${spotbugsPluginVersion}" + id 'application' +} + +application { + mainClass = 'com.scalar.db.dataloader.cli.DataLoaderCli' +} + +archivesBaseName = "scalardb-data-loader-cli" + +dependencies { + implementation project(':core') + implementation project(':data-loader:core') + implementation "org.slf4j:slf4j-simple:${slf4jVersion}" + implementation "info.picocli:picocli:${picocliVersion}" + + // for SpotBugs + compileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}" + testCompileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}" + + // for Error Prone + errorprone "com.google.errorprone:error_prone_core:${errorproneVersion}" + errorproneJavac "com.google.errorprone:javac:${errorproneJavacVersion}" +} + +javadoc { + title = "ScalarDB Data Loader CLI" +} + +// Build a fat jar +shadowJar { + archiveClassifier.set("") + manifest { + attributes 'Main-Class': 'com.scalar.db.dataloader.DataLoaderCli' + } + mergeServiceFiles() +} + +spotless { + java { + target 'src/*/java/**/*.java' + importOrder() + removeUnusedImports() + googleJavaFormat(googleJavaFormatVersion) + } +} + +spotbugsMain.reports { + html.enabled = true +} +spotbugsMain.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml") + +spotbugsTest.reports { + html.enabled = true +} +spotbugsTest.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml") diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/DataLoaderCli.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/DataLoaderCli.java new file mode 100644 index 0000000000..821b853939 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/DataLoaderCli.java @@ -0,0 +1,27 @@ +package com.scalar.db.dataloader.cli; + +import com.scalar.db.dataloader.cli.command.dataexport.ExportCommand; +import com.scalar.db.dataloader.cli.command.dataimport.ImportCommand; +import picocli.CommandLine; + +/** The main class to start the ScalarDB Data loader CLI tool */ +@CommandLine.Command( + description = "ScalarDB Data Loader CLI", + mixinStandardHelpOptions = true, + version = "1.0", + subcommands = {ImportCommand.class, ExportCommand.class}) +public class DataLoaderCli { + + /** + * Main method to start the ScalarDB Data Loader CLI tool + * + * @param args the command line arguments + */ + public static void main(String[] args) { + int exitCode = + new CommandLine(new DataLoaderCli()) + .setCaseInsensitiveEnumValuesAllowed(true) + .execute(args); + System.exit(exitCode); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverter.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverter.java new file mode 100644 index 0000000000..9022e4cf0c --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverter.java @@ -0,0 +1,32 @@ +package com.scalar.db.dataloader.cli.command; + +import com.scalar.db.dataloader.core.ColumnKeyValue; +import picocli.CommandLine; + +/** + * Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object. The + * string format should be "key=value". + */ +public class ColumnKeyValueConverter implements CommandLine.ITypeConverter { + + /** + * Converts a string representation of a key-value pair into a {@link ColumnKeyValue} object. + * + * @param keyValue the string representation of the key-value pair in the format "key=value" + * @return a {@link ColumnKeyValue} object representing the key-value pair + * @throws IllegalArgumentException if the input string is not in the expected format + */ + @Override + public ColumnKeyValue convert(String keyValue) { + if (keyValue == null) { + throw new IllegalArgumentException("Key-value cannot be null"); + } + String[] parts = keyValue.split("=", 2); + if (parts.length != 2 || parts[0].trim().isEmpty() || parts[1].trim().isEmpty()) { + throw new IllegalArgumentException("Invalid key-value format: " + keyValue); + } + String columnName = parts[0].trim(); + String value = parts[1].trim(); + return new ColumnKeyValue(columnName, value); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java new file mode 100644 index 0000000000..66bdfb8f26 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java @@ -0,0 +1,70 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import com.scalar.db.dataloader.cli.exception.DirectoryValidationException; +import com.scalar.db.dataloader.cli.exception.InvalidFileExtensionException; +import com.scalar.db.dataloader.cli.util.DirectoryUtils; +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import javax.annotation.Nullable; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import picocli.CommandLine; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Spec; + +@CommandLine.Command(name = "export", description = "Export data from a ScalarDB table") +public class ExportCommand extends ExportCommandOptions implements Callable { + + private static final List ALLOWED_EXTENSIONS = Arrays.asList("csv", "json", "jsonl"); + + @Spec CommandSpec spec; + + @Override + public Integer call() throws Exception { + validateOutputDirectory(outputFilePath); + return 0; + } + + private void validateOutputDirectory(@Nullable String path) + throws DirectoryValidationException, InvalidFileExtensionException { + if (path == null || path.isEmpty()) { + // It is ok for the output file path to be null or empty as a default file name will be used + // if not provided + return; + } + + File file = new File(path); + + if (file.isDirectory()) { + validateDirectory(path); + } else { + validateFileExtension(file.getName()); + validateDirectory(file.getParent()); + } + } + + private void validateDirectory(String directoryPath) throws DirectoryValidationException { + // If the directory path is null or empty, use the current working directory + if (directoryPath == null || directoryPath.isEmpty()) { + DirectoryUtils.validateTargetDirectory(DirectoryUtils.getCurrentWorkingDirectory()); + } else { + DirectoryUtils.validateTargetDirectory(directoryPath); + } + } + + private void validateFileExtension(String filename) throws InvalidFileExtensionException { + String extension = FilenameUtils.getExtension(filename); + if (StringUtils.isBlank(extension)) { + throw new InvalidFileExtensionException( + String.format("No file extension was found on the provided file name %s.", filename)); + } + if (!ALLOWED_EXTENSIONS.contains(extension.toLowerCase())) { + throw new InvalidFileExtensionException( + String.format( + "Invalid file extension: %s. Allowed extensions are: %s", + extension, String.join(", ", ALLOWED_EXTENSIONS))); + } + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java new file mode 100644 index 0000000000..b5ac608f27 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java @@ -0,0 +1,14 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import picocli.CommandLine; + +/** A class to represent the command options for the export command. */ +public class ExportCommandOptions { + + @CommandLine.Option( + names = {"--output-file", "-o"}, + paramLabel = "", + description = + "Path and name of the output file for the exported data (default: .)") + protected String outputFilePath; +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java new file mode 100644 index 0000000000..f7eb9a1198 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java @@ -0,0 +1,14 @@ +package com.scalar.db.dataloader.cli.command.dataimport; + +import java.util.concurrent.Callable; +import picocli.CommandLine; + +@CommandLine.Command(name = "import", description = "Import data into a ScalarDB table") +public class ImportCommand extends ImportCommandOptions implements Callable { + @CommandLine.Spec CommandLine.Model.CommandSpec spec; + + @Override + public Integer call() throws Exception { + return 0; + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommandOptions.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommandOptions.java new file mode 100644 index 0000000000..ab3fa54d45 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommandOptions.java @@ -0,0 +1,3 @@ +package com.scalar.db.dataloader.cli.command.dataimport; + +public class ImportCommandOptions {} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/DirectoryValidationException.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/DirectoryValidationException.java new file mode 100644 index 0000000000..c980a969fe --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/DirectoryValidationException.java @@ -0,0 +1,9 @@ +package com.scalar.db.dataloader.cli.exception; + +/** Exception thrown when there is an error validating a directory. */ +public class DirectoryValidationException extends Exception { + + public DirectoryValidationException(String message) { + super(message); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/InvalidFileExtensionException.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/InvalidFileExtensionException.java new file mode 100644 index 0000000000..7be7cbfa43 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/exception/InvalidFileExtensionException.java @@ -0,0 +1,9 @@ +package com.scalar.db.dataloader.cli.exception; + +/** An exception thrown when the file extension is invalid. */ +public class InvalidFileExtensionException extends Exception { + + public InvalidFileExtensionException(String message) { + super(message); + } +} diff --git a/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/DirectoryUtils.java b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/DirectoryUtils.java new file mode 100644 index 0000000000..f2cfe0fe87 --- /dev/null +++ b/data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/util/DirectoryUtils.java @@ -0,0 +1,62 @@ +package com.scalar.db.dataloader.cli.util; + +import com.scalar.db.dataloader.cli.exception.DirectoryValidationException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.commons.lang3.StringUtils; + +/** Utility class for validating and handling directories. */ +public final class DirectoryUtils { + + private DirectoryUtils() { + // restrict instantiation + } + + /** + * Validates the provided directory path. Ensures that the directory exists and is writable. If + * the directory doesn't exist, a creation attempt is made. + * + * @param directoryPath the directory path to validate + * @throws DirectoryValidationException if the directory is not writable or cannot be created + */ + public static void validateTargetDirectory(String directoryPath) + throws DirectoryValidationException { + if (StringUtils.isBlank(directoryPath)) { + throw new IllegalArgumentException("Directory path cannot be null or empty."); + } + + Path path = Paths.get(directoryPath); + + if (Files.exists(path)) { + // Check if the provided directory is writable + if (!Files.isWritable(path)) { + throw new DirectoryValidationException( + String.format( + "The directory '%s' does not have write permissions. Please ensure that the current user has write access to the directory.", + path.toAbsolutePath())); + } + + } else { + // Create the directory if it doesn't exist + try { + Files.createDirectories(path); + } catch (IOException e) { + throw new DirectoryValidationException( + String.format( + "Failed to create the directory '%s'. Please check if you have sufficient permissions and if there are any file system restrictions. Details: %s", + path.toAbsolutePath(), e.getMessage())); + } + } + } + + /** + * Returns the current working directory. + * + * @return the current working directory + */ + public static String getCurrentWorkingDirectory() { + return Paths.get(System.getProperty("user.dir")).toAbsolutePath().toString(); + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverterTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverterTest.java new file mode 100644 index 0000000000..cbbf5a7693 --- /dev/null +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/ColumnKeyValueConverterTest.java @@ -0,0 +1,62 @@ +package com.scalar.db.dataloader.cli.command; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.scalar.db.dataloader.core.ColumnKeyValue; +import org.junit.jupiter.api.Test; + +class ColumnKeyValueConverterTest { + + private final ColumnKeyValueConverter converter = new ColumnKeyValueConverter(); + + @Test + void convert_ValidInput_ReturnsColumnKeyValue() { + String input = "name=John Doe"; + ColumnKeyValue expected = new ColumnKeyValue("name", "John Doe"); + ColumnKeyValue result = converter.convert(input); + assertEquals(expected.getColumnName(), result.getColumnName()); + assertEquals(expected.getColumnValue(), result.getColumnValue()); + } + + @Test + void convert_ValidInputWithExtraSpaces_ReturnsColumnKeyValue() { + String input = " age = 25 "; + ColumnKeyValue expected = new ColumnKeyValue("age", "25"); + ColumnKeyValue result = converter.convert(input); + assertEquals(expected.getColumnName(), result.getColumnName()); + assertEquals(expected.getColumnValue(), result.getColumnValue()); + } + + @Test + void convert_InvalidInputMissingValue_ThrowsIllegalArgumentException() { + String input = "name="; + assertThrows(IllegalArgumentException.class, () -> converter.convert(input)); + } + + @Test + void convert_InvalidInputMissingKey_ThrowsIllegalArgumentException() { + String input = "=John Doe"; + assertThrows(IllegalArgumentException.class, () -> converter.convert(input)); + } + + @Test + void convert_InvalidInputMissingEquals_ThrowsIllegalArgumentException() { + String input = "nameJohn Doe"; + assertThrows(IllegalArgumentException.class, () -> converter.convert(input)); + } + + @Test + void convert_ValidInputMultipleEquals_Returns() { + String input = "name=John=Doe"; + ColumnKeyValue expected = new ColumnKeyValue("name", "John=Doe"); + ColumnKeyValue result = converter.convert(input); + assertEquals(expected.getColumnName(), result.getColumnName()); + assertEquals(expected.getColumnValue(), result.getColumnValue()); + } + + @Test + void convert_NullValue_ThrowsIllegalArgumentException() { + assertThrows(IllegalArgumentException.class, () -> converter.convert(null)); + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java new file mode 100644 index 0000000000..538de9f404 --- /dev/null +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandTest.java @@ -0,0 +1,112 @@ +package com.scalar.db.dataloader.cli.command.dataexport; + +import static org.junit.jupiter.api.Assertions.*; + +import com.scalar.db.dataloader.cli.exception.InvalidFileExtensionException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +class ExportCommandTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExportCommandTest.class); + @TempDir Path tempDir; + + private ExportCommand exportCommand; + + @BeforeEach + void setUp() { + exportCommand = new ExportCommand(); + CommandLine cmd = new CommandLine(exportCommand); + exportCommand.spec = cmd.getCommandSpec(); + } + + @AfterEach + public void cleanup() throws IOException { + cleanUpTempDir(); + } + + @Test + void call_WithValidOutputDirectory_ShouldReturnZero() throws Exception { + Path configFile = tempDir.resolve("config.properties"); + Files.createFile(configFile); + + Path outputDir = tempDir.resolve("output"); + Files.createDirectory(outputDir); + + exportCommand.outputFilePath = outputDir.toString(); + + assertEquals(0, exportCommand.call()); + } + + @Test + void call_WithInvalidOutputDirectory_ShouldThrowInvalidFileExtensionException() + throws IOException { + Path configFile = tempDir.resolve("config.properties"); + Files.createFile(configFile); + + Path outputDir = tempDir.resolve("output"); + outputDir.toFile().setWritable(false); + + exportCommand.outputFilePath = outputDir.toString(); + + assertThrows(InvalidFileExtensionException.class, () -> exportCommand.call()); + } + + @Test + void call_WithValidOutputFile_ShouldReturnZero() throws Exception { + Path configFile = tempDir.resolve("config.properties"); + Files.createFile(configFile); + + Path outputFile = tempDir.resolve("output.csv"); + + exportCommand.outputFilePath = outputFile.toString(); + + assertEquals(0, exportCommand.call()); + } + + @Test + void call_WithValidOutputFileInCurrentDirectory_ShouldReturnZero() throws Exception { + Path configFile = tempDir.resolve("config.properties"); + Files.createFile(configFile); + + Path outputFile = Paths.get("output.csv"); + + exportCommand.outputFilePath = outputFile.toString(); + + assertEquals(0, exportCommand.call()); + } + + @Test + void call_WithValidOutputFileWithoutDirectory_ShouldReturnZero() throws Exception { + Path configFile = tempDir.resolve("config.properties"); + Files.createFile(configFile); + + exportCommand.outputFilePath = "output.csv"; + + assertEquals(0, exportCommand.call()); + } + + private void cleanUpTempDir() throws IOException { + try (Stream paths = Files.list(tempDir)) { + paths.forEach(this::deleteFile); + } + } + + private void deleteFile(Path file) { + try { + Files.deleteIfExists(file); + } catch (IOException e) { + LOGGER.error("Failed to delete file: {}", file, e); + } + } +} diff --git a/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java new file mode 100644 index 0000000000..60c0eafa60 --- /dev/null +++ b/data-loader/cli/src/test/java/com/scalar/db/dataloader/cli/util/DirectoryUtilsTest.java @@ -0,0 +1,87 @@ +package com.scalar.db.dataloader.cli.util; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.scalar.db.dataloader.cli.exception.DirectoryValidationException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** This class tests the DirectoryValidationUtil class. */ +class DirectoryUtilsTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(DirectoryUtilsTest.class); + + @TempDir Path tempDir; + + @AfterEach + public void cleanup() throws IOException { + cleanUpTempDir(); + } + + @Test + void validateTargetDirectory_ValidDirectory_NoExceptionThrown() + throws DirectoryValidationException { + DirectoryUtils.validateTargetDirectory(tempDir.toString()); + } + + @Test + void validateTargetDirectory_DirectoryDoesNotExist_CreatesDirectory() + throws DirectoryValidationException { + Path newDirectory = Paths.get(tempDir.toString(), "newDir"); + DirectoryUtils.validateTargetDirectory(newDirectory.toString()); + assertTrue(Files.exists(newDirectory)); + } + + @Test + void validateTargetDirectory_DirectoryNotWritable_ThrowsException() throws IOException { + Path readOnlyDirectory = Files.createDirectory(Paths.get(tempDir.toString(), "readOnlyDir")); + readOnlyDirectory.toFile().setWritable(false); + + assertThrows( + DirectoryValidationException.class, + () -> { + DirectoryUtils.validateTargetDirectory(readOnlyDirectory.toString()); + }); + } + + @Test + void validateTargetDirectory_NullDirectory_ThrowsException() { + assertThrows( + IllegalArgumentException.class, + () -> { + DirectoryUtils.validateTargetDirectory(null); + }); + } + + @Test + void validateTargetDirectory_EmptyDirectory_ThrowsException() { + assertThrows( + IllegalArgumentException.class, + () -> { + DirectoryUtils.validateTargetDirectory(""); + }); + } + + private void cleanUpTempDir() throws IOException { + try (Stream paths = Files.list(tempDir)) { + paths.forEach(this::deleteFile); + } + } + + private void deleteFile(Path file) { + try { + Files.deleteIfExists(file); + } catch (IOException e) { + LOGGER.error("Failed to delete file: {}", file, e); + } + } +} diff --git a/data-loader/core/build.gradle b/data-loader/core/build.gradle new file mode 100644 index 0000000000..28a2dba4de --- /dev/null +++ b/data-loader/core/build.gradle @@ -0,0 +1,43 @@ +plugins { + id 'net.ltgt.errorprone' version "${errorpronePluginVersion}" + id 'com.github.johnrengelman.shadow' version "${shadowPluginVersion}" + id 'com.github.spotbugs' version "${spotbugsPluginVersion}" +} + +archivesBaseName = "scalardb-data-loader-core" + +dependencies { + // ScalarDB core + implementation project(':core') + + // for SpotBugs + compileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}" + testCompileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}" + + // for Error Prone + errorprone "com.google.errorprone:error_prone_core:${errorproneVersion}" + errorproneJavac "com.google.errorprone:javac:${errorproneJavacVersion}" +} + +javadoc { + title = "ScalarDB Data Loader Core" +} + +spotless { + java { + target 'src/*/java/**/*.java' + importOrder() + removeUnusedImports() + googleJavaFormat(googleJavaFormatVersion) + } +} + +spotbugsMain.reports { + html.enabled = true +} +spotbugsMain.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml") + +spotbugsTest.reports { + html.enabled = true +} +spotbugsTest.excludeFilter = file("${project.rootDir}/gradle/spotbugs-exclude.xml") diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/ColumnKeyValue.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/ColumnKeyValue.java new file mode 100644 index 0000000000..03acab3b35 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/ColumnKeyValue.java @@ -0,0 +1,36 @@ +package com.scalar.db.dataloader.core; + +/** Represents a key-value pair for a column and its corresponding value. */ +public class ColumnKeyValue { + private final String columnName; + private final String columnValue; + + /** + * Class constructor + * + * @param columnName the column name + * @param columnValue the column value + */ + public ColumnKeyValue(String columnName, String columnValue) { + this.columnName = columnName; + this.columnValue = columnValue; + } + + /** + * Gets the column name + * + * @return the column name + */ + public String getColumnName() { + return columnName; + } + + /** + * Gets the column value + * + * @return the column value + */ + public String getColumnValue() { + return columnValue; + } +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/FileFormat.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/FileFormat.java new file mode 100644 index 0000000000..8a7b2a6113 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/FileFormat.java @@ -0,0 +1,8 @@ +package com.scalar.db.dataloader.core; + +/** The available input and output formats for the data loader import and export commands */ +public enum FileFormat { + JSON, + JSONL, + CSV +} diff --git a/settings.gradle b/settings.gradle index 1071d026c6..f23e9c4670 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,3 +4,5 @@ include 'rpc' include 'server' include 'schema-loader' include 'integration-test' +include 'data-loader:core' +include 'data-loader:cli'