Skip to content

Commit

Permalink
refactor: use commons module in simulator (#271)
Browse files Browse the repository at this point in the history 
Signed-off-by: Atanas Atanasov <[email protected]>
  • Loading branch information
@ata-nas
ata-nas authored Oct 28, 2024
1 parent d829247 commit 88ef1f8
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 125 deletions.
51 changes: 41 additions & 10 deletions common/src/test/java/com/hedera/block/common/utils/FileUtilitiesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@
import org.junit.jupiter.params.provider.MethodSource;

class FileUtilitiesTest {
private static final String FILE_WITH_UNRECOGNIZED_EXTENSION = "src/test/resources/nonexistent.unrecognized";

@Test
void test_createPathIfNotExists_CreatesDirIfDoesNotExist(@TempDir final Path tempDir)
throws IOException {
void test_createPathIfNotExists_CreatesDirIfDoesNotExist(@TempDir final Path tempDir) throws IOException {
final String newDir = "newDir";
final Path toCreate = tempDir.resolve(newDir);

Expand All @@ -51,8 +52,7 @@ void test_createPathIfNotExists_CreatesDirIfDoesNotExist(@TempDir final Path tem
}

@Test
void test_createPathIfNotExists_DoesNotCreateDirIfExists(@TempDir final Path tempDir)
throws IOException {
void test_createPathIfNotExists_DoesNotCreateDirIfExists(@TempDir final Path tempDir) throws IOException {
final String newDir = "newDir";
final Path toCreate = tempDir.resolve(newDir);

Expand All @@ -78,8 +78,7 @@ void test_createPathIfNotExists_DoesNotCreateDirIfExists(@TempDir final Path tem
}

@Test
void test_createPathIfNotExists_CreatesFileIfDoesNotExist(@TempDir final Path tempDir)
throws IOException {
void test_createPathIfNotExists_CreatesFileIfDoesNotExist(@TempDir final Path tempDir) throws IOException {
final String newFile = "newFile";
final Path toCreate = tempDir.resolve(newFile);

Expand All @@ -96,8 +95,7 @@ void test_createPathIfNotExists_CreatesFileIfDoesNotExist(@TempDir final Path te
}

@Test
void test_createPathIfNotExists_DoesNotCreateFileIfExists(@TempDir final Path tempDir)
throws IOException {
void test_createPathIfNotExists_DoesNotCreateFileIfExists(@TempDir final Path tempDir) throws IOException {
final String newFile = "newFile";
final Path toCreate = tempDir.resolve(newFile);

Expand Down Expand Up @@ -156,12 +154,46 @@ void test_readFileBytesUnsafe_ReturnsByteArrayWithValidContentForValidFile(
.isEqualTo(expectedContent);
}

@Test
void test_readFileBytesUnsafe_ReturnsNullByteArrayWhenExtensionIsNotRecognized() throws IOException {
final byte[] actualContent = FileUtilities.readFileBytesUnsafe(Path.of(FILE_WITH_UNRECOGNIZED_EXTENSION));
assertThat(actualContent).isNull();
}

@ParameterizedTest
@MethodSource("invalidFiles")
void test_readFileBytesUnsafe_ThrowsIOExceptionForInvalidGzipFile(final Path filePath) {
assertThatIOException().isThrownBy(() -> FileUtilities.readFileBytesUnsafe(filePath));
}

@ParameterizedTest
@MethodSource({"validGzipFiles", "validBlkFiles"})
void test_readFileBytesUnsafe_ReturnsByteArrayWithValidContentForValidFileWithGivenExtension(
final Path filePath, final String expectedContent) throws IOException {
final byte[] actualContent = FileUtilities.readFileBytesUnsafe(filePath, ".blk", ".gz");
assertThat(actualContent)
.isNotNull()
.isNotEmpty()
.asString()
.isNotNull()
.isNotBlank()
.isEqualTo(expectedContent);
}

@Test
void test_readFileBytesUnsafe_ReturnsNullByteArrayWhenExtensionIsNotRecognizedWithGivenExtension()
throws IOException {
final byte[] actualContent =
FileUtilities.readFileBytesUnsafe(Path.of(FILE_WITH_UNRECOGNIZED_EXTENSION), ".blk", ".gz");
assertThat(actualContent).isNull();
}

@ParameterizedTest
@MethodSource("invalidFiles")
void test_readFileBytesUnsafe_ThrowsIOExceptionForInvalidGzipFileWithGivenExtension(final Path filePath) {
assertThatIOException().isThrownBy(() -> FileUtilities.readFileBytesUnsafe(filePath, ".blk", ".gz"));
}

private static Stream<Arguments> validGzipFiles() {
return Stream.of(
Arguments.of("src/test/resources/valid1.txt.gz", "valid1"),
Expand All @@ -176,7 +208,6 @@ private static Stream<Arguments> validBlkFiles() {

private static Stream<Arguments> invalidFiles() {
return Stream.of(
Arguments.of("src/test/resources/invalid1.gz"),
Arguments.of("src/test/resources/nonexistent.gz"));
Arguments.of("src/test/resources/invalid1.gz"), Arguments.of("src/test/resources/nonexistent.gz"));
}
}
20 changes: 10 additions & 10 deletions simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.hedera.block.simulator;

import static com.hedera.block.common.constants.StringsConstants.APPLICATION_PROPERTIES;
import static java.lang.System.Logger.Level.INFO;

import com.hedera.block.simulator.exception.BlockSimulatorParsingException;
Expand Down Expand Up @@ -43,24 +44,23 @@ private BlockStreamSimulator() {}
* @throws InterruptedException if the thread is interrupted
* @throws BlockSimulatorParsingException if a parse error occurs
*/
public static void main(String[] args)
public static void main(final String[] args)
throws IOException, InterruptedException, BlockSimulatorParsingException {

LOGGER.log(INFO, "Starting Block Stream Simulator");

ConfigurationBuilder configurationBuilder =
ConfigurationBuilder.create()
.withSource(SystemEnvironmentConfigSource.getInstance())
.withSource(SystemPropertiesConfigSource.getInstance())
.withSource(new ClasspathFileConfigSource(Path.of("app.properties")))
.autoDiscoverExtensions();
final ConfigurationBuilder configurationBuilder = ConfigurationBuilder.create()
.withSource(SystemEnvironmentConfigSource.getInstance())
.withSource(SystemPropertiesConfigSource.getInstance())
.withSource(new ClasspathFileConfigSource(Path.of(APPLICATION_PROPERTIES)))
.autoDiscoverExtensions();

Configuration configuration = configurationBuilder.build();
final Configuration configuration = configurationBuilder.build();

BlockStreamSimulatorInjectionComponent DIComponent =
final BlockStreamSimulatorInjectionComponent DIComponent =
DaggerBlockStreamSimulatorInjectionComponent.factory().create(configuration);

BlockStreamSimulatorApp blockStreamSimulatorApp = DIComponent.getBlockStreamSimulatorApp();
final BlockStreamSimulatorApp blockStreamSimulatorApp = DIComponent.getBlockStreamSimulatorApp();
blockStreamSimulatorApp.start();
}
}
11 changes: 5 additions & 6 deletions simulator/src/main/java/com/hedera/block/simulator/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,13 @@
package com.hedera.block.simulator;

/** The Constants class defines the constants for the block simulator. */
public class Constants {

/** Constructor to prevent instantiation. this is only a utility class */
private Constants() {}

public final class Constants {
/** The file extension for block files. */
public static final String RECORD_EXTENSION = "blk";
public static final String RECORD_EXTENSION = ".blk";

/** postfix for gzip files */
public static final String GZ_EXTENSION = ".gz";

/** Constructor to prevent instantiation. this is only a utility class */
private Constants() {}
}
39 changes: 19 additions & 20 deletions ...ator/src/main/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@

package com.hedera.block.simulator.generator;

import static com.hedera.block.simulator.generator.Utils.readFileBytes;
import static com.hedera.block.simulator.Constants.GZ_EXTENSION;
import static com.hedera.block.simulator.Constants.RECORD_EXTENSION;
import static java.lang.System.Logger.Level.DEBUG;
import static java.lang.System.Logger.Level.ERROR;
import static java.lang.System.Logger.Level.INFO;

import com.hedera.block.common.utils.FileUtilities;
import com.hedera.block.simulator.config.data.BlockGeneratorConfig;
import com.hedera.block.simulator.config.types.GenerationMode;
import com.hedera.hapi.block.stream.Block;
Expand Down Expand Up @@ -106,32 +108,29 @@ public Block getNextBlock() {
}

private void loadBlocks() throws IOException, ParseException {
Path rootPath = Path.of(rootFolder);
final Path rootPath = Path.of(rootFolder);

try (Stream<Path> blockDirs = Files.list(rootPath).filter(Files::isDirectory)) {
List<Path> sortedBlockDirs =
try (final Stream<Path> blockDirs = Files.list(rootPath).filter(Files::isDirectory)) {
final List<Path> sortedBlockDirs =
blockDirs.sorted(Comparator.comparing(Path::getFileName)).toList();

for (Path blockDirPath : sortedBlockDirs) {
List<BlockItem> parsedBlockItems = new ArrayList<>();

try (Stream<Path> blockItems =
Files.list(blockDirPath).filter(Files::isRegularFile)) {
List<Path> sortedBlockItems =
blockItems
.sorted(
Comparator.comparing(
BlockAsDirBlockStreamManager
::extractNumberFromPath))
.toList();

for (Path pathBlockItem : sortedBlockItems) {
byte[] blockItemBytes = readFileBytes(pathBlockItem);
for (final Path blockDirPath : sortedBlockDirs) {
final List<BlockItem> parsedBlockItems = new ArrayList<>();

try (final Stream<Path> blockItems = Files.list(blockDirPath).filter(Files::isRegularFile)) {
final Comparator<Path> comparator =
Comparator.comparing(BlockAsDirBlockStreamManager::extractNumberFromPath);
final List<Path> sortedBlockItems =
blockItems.sorted(comparator).toList();

for (final Path pathBlockItem : sortedBlockItems) {
final byte[] blockItemBytes =
FileUtilities.readFileBytesUnsafe(pathBlockItem, RECORD_EXTENSION, GZ_EXTENSION);
// if null means the file is not a block item and we can skip the file.
if (blockItemBytes == null) {
continue;
}
BlockItem blockItem = BlockItem.PROTOBUF.parse(Bytes.wrap(blockItemBytes));
final BlockItem blockItem = BlockItem.PROTOBUF.parse(Bytes.wrap(blockItemBytes));
parsedBlockItems.add(blockItem);
}
}
Expand Down
16 changes: 9 additions & 7 deletions ...tor/src/main/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@

package com.hedera.block.simulator.generator;

import static com.hedera.block.simulator.generator.Utils.readFileBytes;
import static com.hedera.block.simulator.Constants.GZ_EXTENSION;
import static com.hedera.block.simulator.Constants.RECORD_EXTENSION;
import static java.lang.System.Logger.Level.DEBUG;
import static java.lang.System.Logger.Level.ERROR;
import static java.lang.System.Logger.Level.INFO;

import com.hedera.block.common.utils.FileUtilities;
import com.hedera.block.simulator.config.data.BlockGeneratorConfig;
import com.hedera.block.simulator.config.types.GenerationMode;
import com.hedera.hapi.block.stream.Block;
Expand Down Expand Up @@ -100,22 +102,22 @@ public Block getNextBlock() {

private void loadBlocks() throws IOException, ParseException {

Path rootPath = Path.of(rootFolder);
final Path rootPath = Path.of(rootFolder);

try (Stream<Path> blockFiles = Files.list(rootPath)) {
try (final Stream<Path> blockFiles = Files.list(rootPath)) {

List<Path> sortedBlockFiles =
final List<Path> sortedBlockFiles =
blockFiles.sorted(Comparator.comparing(Path::getFileName)).toList();

for (Path blockPath : sortedBlockFiles) {
for (final Path blockPath : sortedBlockFiles) {

byte[] blockBytes = readFileBytes(blockPath);
final byte[] blockBytes = FileUtilities.readFileBytesUnsafe(blockPath, RECORD_EXTENSION, GZ_EXTENSION);
// skip if block is null, usually due to SO files like .DS_STORE
if (blockBytes == null) {
continue;
}

Block block = Block.PROTOBUF.parse(Bytes.wrap(blockBytes));
final Block block = Block.PROTOBUF.parse(Bytes.wrap(blockBytes));
blocks.add(block);
LOGGER.log(DEBUG, "Loaded block: " + blockPath);
}
Expand Down
31 changes: 20 additions & 11 deletions simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSets.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@

package com.hedera.block.simulator.generator;

import static com.hedera.block.simulator.generator.Utils.readFileBytes;
import static com.hedera.block.simulator.Constants.GZ_EXTENSION;
import static com.hedera.block.simulator.Constants.RECORD_EXTENSION;
import static java.lang.System.Logger.Level.INFO;

import com.hedera.block.common.utils.FileUtilities;
import com.hedera.block.simulator.config.data.BlockGeneratorConfig;
import com.hedera.block.simulator.config.types.GenerationMode;
import com.hedera.block.simulator.exception.BlockSimulatorParsingException;
Expand All @@ -27,8 +29,10 @@
import com.hedera.pbj.runtime.ParseException;
import com.hedera.pbj.runtime.io.buffer.Bytes;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
import javax.inject.Inject;

/** A block stream manager that reads blocks from files in a directory. */
Expand Down Expand Up @@ -78,22 +82,27 @@ public BlockItem getNextBlockItem() throws IOException, BlockSimulatorParsingExc
public Block getNextBlock() throws IOException, BlockSimulatorParsingException {
currentBlockIndex++;

String nextBlockFileName = String.format(formatString, currentBlockIndex);
File blockFile = new File(blockstreamPath, nextBlockFileName);

if (!blockFile.exists()) {
final String nextBlockFileName = String.format(formatString, currentBlockIndex);
final Path localBlockStreamPath = Path.of(blockstreamPath).resolve(nextBlockFileName);
if (!Files.exists(localBlockStreamPath)) {
return null;
}

try {
byte[] blockBytes = readFileBytes(blockFile.toPath());
final byte[] blockBytes =
FileUtilities.readFileBytesUnsafe(localBlockStreamPath, RECORD_EXTENSION, GZ_EXTENSION);

if (Objects.isNull(blockBytes)) {
throw new NullPointerException(
"Unable to read block file [%s]! Most likely not found with the extensions '%s' or '%s'"
.formatted(localBlockStreamPath, RECORD_EXTENSION, GZ_EXTENSION));
}

LOGGER.log(INFO, "Loading block: " + blockFile.getName());
LOGGER.log(INFO, "Loading block: " + localBlockStreamPath.getFileName());

Block block = Block.PROTOBUF.parse(Bytes.wrap(blockBytes));
final Block block = Block.PROTOBUF.parse(Bytes.wrap(blockBytes));
LOGGER.log(INFO, "block loaded with items size= " + block.items().size());
return block;
} catch (ParseException e) {
} catch (final ParseException e) {
throw new BlockSimulatorParsingException(e.getMessage());
}
}
Expand Down
61 changes: 0 additions & 61 deletions simulator/src/main/java/com/hedera/block/simulator/generator/Utils.java
View file

This file was deleted.

0 comments on commit 88ef1f8

Please sign in to comment.