diff --git a/CMakeLists.txt b/CMakeLists.txt index 098e3c0..599b611 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,9 +18,16 @@ if(TESTS) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) enable_testing() + add_executable(binfmt_tests tests.cpp) + add_executable(binfmt_benchmarks benchmarks.cpp) + target_compile_definitions(binfmt_tests PRIVATE -DTESTS) + target_compile_definitions(binfmt_benchmarks PRIVATE -DTESTS) + target_link_libraries(binfmt_tests gtest_main) + target_link_libraries(binfmt_benchmarks gtest_main) + include(GoogleTest) gtest_discover_tests(binfmt_tests) endif() diff --git a/README.md b/README.md index 4bc765c..ec54fb1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,27 @@ [![CodeFactor](https://www.codefactor.io/repository/github/nbdy/binfmt/badge)](https://www.codefactor.io/repository/github/nbdy/binfmt)
`A header only framework for binary file formats` +## benchmarks +### single insert (append EntryType) +|Number|Time (ms)| +|------|---------| +|1k |1290 | +|10k |13260 | +|100k |toolong | + +### vector insert (append std::vector) +|Number|Time (ms)| +|------|---------| +|1k |< 0 s | +|1M |~ 2 s | + + +### read +|Number|Time (ms)| +|------|---------| +|1M |< 0 s | + + ## Minimal example ```c++ #include diff --git a/benchmarks.cpp b/benchmarks.cpp new file mode 100644 index 0000000..b9c511d --- /dev/null +++ b/benchmarks.cpp @@ -0,0 +1,155 @@ +// +// Created by nbdy on 02.09.21. +// + +#include +#include +#include "binfmt.h" + +#define TEST_DIRECTORY "/tmp/xxxxxxxxxx__xxxxxxxxxxx__xx" +#define TEST_BINARY_FILE "/tmp/xxxxxxx__xxxxxxxxx.bin" +#define TEST_MAX_ENTRIES 1000000000 + +#define TIMESTAMP std::chrono::high_resolution_clock::to_time_t(std::chrono::high_resolution_clock::now()) + +std::string TIMEIT_NAME = "TIMEIT"; +time_t TIMEIT_START_TIMESTAMP = 0; +time_t TIMEIT_END_TIMESTAMP = 0; +time_t TIMEIT_DIFF = 0; + +#define TIMEIT_START(name) \ +TIMEIT_NAME = name; \ +TIMEIT_START_TIMESTAMP = TIMESTAMP; + +#define TIMEIT_END \ +TIMEIT_END_TIMESTAMP = TIMESTAMP; + +#define TIMEIT_RESULT \ +TIMEIT_DIFF = TIMEIT_END_TIMESTAMP - TIMEIT_START_TIMESTAMP; \ +std::cout << TIMEIT_NAME << ": " << std::to_string(TIMEIT_DIFF) << " s" << std::endl; + +struct TestBinaryFileHeader : public BinaryFileHeaderBase { + TestBinaryFileHeader(): BinaryFileHeaderBase(0x7357, 0x8888) {} +}; + +struct TestBinaryEntry { + uint32_t uNumber; + int32_t iNumber; + float fNumber; + char cChar; +}; + +typedef BinaryEntryContainer TestBinaryEntryContainer; +typedef BinaryFile TestBinaryFile; + + +TestBinaryFile getRandomTestFile() { + TestBinaryFile r(TEST_BINARY_FILE, TestBinaryFileHeader{}); + EXPECT_TRUE(Fs::exists(r.getFilePath())); + return r; +} + +char generateRandomChar(){ + return 'A' + std::rand() % 24; // NOLINT(cert-msc50-cpp,cppcoreguidelines-narrowing-conversions) +} + +int generateRandomInteger() { + return std::rand() % 10000000; // NOLINT(cert-msc50-cpp) +} + +float generateRandomFloat() { + return static_cast(std::rand()) / static_cast(RAND_MAX); // NOLINT(cert-msc50-cpp) +} + +TestBinaryEntryContainer generateRandomTestEntryContainer() { + return TestBinaryEntryContainer(TestBinaryEntry { + static_cast(generateRandomInteger()), + generateRandomInteger(), + generateRandomFloat(), + generateRandomChar() + }); +} + +void cleanupTestFile(TestBinaryFile f) { + f.deleteFile(); + EXPECT_FALSE(Fs::exists(f.getFilePath())); +} + +TEST(BinaryFile, test1kInsert) { + auto t = getRandomTestFile(); + std::vector entries; + for(uint32_t i = 0; i < 1000; i++) { + entries.push_back(generateRandomTestEntryContainer().entry); + } + TIMEIT_START("1kWrite") + auto r = t.append(entries); + TIMEIT_END + TIMEIT_RESULT + EXPECT_TRUE(r.ok); + EXPECT_EQ(t.getFileSize(), 1000 * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader)); + cleanupTestFile(t); +} + +TEST(BinaryFile, test10kInsert) { + auto t = getRandomTestFile(); + std::vector entries; + for(uint32_t i = 0; i < 10000; i++) { + entries.push_back(generateRandomTestEntryContainer().entry); + } + TIMEIT_START("10kWrite") + auto r = t.append(entries); + TIMEIT_END + TIMEIT_RESULT + EXPECT_TRUE(r.ok); + EXPECT_EQ(t.getFileSize(), 10000 * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader)); + int ec = 0; + std::vector allEntries; + EXPECT_TRUE(t.getAllEntries(allEntries)); + for(const auto& entry : allEntries) { + EXPECT_EQ(entry.checksum, TestBinaryEntryContainer(entries[ec]).checksum); + ec++; + } + cleanupTestFile(t); +} + +TEST(BinaryFile, test100kInsert) { + auto t = getRandomTestFile(); + std::vector entries; + for(uint32_t i = 0; i < 100000; i++) { + entries.push_back(generateRandomTestEntryContainer().entry); + } + TIMEIT_START("100kWrite") + auto r = t.append(entries); + TIMEIT_END + TIMEIT_RESULT + EXPECT_TRUE(r.ok); + EXPECT_EQ(t.getFileSize(), 100000 * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader)); + cleanupTestFile(t); +} + +TEST(BinaryFile, test1MInsert) { + auto t = getRandomTestFile(); + std::vector entries; + uint32_t insertCount = 10000000; + std::cout << "Generating file of size " << std::to_string(insertCount * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader)) << std::endl; + for(uint32_t i = 0; i < insertCount; i++) { + entries.push_back(generateRandomTestEntryContainer().entry); + } + TIMEIT_START("1MWrite") + auto r = t.append(entries); + TIMEIT_END + TIMEIT_RESULT + EXPECT_TRUE(r.ok); + EXPECT_EQ(t.getFileSize(), insertCount * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader)); + int ec = 0; + std::vector allEntries; + TIMEIT_START("1MRead") + EXPECT_TRUE(t.getAllEntries(allEntries)); + TIMEIT_END + TIMEIT_RESULT + for(const auto& entry : allEntries) { + EXPECT_EQ(entry.checksum, TestBinaryEntryContainer(entries[ec]).checksum); + ec++; + } + //cleanupTestFile(t); +} \ No newline at end of file diff --git a/binfmt.h b/binfmt.h index 5249a8e..68eb346 100644 --- a/binfmt.h +++ b/binfmt.h @@ -145,7 +145,13 @@ struct FileUtils { } if (Fs::exists(FilePath) || Create) { r = std::fopen(FilePath.c_str(), Create ? "wbe" : "r+be"); - if (fseek(r, offset, SEEK_SET) != 0) { + auto fn = fileno(r); + if(lockf(fn, F_LOCK, 0) != 0) { + (void) CloseBinaryFile(r); + r = nullptr; + } + auto sr = fseek(r, offset, SEEK_SET); + if (sr != 0) { (void) CloseBinaryFile(r); r = nullptr; } @@ -159,7 +165,9 @@ struct FileUtils { * @return */ static bool CloseBinaryFile(FILE* fp) { - auto r = syncfs(fileno(fp)); + auto fn = fileno(fp); + (void) lockf(fn, F_ULOCK, 0); + auto r = syncfs(fn); return fclose(fp) == 0 && r == 0; } @@ -231,6 +239,26 @@ struct FileUtils { return r == 1; } + /*! + * Write data entry to file. Skips size of HeaderType in bytes. + * @tparam EntryType + * @param FilePath + * @param entry + * @return false if file does not exist, seek to offset failed or we can't write the object + */ + template + static bool WriteDataVector(const std::string& FilePath, std::vector entries, uint32_t offset = 0) { + uint32_t o = sizeof(HeaderType) + offset * sizeof(EntryType); + FILE* fp = OpenBinaryFile(FilePath, false, o); + if (fp == nullptr) { + std::cout << strerror(errno) << std::endl; + return false; + } + auto r = fwrite(&entries[0], sizeof(EntryType), entries.size(), fp); + (void) CloseBinaryFile(fp); + return r == entries.size(); + } + /*! * * @tparam HeaderType @@ -482,6 +510,27 @@ class BinaryFile { return AppendResult{rewind, ok, m_uCurrentAppendOffset}; } + AppendResult append(std::vector entries) { + AppendResult r { + false, true, 0 + }; + bool rewind = false; + if(getFileSize() + sizeof(ContainerType) * entries.size() > m_uMaxFileSize) { + r.rewind = true; + auto sizeLeft = m_uMaxFileSize - getFileSize(); + uint32_t entriesLeft = sizeLeft / sizeof(ContainerType) - 1; + std::vector tmpEntries(entries.begin(), entries.begin() + entriesLeft); + entries = std::vector(entries.begin() + entriesLeft + 1, entries.end()); + r = append(tmpEntries); + } + std::vector containers; + for(const auto& entry : entries) { + containers.push_back(ContainerType(entry)); + } + r.ok = FileUtils::WriteDataVector(m_sFilePath, containers); + return r; + } + bool setEntryAt(EntryType entry, uint32_t position) { return FileUtils::WriteData(m_sFilePath, ContainerType(entry), position); } diff --git a/tests.cpp b/tests.cpp index 36ae319..c022f6b 100644 --- a/tests.cpp +++ b/tests.cpp @@ -3,11 +3,14 @@ // #include "gtest/gtest.h" +#include #include "binfmt.h" +#include #define TEST_DIRECTORY "/tmp/xxxxxxxxxx__xxxxxxxxxxx__xx" #define TEST_BINARY_FILE "/tmp/xxxxxxx__xxxxxxxxx.bin" -#define TEST_MAX_ENTRIES 50 +#define TEST_MAX_ENTRIES 1000 + struct TestBinaryFileHeader : public BinaryFileHeaderBase { TestBinaryFileHeader(): BinaryFileHeaderBase(0x7357, 0x8888) {} @@ -63,7 +66,7 @@ std::vector appendRandomAmountOfEntriesV(TestBinaryFil return r; } -std::vector appendExactAmountOfEntriesV(TestBinaryFile f, int count = 42) { +std::vector appendExactAmountOfEntriesV(TestBinaryFile f, uint32_t count = 42) { std::vector r; for(int i = 0; i < count; i++) { auto v = generateRandomTestEntryContainer(); @@ -286,3 +289,4 @@ TEST(BinaryFile, testRemoveEntryAt) { EXPECT_EQ(t0.checksum, t1.checksum); cleanupTestFile(t); } +