Skip to content

Commit

Permalink
DataSvc Recovery (#178)
Browse files Browse the repository at this point in the history
  • Loading branch information
@yamingk
yamingk authored Sep 19, 2023
1 parent edf8932 commit e203590
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 57 deletions.
2 changes: 1 addition & 1 deletion conanfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

class HomestoreConan(ConanFile):
name = "homestore"
version = "4.3.1"
version = "4.3.2"

homepage = "https://github.com/eBay/Homestore"
description = "HomeStore Storage Engine"
Expand Down
1 change: 1 addition & 0 deletions src/include/homestore/blk.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ using chunk_num_t = uint16_t;
using blk_count_t = uint16_t;
using blk_num_t = uint32_t;
using blk_temp_t = uint16_t;
using allocator_id_t = chunk_num_t;

static constexpr size_t max_addressable_chunks() { return 1UL << (8 * sizeof(chunk_num_t)); }
static constexpr size_t max_blks_per_chunk() { return 1UL << (8 * sizeof(blk_num_t)); }
Expand Down
30 changes: 15 additions & 15 deletions src/lib/blkalloc/append_blk_allocator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

namespace homestore {

AppendBlkAllocator::AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, chunk_num_t id) :
AppendBlkAllocator::AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, allocator_id_t id) :
BlkAllocator{cfg, id}, m_metrics{get_name().c_str()} {
// TODO: try to make all append_blk_allocator instances use same client type to reduce metablk's cache footprint;
meta_service().register_handler(
Expand All @@ -32,28 +32,28 @@ AppendBlkAllocator::AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_form
if (need_format) {
m_freeable_nblks = available_blks();
m_last_append_offset = 0;
}

for (uint8_t i = 0; i < m_sb.size(); ++i) {
m_sb[i].set_name(get_name());
m_sb[i].create(sizeof(append_blkalloc_ctx));
m_sb[i]->is_dirty = false;
m_sb[i]->allocator_id = id;
m_sb[i]->last_append_offset = 0;
m_sb[i]->freeable_nblks = m_freeable_nblks;
}
// for both fresh start and recovery, firstly init m_sb fields;
for (uint8_t i = 0; i < m_sb.size(); ++i) {
m_sb[i].set_name(get_name());
m_sb[i].create(sizeof(append_blkalloc_ctx));
m_sb[i]->is_dirty = false;
m_sb[i]->allocator_id = id;
m_sb[i]->last_append_offset = 0;
m_sb[i]->freeable_nblks = m_freeable_nblks;
}

// for recovery boot, fields should be recovered from metablks;
// for recovery boot, fields will also be recovered from metablks;
}

void AppendBlkAllocator::on_meta_blk_found(const sisl::byte_view& buf, void* meta_cookie) {
// TODO: also needs to initialize base class blkallocator for recovery path;
// load all dirty buffer from the same starting point;
m_sb[0].load(buf, meta_cookie);
for (uint8_t i = 1; i < m_sb.size(); ++i) {
for (uint8_t i = 0; i < m_sb.size(); ++i) {
m_sb[i].load(buf, meta_cookie);
}

// recover in-memory counter/offset from metablk;
m_last_append_offset = m_sb[0]->last_append_offset;
m_freeable_nblks = m_sb[0]->freeable_nblks;

Expand Down Expand Up @@ -125,11 +125,11 @@ BlkAllocStatus AppendBlkAllocator::alloc(blk_count_t nblks, const blk_alloc_hint
// cp_flush should not block alloc/free;
//
void AppendBlkAllocator::cp_flush(CP* cp) {
const auto idx = cp->id();
const auto idx = cp->id() % MAX_CP_COUNT;
// check if current cp's context has dirty buffer already
if (m_sb[idx]->is_dirty) {
// write to metablk;
m_sb[cp->id()].write();
m_sb[idx].write();

// clear this dirty buff's dirty flag;
clear_dirty_offset(idx);
Expand Down
4 changes: 2 additions & 2 deletions src/lib/blkalloc/append_blk_allocator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ struct append_blkalloc_ctx {
uint64_t magic{append_blkalloc_sb_magic};
uint32_t version{append_blkalloc_sb_version};
bool is_dirty; // this field is needed for cp_flush, but not necessarily needed for persistence;
uint64_t allocator_id;
allocator_id_t allocator_id;
blk_num_t freeable_nblks;
blk_num_t last_append_offset;
};
Expand Down Expand Up @@ -67,7 +67,7 @@ class AppendBlkAllocMetrics : public sisl::MetricsGroup {
//
class AppendBlkAllocator : public BlkAllocator {
public:
AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, chunk_num_t id = 0);
AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, allocator_id_t id = 0);

AppendBlkAllocator(const AppendBlkAllocator&) = delete;
AppendBlkAllocator(AppendBlkAllocator&&) noexcept = delete;
Expand Down
34 changes: 20 additions & 14 deletions src/lib/device/virtual_dev.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,22 +183,28 @@ BlkAllocStatus VirtualDev::alloc_blks(blk_count_t nblks, blk_alloc_hints const&
BlkAllocStatus status;
Chunk* chunk;
size_t attempt{0};

do {
chunk = m_chunk_selector->select_chunk(nblks, hints).get();
if (chunk == nullptr) {
status = BlkAllocStatus::SPACE_FULL;
break;
}

if (hints.chunk_id_hint) {
// this is a target-chunk allocation;
chunk = m_dmgr.get_chunk_mutable(*(hints.chunk_id_hint));
status = alloc_blks_from_chunk(nblks, hints, out_blkid, chunk);
if ((status == BlkAllocStatus::SUCCESS) || !hints.can_look_for_other_chunk ||
(status == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) {
break;
}
} while (++attempt < m_all_chunks.size());
// don't look for other chunks because user wants allocation on chunk_id_hint only;
} else {
do {
chunk = m_chunk_selector->select_chunk(nblks, hints).get();
if (chunk == nullptr) {
status = BlkAllocStatus::SPACE_FULL;
break;
}

status = alloc_blks_from_chunk(nblks, hints, out_blkid, chunk);
if ((status == BlkAllocStatus::SUCCESS) || !hints.can_look_for_other_chunk ||
(status == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) {
break;
}
} while (++attempt < m_all_chunks.size());
}

if ((status != BlkAllocStatus::SUCCESS) || (status != BlkAllocStatus::PARTIAL)) {
if ((status != BlkAllocStatus::SUCCESS) && !((status == BlkAllocStatus::PARTIAL) && hints.partial_alloc_ok)) {
LOGERROR("nblks={} failed to alloc after trying to alloc on every chunks {} and devices {}.", nblks);
COUNTER_INCREMENT(m_metrics, vdev_num_alloc_failure, 1);
}
Expand Down
78 changes: 53 additions & 25 deletions src/tests/test_append_blkalloc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,22 @@ class AppendBlkAllocatorTest : public testing::Test {
virtual void SetUp() override {
test_common::HSTestHelper::set_data_svc_allocator(homestore::blk_allocator_type_t::append);
test_common::HSTestHelper::set_data_svc_chunk_selector(homestore::chunk_selector_type_t::HEAP);

test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0 /* meta */, 0 /* data_log */,
0 /* ctrl_log */, 80.0 /* data */, 0 /* index */, nullptr,
false /* recovery */, nullptr, false /* default ds type */);
}

virtual void TearDown() override { test_common::HSTestHelper::shutdown_homestore(); }

void restart_homestore() {
test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0, 0, 0, 80.0, 0,
nullptr /* before_svc_start_cb */, true /* restart */,
nullptr /* indx_svc_cb */, false /* default ds type */);
}

void reset_io_job_done() { m_io_job_done = false; }

void finish_and_notify() {
{
std::lock_guard lk(this->m_mtx);
Expand Down Expand Up @@ -186,11 +200,6 @@ class AppendBlkAllocatorTest : public testing::Test {
};

TEST_F(AppendBlkAllocatorTest, TestBasicWrite) {
LOGINFO("Step 0: Starting homestore.");
test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0 /* meta */, 0 /* data_log */,
0 /* ctrl_log */, 80.0 /* data */, 0 /* index */, nullptr,
false /* recovery */, nullptr, false /* default ds type */);

// start io in worker thread;
const auto io_size = 4 * Ki;
LOGINFO("Step 1: run on worker thread to schedule write for {} Bytes.", io_size);
Expand All @@ -200,59 +209,78 @@ TEST_F(AppendBlkAllocatorTest, TestBasicWrite) {
wait_for_all_io_complete();

LOGINFO("Step 3: I/O completed, do shutdown.");
test_common::HSTestHelper::shutdown_homestore();
}

TEST_F(AppendBlkAllocatorTest, TestWriteThenReadVerify) {
LOGINFO("Step 0: Starting homestore.");
test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0, 0, 0, 80.0, 0, nullptr, false, nullptr,
false /* default ds type */);

// start io in worker thread;
auto io_size = 4 * Ki;
LOGINFO("Step 1: run on worker thread to schedule write for {} Bytes.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [this, io_size]() { this->write_io_verify(io_size); });

LOGINFO("Step 3: Wait for I/O to complete.");
LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 4: I/O completed, do shutdown.");
test_common::HSTestHelper::shutdown_homestore();
LOGINFO("Step 3: I/O completed, do shutdown.");
}

TEST_F(AppendBlkAllocatorTest, TestWriteThenFreeBlk) {
LOGINFO("Step 0: Starting homestore.");
test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0, 0, 0, 80.0, 0, nullptr, false, nullptr,
false /* default ds type */);

// start io in worker thread;
auto io_size = 4 * Mi;
LOGINFO("Step 1: run on worker thread to schedule write for {} Bytes, then free blk.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker,
[this, io_size]() { this->write_io_free_blk(io_size); });

LOGINFO("Step 3: Wait for I/O to complete.");
LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 4: I/O completed, do shutdown.");
test_common::HSTestHelper::shutdown_homestore();
LOGINFO("Step 3: I/O completed, do shutdown.");
}

TEST_F(AppendBlkAllocatorTest, TestCPFlush) {
LOGINFO("Step 0: Starting homestore.");
test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0, 0, 0, 80.0, 0, nullptr, false, nullptr,
false /* default ds type */);
const auto io_size = 4 * Ki;
LOGINFO("Step 1: run on worker thread to schedule write for {} Bytes.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [this, io_size]() { this->write_io(io_size); });

LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 3: I/O completed, trigger_cp and wait.");
test_common::HSTestHelper::trigger_cp(true /* wait */);

LOGINFO("Step 4: cp completed, do shutdown.");
}

TEST_F(AppendBlkAllocatorTest, TestWriteThenRecovey) {
// start io in worker thread;
auto io_size = 4 * Mi;
LOGINFO("Step 1: run on worker thread to schedule write for {} Bytes, then free blk.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker,
[this, io_size]() { this->write_io_free_blk(io_size); });

LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 3: I/O completed, trigger_cp and wait.");
test_common::HSTestHelper::trigger_cp(true /* wait */);

LOGINFO("Step 4: cp completed, restart homestore.");
this->restart_homestore();

std::this_thread::sleep_for(std::chrono::seconds{3});
LOGINFO("Step 5: Restarted homestore with data service recovered");

this->reset_io_job_done();

LOGINFO("Step 6: run on worker thread to schedule write for {} Bytes.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [this, io_size]() { this->write_io(io_size); });

LOGINFO("Step 7: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 8: I/O completed, trigger_cp and wait.");
test_common::HSTestHelper::trigger_cp(true /* wait */);

LOGINFO("Step 4: I/O completed, do shutdown.");
test_common::HSTestHelper::shutdown_homestore();
LOGINFO("Step 9: do shutdown. ");
}

SISL_OPTION_GROUP(test_append_blkalloc,
Expand Down

0 comments on commit e203590

Please sign in to comment.