Skip to content

Commit

Permalink
DataService Recovery
Browse files Browse the repository at this point in the history
  • Loading branch information
@yamingk
yamingk committed Sep 18, 2023
1 parent 6db84f2 commit e20b02f
Show file tree
Hide file tree
Showing 6 changed files with 80 additions and 28 deletions.
2 changes: 1 addition & 1 deletion conanfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

class HomestoreConan(ConanFile):
name = "homestore"
version = "4.3.0"
version = "4.3.1"

homepage = "https://github.com/eBay/Homestore"
description = "HomeStore Storage Engine"
Expand Down
1 change: 1 addition & 0 deletions src/include/homestore/blk.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ using chunk_num_t = uint16_t;
using blk_count_t = uint16_t;
using blk_num_t = uint32_t;
using blk_temp_t = uint16_t;
using allocator_id_t = chunk_num_t;

static constexpr size_t max_addressable_chunks() { return 1UL << (8 * sizeof(chunk_num_t)); }
static constexpr size_t max_blks_per_chunk() { return 1UL << (8 * sizeof(blk_num_t)); }
Expand Down
11 changes: 5 additions & 6 deletions src/lib/blkalloc/append_blk_allocator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

namespace homestore {

AppendBlkAllocator::AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, chunk_num_t id) :
AppendBlkAllocator::AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, allocator_id_t id) :
BlkAllocator{cfg, id}, m_metrics{get_name().c_str()} {
// TODO: try to make all append_blk_allocator instances use same client type to reduce metablk's cache footprint;
meta_service().register_handler(
Expand All @@ -47,13 +47,12 @@ AppendBlkAllocator::AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_form
}

void AppendBlkAllocator::on_meta_blk_found(const sisl::byte_view& buf, void* meta_cookie) {
// TODO: also needs to initialize base class blkallocator for recovery path;
// load all dirty buffer from the same starting point;
m_sb[0].load(buf, meta_cookie);
for (uint8_t i = 1; i < m_sb.size(); ++i) {
for (uint8_t i = 0; i < m_sb.size(); ++i) {
m_sb[i].load(buf, meta_cookie);
}

// recover in-memory counter/offset from metablk;
m_last_append_offset = m_sb[0]->last_append_offset;
m_freeable_nblks = m_sb[0]->freeable_nblks;

Expand Down Expand Up @@ -125,11 +124,11 @@ BlkAllocStatus AppendBlkAllocator::alloc(blk_count_t nblks, const blk_alloc_hint
// cp_flush should not block alloc/free;
//
void AppendBlkAllocator::cp_flush(CP* cp) {
const auto idx = cp->id();
const auto idx = cp->id() % MAX_CP_COUNT;
// check if current cp's context has dirty buffer already
if (m_sb[idx]->is_dirty) {
// write to metablk;
m_sb[cp->id()].write();
m_sb[idx].write();

// clear this dirty buff's dirty flag;
clear_dirty_offset(idx);
Expand Down
4 changes: 2 additions & 2 deletions src/lib/blkalloc/append_blk_allocator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ struct append_blkalloc_ctx {
uint64_t magic{append_blkalloc_sb_magic};
uint32_t version{append_blkalloc_sb_version};
bool is_dirty; // this field is needed for cp_flush, but not necessarily needed for persistence;
uint64_t allocator_id;
allocator_id_t allocator_id;
blk_num_t freeable_nblks;
blk_num_t last_append_offset;
};
Expand Down Expand Up @@ -67,7 +67,7 @@ class AppendBlkAllocMetrics : public sisl::MetricsGroup {
//
class AppendBlkAllocator : public BlkAllocator {
public:
AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, chunk_num_t id = 0);
AppendBlkAllocator(const BlkAllocConfig& cfg, bool need_format, allocator_id_t id = 0);

AppendBlkAllocator(const AppendBlkAllocator&) = delete;
AppendBlkAllocator(AppendBlkAllocator&&) noexcept = delete;
Expand Down
37 changes: 23 additions & 14 deletions src/lib/device/virtual_dev.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,21 +184,28 @@ BlkAllocStatus VirtualDev::alloc_blks(blk_count_t nblks, blk_alloc_hints const&
Chunk* chunk;
size_t attempt{0};

do {
chunk = m_chunk_selector->select_chunk(nblks, hints).get();
if (chunk == nullptr) {
status = BlkAllocStatus::SPACE_FULL;
break;
}

status = alloc_blks_from_chunk(nblks, hints, out_blkid, chunk);
if ((status == BlkAllocStatus::SUCCESS) || !hints.can_look_for_other_chunk ||
(status == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) {
break;
}
} while (++attempt < m_all_chunks.size());
if (hints.chunk_id_hint) {
// this is a target-chunk allocation;
chunk = m_dmgr.get_chunk_mutable(*(hints.chunk_id_hint));
status = alloc_blk_from_chunk(nblks, hints, out_blkid, chunk);
// don't look for other chunks because user wants allocation on chunk_id_hint only;
} else {
do {
chunk = m_chunk_selector->select_chunk(nblks, hints).get();
if (chunk == nullptr) {
status = BlkAllocStatus::SPACE_FULL;
break;
}

status = alloc_blks_from_chunk(nblks, hints, out_blkid, chunk);
if ((status == BlkAllocStatus::SUCCESS) || !hints.can_look_for_other_chunk ||
(status == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) {
break;
}
} while (++attempt < m_all_chunks.size());
}

if ((status != BlkAllocStatus::SUCCESS) || (status != BlkAllocStatus::PARTIAL)) {
if ((status != BlkAllocStatus::SUCCESS) && !((status == BlkAllocStatus::PARTIAL) && hints.partial_alloc_ok) {
LOGERROR("nblks={} failed to alloc after trying to alloc on every chunks {} and devices {}.", nblks);
COUNTER_INCREMENT(m_metrics, vdev_num_alloc_failure, 1);
}
Expand All @@ -221,6 +228,8 @@ BlkAllocStatus VirtualDev::alloc_blks(blk_count_t nblks, blk_alloc_hints const&
blk_count_t nblks_remain = nblks;
BlkAllocStatus status;

HS_REL_ASSERT(h.chunk_id_hint, "Not expecting to received targetd chunk allocation in this function. ");

do {
out_blkids.emplace_back(); // Put an empty MultiBlkId and use that for allocating them
BlkId& out_bid = out_blkids.back();
Expand Down
53 changes: 48 additions & 5 deletions src/tests/test_append_blkalloc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ class AppendBlkAllocatorTest : public testing::Test {
m_cv.wait(lk, [this] { return this->m_io_job_done; });
}

void reset_io_job_done() { m_io_job_done = false; }

void write_io_verify(const uint64_t io_size) {
auto sg_write_ptr = std::make_shared< sisl::sg_list >();
auto sg_read_ptr = std::make_shared< sisl::sg_list >();
Expand Down Expand Up @@ -213,10 +215,10 @@ TEST_F(AppendBlkAllocatorTest, TestWriteThenReadVerify) {
LOGINFO("Step 1: run on worker thread to schedule write for {} Bytes.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [this, io_size]() { this->write_io_verify(io_size); });

LOGINFO("Step 3: Wait for I/O to complete.");
LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 4: I/O completed, do shutdown.");
LOGINFO("Step 3: I/O completed, do shutdown.");
test_common::HSTestHelper::shutdown_homestore();
}

Expand All @@ -231,10 +233,10 @@ TEST_F(AppendBlkAllocatorTest, TestWriteThenFreeBlk) {
iomanager.run_on_forget(iomgr::reactor_regex::random_worker,
[this, io_size]() { this->write_io_free_blk(io_size); });

LOGINFO("Step 3: Wait for I/O to complete.");
LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 4: I/O completed, do shutdown.");
LOGINFO("Step 3: I/O completed, do shutdown.");
test_common::HSTestHelper::shutdown_homestore();
}

Expand All @@ -249,9 +251,50 @@ TEST_F(AppendBlkAllocatorTest, TestCPFlush) {
LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 3: I/O completed, trigger_cp and wait.");
test_common::HSTestHelper::trigger_cp(true /* wait */);

LOGINFO("Step 4: cp completed, do shutdown.");
test_common::HSTestHelper::shutdown_homestore();
}

TEST_F(AppendBlkAllocatorTest, TestWriteThenRecovey) {
LOGINFO("Step 0: Starting homestore.");
test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0, 0, 0, 80.0, 0, nullptr, false, nullptr,
false /* default ds type */);

// start io in worker thread;
auto io_size = 4 * Mi;
LOGINFO("Step 1: run on worker thread to schedule write for {} Bytes, then free blk.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker,
[this, io_size]() { this->write_io_free_blk(io_size); });

LOGINFO("Step 2: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 3: I/O completed, trigger_cp and wait.");
test_common::HSTestHelper::trigger_cp(true /* wait */);

LOGINFO("Step 4: cp completed, restart homestore.");
test_common::HSTestHelper::start_homestore("test_append_blkalloc", 5.0, 0, 0, 80.0, 0,
nullptr /* before_svc_start_cb */, true /* restart */,
nullptr /* indx_svc_cb */, false /* default ds type */);

std::this_thread::sleep_for(std::chrono::seconds{3});
LOGINFO("Step 5: Restarted homestore with data service recovered");

this->reset_io_job_done();

LOGINFO("Step 6: run on worker thread to schedule write for {} Bytes.", io_size);
iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [this, io_size]() { this->write_io(io_size); });

LOGINFO("Step 7: Wait for I/O to complete.");
wait_for_all_io_complete();

LOGINFO("Step 8: I/O completed, trigger_cp and wait.");
test_common::HSTestHelper::trigger_cp(true /* wait */);

LOGINFO("Step 4: I/O completed, do shutdown.");
LOGINFO("Step 9: do shutdown. ");
test_common::HSTestHelper::shutdown_homestore();
}

Expand Down

0 comments on commit e20b02f

Please sign in to comment.