write buffer pool

This commit is contained in:
2024-04-25 11:03:28 +00:00
parent 36a615aaad
commit 28bdaea8eb
8 changed files with 493 additions and 3 deletions

View File

@ -1 +1 @@
add_library(bpt STATIC src/disk_manager.cpp src/replacer.cpp)
add_library(bpt STATIC src/disk_manager.cpp src/replacer.cpp src/buffer_pool_manager.cpp)

View File

@ -0,0 +1,174 @@
#ifndef BUFFER_POOL_MANAGER_H
#define BUFFER_POOL_MANAGER_H
#include <cstddef>
#include <list>
#include <memory>
#include <mutex>
#include <shared_mutex>
#include <unordered_map>
#include "bpt/config.h"
#include "bpt/disk_manager.h"
#include "bpt/replacer.h"
class BufferPoolManager;
class Page {
public:
Page();
~Page();
friend BufferPoolManager;
void ResetMemory();
char *GetData();
private:
std::shared_mutex latch_;
char *mem;
bool is_dirty_;
size_t pin_count_;
page_id_t page_id_;
};
class BufferPoolManager {
public:
BufferPoolManager() = delete;
BufferPoolManager(const BufferPoolManager &) = delete;
BufferPoolManager(BufferPoolManager &&) = delete;
explicit BufferPoolManager(size_t pool_size, size_t replacer_k, DiskManager *disk_manager);
BufferPoolManager &operator=(const BufferPoolManager &) = delete;
BufferPoolManager &operator=(BufferPoolManager &&) = delete;
~BufferPoolManager();
/**
* @brief Allocate a page on disk. Caller should acquire the latch before calling this function.
* @return the id of the allocated page
*/
auto AllocatePage() -> page_id_t;
/**
* @brief Deallocate a page on disk. Caller should acquire the latch before calling this function.
* @param page_id id of the page to deallocate
*/
void DeallocatePage(page_id_t page_id);
/** @brief Return the size (number of frames) of the buffer pool. */
auto GetPoolSize() -> size_t;
/** @brief Return the pointer to all the pages in the buffer pool. */
auto GetPages() -> Page *;
/**
* @brief Create a new page in the buffer pool. Set page_id to the new page's id, or nullptr if all frames
* are currently in use and not evictable (in another word, pinned).
*
* You should pick the replacement frame from either the free list or the replacer (always find from the free list
* first), and then call the AllocatePage() method to get a new page id. If the replacement frame has a dirty page,
* you should write it back to the disk first. You also need to reset the memory and metadata for the new page.
*
* Remember to "Pin" the frame by calling replacer.SetEvictable(frame_id, false)
* so that the replacer wouldn't evict the frame before the buffer pool manager "Unpin"s it.
* Also, remember to record the access history of the frame in the replacer for the lru-k algorithm to work.
*
* @param[out] page_id id of created page
* @return nullptr if no new pages could be created, otherwise pointer to new page
*/
auto NewPage(page_id_t *page_id) -> Page *;
/**
* TODO(P1): Add implementation
*
* @brief PageGuard wrapper for NewPage
*
* Functionality should be the same as NewPage, except that
* instead of returning a pointer to a page, you return a
* BasicPageGuard structure.
*
* @param[out] page_id, the id of the new page
* @return BasicPageGuard holding a new page
*/
// auto NewPageGuarded(page_id_t *page_id) -> BasicPageGuard;
/**
* @brief Fetch the requested page from the buffer pool. Return nullptr if page_id needs to be fetched from the disk
* but all frames are currently in use and not evictable (in another word, pinned).
*
* First search for page_id in the buffer pool. If not found, pick a replacement frame from either the free list or
* the replacer (always find from the free list first), read the page from disk by calling disk_manager_->ReadPage(),
* and replace the old page in the frame. Similar to NewPage(), if the old page is dirty, you need to write it back
* to disk and update the metadata of the new page
*
* In addition, remember to disable eviction and record the access history of the frame like you did for NewPage().
*
* @param page_id id of page to be fetched
* @param access_type type of access to the page, only needed for leaderboard tests.
* @return nullptr if page_id cannot be fetched, otherwise pointer to the requested page
*/
auto FetchPage(page_id_t page_id) -> Page *;
/**
* TODO(P1): Add implementation
*
* @brief PageGuard wrappers for FetchPage
*
* Functionality should be the same as FetchPage, except
* that, depending on the function called, a guard is returned.
* If FetchPageRead or FetchPageWrite is called, it is expected that
* the returned page already has a read or write latch held, respectively.
*
* @param page_id, the id of the page to fetch
* @return PageGuard holding the fetched page
*/
// auto FetchPageBasic(page_id_t page_id) -> BasicPageGuard;
// auto FetchPageRead(page_id_t page_id) -> ReadPageGuard;
// auto FetchPageWrite(page_id_t page_id) -> WritePageGuard;
/**
* TODO(P1): Add implementation
*
* @brief Unpin the target page from the buffer pool. If page_id is not in the buffer pool or its pin count is already
* 0, return false.
*
* Decrement the pin count of a page. If the pin count reaches 0, the frame should be evictable by the replacer.
* Also, set the dirty flag on the page to indicate if the page was modified.
*
* @param page_id id of page to be unpinned
* @param is_dirty true if the page should be marked as dirty, false otherwise
* @param access_type type of access to the page, only needed for leaderboard tests.
* @return false if the page is not in the page table or its pin count is <= 0 before this call, true otherwise
*/
auto UnpinPage(page_id_t page_id, bool is_dirty) -> bool;
/**
* @brief Flush the target page to disk.
*
* Use the DiskManager::WritePage() method to flush a page to disk, REGARDLESS of the dirty flag.
* Unset the dirty flag of the page after flushing.
*
* @param page_id id of page to be flushed, cannot be INVALID_PAGE_ID
* @return false if the page could not be found in the page table, true otherwise
*/
auto FlushPage(page_id_t page_id) -> bool;
/**
* @brief Flush all the pages in the buffer pool to disk.
*/
void FlushAllPages();
/**
* @brief Delete a page from the buffer pool. If page_id is not in the buffer pool, do nothing and return true. If the
* page is pinned and cannot be deleted, return false immediately.
*
* After deleting the page from the page table, stop tracking the frame in the replacer and add the frame
* back to the free list. Also, reset the page's memory and metadata. Finally, you should call DeallocatePage() to
* imitate freeing the page on the disk.
*
* @param page_id id of page to be deleted
* @return false if the page exists but could not be deleted, true if the page didn't exist or deletion succeeded
*/
auto DeletePage(page_id_t page_id) -> bool;
private:
const size_t pool_size;
const size_t replacer_k;
LRUKReplacer replacer;
DiskManager *disk_manager;
std::mutex latch;
Page *pages_;
std::unordered_map<page_id_t, frame_id_t> page_table_;
std::list<frame_id_t> free_list_;
};
#endif

View File

@ -3,7 +3,6 @@
#include <cstdio>
#include <string>
#include "config.h"
extern const size_t kPageSize;
class DiskManager {
/**
* The Data Structure on Disk:

View File

@ -6,6 +6,8 @@
class LRUKReplacer {
public:
LRUKReplacer() = delete;
LRUKReplacer(const LRUKReplacer &) = delete;
LRUKReplacer(LRUKReplacer &&) = delete;
explicit LRUKReplacer(size_t max_frame_count, size_t k_value);
~LRUKReplacer();
bool TryEvictLeastImportant(frame_id_t &frame_id);
@ -13,6 +15,7 @@ class LRUKReplacer {
void SetEvictable(frame_id_t frame_id, bool evitable);
bool TryEvictExactFrame(frame_id_t frame_id);
LRUKReplacer &operator=(const LRUKReplacer &) = delete;
LRUKReplacer &operator=(LRUKReplacer &&) = delete;
size_t GetCurrentEvitableCount();
private:

View File

@ -0,0 +1,169 @@
#include "bpt/buffer_pool_manager.h"
#include <cstring>
#include <mutex>
#include "bpt/config.h"
Page::Page() : mem(new char[kPageSize]) {}
Page::~Page() { delete[] mem; }
void Page::ResetMemory() { memset(mem, 0, kPageSize); }
char *Page::GetData() { return mem; }
BufferPoolManager::BufferPoolManager(size_t pool_size, size_t replacer_k, DiskManager *disk_manager)
: pool_size(pool_size),
replacer_k(replacer_k),
replacer(pool_size, replacer_k),
disk_manager(disk_manager) { // we allocate a consecutive memory space for the buffer pool
pages_ = new Page[pool_size];
// Initially, every page is in the free list.
for (size_t i = 0; i < pool_size; ++i) {
free_list_.emplace_back(static_cast<int>(i));
}
}
BufferPoolManager::~BufferPoolManager() { delete[] pages_; }
page_id_t BufferPoolManager::AllocatePage() {
page_id_t page_id = disk_manager->AllocNewEmptyPageId();
return page_id;
}
void BufferPoolManager::DeallocatePage(page_id_t page_id) {
disk_manager->DeallocatePage(page_id);
}
size_t BufferPoolManager::GetPoolSize() { return pool_size; }
Page *BufferPoolManager::GetPages() { return pages_; }
auto BufferPoolManager::NewPage(page_id_t *page_id) -> Page * {
std::lock_guard<std::mutex> guard(latch);
if (!free_list_.empty()) {
int internal_page_object_offset = free_list_.front();
free_list_.pop_front();
Page *page = &pages_[internal_page_object_offset];
*page_id = AllocatePage();
page_table_.insert({*page_id, internal_page_object_offset});
page->is_dirty_ = false;
page->page_id_ = *page_id;
page->pin_count_ = 1;
// page->ResetMemory();
replacer.RecordAccess(internal_page_object_offset);
replacer.SetEvictable(internal_page_object_offset, false);
return page;
}
frame_id_t victim_frame_id;
if (!replacer.TryEvictLeastImportant(victim_frame_id)) {
return nullptr;
}
Page *victim_page_ptr = &pages_[victim_frame_id];
if (victim_page_ptr->is_dirty_) {
disk_manager->WritePage(victim_page_ptr->page_id_, victim_page_ptr->GetData());
}
*page_id = AllocatePage();
page_table_.erase(victim_page_ptr->page_id_);
page_table_.insert({*page_id, victim_frame_id});
victim_page_ptr->is_dirty_ = false;
victim_page_ptr->pin_count_ = 1;
victim_page_ptr->page_id_ = *page_id;
victim_page_ptr->ResetMemory();
replacer.RecordAccess(victim_frame_id);
replacer.SetEvictable(victim_frame_id, false);
return victim_page_ptr;
}
auto BufferPoolManager::FetchPage(page_id_t page_id) -> Page * {
std::lock_guard<std::mutex> guard(latch);
if (page_table_.find(page_id) != page_table_.end()) {
frame_id_t frame_id = page_table_[page_id];
Page *page = &pages_[frame_id];
page->pin_count_++;
replacer.RecordAccess(frame_id);
replacer.SetEvictable(frame_id, false);
return page;
}
if (!free_list_.empty()) {
int internal_page_object_offset = free_list_.front();
free_list_.pop_front();
Page *page = &pages_[internal_page_object_offset];
page_table_.insert({page_id, internal_page_object_offset});
page->is_dirty_ = false;
page->page_id_ = page_id;
page->pin_count_ = 1;
replacer.RecordAccess(internal_page_object_offset);
replacer.SetEvictable(internal_page_object_offset, false);
disk_manager->ReadPage(page_id, page->GetData());
return page;
}
frame_id_t victim_frame_id;
if (!replacer.TryEvictLeastImportant(victim_frame_id)) {
return nullptr;
}
Page *victim_page_ptr = &pages_[victim_frame_id];
if (victim_page_ptr->is_dirty_) {
disk_manager->WritePage(victim_page_ptr->page_id_, victim_page_ptr->GetData());
}
page_table_.erase(victim_page_ptr->page_id_);
page_table_.insert({page_id, victim_frame_id});
victim_page_ptr->is_dirty_ = false;
victim_page_ptr->pin_count_ = 1;
victim_page_ptr->page_id_ = page_id;
replacer.RecordAccess(victim_frame_id);
replacer.SetEvictable(victim_frame_id, false);
disk_manager->ReadPage(page_id, victim_page_ptr->GetData());
return victim_page_ptr;
}
auto BufferPoolManager::UnpinPage(page_id_t page_id, bool is_dirty) -> bool {
std::lock_guard<std::mutex> guard(latch);
if (page_table_.find(page_id) == page_table_.end()) {
return false;
}
frame_id_t frame_id = page_table_[page_id];
Page *cur_page = &pages_[frame_id];
if (cur_page->pin_count_ <= 0) {
return false;
}
cur_page->pin_count_--;
if (cur_page->pin_count_ == 0) {
replacer.SetEvictable(frame_id, true);
}
if (is_dirty) {
cur_page->is_dirty_ = true;
}
return true;
}
auto BufferPoolManager::FlushPage(page_id_t page_id) -> bool {
std::lock_guard<std::mutex> guard(latch);
frame_id_t frame_id = page_table_[page_id];
if (page_table_.find(page_id) == page_table_.end()) {
return false;
}
Page *cur_page = &pages_[frame_id];
disk_manager->WritePage(page_id, cur_page->GetData());
cur_page->is_dirty_ = false;
return true;
}
void BufferPoolManager::FlushAllPages() {
for (auto &pair : page_table_) {
FlushPage(pair.first);
}
}
auto BufferPoolManager::DeletePage(page_id_t page_id) -> bool {
std::lock_guard<std::mutex> guard(latch);
if (page_table_.find(page_id) == page_table_.end()) {
return true;
}
frame_id_t frame_id = page_table_[page_id];
Page *page = &pages_[frame_id];
if (page->pin_count_ > 0) {
return false;
}
page_table_.erase(page_id);
replacer.TryEvictExactFrame(frame_id);
free_list_.push_back(frame_id);
page->is_dirty_ = false;
page->pin_count_ = 0;
page->page_id_ = 0;
page->ResetMemory();
DeallocatePage(page_id);
return true;
}

View File

@ -2,6 +2,7 @@
#include <cstring>
#include <exception>
#include <stdexcept>
const size_t kPageSize = 4096;
DiskManager::DiskManager(const std::string &file_path_)
: file_path(file_path_),
first_empty_page_id(0),

View File

@ -5,3 +5,5 @@ if(OJ_TEST_BPT)
endif()
add_executable(replacer_test replacer_test.cpp)
target_link_libraries(replacer_test bpt GTest::gtest_main)
add_executable(buffer_pool_manager_test buffer_pool_manager_test.cpp)
target_link_libraries(buffer_pool_manager_test bpt GTest::gtest_main)

View File

@ -0,0 +1,142 @@
#include "bpt/buffer_pool_manager.h"
#include <gtest/gtest.h>
#include <cstdio>
#include <cstring>
#include <memory>
#include <random>
#include <string>
#include "bpt/config.h"
// Demonstrate some basic assertions.
TEST(HelloTest, BasicAssertions) {
// Expect two strings not to be equal.
EXPECT_STRNE("hello", "world");
// Expect equality.
EXPECT_EQ(7 * 6, 42);
}
TEST(Basic, BasicTest) {
DiskManager disk_manager("/tmp/test.db");
BufferPoolManager buffer_pool_manager(10, 3, &disk_manager);
}
TEST(BufferPoolManagerTest, DISABLED_BinaryDataTest) {
const std::string db_name = "test.db";
const size_t buffer_pool_size = 10;
const size_t k = 5;
std::random_device r;
std::default_random_engine rng(r());
std::uniform_int_distribution<char> uniform_dist(0);
auto *disk_manager = new DiskManager(db_name);
auto *bpm = new BufferPoolManager(buffer_pool_size, k, disk_manager);
page_id_t page_id_temp;
auto *page0 = bpm->NewPage(&page_id_temp);
// Scenario: The buffer pool is empty. We should be able to create a new page.
ASSERT_NE(nullptr, page0);
EXPECT_EQ(1, page_id_temp);
char random_binary_data[kPageSize];
// Generate random binary data
for (char &i : random_binary_data) {
i = uniform_dist(rng);
}
// Insert terminal characters both in the middle and at end
random_binary_data[kPageSize / 2] = '\0';
random_binary_data[kPageSize - 1] = '\0';
// Scenario: Once we have a page, we should be able to read and write content.
std::memcpy(page0->GetData(), random_binary_data, kPageSize);
EXPECT_EQ(0, std::memcmp(page0->GetData(), random_binary_data, kPageSize));
// Scenario: We should be able to create new pages until we fill up the buffer pool.
for (size_t i = 1; i < buffer_pool_size; ++i) {
EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp));
}
// Scenario: Once the buffer pool is full, we should not be able to create any new pages.
for (size_t i = buffer_pool_size; i < buffer_pool_size * 2; ++i) {
EXPECT_EQ(nullptr, bpm->NewPage(&page_id_temp));
}
// Scenario: After unpinning pages {0, 1, 2, 3, 4} we should be able to create 5 new pages
for (int i = 1; i <= 5; ++i) {
EXPECT_EQ(true, bpm->UnpinPage(i, true));
bpm->FlushPage(i);
}
for (int i = 1; i <= 5; ++i) {
EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp));
bpm->UnpinPage(page_id_temp, false);
}
// Scenario: We should be able to fetch the data we wrote a while ago.
page0 = bpm->FetchPage(1);
EXPECT_EQ(0, memcmp(page0->GetData(), random_binary_data, kPageSize));
EXPECT_EQ(true, bpm->UnpinPage(1, true));
// Shutdown the disk manager and remove the temporary file we created.
disk_manager->Close();
remove("test.db");
delete bpm;
delete disk_manager;
}
// NOLINTNEXTLINE
TEST(BufferPoolManagerTest, DISABLED_SampleTest) {
const std::string db_name = "test.db";
const size_t buffer_pool_size = 10;
const size_t k = 5;
auto *disk_manager = new DiskManager(db_name);
auto *bpm = new BufferPoolManager(buffer_pool_size, k, disk_manager);
page_id_t page_id_temp;
auto *page0 = bpm->NewPage(&page_id_temp);
// Scenario: The buffer pool is empty. We should be able to create a new page.
ASSERT_NE(nullptr, page0);
EXPECT_EQ(1, page_id_temp);
// Scenario: Once we have a page, we should be able to read and write content.
snprintf(page0->GetData(), kPageSize, "Hello");
EXPECT_EQ(0, strcmp(page0->GetData(), "Hello"));
// Scenario: We should be able to create new pages until we fill up the buffer pool.
for (size_t i = 2; i <= buffer_pool_size; ++i) {
EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp));
}
// Scenario: Once the buffer pool is full, we should not be able to create any new pages.
for (size_t i = buffer_pool_size + 1; i <= buffer_pool_size * 2; ++i) {
EXPECT_EQ(nullptr, bpm->NewPage(&page_id_temp));
}
// Scenario: After unpinning pages {1, 2, 3, 4, 5} and pinning another 4 new pages,
// there would still be one buffer page left for reading page 0.
for (int i = 1; i <= 5; ++i) {
EXPECT_EQ(true, bpm->UnpinPage(i, true));
}
for (int i = 1; i <= 4; ++i) {
EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp));
}
// Scenario: We should be able to fetch the data we wrote a while ago.
page0 = bpm->FetchPage(1);
EXPECT_EQ(0, strcmp(page0->GetData(), "Hello"));
// Scenario: If we unpin page 0 and then make a new page, all the buffer pages should
// now be pinned. Fetching page 0 should fail.
EXPECT_EQ(true, bpm->UnpinPage(1, true));
EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp));
EXPECT_EQ(nullptr, bpm->FetchPage(1));
// Shutdown the disk manager and remove the temporary file we created.
disk_manager->Close();
remove("test.db");
delete bpm;
delete disk_manager;
}