diff --git a/bpt/CMakeLists.txt b/bpt/CMakeLists.txt index 4bf7665..fae0699 100644 --- a/bpt/CMakeLists.txt +++ b/bpt/CMakeLists.txt @@ -1 +1 @@ -add_library(bpt STATIC src/disk_manager.cpp src/replacer.cpp) \ No newline at end of file +add_library(bpt STATIC src/disk_manager.cpp src/replacer.cpp src/buffer_pool_manager.cpp) \ No newline at end of file diff --git a/bpt/include/bpt/buffer_pool_manager.h b/bpt/include/bpt/buffer_pool_manager.h new file mode 100644 index 0000000..741b62e --- /dev/null +++ b/bpt/include/bpt/buffer_pool_manager.h @@ -0,0 +1,174 @@ +#ifndef BUFFER_POOL_MANAGER_H +#define BUFFER_POOL_MANAGER_H +#include +#include +#include +#include +#include +#include +#include "bpt/config.h" +#include "bpt/disk_manager.h" +#include "bpt/replacer.h" +class BufferPoolManager; +class Page { + public: + Page(); + ~Page(); + friend BufferPoolManager; + void ResetMemory(); + char *GetData(); + + private: + std::shared_mutex latch_; + char *mem; + bool is_dirty_; + size_t pin_count_; + page_id_t page_id_; +}; +class BufferPoolManager { + public: + BufferPoolManager() = delete; + BufferPoolManager(const BufferPoolManager &) = delete; + BufferPoolManager(BufferPoolManager &&) = delete; + explicit BufferPoolManager(size_t pool_size, size_t replacer_k, DiskManager *disk_manager); + BufferPoolManager &operator=(const BufferPoolManager &) = delete; + BufferPoolManager &operator=(BufferPoolManager &&) = delete; + ~BufferPoolManager(); + /** + * @brief Allocate a page on disk. Caller should acquire the latch before calling this function. + * @return the id of the allocated page + */ + auto AllocatePage() -> page_id_t; + + /** + * @brief Deallocate a page on disk. Caller should acquire the latch before calling this function. + * @param page_id id of the page to deallocate + */ + void DeallocatePage(page_id_t page_id); + /** @brief Return the size (number of frames) of the buffer pool. */ + auto GetPoolSize() -> size_t; + + /** @brief Return the pointer to all the pages in the buffer pool. */ + auto GetPages() -> Page *; + + /** + * @brief Create a new page in the buffer pool. Set page_id to the new page's id, or nullptr if all frames + * are currently in use and not evictable (in another word, pinned). + * + * You should pick the replacement frame from either the free list or the replacer (always find from the free list + * first), and then call the AllocatePage() method to get a new page id. If the replacement frame has a dirty page, + * you should write it back to the disk first. You also need to reset the memory and metadata for the new page. + * + * Remember to "Pin" the frame by calling replacer.SetEvictable(frame_id, false) + * so that the replacer wouldn't evict the frame before the buffer pool manager "Unpin"s it. + * Also, remember to record the access history of the frame in the replacer for the lru-k algorithm to work. + * + * @param[out] page_id id of created page + * @return nullptr if no new pages could be created, otherwise pointer to new page + */ + auto NewPage(page_id_t *page_id) -> Page *; + + /** + * TODO(P1): Add implementation + * + * @brief PageGuard wrapper for NewPage + * + * Functionality should be the same as NewPage, except that + * instead of returning a pointer to a page, you return a + * BasicPageGuard structure. + * + * @param[out] page_id, the id of the new page + * @return BasicPageGuard holding a new page + */ + // auto NewPageGuarded(page_id_t *page_id) -> BasicPageGuard; + + /** + * @brief Fetch the requested page from the buffer pool. Return nullptr if page_id needs to be fetched from the disk + * but all frames are currently in use and not evictable (in another word, pinned). + * + * First search for page_id in the buffer pool. If not found, pick a replacement frame from either the free list or + * the replacer (always find from the free list first), read the page from disk by calling disk_manager_->ReadPage(), + * and replace the old page in the frame. Similar to NewPage(), if the old page is dirty, you need to write it back + * to disk and update the metadata of the new page + * + * In addition, remember to disable eviction and record the access history of the frame like you did for NewPage(). + * + * @param page_id id of page to be fetched + * @param access_type type of access to the page, only needed for leaderboard tests. + * @return nullptr if page_id cannot be fetched, otherwise pointer to the requested page + */ + auto FetchPage(page_id_t page_id) -> Page *; + + /** + * TODO(P1): Add implementation + * + * @brief PageGuard wrappers for FetchPage + * + * Functionality should be the same as FetchPage, except + * that, depending on the function called, a guard is returned. + * If FetchPageRead or FetchPageWrite is called, it is expected that + * the returned page already has a read or write latch held, respectively. + * + * @param page_id, the id of the page to fetch + * @return PageGuard holding the fetched page + */ + // auto FetchPageBasic(page_id_t page_id) -> BasicPageGuard; + // auto FetchPageRead(page_id_t page_id) -> ReadPageGuard; + // auto FetchPageWrite(page_id_t page_id) -> WritePageGuard; + + /** + * TODO(P1): Add implementation + * + * @brief Unpin the target page from the buffer pool. If page_id is not in the buffer pool or its pin count is already + * 0, return false. + * + * Decrement the pin count of a page. If the pin count reaches 0, the frame should be evictable by the replacer. + * Also, set the dirty flag on the page to indicate if the page was modified. + * + * @param page_id id of page to be unpinned + * @param is_dirty true if the page should be marked as dirty, false otherwise + * @param access_type type of access to the page, only needed for leaderboard tests. + * @return false if the page is not in the page table or its pin count is <= 0 before this call, true otherwise + */ + auto UnpinPage(page_id_t page_id, bool is_dirty) -> bool; + + /** + * @brief Flush the target page to disk. + * + * Use the DiskManager::WritePage() method to flush a page to disk, REGARDLESS of the dirty flag. + * Unset the dirty flag of the page after flushing. + * + * @param page_id id of page to be flushed, cannot be INVALID_PAGE_ID + * @return false if the page could not be found in the page table, true otherwise + */ + auto FlushPage(page_id_t page_id) -> bool; + + /** + * @brief Flush all the pages in the buffer pool to disk. + */ + void FlushAllPages(); + + /** + * @brief Delete a page from the buffer pool. If page_id is not in the buffer pool, do nothing and return true. If the + * page is pinned and cannot be deleted, return false immediately. + * + * After deleting the page from the page table, stop tracking the frame in the replacer and add the frame + * back to the free list. Also, reset the page's memory and metadata. Finally, you should call DeallocatePage() to + * imitate freeing the page on the disk. + * + * @param page_id id of page to be deleted + * @return false if the page exists but could not be deleted, true if the page didn't exist or deletion succeeded + */ + auto DeletePage(page_id_t page_id) -> bool; + + private: + const size_t pool_size; + const size_t replacer_k; + LRUKReplacer replacer; + DiskManager *disk_manager; + std::mutex latch; + Page *pages_; + std::unordered_map page_table_; + std::list free_list_; +}; +#endif \ No newline at end of file diff --git a/bpt/include/bpt/disk_manager.h b/bpt/include/bpt/disk_manager.h index 57bff04..232e68f 100644 --- a/bpt/include/bpt/disk_manager.h +++ b/bpt/include/bpt/disk_manager.h @@ -3,7 +3,6 @@ #include #include #include "config.h" -extern const size_t kPageSize; class DiskManager { /** * The Data Structure on Disk: diff --git a/bpt/include/bpt/replacer.h b/bpt/include/bpt/replacer.h index 9bbca16..9b28e98 100644 --- a/bpt/include/bpt/replacer.h +++ b/bpt/include/bpt/replacer.h @@ -6,6 +6,8 @@ class LRUKReplacer { public: LRUKReplacer() = delete; + LRUKReplacer(const LRUKReplacer &) = delete; + LRUKReplacer(LRUKReplacer &&) = delete; explicit LRUKReplacer(size_t max_frame_count, size_t k_value); ~LRUKReplacer(); bool TryEvictLeastImportant(frame_id_t &frame_id); @@ -13,6 +15,7 @@ class LRUKReplacer { void SetEvictable(frame_id_t frame_id, bool evitable); bool TryEvictExactFrame(frame_id_t frame_id); LRUKReplacer &operator=(const LRUKReplacer &) = delete; + LRUKReplacer &operator=(LRUKReplacer &&) = delete; size_t GetCurrentEvitableCount(); private: diff --git a/bpt/src/buffer_pool_manager.cpp b/bpt/src/buffer_pool_manager.cpp new file mode 100644 index 0000000..36e142b --- /dev/null +++ b/bpt/src/buffer_pool_manager.cpp @@ -0,0 +1,169 @@ +#include "bpt/buffer_pool_manager.h" +#include +#include +#include "bpt/config.h" +Page::Page() : mem(new char[kPageSize]) {} +Page::~Page() { delete[] mem; } +void Page::ResetMemory() { memset(mem, 0, kPageSize); } +char *Page::GetData() { return mem; } +BufferPoolManager::BufferPoolManager(size_t pool_size, size_t replacer_k, DiskManager *disk_manager) + : pool_size(pool_size), + replacer_k(replacer_k), + replacer(pool_size, replacer_k), + disk_manager(disk_manager) { // we allocate a consecutive memory space for the buffer pool + pages_ = new Page[pool_size]; + + // Initially, every page is in the free list. + for (size_t i = 0; i < pool_size; ++i) { + free_list_.emplace_back(static_cast(i)); + } +} +BufferPoolManager::~BufferPoolManager() { delete[] pages_; } + +page_id_t BufferPoolManager::AllocatePage() { + page_id_t page_id = disk_manager->AllocNewEmptyPageId(); + return page_id; +} + +void BufferPoolManager::DeallocatePage(page_id_t page_id) { + disk_manager->DeallocatePage(page_id); +} + +size_t BufferPoolManager::GetPoolSize() { return pool_size; } +Page *BufferPoolManager::GetPages() { return pages_; } +auto BufferPoolManager::NewPage(page_id_t *page_id) -> Page * { + std::lock_guard guard(latch); + if (!free_list_.empty()) { + int internal_page_object_offset = free_list_.front(); + free_list_.pop_front(); + Page *page = &pages_[internal_page_object_offset]; + *page_id = AllocatePage(); + page_table_.insert({*page_id, internal_page_object_offset}); + page->is_dirty_ = false; + page->page_id_ = *page_id; + page->pin_count_ = 1; + // page->ResetMemory(); + replacer.RecordAccess(internal_page_object_offset); + replacer.SetEvictable(internal_page_object_offset, false); + return page; + } + frame_id_t victim_frame_id; + if (!replacer.TryEvictLeastImportant(victim_frame_id)) { + return nullptr; + } + Page *victim_page_ptr = &pages_[victim_frame_id]; + if (victim_page_ptr->is_dirty_) { + disk_manager->WritePage(victim_page_ptr->page_id_, victim_page_ptr->GetData()); + } + *page_id = AllocatePage(); + page_table_.erase(victim_page_ptr->page_id_); + page_table_.insert({*page_id, victim_frame_id}); + victim_page_ptr->is_dirty_ = false; + victim_page_ptr->pin_count_ = 1; + victim_page_ptr->page_id_ = *page_id; + victim_page_ptr->ResetMemory(); + replacer.RecordAccess(victim_frame_id); + replacer.SetEvictable(victim_frame_id, false); + return victim_page_ptr; +} + +auto BufferPoolManager::FetchPage(page_id_t page_id) -> Page * { + std::lock_guard guard(latch); + if (page_table_.find(page_id) != page_table_.end()) { + frame_id_t frame_id = page_table_[page_id]; + Page *page = &pages_[frame_id]; + page->pin_count_++; + replacer.RecordAccess(frame_id); + replacer.SetEvictable(frame_id, false); + return page; + } + if (!free_list_.empty()) { + int internal_page_object_offset = free_list_.front(); + free_list_.pop_front(); + Page *page = &pages_[internal_page_object_offset]; + page_table_.insert({page_id, internal_page_object_offset}); + page->is_dirty_ = false; + page->page_id_ = page_id; + page->pin_count_ = 1; + replacer.RecordAccess(internal_page_object_offset); + replacer.SetEvictable(internal_page_object_offset, false); + disk_manager->ReadPage(page_id, page->GetData()); + return page; + } + frame_id_t victim_frame_id; + if (!replacer.TryEvictLeastImportant(victim_frame_id)) { + return nullptr; + } + Page *victim_page_ptr = &pages_[victim_frame_id]; + if (victim_page_ptr->is_dirty_) { + disk_manager->WritePage(victim_page_ptr->page_id_, victim_page_ptr->GetData()); + } + page_table_.erase(victim_page_ptr->page_id_); + page_table_.insert({page_id, victim_frame_id}); + victim_page_ptr->is_dirty_ = false; + victim_page_ptr->pin_count_ = 1; + victim_page_ptr->page_id_ = page_id; + replacer.RecordAccess(victim_frame_id); + replacer.SetEvictable(victim_frame_id, false); + disk_manager->ReadPage(page_id, victim_page_ptr->GetData()); + return victim_page_ptr; +} + +auto BufferPoolManager::UnpinPage(page_id_t page_id, bool is_dirty) -> bool { + std::lock_guard guard(latch); + if (page_table_.find(page_id) == page_table_.end()) { + return false; + } + frame_id_t frame_id = page_table_[page_id]; + Page *cur_page = &pages_[frame_id]; + if (cur_page->pin_count_ <= 0) { + return false; + } + cur_page->pin_count_--; + if (cur_page->pin_count_ == 0) { + replacer.SetEvictable(frame_id, true); + } + if (is_dirty) { + cur_page->is_dirty_ = true; + } + return true; +} + +auto BufferPoolManager::FlushPage(page_id_t page_id) -> bool { + std::lock_guard guard(latch); + frame_id_t frame_id = page_table_[page_id]; + if (page_table_.find(page_id) == page_table_.end()) { + return false; + } + Page *cur_page = &pages_[frame_id]; + disk_manager->WritePage(page_id, cur_page->GetData()); + cur_page->is_dirty_ = false; + return true; +} + +void BufferPoolManager::FlushAllPages() { + for (auto &pair : page_table_) { + FlushPage(pair.first); + } +} + +auto BufferPoolManager::DeletePage(page_id_t page_id) -> bool { + std::lock_guard guard(latch); + if (page_table_.find(page_id) == page_table_.end()) { + return true; + } + frame_id_t frame_id = page_table_[page_id]; + Page *page = &pages_[frame_id]; + if (page->pin_count_ > 0) { + return false; + } + page_table_.erase(page_id); + replacer.TryEvictExactFrame(frame_id); + free_list_.push_back(frame_id); + page->is_dirty_ = false; + page->pin_count_ = 0; + page->page_id_ = 0; + page->ResetMemory(); + DeallocatePage(page_id); + return true; +} \ No newline at end of file diff --git a/bpt/src/disk_manager.cpp b/bpt/src/disk_manager.cpp index 6669c63..138c7a6 100644 --- a/bpt/src/disk_manager.cpp +++ b/bpt/src/disk_manager.cpp @@ -2,6 +2,7 @@ #include #include #include +const size_t kPageSize = 4096; DiskManager::DiskManager(const std::string &file_path_) : file_path(file_path_), first_empty_page_id(0), diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4bfa0d3..8413aba 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,4 +4,6 @@ if(OJ_TEST_BPT) set_target_properties(code PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) endif() add_executable(replacer_test replacer_test.cpp) -target_link_libraries(replacer_test bpt GTest::gtest_main) \ No newline at end of file +target_link_libraries(replacer_test bpt GTest::gtest_main) +add_executable(buffer_pool_manager_test buffer_pool_manager_test.cpp) +target_link_libraries(buffer_pool_manager_test bpt GTest::gtest_main) \ No newline at end of file diff --git a/test/buffer_pool_manager_test.cpp b/test/buffer_pool_manager_test.cpp new file mode 100644 index 0000000..d37eb3c --- /dev/null +++ b/test/buffer_pool_manager_test.cpp @@ -0,0 +1,142 @@ +#include "bpt/buffer_pool_manager.h" +#include +#include +#include +#include +#include +#include +#include "bpt/config.h" +// Demonstrate some basic assertions. +TEST(HelloTest, BasicAssertions) { + // Expect two strings not to be equal. + EXPECT_STRNE("hello", "world"); + // Expect equality. + EXPECT_EQ(7 * 6, 42); +} + +TEST(Basic, BasicTest) { + DiskManager disk_manager("/tmp/test.db"); + BufferPoolManager buffer_pool_manager(10, 3, &disk_manager); +} + +TEST(BufferPoolManagerTest, DISABLED_BinaryDataTest) { + const std::string db_name = "test.db"; + const size_t buffer_pool_size = 10; + const size_t k = 5; + + std::random_device r; + std::default_random_engine rng(r()); + std::uniform_int_distribution uniform_dist(0); + + auto *disk_manager = new DiskManager(db_name); + auto *bpm = new BufferPoolManager(buffer_pool_size, k, disk_manager); + + page_id_t page_id_temp; + auto *page0 = bpm->NewPage(&page_id_temp); + + // Scenario: The buffer pool is empty. We should be able to create a new page. + ASSERT_NE(nullptr, page0); + EXPECT_EQ(1, page_id_temp); + + char random_binary_data[kPageSize]; + // Generate random binary data + for (char &i : random_binary_data) { + i = uniform_dist(rng); + } + + // Insert terminal characters both in the middle and at end + random_binary_data[kPageSize / 2] = '\0'; + random_binary_data[kPageSize - 1] = '\0'; + + // Scenario: Once we have a page, we should be able to read and write content. + std::memcpy(page0->GetData(), random_binary_data, kPageSize); + EXPECT_EQ(0, std::memcmp(page0->GetData(), random_binary_data, kPageSize)); + + // Scenario: We should be able to create new pages until we fill up the buffer pool. + for (size_t i = 1; i < buffer_pool_size; ++i) { + EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp)); + } + + // Scenario: Once the buffer pool is full, we should not be able to create any new pages. + for (size_t i = buffer_pool_size; i < buffer_pool_size * 2; ++i) { + EXPECT_EQ(nullptr, bpm->NewPage(&page_id_temp)); + } + + // Scenario: After unpinning pages {0, 1, 2, 3, 4} we should be able to create 5 new pages + for (int i = 1; i <= 5; ++i) { + EXPECT_EQ(true, bpm->UnpinPage(i, true)); + bpm->FlushPage(i); + } + for (int i = 1; i <= 5; ++i) { + EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp)); + bpm->UnpinPage(page_id_temp, false); + } + // Scenario: We should be able to fetch the data we wrote a while ago. + page0 = bpm->FetchPage(1); + EXPECT_EQ(0, memcmp(page0->GetData(), random_binary_data, kPageSize)); + EXPECT_EQ(true, bpm->UnpinPage(1, true)); + + // Shutdown the disk manager and remove the temporary file we created. + disk_manager->Close(); + remove("test.db"); + + delete bpm; + delete disk_manager; +} + +// NOLINTNEXTLINE +TEST(BufferPoolManagerTest, DISABLED_SampleTest) { + const std::string db_name = "test.db"; + const size_t buffer_pool_size = 10; + const size_t k = 5; + + auto *disk_manager = new DiskManager(db_name); + auto *bpm = new BufferPoolManager(buffer_pool_size, k, disk_manager); + + page_id_t page_id_temp; + auto *page0 = bpm->NewPage(&page_id_temp); + + // Scenario: The buffer pool is empty. We should be able to create a new page. + ASSERT_NE(nullptr, page0); + EXPECT_EQ(1, page_id_temp); + + // Scenario: Once we have a page, we should be able to read and write content. + snprintf(page0->GetData(), kPageSize, "Hello"); + EXPECT_EQ(0, strcmp(page0->GetData(), "Hello")); + + // Scenario: We should be able to create new pages until we fill up the buffer pool. + for (size_t i = 2; i <= buffer_pool_size; ++i) { + EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp)); + } + + // Scenario: Once the buffer pool is full, we should not be able to create any new pages. + for (size_t i = buffer_pool_size + 1; i <= buffer_pool_size * 2; ++i) { + EXPECT_EQ(nullptr, bpm->NewPage(&page_id_temp)); + } + + // Scenario: After unpinning pages {1, 2, 3, 4, 5} and pinning another 4 new pages, + // there would still be one buffer page left for reading page 0. + for (int i = 1; i <= 5; ++i) { + EXPECT_EQ(true, bpm->UnpinPage(i, true)); + } + for (int i = 1; i <= 4; ++i) { + EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp)); + } + + // Scenario: We should be able to fetch the data we wrote a while ago. + page0 = bpm->FetchPage(1); + EXPECT_EQ(0, strcmp(page0->GetData(), "Hello")); + + // Scenario: If we unpin page 0 and then make a new page, all the buffer pages should + // now be pinned. Fetching page 0 should fail. + EXPECT_EQ(true, bpm->UnpinPage(1, true)); + EXPECT_NE(nullptr, bpm->NewPage(&page_id_temp)); + EXPECT_EQ(nullptr, bpm->FetchPage(1)); + + // Shutdown the disk manager and remove the temporary file we created. + disk_manager->Close(); + remove("test.db"); + + delete bpm; + delete disk_manager; +} \ No newline at end of file