ZYM/BH-TicketSystem
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ready to write delete

Browse Source
This commit is contained in:
happyZYM
2024-04-28 15:37:51 +00:00
parent 81d96aadaf
commit 1632870656
9 changed files with 150 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@ -32,14 +32,14 @@ else()
add_definitions(-DGIT_COMMIT_HASH="[developing]") add_definitions(-DGIT_COMMIT_HASH="[developing]")
endif() endif()
# 设置一个布尔类型的选项，用于控制是否启用日志功能 # 设置一个布尔类型的选项，用于控制是否启用高级功能，如程序日志、并发、快照等
option(ENABLE_LOG "Enable logging" OFF) option(ENABLE_ADVANCED_FEATURE "Enable advanced features" OFF)
option(OJ_TEST_BPT "Enable OJ test for B+ Tree" ON) option(OJ_TEST_BPT "Enable OJ test for B+ Tree" ON)
option(OJ_TEST_BACKEND "Enable OJ test for backend" OFF) option(OJ_TEST_BACKEND "Enable OJ test for backend" OFF)
# 如果 ENABLE_LOG 选项为 ON，则定义 ENABLE_LOG 宏 # 如果 ENABLE_ADVANCED_FEATURE 选项为 ON，则定义 ENABLE_ADVANCED_FEATURE 宏
if (ENABLE_LOG) if (ENABLE_ADVANCED_FEATURE)
add_definitions(-DENABLE_LOG) add_definitions(-DENABLE_ADVANCED_FEATURE)
endif() endif()
include(FetchContent) include(FetchContent)

46
bpt/include/bpt/bpt.hpp
View File

@ -88,8 +88,8 @@ class BPlusTreeIndexer {
.second; .second;
} }
// Then, use memmove to move the key_point pairs // Then, use memmove to move the key_point pairs
fprintf(stderr, "parent_page_guard.template As<PageType>()->data.key_count = %d\n", // fprintf(stderr, "parent_page_guard.template As<PageType>()->data.key_count = %d\n",
(int)parent_page_guard.template As<PageType>()->data.key_count); // (int)parent_page_guard.template As<PageType>()->data.key_count);
if (pos.path[pos.path.size() - 2].second < parent_page_guard.template As<PageType>()->data.key_count) { if (pos.path[pos.path.size() - 2].second < parent_page_guard.template As<PageType>()->data.key_count) {
memmove(parent_page_guard.template AsMut<PageType>()->data.p_data + pos.path[pos.path.size() - 2].second + 1, memmove(parent_page_guard.template AsMut<PageType>()->data.p_data + pos.path[pos.path.size() - 2].second + 1,
parent_page_guard.template As<PageType>()->data.p_data + pos.path[pos.path.size() - 2].second, parent_page_guard.template As<PageType>()->data.p_data + pos.path[pos.path.size() - 2].second,
@ -178,10 +178,10 @@ class BPlusTreeIndexer {
new_page_guard.template AsMut<PageType>()->data.key_count = _ActualDataType::kMinNumberOfKeysForLeaf; new_page_guard.template AsMut<PageType>()->data.key_count = _ActualDataType::kMinNumberOfKeysForLeaf;
page_guard.template AsMut<PageType>()->data.key_count -= _ActualDataType::kMinNumberOfKeysForLeaf - 1; page_guard.template AsMut<PageType>()->data.key_count -= _ActualDataType::kMinNumberOfKeysForLeaf - 1;
} }
fprintf(stderr, "Evicting page %d\n", (int)pos.path.back().first.PageId()); // fprintf(stderr, "Evicting page %d\n", (int)pos.path.back().first.PageId());
fprintf(stderr, "page id of page_guard = %d\n", (int)page_guard.PageId()); // fprintf(stderr, "page id of page_guard = %d\n", (int)page_guard.PageId());
pos.path.pop_back(); pos.path.pop_back();
fprintf(stderr, "the page id of the res page in pos %d\n", (int)pos.path.back().first.PageId()); // fprintf(stderr, "the page id of the res page in pos %d\n", (int)pos.path.back().first.PageId());
if (pos.path.size() == 1) { if (pos.path.size() == 1) {
// we have split the root page, update and quit // we have split the root page, update and quit
page_guard.template AsMut<PageType>()->data.page_status &= ~PageStatusType::ROOT; page_guard.template AsMut<PageType>()->data.page_status &= ~PageStatusType::ROOT;
@ -199,7 +199,7 @@ class BPlusTreeIndexer {
} }
void InsertEntryAt(PositionSignType &pos, const KeyType &key, b_plus_tree_value_index_t value, void InsertEntryAt(PositionSignType &pos, const KeyType &key, b_plus_tree_value_index_t value,
bool is_fixing_up_recursive = false) { bool is_fixing_up_recursive = false) {
fprintf(stderr, "_ActualDataType::kMaxKeyCount = %d\n", (int)_ActualDataType::kMaxKeyCount); // fprintf(stderr, "_ActualDataType::kMaxKeyCount = %d\n", (int)_ActualDataType::kMaxKeyCount);
if (siz == 0) { if (siz == 0) {
// special case for the first entry // special case for the first entry
BasicPageGuard new_page_guard = bpm->NewPageGuarded(&root_page_id); BasicPageGuard new_page_guard = bpm->NewPageGuarded(&root_page_id);
@ -218,8 +218,8 @@ class BPlusTreeIndexer {
(page_guard.template As<PageType>()->data.key_count - pos.path.back().second) * sizeof(key_index_pair_t)); (page_guard.template As<PageType>()->data.key_count - pos.path.back().second) * sizeof(key_index_pair_t));
page_guard.template AsMut<PageType>()->data.p_data[pos.path.back().second] = std::make_pair(key, value); page_guard.template AsMut<PageType>()->data.p_data[pos.path.back().second] = std::make_pair(key, value);
page_guard.template AsMut<PageType>()->data.key_count++; page_guard.template AsMut<PageType>()->data.key_count++;
fprintf(stderr, "page_guard.template As<PageType>()->data.key_count = %d\n", // fprintf(stderr, "page_guard.template As<PageType>()->data.key_count = %d\n",
(int)page_guard.template As<PageType>()->data.key_count); // (int)page_guard.template As<PageType>()->data.key_count);
if (!is_fixing_up_recursive) ++siz; if (!is_fixing_up_recursive) ++siz;
return; return;
} }
@ -279,8 +279,8 @@ class BPlusTreeIndexer {
page_guard.PageId()); page_guard.PageId());
new_root_page_guard.AsMut<PageType>()->data.p_data[1] = std::make_pair(KeyType(), new_page_id); new_root_page_guard.AsMut<PageType>()->data.p_data[1] = std::make_pair(KeyType(), new_page_id);
if (!is_fixing_up_recursive) ++siz; if (!is_fixing_up_recursive) ++siz;
fprintf(stderr, "new_page_guard.AsMut<PageType>()->data.key_count = %d\n", // fprintf(stderr, "new_page_guard.AsMut<PageType>()->data.key_count = %d\n",
(int)new_page_guard.AsMut<PageType>()->data.key_count); // (int)new_page_guard.AsMut<PageType>()->data.key_count);
return; return;
} }
assert(pos.path.size() >= 2); assert(pos.path.size() >= 2);
@ -302,7 +302,7 @@ class BPlusTreeIndexer {
page_guard.template As<PageType>()->data.p_data[page_guard.template As<PageType>()->data.key_count - 1].first; page_guard.template As<PageType>()->data.p_data[page_guard.template As<PageType>()->data.key_count - 1].first;
pos.path[pos.path.size() - 2].second++; pos.path[pos.path.size() - 2].second++;
pos.path.pop_back(); pos.path.pop_back();
fprintf(stderr, "begin processing recursively\n"); // fprintf(stderr, "begin processing recursively\n");
InsertEntryAt(pos, InsertEntryAt(pos,
new_page_guard.template As<PageType>() new_page_guard.template As<PageType>()
->data.p_data[new_page_guard.template As<PageType>()->data.key_count - 1] ->data.p_data[new_page_guard.template As<PageType>()->data.key_count - 1]
@ -346,11 +346,15 @@ class BPlusTreeIndexer {
public: public:
const KeyType &GetKey() const { const KeyType &GetKey() const {
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_lock<std::shared_mutex> lock_guard(domain->latch); std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
#endif
return guard.As<PageType>()->data.p_data[internal_offset].first; return guard.As<PageType>()->data.p_data[internal_offset].first;
} }
const b_plus_tree_value_index_t &GetValue() { const b_plus_tree_value_index_t &GetValue() {
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_lock<std::shared_mutex> lock_guard(domain->latch); std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
#endif
return guard.As<PageType>()->data.p_data[internal_offset].second; return guard.As<PageType>()->data.p_data[internal_offset].second;
} }
bool operator==(iterator &that) { bool operator==(iterator &that) {
@ -358,7 +362,9 @@ class BPlusTreeIndexer {
(is_end || (guard.PageId() == that.guard.PageId() && internal_offset == that.internal_offset)); (is_end || (guard.PageId() == that.guard.PageId() && internal_offset == that.internal_offset));
} }
void SetValue(b_plus_tree_value_index_t new_value) { void SetValue(b_plus_tree_value_index_t new_value) {
#ifdef ENABLE_ADVANCED_FEATURE
std::unique_lock<std::shared_mutex> lock_guard(domain->latch); std::unique_lock<std::shared_mutex> lock_guard(domain->latch);
#endif
guard.AsMut<PageType>()->data.p_data[internal_offset].second = new_value; guard.AsMut<PageType>()->data.p_data[internal_offset].second = new_value;
} }
// only support ++it // only support ++it
@ -386,11 +392,15 @@ class BPlusTreeIndexer {
public: public:
const KeyType &GetKey() { const KeyType &GetKey() {
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_lock<std::shared_mutex> lock_guard(domain->latch); std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
#endif
return guard.As<PageType>()->data.p_data[internal_offset].first; return guard.As<PageType>()->data.p_data[internal_offset].first;
} }
const b_plus_tree_value_index_t &GetValue() { const b_plus_tree_value_index_t &GetValue() {
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_lock<std::shared_mutex> lock_guard(domain->latch); std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
#endif
return guard.As<PageType>()->data.p_data[internal_offset].second; return guard.As<PageType>()->data.p_data[internal_offset].second;
} }
bool operator==(const_iterator &that) { bool operator==(const_iterator &that) {
@ -438,7 +448,9 @@ class BPlusTreeIndexer {
return res; return res;
} }
iterator lower_bound(const KeyType &key) { // Finish Design iterator lower_bound(const KeyType &key) { // Finish Design
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_lock<std::shared_mutex> guard(latch); std::shared_lock<std::shared_mutex> guard(latch);
#endif
PositionSignType pos(std::move(FindPosition(key))); PositionSignType pos(std::move(FindPosition(key)));
iterator res; iterator res;
res.domain = this; res.domain = this;
@ -449,7 +461,9 @@ class BPlusTreeIndexer {
return res; return res;
} }
const_iterator lower_bound_const(const KeyType &key) { // Finish Design const_iterator lower_bound_const(const KeyType &key) { // Finish Design
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_lock<std::shared_mutex> guard(latch); std::shared_lock<std::shared_mutex> guard(latch);
#endif
PositionSignType pos(std::move(FindPosition(key))); PositionSignType pos(std::move(FindPosition(key)));
const_iterator res; const_iterator res;
res.domain = this; res.domain = this;
@ -460,14 +474,18 @@ class BPlusTreeIndexer {
return res; return res;
} }
b_plus_tree_value_index_t Get(const KeyType &key) { // Finish Design b_plus_tree_value_index_t Get(const KeyType &key) { // Finish Design
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_lock<std::shared_mutex> guard(latch); std::shared_lock<std::shared_mutex> guard(latch);
#endif
auto it = lower_bound_const(key); auto it = lower_bound_const(key);
if (it == end_const()) return kInvalidValueIndex; if (it == end_const()) return kInvalidValueIndex;
if (key_cmp(key, it.GetKey())) return kInvalidValueIndex; if (key_cmp(key, it.GetKey())) return kInvalidValueIndex;
return it.GetValue(); return it.GetValue();
} }
bool Put(const KeyType &key, b_plus_tree_value_index_t value) { // Finish Design bool Put(const KeyType &key, b_plus_tree_value_index_t value) { // Finish Design
#ifdef ENABLE_ADVANCED_FEATURE
std::unique_lock<std::shared_mutex> guard(latch); std::unique_lock<std::shared_mutex> guard(latch);
#endif
PositionSignType pos(std::move(FindPosition(key))); PositionSignType pos(std::move(FindPosition(key)));
if (!pos.is_end && if (!pos.is_end &&
!key_cmp(key, pos.path.back().first.template As<PageType>()->data.p_data[pos.path.back().second].first)) { !key_cmp(key, pos.path.back().first.template As<PageType>()->data.p_data[pos.path.back().second].first)) {
@ -478,7 +496,9 @@ class BPlusTreeIndexer {
return true; return true;
} }
bool Remove(const KeyType &key) { // Finish Design bool Remove(const KeyType &key) { // Finish Design
#ifdef ENABLE_ADVANCED_FEATURE
std::unique_lock<std::shared_mutex> guard(latch); std::unique_lock<std::shared_mutex> guard(latch);
#endif
PositionSignType pos(std::move(FindPosition(key))); PositionSignType pos(std::move(FindPosition(key)));
if (pos.is_end) return false; if (pos.is_end) return false;
if (key_cmp(key, pos.path.back().first.template As<PageType>()->data.p_data[pos.path.back().second].first)) if (key_cmp(key, pos.path.back().first.template As<PageType>()->data.p_data[pos.path.back().second].first))
@ -488,7 +508,9 @@ class BPlusTreeIndexer {
} }
size_t Size() { return siz; } // Finish Design size_t Size() { return siz; } // Finish Design
void Flush() { // Finish Design void Flush() { // Finish Design
#ifdef ENABLE_ADVANCED_FEATURE
std::unique_lock<std::shared_mutex> guard(latch); std::unique_lock<std::shared_mutex> guard(latch);
#endif
memcpy(raw_data_memory, &root_page_id, sizeof(page_id_t)); memcpy(raw_data_memory, &root_page_id, sizeof(page_id_t));
memcpy(raw_data_memory + sizeof(page_id_t), &siz, sizeof(bpt_size_t)); memcpy(raw_data_memory + sizeof(page_id_t), &siz, sizeof(bpt_size_t));
bpm->FlushAllPages(); bpm->FlushAllPages();
@ -500,7 +522,9 @@ class BPlusTreeIndexer {
bpt_size_t siz; // stored in the next 8 (4-11) bytes of RawDatMemory, this directly operates on the buf bpt_size_t siz; // stored in the next 8 (4-11) bytes of RawDatMemory, this directly operates on the buf
// maintained by DiskManager, BufferPoolManager only passes the pointer to it // maintained by DiskManager, BufferPoolManager only passes the pointer to it
static KeyComparator key_cmp; static KeyComparator key_cmp;
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_mutex latch; std::shared_mutex latch;
#endif
BufferPoolManager *bpm; BufferPoolManager *bpm;
char *raw_data_memory; char *raw_data_memory;
}; };

6
bpt/include/bpt/buffer_pool_manager.h
View File

@ -18,6 +18,7 @@ class Page {
void ResetMemory(); void ResetMemory();
char *GetData(); char *GetData();
page_id_t GetPageId(); page_id_t GetPageId();
#ifdef ENABLE_ADVANCED_FEATURE
/** Acquire the page write latch. */ /** Acquire the page write latch. */
inline void WLatch() { rwlatch_.lock(); } inline void WLatch() { rwlatch_.lock(); }
@ -29,11 +30,14 @@ class Page {
/** Release the page read latch. */ /** Release the page read latch. */
inline void RUnlatch() { rwlatch_.unlock_shared(); } inline void RUnlatch() { rwlatch_.unlock_shared(); }
#endif
inline size_t GetPinCount() { return pin_count_; } inline size_t GetPinCount() { return pin_count_; }
private: private:
#ifdef ENABLE_ADVANCED_FEATURE
std::shared_mutex rwlatch_; std::shared_mutex rwlatch_;
#endif
char *mem; char *mem;
bool is_dirty_; bool is_dirty_;
size_t pin_count_; size_t pin_count_;
@ -387,7 +391,9 @@ class BufferPoolManager {
const size_t replacer_k; const size_t replacer_k;
LRUKReplacer replacer; LRUKReplacer replacer;
DiskManager *disk_manager; DiskManager *disk_manager;
#ifdef ENABLE_ADVANCED_FEATURE
std::mutex latch; std::mutex latch;
#endif
Page *pages_; Page *pages_;
std::unordered_map<page_id_t, frame_id_t> page_table_; std::unordered_map<page_id_t, frame_id_t> page_table_;
std::list<frame_id_t> free_list_; std::list<frame_id_t> free_list_;

2
bpt/include/bpt/replacer.h
View File

@ -69,7 +69,9 @@ class LRUKReplacer {
size_t current_evitable_count_{0}; size_t current_evitable_count_{0};
size_t max_frame_count; size_t max_frame_count;
size_t k_value; size_t k_value;
#ifdef ENABLE_ADVANCED_FEATURE
std::mutex latch; std::mutex latch;
#endif
LRUKRecord *hash_for_record; LRUKRecord *hash_for_record;
}; };
#endif #endif

22
bpt/src/buffer_pool_manager.cpp
View File

@ -48,17 +48,21 @@ auto ReadPageGuard::operator=(ReadPageGuard &&that) noexcept -> ReadPageGuard &
if (this == &that) { if (this == &that) {
return *this; return *this;
} }
#ifdef ENABLE_ADVANCED_FEATURE
if (guard_.page_ != nullptr) { if (guard_.page_ != nullptr) {
guard_.page_->RUnlatch(); guard_.page_->RUnlatch();
} }
#endif
guard_ = std::move(that.guard_); guard_ = std::move(that.guard_);
return *this; return *this;
} }
void ReadPageGuard::Drop() { void ReadPageGuard::Drop() {
#ifdef ENABLE_ADVANCED_FEATURE
if (guard_.page_ != nullptr) { if (guard_.page_ != nullptr) {
guard_.page_->RUnlatch(); guard_.page_->RUnlatch();
} }
#endif
guard_.Drop(); guard_.Drop();
} }
@ -70,17 +74,21 @@ auto WritePageGuard::operator=(WritePageGuard &&that) noexcept -> WritePageGuard
if (this == &that) { if (this == &that) {
return *this; return *this;
} }
#ifdef ENABLE_ADVANCED_FEATURE
if (guard_.page_ != nullptr) { if (guard_.page_ != nullptr) {
guard_.page_->WUnlatch(); guard_.page_->WUnlatch();
} }
#endif
guard_ = std::move(that.guard_); guard_ = std::move(that.guard_);
return *this; return *this;
} }
void WritePageGuard::Drop() { void WritePageGuard::Drop() {
#ifdef ENABLE_ADVANCED_FEATURE
if (guard_.page_ != nullptr) { if (guard_.page_ != nullptr) {
guard_.page_->WUnlatch(); guard_.page_->WUnlatch();
} }
#endif
guard_.Drop(); guard_.Drop();
} }
@ -112,7 +120,9 @@ void BufferPoolManager::DeallocatePage(page_id_t page_id) { disk_manager->Deallo
size_t BufferPoolManager::GetPoolSize() { return pool_size; } size_t BufferPoolManager::GetPoolSize() { return pool_size; }
Page *BufferPoolManager::GetPages() { return pages_; } Page *BufferPoolManager::GetPages() { return pages_; }
auto BufferPoolManager::NewPage(page_id_t *page_id) -> Page * { auto BufferPoolManager::NewPage(page_id_t *page_id) -> Page * {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
if (!free_list_.empty()) { if (!free_list_.empty()) {
int internal_page_object_offset = free_list_.front(); int internal_page_object_offset = free_list_.front();
free_list_.pop_front(); free_list_.pop_front();
@ -148,7 +158,9 @@ auto BufferPoolManager::NewPage(page_id_t *page_id) -> Page * {
} }
auto BufferPoolManager::FetchPage(page_id_t page_id) -> Page * { auto BufferPoolManager::FetchPage(page_id_t page_id) -> Page * {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
if (page_table_.find(page_id) != page_table_.end()) { if (page_table_.find(page_id) != page_table_.end()) {
frame_id_t frame_id = page_table_[page_id]; frame_id_t frame_id = page_table_[page_id];
Page *page = &pages_[frame_id]; Page *page = &pages_[frame_id];
@ -190,7 +202,9 @@ auto BufferPoolManager::FetchPage(page_id_t page_id) -> Page * {
} }
auto BufferPoolManager::UnpinPage(page_id_t page_id, bool is_dirty) -> bool { auto BufferPoolManager::UnpinPage(page_id_t page_id, bool is_dirty) -> bool {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
if (page_table_.find(page_id) == page_table_.end()) { if (page_table_.find(page_id) == page_table_.end()) {
return false; return false;
} }
@ -210,7 +224,9 @@ auto BufferPoolManager::UnpinPage(page_id_t page_id, bool is_dirty) -> bool {
} }
auto BufferPoolManager::FlushPage(page_id_t page_id) -> bool { auto BufferPoolManager::FlushPage(page_id_t page_id) -> bool {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
frame_id_t frame_id = page_table_[page_id]; frame_id_t frame_id = page_table_[page_id];
if (page_table_.find(page_id) == page_table_.end()) { if (page_table_.find(page_id) == page_table_.end()) {
return false; return false;
@ -229,7 +245,9 @@ void BufferPoolManager::FlushAllPages() {
} }
auto BufferPoolManager::DeletePage(page_id_t page_id) -> bool { auto BufferPoolManager::DeletePage(page_id_t page_id) -> bool {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
if (page_table_.find(page_id) == page_table_.end()) { if (page_table_.find(page_id) == page_table_.end()) {
return true; return true;
} }
@ -258,18 +276,22 @@ auto BufferPoolManager::FetchPageBasic(page_id_t page_id) -> BasicPageGuard {
auto BufferPoolManager::FetchPageRead(page_id_t page_id) -> ReadPageGuard { auto BufferPoolManager::FetchPageRead(page_id_t page_id) -> ReadPageGuard {
Page *page = FetchPage(page_id); Page *page = FetchPage(page_id);
if (page == nullptr) throw std::runtime_error("Buffer Pool is full!"); if (page == nullptr) throw std::runtime_error("Buffer Pool is full!");
#ifdef ENABLE_ADVANCED_FEATURE
if (page != nullptr) { if (page != nullptr) {
page->RLatch(); page->RLatch();
} }
#endif
return {this, page}; return {this, page};
} }
auto BufferPoolManager::FetchPageWrite(page_id_t page_id) -> WritePageGuard { auto BufferPoolManager::FetchPageWrite(page_id_t page_id) -> WritePageGuard {
Page *page = FetchPage(page_id); Page *page = FetchPage(page_id);
if (page == nullptr) throw std::runtime_error("Buffer Pool is full!"); if (page == nullptr) throw std::runtime_error("Buffer Pool is full!");
#ifdef ENABLE_ADVANCED_FEATURE
if (page != nullptr) { if (page != nullptr) {
page->WLatch(); page->WLatch();
} }
#endif
return {this, page}; return {this, page};
} }

16
bpt/src/replacer.cpp
View File

@ -37,7 +37,9 @@ LRUKReplacer::~LRUKReplacer() {
} }
void LRUKReplacer::SetEvictable(frame_id_t frame_id, bool evitable) { void LRUKReplacer::SetEvictable(frame_id_t frame_id, bool evitable) {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
if (!hash_for_record[frame_id].active) { if (!hash_for_record[frame_id].active) {
return; return;
} }
@ -72,7 +74,9 @@ LRUKReplacer::MainChainNodeType *LRUKReplacer::AddRecordToMainChain(frame_id_t f
} }
bool LRUKReplacer::TryEvictExactFrame(frame_id_t frame_id) { bool LRUKReplacer::TryEvictExactFrame(frame_id_t frame_id) {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
if (!hash_for_record[frame_id].active) { if (!hash_for_record[frame_id].active) {
return false; return false;
} }
@ -90,16 +94,22 @@ bool LRUKReplacer::TryEvictExactFrame(frame_id_t frame_id) {
} }
bool LRUKReplacer::TryEvictLeastImportant(frame_id_t &frame_id) { bool LRUKReplacer::TryEvictLeastImportant(frame_id_t &frame_id) {
#ifdef ENABLE_ADVANCED_FEATURE
latch.lock(); latch.lock();
#endif
if (current_evitable_count_ == 0) { if (current_evitable_count_ == 0) {
#ifdef ENABLE_ADVANCED_FEATURE
latch.unlock(); latch.unlock();
#endif
return false; return false;
} }
LRUChainNodeType *node = LRU_chain_head_guard->next; LRUChainNodeType *node = LRU_chain_head_guard->next;
while (node != LRU_chain_tail_guard) { while (node != LRU_chain_tail_guard) {
frame_id = node->frame_id; frame_id = node->frame_id;
if (hash_for_record[frame_id].evitable) { if (hash_for_record[frame_id].evitable) {
#ifdef ENABLE_ADVANCED_FEATURE
latch.unlock(); latch.unlock();
#endif
return TryEvictExactFrame(frame_id); return TryEvictExactFrame(frame_id);
} }
node = node->next; node = node->next;
@ -108,17 +118,23 @@ bool LRUKReplacer::TryEvictLeastImportant(frame_id_t &frame_id) {
while (main_chain_node != LRUK_chain_tail_guard) { while (main_chain_node != LRUK_chain_tail_guard) {
frame_id = main_chain_node->frame_id; frame_id = main_chain_node->frame_id;
if (hash_for_record[frame_id].evitable) { if (hash_for_record[frame_id].evitable) {
#ifdef ENABLE_ADVANCED_FEATURE
latch.unlock(); latch.unlock();
#endif
return TryEvictExactFrame(frame_id); return TryEvictExactFrame(frame_id);
} }
main_chain_node = main_chain_node->next; main_chain_node = main_chain_node->next;
} }
#ifdef ENABLE_ADVANCED_FEATURE
latch.unlock(); latch.unlock();
#endif
return false; return false;
} }
void LRUKReplacer::RecordAccess(frame_id_t frame_id) { void LRUKReplacer::RecordAccess(frame_id_t frame_id) {
#ifdef ENABLE_ADVANCED_FEATURE
std::lock_guard<std::mutex> guard(latch); std::lock_guard<std::mutex> guard(latch);
#endif
current_timestamp_++; current_timestamp_++;
if (!hash_for_record[frame_id].active) { if (!hash_for_record[frame_id].active) {
hash_for_record[frame_id].active = true; hash_for_record[frame_id].active = true;

2
design.md
View File

@ -3,7 +3,7 @@
- 空间回收 - 空间回收
- 快照：贯通于数据库系统和火车票系统整体，以文件为单位夹打快照（类似于git，在火车票系统后端处于非活动状态时操作，比对stage区和版本库中的最后一次commit，然后打一个新的commit进去），额外消耗空间为 当前文件实际大小 + 压缩后的 当前文件实际大小+变化量，使用zstd算法压缩。交互方式：`./core-cli snapshot [options]`。而stage功能内置于DiskManager，当收到信号后，会把工作文件夹的变化打进stage区。 - 快照：贯通于数据库系统和火车票系统整体，以文件为单位夹打快照（类似于git，在火车票系统后端处于非活动状态时操作，比对stage区和版本库中的最后一次commit，然后打一个新的commit进去），额外消耗空间为 当前文件实际大小 + 压缩后的 当前文件实际大小+变化量，使用zstd算法压缩。交互方式：`./core-cli snapshot [options]`。而stage功能内置于DiskManager，当收到信号后，会把工作文件夹的变化打进stage区。
- 并发：内置于数据库系统，基于`std::shared_mutex`的简单并发，可以真正意义上支持读操作的并发，但写操作会独占数据库的控制权。（但火车票系统会直接在整个业务层面上加读写锁，因此不会直接使用数据库系统的并发安全）。 - 并发：内置于数据库系统，基于`std::shared_mutex`的简单并发，可以真正意义上支持读操作的并发，但写操作会独占数据库的控制权。（但火车票系统会直接在整个业务层面上加读写锁，因此不会直接使用数据库系统的并发安全）。
- 容错：不内置于数据库系统，由火车票系统针对实际业务逻辑记录日志。在文件系统层级上修复完损伤后，运行`./core-cli fsck`检查是否有可能有损坏，借助快照系统和日志修复可能的损伤。具体而言，每条指令视为一个事务，每隔1e3~1e4个事务之后，Flush数据库，调用快照系统，把数据库文件塞进stage区域（直接由DiskManager异步完成，不会阻塞数据库运行），并在事务日志里记录“截至当前已存档”。当需要修复时，先借助快照系统恢复到最近的快照（或从stage区恢复），然后把未反映进该checkpoint的数量较少的事务再重新操作一下（考虑到后端的执行速率，重新执行1e3到1e4个事务的代价是可以接受的）。此时，恢复的即时性就由新增事务多长时间内会实际存入磁盘决定，单独开启一个线程，以最快的可能速度往某个单独的日志文件末尾追加。 - 容错：commit功能不内置于数据库系统，由火车票系统针对实际业务逻辑记录日志。在文件系统层级上修复完损伤后，运行`./core-cli fsck`检查是否有可能有损坏，借助快照系统和日志修复可能的损伤。具体而言，每条指令视为一个事务，每隔1e3~1e4个事务之后，Flush数据库，调用快照系统，把数据库文件塞进stage区域（直接由DiskManager异步完成，不会阻塞数据库运行），并在事务日志里记录“截至当前已存档”。当需要修复时，先借助快照系统恢复到最近的快照（或从stage区恢复），然后把未反映进该checkpoint的数量较少的事务再重新操作一下（考虑到后端的执行速率，重新执行1e3到1e4个事务的代价是可以接受的）。此时，恢复的即时性就由新增事务多长时间内会实际存入磁盘决定，单独开启一个线程，以最快的可能速度往某个单独的日志文件末尾追加。
- 前端：一个使用正经框架写的简洁美观的UI，无响应式设计。 - 前端：一个使用正经框架写的简洁美观的UI，无响应式设计。
## 快照系统 ## 快照系统

2
src/main.cpp
View File

@ -8,7 +8,7 @@ const bool optimize_enabled = __OPTIMIZE__;
#else #else
const bool optimize_enabled = false; const bool optimize_enabled = false;
#endif #endif
#ifndef ENABLE_LOG #ifndef ENABLE_ADVANCED_FEATURE
const bool global_log_enabled = false; const bool global_log_enabled = false;
#else #else
const bool global_log_enabled = true; const bool global_log_enabled = true;

86
test/bpt_basic_test.cpp
View File

@ -216,8 +216,8 @@ TEST(BasicTest, Split_in_Put_Simple_3) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 16; const int str_len = 16;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
remove("/tmp/bpt5.db"); remove("/tmp/bpt5.db");
DiskManager *dm = new DiskManager("/tmp/bpt5.db"); DiskManager *dm = new DiskManager("/tmp/bpt5.db");
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm); BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
@ -260,8 +260,8 @@ TEST(HarderTest, Split_in_Put_Harder_1) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 1360 - 4; const int str_len = 1360 - 4;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
remove("/tmp/bpt6.db"); remove("/tmp/bpt6.db");
DiskManager *dm = new DiskManager("/tmp/bpt6.db"); DiskManager *dm = new DiskManager("/tmp/bpt6.db");
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm); BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
@ -304,8 +304,8 @@ TEST(HarderTest, Split_in_Put_Harder_2) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 2030; const int str_len = 2030;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
remove("/tmp/bpt7.db"); remove("/tmp/bpt7.db");
DiskManager *dm = new DiskManager("/tmp/bpt7.db"); DiskManager *dm = new DiskManager("/tmp/bpt7.db");
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm); BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
@ -348,8 +348,8 @@ TEST(HarderTest, Split_in_Put_Harder_3) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 800; const int str_len = 800;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
const std::string db_file_name = "/tmp/bpt8.db"; const std::string db_file_name = "/tmp/bpt8.db";
std::vector<KeyType> keys; std::vector<KeyType> keys;
const int ops = 1000; const int ops = 1000;
@ -364,7 +364,7 @@ TEST(HarderTest, Split_in_Put_Harder_3) {
} }
// sort(keys.begin(), keys.end()); // sort(keys.begin(), keys.end());
std::shuffle(keys.begin(), keys.end(), rnd); std::shuffle(keys.begin(), keys.end(), rnd);
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data); // for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
{ {
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm); BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
for (int i = 1; i <= ops; i++) { for (int i = 1; i <= ops; i++) {
@ -394,8 +394,8 @@ TEST(HarderTest, Split_in_Put_Harder_4) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 800; const int str_len = 800;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
const std::string db_file_name = "/tmp/bpt9.db"; const std::string db_file_name = "/tmp/bpt9.db";
std::vector<KeyType> keys; std::vector<KeyType> keys;
const int ops = 1000; const int ops = 1000;
@ -410,7 +410,7 @@ TEST(HarderTest, Split_in_Put_Harder_4) {
} }
sort(keys.begin(), keys.end()); sort(keys.begin(), keys.end());
// std::shuffle(keys.begin(), keys.end(), rnd); // std::shuffle(keys.begin(), keys.end(), rnd);
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data); // for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
{ {
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm); BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
for (int i = 1; i <= ops; i++) { for (int i = 1; i <= ops; i++) {
@ -440,8 +440,8 @@ TEST(HarderTest, Split_in_Put_Harder_5) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 800; const int str_len = 800;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
const std::string db_file_name = "/tmp/bpt10.db"; const std::string db_file_name = "/tmp/bpt10.db";
std::vector<KeyType> keys; std::vector<KeyType> keys;
const int ops = 15 + rnd() % 20; const int ops = 15 + rnd() % 20;
@ -456,7 +456,7 @@ TEST(HarderTest, Split_in_Put_Harder_5) {
} }
// sort(keys.begin(), keys.end()); // sort(keys.begin(), keys.end());
std::shuffle(keys.begin(), keys.end(), rnd); std::shuffle(keys.begin(), keys.end(), rnd);
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data); // for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
{ {
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm); BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
for (int i = 1; i <= ops; i++) { for (int i = 1; i <= ops; i++) {
@ -486,8 +486,8 @@ TEST(HarderTest, Split_in_Put_Harder_6) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 1000; const int str_len = 1000;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
const std::string db_file_name = "/tmp/bpt11.db"; const std::string db_file_name = "/tmp/bpt11.db";
std::vector<KeyType> keys; std::vector<KeyType> keys;
const int ops = 15 + rnd() % 20; const int ops = 15 + rnd() % 20;
@ -502,7 +502,7 @@ TEST(HarderTest, Split_in_Put_Harder_6) {
} }
// sort(keys.begin(), keys.end()); // sort(keys.begin(), keys.end());
std::shuffle(keys.begin(), keys.end(), rnd); std::shuffle(keys.begin(), keys.end(), rnd);
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data); // for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
{ {
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm); BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
for (int i = 1; i <= ops; i++) { for (int i = 1; i <= ops; i++) {
@ -532,8 +532,8 @@ TEST(HarderTest, Split_in_Put_Harder_7) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 2000; const int str_len = 2000;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
const std::string db_file_name = "/tmp/bpt12.db"; const std::string db_file_name = "/tmp/bpt12.db";
std::vector<KeyType> keys; std::vector<KeyType> keys;
const int ops = 15 + rnd() % 20; const int ops = 15 + rnd() % 20;
@ -548,7 +548,7 @@ TEST(HarderTest, Split_in_Put_Harder_7) {
} }
// sort(keys.begin(), keys.end()); // sort(keys.begin(), keys.end());
std::shuffle(keys.begin(), keys.end(), rnd); std::shuffle(keys.begin(), keys.end(), rnd);
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data); // for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
{ {
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm); BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
for (int i = 1; i <= ops; i++) { for (int i = 1; i <= ops; i++) {
@ -578,8 +578,8 @@ TEST(HarderTest, Split_in_Put_Harder_8) {
std::mt19937 rnd(RndSeed); std::mt19937 rnd(RndSeed);
const int str_len = 1300; const int str_len = 1300;
typedef bpt_basic_test::FixLengthString<str_len> KeyType; typedef bpt_basic_test::FixLengthString<str_len> KeyType;
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n", // fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
sizeof(std::pair<KeyType, default_numeric_index_t>)); // sizeof(std::pair<KeyType, default_numeric_index_t>));
const std::string db_file_name = "/tmp/bpt13.db"; const std::string db_file_name = "/tmp/bpt13.db";
std::vector<KeyType> keys; std::vector<KeyType> keys;
const int ops = 15 + rnd() % 20; const int ops = 15 + rnd() % 20;
@ -594,7 +594,7 @@ TEST(HarderTest, Split_in_Put_Harder_8) {
} }
// sort(keys.begin(), keys.end()); // sort(keys.begin(), keys.end());
std::shuffle(keys.begin(), keys.end(), rnd); std::shuffle(keys.begin(), keys.end(), rnd);
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data); // for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
{ {
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm); BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
for (int i = 1; i <= ops; i++) { for (int i = 1; i <= ops; i++) {
@ -617,4 +617,42 @@ TEST(HarderTest, Split_in_Put_Harder_8) {
} }
delete bpm; delete bpm;
delete dm; delete dm;
}
TEST(HarderTest, Split_in_Put_Harder_9) {
std::vector<std::pair<int, int>> entries;
const int kNumberOfKeys = 100000;
const unsigned int RndSeed = testing::GTEST_FLAG(random_seed);
std::mt19937 rnd(RndSeed);
for (int i = 0; i < kNumberOfKeys; i++) {
entries.push_back({i + 3, rnd()});
}
std::shuffle(entries.begin(), entries.end(), rnd);
remove("/tmp/bpt14.db");
DiskManager *dm = new DiskManager("/tmp/bpt14.db");
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
{
BPlusTreeIndexer<long long, std::less<long long>> bpt(bpm);
for (int i = 0; i < kNumberOfKeys; i++) {
bpt.Put(entries[i].first, entries[i].second);
ASSERT_EQ(bpt.Get(entries[i].first), entries[i].second);
}
std::shuffle(entries.begin(), entries.end(), rnd);
for (int i = 0; i < kNumberOfKeys; i++) {
ASSERT_EQ(bpt.Get(entries[i].first), entries[i].second);
}
}
delete bpm;
delete dm;
dm = new DiskManager("/tmp/bpt14.db");
bpm = new BufferPoolManager(20, 3, dm);
std::shuffle(entries.begin(), entries.end(), rnd);
{
BPlusTreeIndexer<long long, std::less<long long>> bpt(bpm);
for (int i = 0; i < kNumberOfKeys; i++) {
ASSERT_EQ(bpt.Get(entries[i].first), entries[i].second);
}
}
delete bpm;
delete dm;
} }