ready to write delete
This commit is contained in:
@ -32,14 +32,14 @@ else()
|
||||
add_definitions(-DGIT_COMMIT_HASH="[developing]")
|
||||
endif()
|
||||
|
||||
# 设置一个布尔类型的选项,用于控制是否启用日志功能
|
||||
option(ENABLE_LOG "Enable logging" OFF)
|
||||
# 设置一个布尔类型的选项,用于控制是否启用高级功能,如程序日志、并发、快照等
|
||||
option(ENABLE_ADVANCED_FEATURE "Enable advanced features" OFF)
|
||||
option(OJ_TEST_BPT "Enable OJ test for B+ Tree" ON)
|
||||
option(OJ_TEST_BACKEND "Enable OJ test for backend" OFF)
|
||||
|
||||
# 如果 ENABLE_LOG 选项为 ON,则定义 ENABLE_LOG 宏
|
||||
if (ENABLE_LOG)
|
||||
add_definitions(-DENABLE_LOG)
|
||||
# 如果 ENABLE_ADVANCED_FEATURE 选项为 ON,则定义 ENABLE_ADVANCED_FEATURE 宏
|
||||
if (ENABLE_ADVANCED_FEATURE)
|
||||
add_definitions(-DENABLE_ADVANCED_FEATURE)
|
||||
endif()
|
||||
|
||||
include(FetchContent)
|
||||
|
@ -88,8 +88,8 @@ class BPlusTreeIndexer {
|
||||
.second;
|
||||
}
|
||||
// Then, use memmove to move the key_point pairs
|
||||
fprintf(stderr, "parent_page_guard.template As<PageType>()->data.key_count = %d\n",
|
||||
(int)parent_page_guard.template As<PageType>()->data.key_count);
|
||||
// fprintf(stderr, "parent_page_guard.template As<PageType>()->data.key_count = %d\n",
|
||||
// (int)parent_page_guard.template As<PageType>()->data.key_count);
|
||||
if (pos.path[pos.path.size() - 2].second < parent_page_guard.template As<PageType>()->data.key_count) {
|
||||
memmove(parent_page_guard.template AsMut<PageType>()->data.p_data + pos.path[pos.path.size() - 2].second + 1,
|
||||
parent_page_guard.template As<PageType>()->data.p_data + pos.path[pos.path.size() - 2].second,
|
||||
@ -178,10 +178,10 @@ class BPlusTreeIndexer {
|
||||
new_page_guard.template AsMut<PageType>()->data.key_count = _ActualDataType::kMinNumberOfKeysForLeaf;
|
||||
page_guard.template AsMut<PageType>()->data.key_count -= _ActualDataType::kMinNumberOfKeysForLeaf - 1;
|
||||
}
|
||||
fprintf(stderr, "Evicting page %d\n", (int)pos.path.back().first.PageId());
|
||||
fprintf(stderr, "page id of page_guard = %d\n", (int)page_guard.PageId());
|
||||
// fprintf(stderr, "Evicting page %d\n", (int)pos.path.back().first.PageId());
|
||||
// fprintf(stderr, "page id of page_guard = %d\n", (int)page_guard.PageId());
|
||||
pos.path.pop_back();
|
||||
fprintf(stderr, "the page id of the res page in pos %d\n", (int)pos.path.back().first.PageId());
|
||||
// fprintf(stderr, "the page id of the res page in pos %d\n", (int)pos.path.back().first.PageId());
|
||||
if (pos.path.size() == 1) {
|
||||
// we have split the root page, update and quit
|
||||
page_guard.template AsMut<PageType>()->data.page_status &= ~PageStatusType::ROOT;
|
||||
@ -199,7 +199,7 @@ class BPlusTreeIndexer {
|
||||
}
|
||||
void InsertEntryAt(PositionSignType &pos, const KeyType &key, b_plus_tree_value_index_t value,
|
||||
bool is_fixing_up_recursive = false) {
|
||||
fprintf(stderr, "_ActualDataType::kMaxKeyCount = %d\n", (int)_ActualDataType::kMaxKeyCount);
|
||||
// fprintf(stderr, "_ActualDataType::kMaxKeyCount = %d\n", (int)_ActualDataType::kMaxKeyCount);
|
||||
if (siz == 0) {
|
||||
// special case for the first entry
|
||||
BasicPageGuard new_page_guard = bpm->NewPageGuarded(&root_page_id);
|
||||
@ -218,8 +218,8 @@ class BPlusTreeIndexer {
|
||||
(page_guard.template As<PageType>()->data.key_count - pos.path.back().second) * sizeof(key_index_pair_t));
|
||||
page_guard.template AsMut<PageType>()->data.p_data[pos.path.back().second] = std::make_pair(key, value);
|
||||
page_guard.template AsMut<PageType>()->data.key_count++;
|
||||
fprintf(stderr, "page_guard.template As<PageType>()->data.key_count = %d\n",
|
||||
(int)page_guard.template As<PageType>()->data.key_count);
|
||||
// fprintf(stderr, "page_guard.template As<PageType>()->data.key_count = %d\n",
|
||||
// (int)page_guard.template As<PageType>()->data.key_count);
|
||||
if (!is_fixing_up_recursive) ++siz;
|
||||
return;
|
||||
}
|
||||
@ -279,8 +279,8 @@ class BPlusTreeIndexer {
|
||||
page_guard.PageId());
|
||||
new_root_page_guard.AsMut<PageType>()->data.p_data[1] = std::make_pair(KeyType(), new_page_id);
|
||||
if (!is_fixing_up_recursive) ++siz;
|
||||
fprintf(stderr, "new_page_guard.AsMut<PageType>()->data.key_count = %d\n",
|
||||
(int)new_page_guard.AsMut<PageType>()->data.key_count);
|
||||
// fprintf(stderr, "new_page_guard.AsMut<PageType>()->data.key_count = %d\n",
|
||||
// (int)new_page_guard.AsMut<PageType>()->data.key_count);
|
||||
return;
|
||||
}
|
||||
assert(pos.path.size() >= 2);
|
||||
@ -302,7 +302,7 @@ class BPlusTreeIndexer {
|
||||
page_guard.template As<PageType>()->data.p_data[page_guard.template As<PageType>()->data.key_count - 1].first;
|
||||
pos.path[pos.path.size() - 2].second++;
|
||||
pos.path.pop_back();
|
||||
fprintf(stderr, "begin processing recursively\n");
|
||||
// fprintf(stderr, "begin processing recursively\n");
|
||||
InsertEntryAt(pos,
|
||||
new_page_guard.template As<PageType>()
|
||||
->data.p_data[new_page_guard.template As<PageType>()->data.key_count - 1]
|
||||
@ -346,11 +346,15 @@ class BPlusTreeIndexer {
|
||||
|
||||
public:
|
||||
const KeyType &GetKey() const {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
|
||||
#endif
|
||||
return guard.As<PageType>()->data.p_data[internal_offset].first;
|
||||
}
|
||||
const b_plus_tree_value_index_t &GetValue() {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
|
||||
#endif
|
||||
return guard.As<PageType>()->data.p_data[internal_offset].second;
|
||||
}
|
||||
bool operator==(iterator &that) {
|
||||
@ -358,7 +362,9 @@ class BPlusTreeIndexer {
|
||||
(is_end || (guard.PageId() == that.guard.PageId() && internal_offset == that.internal_offset));
|
||||
}
|
||||
void SetValue(b_plus_tree_value_index_t new_value) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::unique_lock<std::shared_mutex> lock_guard(domain->latch);
|
||||
#endif
|
||||
guard.AsMut<PageType>()->data.p_data[internal_offset].second = new_value;
|
||||
}
|
||||
// only support ++it
|
||||
@ -386,11 +392,15 @@ class BPlusTreeIndexer {
|
||||
|
||||
public:
|
||||
const KeyType &GetKey() {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
|
||||
#endif
|
||||
return guard.As<PageType>()->data.p_data[internal_offset].first;
|
||||
}
|
||||
const b_plus_tree_value_index_t &GetValue() {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_lock<std::shared_mutex> lock_guard(domain->latch);
|
||||
#endif
|
||||
return guard.As<PageType>()->data.p_data[internal_offset].second;
|
||||
}
|
||||
bool operator==(const_iterator &that) {
|
||||
@ -438,7 +448,9 @@ class BPlusTreeIndexer {
|
||||
return res;
|
||||
}
|
||||
iterator lower_bound(const KeyType &key) { // Finish Design
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_lock<std::shared_mutex> guard(latch);
|
||||
#endif
|
||||
PositionSignType pos(std::move(FindPosition(key)));
|
||||
iterator res;
|
||||
res.domain = this;
|
||||
@ -449,7 +461,9 @@ class BPlusTreeIndexer {
|
||||
return res;
|
||||
}
|
||||
const_iterator lower_bound_const(const KeyType &key) { // Finish Design
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_lock<std::shared_mutex> guard(latch);
|
||||
#endif
|
||||
PositionSignType pos(std::move(FindPosition(key)));
|
||||
const_iterator res;
|
||||
res.domain = this;
|
||||
@ -460,14 +474,18 @@ class BPlusTreeIndexer {
|
||||
return res;
|
||||
}
|
||||
b_plus_tree_value_index_t Get(const KeyType &key) { // Finish Design
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_lock<std::shared_mutex> guard(latch);
|
||||
#endif
|
||||
auto it = lower_bound_const(key);
|
||||
if (it == end_const()) return kInvalidValueIndex;
|
||||
if (key_cmp(key, it.GetKey())) return kInvalidValueIndex;
|
||||
return it.GetValue();
|
||||
}
|
||||
bool Put(const KeyType &key, b_plus_tree_value_index_t value) { // Finish Design
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::unique_lock<std::shared_mutex> guard(latch);
|
||||
#endif
|
||||
PositionSignType pos(std::move(FindPosition(key)));
|
||||
if (!pos.is_end &&
|
||||
!key_cmp(key, pos.path.back().first.template As<PageType>()->data.p_data[pos.path.back().second].first)) {
|
||||
@ -478,7 +496,9 @@ class BPlusTreeIndexer {
|
||||
return true;
|
||||
}
|
||||
bool Remove(const KeyType &key) { // Finish Design
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::unique_lock<std::shared_mutex> guard(latch);
|
||||
#endif
|
||||
PositionSignType pos(std::move(FindPosition(key)));
|
||||
if (pos.is_end) return false;
|
||||
if (key_cmp(key, pos.path.back().first.template As<PageType>()->data.p_data[pos.path.back().second].first))
|
||||
@ -488,7 +508,9 @@ class BPlusTreeIndexer {
|
||||
}
|
||||
size_t Size() { return siz; } // Finish Design
|
||||
void Flush() { // Finish Design
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::unique_lock<std::shared_mutex> guard(latch);
|
||||
#endif
|
||||
memcpy(raw_data_memory, &root_page_id, sizeof(page_id_t));
|
||||
memcpy(raw_data_memory + sizeof(page_id_t), &siz, sizeof(bpt_size_t));
|
||||
bpm->FlushAllPages();
|
||||
@ -500,7 +522,9 @@ class BPlusTreeIndexer {
|
||||
bpt_size_t siz; // stored in the next 8 (4-11) bytes of RawDatMemory, this directly operates on the buf
|
||||
// maintained by DiskManager, BufferPoolManager only passes the pointer to it
|
||||
static KeyComparator key_cmp;
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_mutex latch;
|
||||
#endif
|
||||
BufferPoolManager *bpm;
|
||||
char *raw_data_memory;
|
||||
};
|
||||
|
@ -18,6 +18,7 @@ class Page {
|
||||
void ResetMemory();
|
||||
char *GetData();
|
||||
page_id_t GetPageId();
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
/** Acquire the page write latch. */
|
||||
inline void WLatch() { rwlatch_.lock(); }
|
||||
|
||||
@ -29,11 +30,14 @@ class Page {
|
||||
|
||||
/** Release the page read latch. */
|
||||
inline void RUnlatch() { rwlatch_.unlock_shared(); }
|
||||
#endif
|
||||
|
||||
inline size_t GetPinCount() { return pin_count_; }
|
||||
|
||||
private:
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::shared_mutex rwlatch_;
|
||||
#endif
|
||||
char *mem;
|
||||
bool is_dirty_;
|
||||
size_t pin_count_;
|
||||
@ -387,7 +391,9 @@ class BufferPoolManager {
|
||||
const size_t replacer_k;
|
||||
LRUKReplacer replacer;
|
||||
DiskManager *disk_manager;
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::mutex latch;
|
||||
#endif
|
||||
Page *pages_;
|
||||
std::unordered_map<page_id_t, frame_id_t> page_table_;
|
||||
std::list<frame_id_t> free_list_;
|
||||
|
@ -69,7 +69,9 @@ class LRUKReplacer {
|
||||
size_t current_evitable_count_{0};
|
||||
size_t max_frame_count;
|
||||
size_t k_value;
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::mutex latch;
|
||||
#endif
|
||||
LRUKRecord *hash_for_record;
|
||||
};
|
||||
#endif
|
@ -48,17 +48,21 @@ auto ReadPageGuard::operator=(ReadPageGuard &&that) noexcept -> ReadPageGuard &
|
||||
if (this == &that) {
|
||||
return *this;
|
||||
}
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
if (guard_.page_ != nullptr) {
|
||||
guard_.page_->RUnlatch();
|
||||
}
|
||||
#endif
|
||||
guard_ = std::move(that.guard_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
void ReadPageGuard::Drop() {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
if (guard_.page_ != nullptr) {
|
||||
guard_.page_->RUnlatch();
|
||||
}
|
||||
#endif
|
||||
guard_.Drop();
|
||||
}
|
||||
|
||||
@ -70,17 +74,21 @@ auto WritePageGuard::operator=(WritePageGuard &&that) noexcept -> WritePageGuard
|
||||
if (this == &that) {
|
||||
return *this;
|
||||
}
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
if (guard_.page_ != nullptr) {
|
||||
guard_.page_->WUnlatch();
|
||||
}
|
||||
#endif
|
||||
guard_ = std::move(that.guard_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
void WritePageGuard::Drop() {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
if (guard_.page_ != nullptr) {
|
||||
guard_.page_->WUnlatch();
|
||||
}
|
||||
#endif
|
||||
guard_.Drop();
|
||||
}
|
||||
|
||||
@ -112,7 +120,9 @@ void BufferPoolManager::DeallocatePage(page_id_t page_id) { disk_manager->Deallo
|
||||
size_t BufferPoolManager::GetPoolSize() { return pool_size; }
|
||||
Page *BufferPoolManager::GetPages() { return pages_; }
|
||||
auto BufferPoolManager::NewPage(page_id_t *page_id) -> Page * {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
if (!free_list_.empty()) {
|
||||
int internal_page_object_offset = free_list_.front();
|
||||
free_list_.pop_front();
|
||||
@ -148,7 +158,9 @@ auto BufferPoolManager::NewPage(page_id_t *page_id) -> Page * {
|
||||
}
|
||||
|
||||
auto BufferPoolManager::FetchPage(page_id_t page_id) -> Page * {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
if (page_table_.find(page_id) != page_table_.end()) {
|
||||
frame_id_t frame_id = page_table_[page_id];
|
||||
Page *page = &pages_[frame_id];
|
||||
@ -190,7 +202,9 @@ auto BufferPoolManager::FetchPage(page_id_t page_id) -> Page * {
|
||||
}
|
||||
|
||||
auto BufferPoolManager::UnpinPage(page_id_t page_id, bool is_dirty) -> bool {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
if (page_table_.find(page_id) == page_table_.end()) {
|
||||
return false;
|
||||
}
|
||||
@ -210,7 +224,9 @@ auto BufferPoolManager::UnpinPage(page_id_t page_id, bool is_dirty) -> bool {
|
||||
}
|
||||
|
||||
auto BufferPoolManager::FlushPage(page_id_t page_id) -> bool {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
frame_id_t frame_id = page_table_[page_id];
|
||||
if (page_table_.find(page_id) == page_table_.end()) {
|
||||
return false;
|
||||
@ -229,7 +245,9 @@ void BufferPoolManager::FlushAllPages() {
|
||||
}
|
||||
|
||||
auto BufferPoolManager::DeletePage(page_id_t page_id) -> bool {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
if (page_table_.find(page_id) == page_table_.end()) {
|
||||
return true;
|
||||
}
|
||||
@ -258,18 +276,22 @@ auto BufferPoolManager::FetchPageBasic(page_id_t page_id) -> BasicPageGuard {
|
||||
auto BufferPoolManager::FetchPageRead(page_id_t page_id) -> ReadPageGuard {
|
||||
Page *page = FetchPage(page_id);
|
||||
if (page == nullptr) throw std::runtime_error("Buffer Pool is full!");
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
if (page != nullptr) {
|
||||
page->RLatch();
|
||||
}
|
||||
#endif
|
||||
return {this, page};
|
||||
}
|
||||
|
||||
auto BufferPoolManager::FetchPageWrite(page_id_t page_id) -> WritePageGuard {
|
||||
Page *page = FetchPage(page_id);
|
||||
if (page == nullptr) throw std::runtime_error("Buffer Pool is full!");
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
if (page != nullptr) {
|
||||
page->WLatch();
|
||||
}
|
||||
#endif
|
||||
return {this, page};
|
||||
}
|
||||
|
||||
|
@ -37,7 +37,9 @@ LRUKReplacer::~LRUKReplacer() {
|
||||
}
|
||||
|
||||
void LRUKReplacer::SetEvictable(frame_id_t frame_id, bool evitable) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
if (!hash_for_record[frame_id].active) {
|
||||
return;
|
||||
}
|
||||
@ -72,7 +74,9 @@ LRUKReplacer::MainChainNodeType *LRUKReplacer::AddRecordToMainChain(frame_id_t f
|
||||
}
|
||||
|
||||
bool LRUKReplacer::TryEvictExactFrame(frame_id_t frame_id) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
if (!hash_for_record[frame_id].active) {
|
||||
return false;
|
||||
}
|
||||
@ -90,16 +94,22 @@ bool LRUKReplacer::TryEvictExactFrame(frame_id_t frame_id) {
|
||||
}
|
||||
|
||||
bool LRUKReplacer::TryEvictLeastImportant(frame_id_t &frame_id) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
latch.lock();
|
||||
#endif
|
||||
if (current_evitable_count_ == 0) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
latch.unlock();
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
LRUChainNodeType *node = LRU_chain_head_guard->next;
|
||||
while (node != LRU_chain_tail_guard) {
|
||||
frame_id = node->frame_id;
|
||||
if (hash_for_record[frame_id].evitable) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
latch.unlock();
|
||||
#endif
|
||||
return TryEvictExactFrame(frame_id);
|
||||
}
|
||||
node = node->next;
|
||||
@ -108,17 +118,23 @@ bool LRUKReplacer::TryEvictLeastImportant(frame_id_t &frame_id) {
|
||||
while (main_chain_node != LRUK_chain_tail_guard) {
|
||||
frame_id = main_chain_node->frame_id;
|
||||
if (hash_for_record[frame_id].evitable) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
latch.unlock();
|
||||
#endif
|
||||
return TryEvictExactFrame(frame_id);
|
||||
}
|
||||
main_chain_node = main_chain_node->next;
|
||||
}
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
latch.unlock();
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
void LRUKReplacer::RecordAccess(frame_id_t frame_id) {
|
||||
#ifdef ENABLE_ADVANCED_FEATURE
|
||||
std::lock_guard<std::mutex> guard(latch);
|
||||
#endif
|
||||
current_timestamp_++;
|
||||
if (!hash_for_record[frame_id].active) {
|
||||
hash_for_record[frame_id].active = true;
|
||||
|
@ -3,7 +3,7 @@
|
||||
- 空间回收
|
||||
- 快照:贯通于数据库系统和火车票系统整体,以文件为单位夹打快照(类似于git,在火车票系统后端处于非活动状态时操作,比对stage区和版本库中的最后一次commit,然后打一个新的commit进去),额外消耗空间为 当前文件实际大小 + 压缩后的 当前文件实际大小+变化量,使用zstd算法压缩。交互方式:`./core-cli snapshot [options]`。而stage功能内置于DiskManager,当收到信号后,会把工作文件夹的变化打进stage区。
|
||||
- 并发:内置于数据库系统,基于`std::shared_mutex`的简单并发,可以真正意义上支持读操作的并发,但写操作会独占数据库的控制权。(但火车票系统会直接在整个业务层面上加读写锁,因此不会直接使用数据库系统的并发安全)。
|
||||
- 容错:不内置于数据库系统,由火车票系统针对实际业务逻辑记录日志。在文件系统层级上修复完损伤后,运行`./core-cli fsck`检查是否有可能有损坏,借助快照系统和日志修复可能的损伤。具体而言,每条指令视为一个事务,每隔1e3~1e4个事务之后,Flush数据库,调用快照系统,把数据库文件塞进stage区域(直接由DiskManager异步完成,不会阻塞数据库运行),并在事务日志里记录“截至当前已存档”。当需要修复时,先借助快照系统恢复到最近的快照(或从stage区恢复),然后把未反映进该checkpoint的数量较少的事务再重新操作一下(考虑到后端的执行速率,重新执行1e3到1e4个事务的代价是可以接受的)。此时,恢复的即时性就由新增事务多长时间内会实际存入磁盘决定,单独开启一个线程,以最快的可能速度往某个单独的日志文件末尾追加。
|
||||
- 容错:commit功能不内置于数据库系统,由火车票系统针对实际业务逻辑记录日志。在文件系统层级上修复完损伤后,运行`./core-cli fsck`检查是否有可能有损坏,借助快照系统和日志修复可能的损伤。具体而言,每条指令视为一个事务,每隔1e3~1e4个事务之后,Flush数据库,调用快照系统,把数据库文件塞进stage区域(直接由DiskManager异步完成,不会阻塞数据库运行),并在事务日志里记录“截至当前已存档”。当需要修复时,先借助快照系统恢复到最近的快照(或从stage区恢复),然后把未反映进该checkpoint的数量较少的事务再重新操作一下(考虑到后端的执行速率,重新执行1e3到1e4个事务的代价是可以接受的)。此时,恢复的即时性就由新增事务多长时间内会实际存入磁盘决定,单独开启一个线程,以最快的可能速度往某个单独的日志文件末尾追加。
|
||||
- 前端:一个使用正经框架写的简洁美观的UI,无响应式设计。
|
||||
|
||||
## 快照系统
|
||||
|
@ -8,7 +8,7 @@ const bool optimize_enabled = __OPTIMIZE__;
|
||||
#else
|
||||
const bool optimize_enabled = false;
|
||||
#endif
|
||||
#ifndef ENABLE_LOG
|
||||
#ifndef ENABLE_ADVANCED_FEATURE
|
||||
const bool global_log_enabled = false;
|
||||
#else
|
||||
const bool global_log_enabled = true;
|
||||
|
@ -216,8 +216,8 @@ TEST(BasicTest, Split_in_Put_Simple_3) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 16;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
remove("/tmp/bpt5.db");
|
||||
DiskManager *dm = new DiskManager("/tmp/bpt5.db");
|
||||
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
|
||||
@ -260,8 +260,8 @@ TEST(HarderTest, Split_in_Put_Harder_1) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 1360 - 4;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
remove("/tmp/bpt6.db");
|
||||
DiskManager *dm = new DiskManager("/tmp/bpt6.db");
|
||||
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
|
||||
@ -304,8 +304,8 @@ TEST(HarderTest, Split_in_Put_Harder_2) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 2030;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
remove("/tmp/bpt7.db");
|
||||
DiskManager *dm = new DiskManager("/tmp/bpt7.db");
|
||||
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
|
||||
@ -348,8 +348,8 @@ TEST(HarderTest, Split_in_Put_Harder_3) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 800;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
const std::string db_file_name = "/tmp/bpt8.db";
|
||||
std::vector<KeyType> keys;
|
||||
const int ops = 1000;
|
||||
@ -364,7 +364,7 @@ TEST(HarderTest, Split_in_Put_Harder_3) {
|
||||
}
|
||||
// sort(keys.begin(), keys.end());
|
||||
std::shuffle(keys.begin(), keys.end(), rnd);
|
||||
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
// for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
{
|
||||
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
|
||||
for (int i = 1; i <= ops; i++) {
|
||||
@ -394,8 +394,8 @@ TEST(HarderTest, Split_in_Put_Harder_4) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 800;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
const std::string db_file_name = "/tmp/bpt9.db";
|
||||
std::vector<KeyType> keys;
|
||||
const int ops = 1000;
|
||||
@ -410,7 +410,7 @@ TEST(HarderTest, Split_in_Put_Harder_4) {
|
||||
}
|
||||
sort(keys.begin(), keys.end());
|
||||
// std::shuffle(keys.begin(), keys.end(), rnd);
|
||||
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
// for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
{
|
||||
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
|
||||
for (int i = 1; i <= ops; i++) {
|
||||
@ -440,8 +440,8 @@ TEST(HarderTest, Split_in_Put_Harder_5) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 800;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
const std::string db_file_name = "/tmp/bpt10.db";
|
||||
std::vector<KeyType> keys;
|
||||
const int ops = 15 + rnd() % 20;
|
||||
@ -456,7 +456,7 @@ TEST(HarderTest, Split_in_Put_Harder_5) {
|
||||
}
|
||||
// sort(keys.begin(), keys.end());
|
||||
std::shuffle(keys.begin(), keys.end(), rnd);
|
||||
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
// for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
{
|
||||
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
|
||||
for (int i = 1; i <= ops; i++) {
|
||||
@ -486,8 +486,8 @@ TEST(HarderTest, Split_in_Put_Harder_6) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 1000;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
const std::string db_file_name = "/tmp/bpt11.db";
|
||||
std::vector<KeyType> keys;
|
||||
const int ops = 15 + rnd() % 20;
|
||||
@ -502,7 +502,7 @@ TEST(HarderTest, Split_in_Put_Harder_6) {
|
||||
}
|
||||
// sort(keys.begin(), keys.end());
|
||||
std::shuffle(keys.begin(), keys.end(), rnd);
|
||||
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
// for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
{
|
||||
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
|
||||
for (int i = 1; i <= ops; i++) {
|
||||
@ -532,8 +532,8 @@ TEST(HarderTest, Split_in_Put_Harder_7) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 2000;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
const std::string db_file_name = "/tmp/bpt12.db";
|
||||
std::vector<KeyType> keys;
|
||||
const int ops = 15 + rnd() % 20;
|
||||
@ -548,7 +548,7 @@ TEST(HarderTest, Split_in_Put_Harder_7) {
|
||||
}
|
||||
// sort(keys.begin(), keys.end());
|
||||
std::shuffle(keys.begin(), keys.end(), rnd);
|
||||
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
// for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
{
|
||||
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
|
||||
for (int i = 1; i <= ops; i++) {
|
||||
@ -578,8 +578,8 @@ TEST(HarderTest, Split_in_Put_Harder_8) {
|
||||
std::mt19937 rnd(RndSeed);
|
||||
const int str_len = 1300;
|
||||
typedef bpt_basic_test::FixLengthString<str_len> KeyType;
|
||||
fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
// fprintf(stderr, "sizeof(std::pair<KeyType, default_numeric_index_t>)=%lu\n",
|
||||
// sizeof(std::pair<KeyType, default_numeric_index_t>));
|
||||
const std::string db_file_name = "/tmp/bpt13.db";
|
||||
std::vector<KeyType> keys;
|
||||
const int ops = 15 + rnd() % 20;
|
||||
@ -594,7 +594,7 @@ TEST(HarderTest, Split_in_Put_Harder_8) {
|
||||
}
|
||||
// sort(keys.begin(), keys.end());
|
||||
std::shuffle(keys.begin(), keys.end(), rnd);
|
||||
for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
// for (int i = 1; i <= ops; i++) fprintf(stderr, "key[%d]=%s\n", i - 1, keys[i - 1].data);
|
||||
{
|
||||
BPlusTreeIndexer<KeyType, std::less<KeyType>> bpt(bpm);
|
||||
for (int i = 1; i <= ops; i++) {
|
||||
@ -618,3 +618,41 @@ TEST(HarderTest, Split_in_Put_Harder_8) {
|
||||
delete bpm;
|
||||
delete dm;
|
||||
}
|
||||
|
||||
TEST(HarderTest, Split_in_Put_Harder_9) {
|
||||
std::vector<std::pair<int, int>> entries;
|
||||
const int kNumberOfKeys = 100000;
|
||||
const unsigned int RndSeed = testing::GTEST_FLAG(random_seed);
|
||||
std::mt19937 rnd(RndSeed);
|
||||
for (int i = 0; i < kNumberOfKeys; i++) {
|
||||
entries.push_back({i + 3, rnd()});
|
||||
}
|
||||
std::shuffle(entries.begin(), entries.end(), rnd);
|
||||
remove("/tmp/bpt14.db");
|
||||
DiskManager *dm = new DiskManager("/tmp/bpt14.db");
|
||||
BufferPoolManager *bpm = new BufferPoolManager(20, 3, dm);
|
||||
{
|
||||
BPlusTreeIndexer<long long, std::less<long long>> bpt(bpm);
|
||||
for (int i = 0; i < kNumberOfKeys; i++) {
|
||||
bpt.Put(entries[i].first, entries[i].second);
|
||||
ASSERT_EQ(bpt.Get(entries[i].first), entries[i].second);
|
||||
}
|
||||
std::shuffle(entries.begin(), entries.end(), rnd);
|
||||
for (int i = 0; i < kNumberOfKeys; i++) {
|
||||
ASSERT_EQ(bpt.Get(entries[i].first), entries[i].second);
|
||||
}
|
||||
}
|
||||
delete bpm;
|
||||
delete dm;
|
||||
dm = new DiskManager("/tmp/bpt14.db");
|
||||
bpm = new BufferPoolManager(20, 3, dm);
|
||||
std::shuffle(entries.begin(), entries.end(), rnd);
|
||||
{
|
||||
BPlusTreeIndexer<long long, std::less<long long>> bpt(bpm);
|
||||
for (int i = 0; i < kNumberOfKeys; i++) {
|
||||
ASSERT_EQ(bpt.Get(entries[i].first), entries[i].second);
|
||||
}
|
||||
}
|
||||
delete bpm;
|
||||
delete dm;
|
||||
}
|
Reference in New Issue
Block a user