diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 99f58aeb53a..a79ed47e070 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -958,7 +958,8 @@ ResultType Catalog::AddUniqueConstraint(concurrency::TransactionContext *txn, ->GetSchema(); // Create index - std::stringstream index_name(table_object->GetTableName()); + std::stringstream index_name; + index_name << table_object->GetTableName(); for (auto column_id : column_ids) index_name << "_" + schema->GetColumn(column_id).GetName(); index_name << "_UNIQ"; @@ -1033,7 +1034,8 @@ ResultType Catalog::AddForeignKeyConstraint(concurrency::TransactionContext *txn ->GetTableCatalogEntry(txn, src_table_oid); auto src_schema = src_table->GetSchema(); - std::stringstream index_name(src_table_object->GetTableName()); + std::stringstream index_name; + index_name << src_table_object->GetTableName(); for (auto col_id : src_col_ids) index_name << "_" << src_schema->GetColumn(col_id).GetName(); index_name << "_fkey"; @@ -1660,6 +1662,26 @@ std::shared_ptr Catalog::GetDatabaseCatalogEntry( return database_object; } +/* Get database catalog object from cache (cached_only == true), + * or all the way from storage (cached_only == false) + * throw exception and abort txn if not exists/invisible + * */ +std::unordered_map> +Catalog::GetDatabaseCatalogEntries(concurrency::TransactionContext *txn, + bool cached_only) { + if (txn == nullptr) { + throw CatalogException("Do not have transaction to get database objects"); + } + + if (!cached_only && !txn->catalog_cache.valid_database_catalog_entry_) { + // cache miss get from pg_table + return DatabaseCatalog::GetInstance()->GetDatabaseCatalogEntries(txn); + } + // make sure to check IsValidTableObjects() before getting table objects + PELOTON_ASSERT(txn->catalog_cache.valid_database_catalog_entry_); + return txn->catalog_cache.database_catalog_entries_cache_; +} + /* Check table from pg_table with table_name & schema_name using txn, * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible diff --git a/src/catalog/catalog_cache.cpp b/src/catalog/catalog_cache.cpp index 69fdcbc486e..d6948b495e6 100644 --- a/src/catalog/catalog_cache.cpp +++ b/src/catalog/catalog_cache.cpp @@ -16,6 +16,7 @@ #include "catalog/database_catalog.h" #include "common/logger.h" +#include "concurrency/transaction_context.h" namespace peloton { namespace catalog { @@ -24,27 +25,27 @@ namespace catalog { * @param database_object * @return false only if database_oid already exists in cache */ -bool CatalogCache::InsertDatabaseObject( +bool CatalogCache::InsertDatabaseCatalogEntry( std::shared_ptr database_object) { if (!database_object || database_object->GetDatabaseOid() == INVALID_OID) { return false; // invalid object } // check if already in cache - if (database_objects_cache_.find(database_object->GetDatabaseOid()) != - database_objects_cache_.end()) { - LOG_DEBUG("Database %u already exists in cache!", + if (database_catalog_entries_cache_.find(database_object->GetDatabaseOid()) != + database_catalog_entries_cache_.end()) { + LOG_TRACE("Database %u already exists in cache!", database_object->GetDatabaseOid()); return false; } if (database_name_cache_.find(database_object->GetDatabaseName()) != database_name_cache_.end()) { - LOG_DEBUG("Database %s already exists in cache!", + LOG_TRACE("Database %s already exists in cache!", database_object->GetDatabaseName().c_str()); return false; } - database_objects_cache_.insert( + database_catalog_entries_cache_.insert( std::make_pair(database_object->GetDatabaseOid(), database_object)); database_name_cache_.insert( std::make_pair(database_object->GetDatabaseName(), database_object)); @@ -55,15 +56,15 @@ bool CatalogCache::InsertDatabaseObject( * @param database_oid * @return true if database_oid is found and evicted; false if not found */ -bool CatalogCache::EvictDatabaseObject(oid_t database_oid) { - auto it = database_objects_cache_.find(database_oid); - if (it == database_objects_cache_.end()) { +bool CatalogCache::EvictDatabaseCatalogEntry(oid_t database_oid) { + auto it = database_catalog_entries_cache_.find(database_oid); + if (it == database_catalog_entries_cache_.end()) { return false; // database oid not found in cache } auto database_object = it->second; PELOTON_ASSERT(database_object); - database_objects_cache_.erase(it); + database_catalog_entries_cache_.erase(it); database_name_cache_.erase(database_object->GetDatabaseName()); return true; } @@ -72,7 +73,7 @@ bool CatalogCache::EvictDatabaseObject(oid_t database_oid) { * @param database_name * @return true if database_name is found and evicted; false if not found */ -bool CatalogCache::EvictDatabaseObject(const std::string &database_name) { +bool CatalogCache::EvictDatabaseCatalogEntry(const std::string &database_name) { auto it = database_name_cache_.find(database_name); if (it == database_name_cache_.end()) { return false; // database name not found in cache @@ -81,7 +82,7 @@ bool CatalogCache::EvictDatabaseObject(const std::string &database_name) { auto database_object = it->second; PELOTON_ASSERT(database_object); database_name_cache_.erase(it); - database_objects_cache_.erase(database_object->GetDatabaseOid()); + database_catalog_entries_cache_.erase(database_object->GetDatabaseOid()); return true; } @@ -89,10 +90,10 @@ bool CatalogCache::EvictDatabaseObject(const std::string &database_name) { * @param database_oid * @return database catalog object; if not found return object with invalid oid */ -std::shared_ptr CatalogCache::GetDatabaseObject( - oid_t database_oid) { - auto it = database_objects_cache_.find(database_oid); - if (it == database_objects_cache_.end()) { +std::shared_ptr +CatalogCache::GetDatabaseCatalogEntry(oid_t database_oid) { + auto it = database_catalog_entries_cache_.find(database_oid); + if (it == database_catalog_entries_cache_.end()) { return nullptr; } return it->second; @@ -102,8 +103,8 @@ std::shared_ptr CatalogCache::GetDatabaseObject( * @param database_name * @return database catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetDatabaseObject( - const std::string &database_name) { +std::shared_ptr +CatalogCache::GetDatabaseCatalogEntry(const std::string &database_name) { auto it = database_name_cache_.find(database_name); if (it == database_name_cache_.end()) { return nullptr; @@ -111,14 +112,31 @@ std::shared_ptr CatalogCache::GetDatabaseObject( return it->second; } +/* @brief Get database catalog object from cache, + or all the way from storage + * @param txn if nullptr, return nullptr on a cache miss + * @return Shared pointer to the requested database catalog object + */ +std::unordered_map> +CatalogCache::GetDatabaseCatalogEntries(concurrency::TransactionContext *txn) { + if (!valid_database_catalog_entry_ && txn != nullptr) { + // cache miss get from pg_database + return DatabaseCatalog::GetInstance()->GetDatabaseCatalogEntries(txn); + } + // make sure to check IsValidTableObjects() before getting table objects + PELOTON_ASSERT(valid_database_catalog_entry_); + return database_catalog_entries_cache_; +} + /*@brief search table catalog object from all cached database objects * @param table_oid * @return table catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetCachedTableObject( - oid_t database_oid, oid_t table_oid) { - auto database_object = GetDatabaseObject(database_oid); - if (database_object == nullptr) return nullptr; +std::shared_ptr +CatalogCache::GetCachedTableCatalogEntry(oid_t database_oid, + oid_t table_oid) { + auto database_object = GetDatabaseCatalogEntry(database_oid); + if (database_object == nullptr) return nullptr; auto table_object = database_object->GetTableCatalogEntry(table_oid, true); if (table_object) return table_object; return nullptr; @@ -128,12 +146,13 @@ std::shared_ptr CatalogCache::GetCachedTableObject( * @param index_oid * @return index catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetCachedIndexObject( - oid_t database_oid, oid_t index_oid) { - auto database_object = GetDatabaseObject(database_oid); - if (database_object == nullptr) return nullptr; - auto index_object = database_object->GetCachedIndexCatalogEntry(index_oid); - if (index_object) return index_object; +std::shared_ptr +CatalogCache::GetCachedIndexCatalogEntry(oid_t database_oid, + oid_t index_oid) { + auto database_object = GetDatabaseCatalogEntry(database_oid); + if (database_object == nullptr) return nullptr; + auto index_object = database_object->GetCachedIndexCatalogEntry(index_oid); + if (index_object) return index_object; return nullptr; } @@ -141,14 +160,16 @@ std::shared_ptr CatalogCache::GetCachedIndexObject( * @param index_name * @return index catalog object; if not found return null */ -std::shared_ptr CatalogCache::GetCachedIndexObject(const std::string &database_name, - const std::string &schema_name, - const std::string &index_name) { - auto database_object = GetDatabaseObject(database_name); - if (database_object == nullptr) return nullptr; - auto index_object = - database_object->GetCachedIndexCatalogEntry(index_name, schema_name); - if (index_object) return index_object; +std::shared_ptr +CatalogCache::GetCachedIndexCatalogEntry(const std::string &database_name, + const std::string &index_name, + const std::string &schema_name) { + auto database_object = GetDatabaseCatalogEntry(database_name); + if (database_object == nullptr) return nullptr; + auto index_object = + database_object->GetCachedIndexCatalogEntry(index_name, schema_name); + if (index_object) return index_object; + return nullptr; } diff --git a/src/catalog/database_catalog.cpp b/src/catalog/database_catalog.cpp index d869b20654d..bcb23961210 100644 --- a/src/catalog/database_catalog.cpp +++ b/src/catalog/database_catalog.cpp @@ -26,11 +26,14 @@ namespace peloton { namespace catalog { DatabaseCatalogEntry::DatabaseCatalogEntry(concurrency::TransactionContext *txn, - executor::LogicalTile *tile) - : database_oid_(tile->GetValue(0, DatabaseCatalog::ColumnId::DATABASE_OID) - .GetAs()), - database_name_(tile->GetValue(0, DatabaseCatalog::ColumnId::DATABASE_NAME) - .ToString()), + executor::LogicalTile *tile, + int tupleId) + : database_oid_( + tile->GetValue(tupleId, DatabaseCatalog::ColumnId::DATABASE_OID) + .GetAs()), + database_name_( + tile->GetValue(tupleId, DatabaseCatalog::ColumnId::DATABASE_NAME) + .ToString()), table_catalog_entries_cache_(), table_catalog_entries_cache_by_name(), valid_table_catalog_entries(false), @@ -180,7 +183,7 @@ DatabaseCatalogEntry::GetTableCatalogEntries(const std::string &schema_name) { // insert every table object into cache pg_table->GetTableCatalogEntries(txn_); } - // make sure to check IsValidTableObjects() before getting table objects + // make sure to check IsValidTableCatalogEntries() before getting table objects PELOTON_ASSERT(valid_table_catalog_entries); std::vector> result; for (auto it : table_catalog_entries_cache_) { @@ -206,7 +209,7 @@ DatabaseCatalogEntry::GetTableCatalogEntries(bool cached_only) { ->GetTableCatalog(); return pg_table->GetTableCatalogEntries(txn_); } - // make sure to check IsValidTableObjects() before getting table objects + // make sure to check IsValidTableCatalogEntries() before getting table objects PELOTON_ASSERT(valid_table_catalog_entries); return table_catalog_entries_cache_; } @@ -324,7 +327,7 @@ bool DatabaseCatalog::DeleteDatabase(concurrency::TransactionContext *txn, oid_t values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); // evict cache - txn->catalog_cache.EvictDatabaseObject(database_oid); + txn->catalog_cache.EvictDatabaseCatalogEntry(database_oid); return DeleteWithIndexScan(txn, index_offset, values); } @@ -336,7 +339,7 @@ std::shared_ptr DatabaseCatalog::GetDatabaseCatalogEntry( throw CatalogException("Transaction is invalid!"); } // try get from cache - auto database_object = txn->catalog_cache.GetDatabaseObject(database_oid); + auto database_object = txn->catalog_cache.GetDatabaseCatalogEntry(database_oid); if (database_object) return database_object; // cache miss, get from pg_database @@ -355,7 +358,7 @@ std::shared_ptr DatabaseCatalog::GetDatabaseCatalogEntry( auto database_object = std::make_shared(txn, (*result_tiles)[0].get()); // insert into cache - bool success = txn->catalog_cache.InsertDatabaseObject(database_object); + bool success = txn->catalog_cache.InsertDatabaseCatalogEntry(database_object); PELOTON_ASSERT(success == true); (void)success; return database_object; @@ -379,7 +382,7 @@ std::shared_ptr DatabaseCatalog::GetDatabaseCatalogEntry( throw CatalogException("Transaction is invalid!"); } // try get from cache - auto database_object = txn->catalog_cache.GetDatabaseObject(database_name); + auto database_object = txn->catalog_cache.GetDatabaseCatalogEntry(database_name); if (database_object) return database_object; // cache miss, get from pg_database @@ -400,7 +403,7 @@ std::shared_ptr DatabaseCatalog::GetDatabaseCatalogEntry( std::make_shared(txn, (*result_tiles)[0].get()); if (database_object) { // insert into cache - bool success = txn->catalog_cache.InsertDatabaseObject(database_object); + bool success = txn->catalog_cache.InsertDatabaseCatalogEntry(database_object); PELOTON_ASSERT(success == true); (void)success; } @@ -411,5 +414,35 @@ std::shared_ptr DatabaseCatalog::GetDatabaseCatalogEntry( return nullptr; } +std::unordered_map> +DatabaseCatalog::GetDatabaseCatalogEntries(concurrency::TransactionContext *txn) { + if (txn == nullptr) { + throw CatalogException("Transaction is invalid!"); + } + + // try get from cache + if (txn->catalog_cache.IsValidDatabaseCatalogEntries()) { + return txn->catalog_cache.GetDatabaseCatalogEntries(); + } + + // get from pg_database + std::vector column_ids(all_column_ids_); + auto result_tiles = this->GetResultWithSeqScan(txn, nullptr, column_ids); + + for (auto &tile : (*result_tiles)) { + for (auto tuple_id : *tile) { + auto database_object = + std::make_shared(txn, tile.get(), tuple_id); + if (database_object) { + // insert into cache + txn->catalog_cache.InsertDatabaseCatalogEntry(database_object); + } + } + } + + txn->catalog_cache.SetValidDatabaseCatalogEntries(true); + return txn->catalog_cache.GetDatabaseCatalogEntries(); +} + } // namespace catalog } // namespace peloton diff --git a/src/catalog/index_catalog.cpp b/src/catalog/index_catalog.cpp index 69a2633ed44..2df2d048f24 100644 --- a/src/catalog/index_catalog.cpp +++ b/src/catalog/index_catalog.cpp @@ -186,12 +186,12 @@ bool IndexCatalog::DeleteIndex(concurrency::TransactionContext *txn, std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); - auto index_object = txn->catalog_cache.GetCachedIndexObject(database_oid, - index_oid); + auto index_object = txn->catalog_cache.GetCachedIndexCatalogEntry(database_oid, + index_oid); if (index_object) { auto table_object = - txn->catalog_cache.GetCachedTableObject(database_oid, - index_object->GetTableOid()); + txn->catalog_cache.GetCachedTableCatalogEntry(database_oid, + index_object->GetTableOid()); table_object->EvictAllIndexCatalogEntries(); } @@ -206,8 +206,8 @@ std::shared_ptr IndexCatalog::GetIndexCatalogEntry( throw CatalogException("Transaction is invalid!"); } // try get from cache - auto index_object = txn->catalog_cache.GetCachedIndexObject(database_oid, - index_oid); + auto index_object = txn->catalog_cache.GetCachedIndexCatalogEntry(database_oid, + index_oid); if (index_object) { return index_object; } @@ -254,9 +254,9 @@ std::shared_ptr IndexCatalog::GetIndexCatalogEntry( } // try get from cache auto index_object = - txn->catalog_cache.GetCachedIndexObject(database_name, - schema_name, - index_name); + txn->catalog_cache.GetCachedIndexCatalogEntry(database_name, + schema_name, + index_name); if (index_object) { return index_object; } diff --git a/src/catalog/schema.cpp b/src/catalog/schema.cpp index 3024ef0a1c3..908e5302d09 100644 --- a/src/catalog/schema.cpp +++ b/src/catalog/schema.cpp @@ -21,11 +21,10 @@ namespace peloton { namespace catalog { // Helper function for creating TupleSchema -void Schema::CreateTupleSchema( - const std::vector &column_types, - const std::vector &column_lengths, - const std::vector &column_names, - const std::vector &is_inlined) { +void Schema::CreateTupleSchema(const std::vector &column_types, + const std::vector &column_lengths, + const std::vector &column_names, + const std::vector &is_inlined) { bool tup_is_inlined = true; oid_t num_columns = column_types.size(); oid_t column_offset = 0; diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 14cdd46c7ef..581028a17b1 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -611,8 +611,8 @@ bool TableCatalog::DeleteTable(concurrency::TransactionContext *txn, oid_t table values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); // evict from cache - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableCatalogEntry(database_oid_, + table_oid); if (table_object) { auto database_object = DatabaseCatalog::GetInstance(nullptr, @@ -637,8 +637,8 @@ std::shared_ptr TableCatalog::GetTableCatalogEntry( throw CatalogException("Transaction is invalid!"); } // try get from cache - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableCatalogEntry(database_oid_, + table_oid); if (table_object) return table_object; // cache miss, get from pg_table @@ -691,7 +691,7 @@ std::shared_ptr TableCatalog::GetTableCatalogEntry( throw CatalogException("Transaction is invalid!"); } // try get from cache - auto database_object = txn->catalog_cache.GetDatabaseObject(database_oid_); + auto database_object = txn->catalog_cache.GetDatabaseCatalogEntry(database_oid_); if (database_object) { auto table_object = database_object->GetTableCatalogEntry(table_name, schema_name, true); @@ -798,8 +798,8 @@ bool TableCatalog::UpdateVersionId(concurrency::TransactionContext *txn, type::ValueFactory::GetIntegerValue(update_val).Copy()); // get table object, then evict table object - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableCatalogEntry(database_oid_, + table_oid); if (table_object) { auto database_object = DatabaseCatalog::GetInstance(nullptr, @@ -836,8 +836,8 @@ bool TableCatalog::UpdateDefaultLayoutOid(concurrency::TransactionContext *txn, type::ValueFactory::GetIntegerValue(update_val).Copy()); // get table object, then evict table object - auto table_object = txn->catalog_cache.GetCachedTableObject(database_oid_, - table_oid); + auto table_object = txn->catalog_cache.GetCachedTableCatalogEntry(database_oid_, + table_oid); if (table_object) { auto database_object = DatabaseCatalog::GetInstance(nullptr, diff --git a/src/catalog/trigger_catalog.cpp b/src/catalog/trigger_catalog.cpp index 7c95b705f00..e8cda74368d 100644 --- a/src/catalog/trigger_catalog.cpp +++ b/src/catalog/trigger_catalog.cpp @@ -80,7 +80,7 @@ bool TriggerCatalog::InsertTrigger(concurrency::TransactionContext *txn, std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); - LOG_INFO("type of trigger inserted:%d", trigger_type); + LOG_TRACE("type of trigger inserted:%d", trigger_type); auto val0 = type::ValueFactory::GetIntegerValue(GetNextOid()); auto val1 = type::ValueFactory::GetIntegerValue(table_oid); @@ -182,9 +182,9 @@ std::unique_ptr TriggerCatalog::GetTriggersByType(concurre values); // carefull! the result tile could be null! if (result_tiles == nullptr) { - LOG_INFO("no trigger on table %d", table_oid); + LOG_TRACE("no trigger on table %d", table_oid); } else { - LOG_INFO("size of the result tiles = %lu", result_tiles->size()); + LOG_TRACE("size of the result tiles = %lu", result_tiles->size()); } // create the trigger list diff --git a/src/common/init.cpp b/src/common/init.cpp index c8b87133211..a7d12240baa 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -21,6 +21,7 @@ #include "concurrency/transaction_manager_factory.h" #include "gc/gc_manager_factory.h" #include "index/index.h" +#include "logging/checkpoint_manager_factory.h" #include "settings/settings_manager.h" #include "threadpool/mono_queue_pool.h" #include "tuning/index_tuner.h" @@ -36,6 +37,7 @@ void PelotonInit::Initialize() { LOGGING_THREAD_COUNT = 1; GC_THREAD_COUNT = 1; EPOCH_THREAD_COUNT = 1; + CHECKPOINTING_THREAD_COUNT = 1; // set max thread number. thread_pool.Initialize(0, CONNECTION_THREAD_COUNT + 3); @@ -59,7 +61,8 @@ void PelotonInit::Initialize() { concurrency::EpochManagerFactory::GetInstance().StartEpoch(); // start GC. - gc::GCManagerFactory::Configure(settings::SettingsManager::GetInt(settings::SettingId::gc_num_threads)); + gc::GCManagerFactory::Configure( + settings::SettingsManager::GetInt(settings::SettingId::gc_num_threads)); gc::GCManagerFactory::GetInstance().StartGC(); // start index tuner @@ -81,21 +84,38 @@ void PelotonInit::Initialize() { pg_catalog->Bootstrap(); // Additional catalogs settings::SettingsManager::GetInstance().InitializeCatalog(); - // begin a transaction - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); + // Recover the latest checkpoint and start checkpointing + bool recovered_default_db = false; + if (settings::SettingsManager::GetBool(settings::SettingId::checkpointing)) { + logging::CheckpointManagerFactory::Configure(CHECKPOINTING_THREAD_COUNT, + CheckpointingType::TIMESTAMP); + auto &checkpoint_manager = logging::CheckpointManagerFactory::GetInstance(); + recovered_default_db = checkpoint_manager.DoCheckpointRecovery(); + checkpoint_manager.StartCheckpointing(); + } else { + logging::CheckpointManagerFactory::Configure(0); + } // initialize the catalog and add the default database, so we don't do this on - // the first query - pg_catalog->CreateDatabase(txn, DEFAULT_DB_NAME); + // the first query, if the checkpoint recovery was not completed. + if (recovered_default_db == false) { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + pg_catalog->CreateDatabase(txn, DEFAULT_DB_NAME); + txn_manager.CommitTransaction(txn); + } - txn_manager.CommitTransaction(txn); // Initialize the Statement Cache Manager StatementCacheManager::Init(); } void PelotonInit::Shutdown() { + // shut down checkpoint managere + if (settings::SettingsManager::GetBool(settings::SettingId::checkpointing)) { + logging::CheckpointManagerFactory::GetInstance().StopCheckpointing(); + } + // shut down index tuner if (settings::SettingsManager::GetBool(settings::SettingId::index_tuner)) { auto &index_tuner = tuning::IndexTuner::GetInstance(); diff --git a/src/common/internal_types.cpp b/src/common/internal_types.cpp index 3b0c20900df..29dde8cb740 100644 --- a/src/common/internal_types.cpp +++ b/src/common/internal_types.cpp @@ -37,6 +37,7 @@ size_t CONNECTION_THREAD_COUNT = 1; size_t LOGGING_THREAD_COUNT = 1; size_t GC_THREAD_COUNT = 1; size_t EPOCH_THREAD_COUNT = 1; +size_t CHECKPOINTING_THREAD_COUNT = 1; //===--------------------------------------------------------------------===// // DatePart <--> String Utilities @@ -2725,8 +2726,11 @@ std::string CheckpointingTypeToString(CheckpointingType type) { case CheckpointingType::OFF: { return "OFF"; } - case CheckpointingType::ON: { - return "ON"; + case CheckpointingType::LOGICAL: { + return "LOGICAL"; + } + case CheckpointingType::TIMESTAMP: { + return "TIMESTAMP"; } default: { throw ConversionException(StringUtil::Format( @@ -2743,8 +2747,10 @@ CheckpointingType StringToCheckpointingType(const std::string &str) { return CheckpointingType::INVALID; } else if (upper_str == "OFF") { return CheckpointingType::OFF; - } else if (upper_str == "ON") { - return CheckpointingType::ON; + } else if (upper_str == "LOGICAL") { + return CheckpointingType::LOGICAL; + } else if (upper_str == "TIMESTAMP") { + return CheckpointingType::TIMESTAMP; } else { throw ConversionException(StringUtil::Format( "No CheckpointingType conversion from string '%s'", upper_str.c_str())); diff --git a/src/include/catalog/abstract_catalog.h b/src/include/catalog/abstract_catalog.h index 0d180975309..5530b2deed6 100644 --- a/src/include/catalog/abstract_catalog.h +++ b/src/include/catalog/abstract_catalog.h @@ -31,6 +31,10 @@ namespace expression { class AbstractExpression; } +namespace logging { +class TimestampCheckpointManager; +} + namespace storage { class Database; class DataTable; @@ -40,6 +44,7 @@ class Tuple; namespace catalog { class AbstractCatalog { + public: virtual ~AbstractCatalog() {} diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index 03cc2a1abbb..fc4f2c89478 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -40,6 +40,10 @@ namespace index { class Index; } // namespace index +namespace logging { +class TimestampCheckpointManager; +} // namespace logging + namespace storage { class Database; class DataTable; @@ -78,6 +82,8 @@ struct FunctionData { }; class Catalog { + friend class logging::TimestampCheckpointManager; + public: // Global Singleton static Catalog *GetInstance(); @@ -303,11 +309,17 @@ class Catalog { * get it from storage layer using table_oid, * throw exception and abort txn if not exists/invisible * */ - std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, - const std::string &database_name); + std::shared_ptr + GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + const std::string &database_name); - std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, - oid_t database_oid); + std::shared_ptr + GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + oid_t database_oid); + + std::unordered_map> + GetDatabaseCatalogEntries(concurrency::TransactionContext *txn, + bool cached_only = false); /* Check table from pg_table with table_name using txn, * get it from storage layer using table_oid, @@ -325,7 +337,8 @@ class Catalog { /* * Using database oid to get system catalog object */ - std::shared_ptr GetSystemCatalogs(oid_t database_oid); + std::shared_ptr GetSystemCatalogs(oid_t database_oid); + //===--------------------------------------------------------------------===// // DEPRECATED FUNCTIONS //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/catalog_cache.h b/src/include/catalog/catalog_cache.h index 3f00559a8db..4407549cdd3 100644 --- a/src/include/catalog/catalog_cache.h +++ b/src/include/catalog/catalog_cache.h @@ -19,6 +19,10 @@ namespace peloton { +namespace concurrency { +class TransactionContext; +} + namespace planner { class PlanUtil; } // namespace planner @@ -30,7 +34,8 @@ class TableCatalogEntry; class IndexCatalogEntry; class CatalogCache { - friend class Transaction; + friend class concurrency::TransactionContext; + friend class Catalog; friend class DatabaseCatalog; friend class TableCatalog; friend class IndexCatalog; @@ -40,33 +45,45 @@ class CatalogCache { friend class planner::PlanUtil; public: - CatalogCache() {} + CatalogCache() : valid_database_catalog_entry_(false) {} DISALLOW_COPY(CatalogCache) private: - std::shared_ptr GetDatabaseObject(oid_t database_oid); - std::shared_ptr GetDatabaseObject( + std::shared_ptr GetDatabaseCatalogEntry(oid_t database_oid); + std::shared_ptr GetDatabaseCatalogEntry( const std::string &name); + std::unordered_map> + GetDatabaseCatalogEntries(concurrency::TransactionContext *txn = nullptr); - std::shared_ptr GetCachedTableObject(oid_t database_oid, + std::shared_ptr GetCachedTableCatalogEntry(oid_t database_oid, oid_t table_oid); - std::shared_ptr GetCachedIndexObject(oid_t database_oid, + std::shared_ptr GetCachedIndexCatalogEntry(oid_t database_oid, oid_t index_oid); - std::shared_ptr GetCachedIndexObject(const std::string &database_name, + std::shared_ptr GetCachedIndexCatalogEntry(const std::string &database_name, const std::string &schema_name, const std::string &index_name); // database catalog cache interface - bool InsertDatabaseObject( + bool InsertDatabaseCatalogEntry( std::shared_ptr database_object); - bool EvictDatabaseObject(oid_t database_oid); - bool EvictDatabaseObject(const std::string &database_name); + bool EvictDatabaseCatalogEntry(oid_t database_oid); + bool EvictDatabaseCatalogEntry(const std::string &database_name); + + void SetValidDatabaseCatalogEntries(bool valid = true) { + valid_database_catalog_entry_ = valid; + } + bool IsValidDatabaseCatalogEntries() { + // return true if this catalog cache contains all database + // objects within the database + return valid_database_catalog_entry_; + } // cache for database catalog object std::unordered_map> - database_objects_cache_; + database_catalog_entries_cache_; std::unordered_map> database_name_cache_; + bool valid_database_catalog_entry_; }; } // namespace catalog diff --git a/src/include/catalog/constraint.h b/src/include/catalog/constraint.h index 72c5076a1ba..7e5bb626bef 100644 --- a/src/include/catalog/constraint.h +++ b/src/include/catalog/constraint.h @@ -10,7 +10,6 @@ // //===----------------------------------------------------------------------===// - #pragma once #include diff --git a/src/include/catalog/database_catalog.h b/src/include/catalog/database_catalog.h index 91267c3d995..50ae45bba1a 100644 --- a/src/include/catalog/database_catalog.h +++ b/src/include/catalog/database_catalog.h @@ -43,7 +43,8 @@ class DatabaseCatalogEntry { public: DatabaseCatalogEntry(concurrency::TransactionContext *txn, - executor::LogicalTile *tile); + executor::LogicalTile *tile, + int tupleId = 0); void EvictAllTableCatalogEntries(); @@ -105,6 +106,7 @@ class DatabaseCatalog : public AbstractCatalog { friend class TableCatalog; friend class CatalogCache; friend class Catalog; + friend class logging::TimestampCheckpointManager; public: ~DatabaseCatalog(); @@ -132,11 +134,16 @@ class DatabaseCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// - std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, - oid_t database_oid); + std::shared_ptr + GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + oid_t database_oid); - std::shared_ptr GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, - const std::string &database_name); + std::shared_ptr + GetDatabaseCatalogEntry(concurrency::TransactionContext *txn, + const std::string &database_name); + + std::unordered_map> + GetDatabaseCatalogEntries(concurrency::TransactionContext *txn); DatabaseCatalog(concurrency::TransactionContext *txn, storage::Database *pg_catalog, diff --git a/src/include/catalog/database_metrics_catalog.h b/src/include/catalog/database_metrics_catalog.h index bb72bc540dc..a030f7ab625 100644 --- a/src/include/catalog/database_metrics_catalog.h +++ b/src/include/catalog/database_metrics_catalog.h @@ -35,6 +35,8 @@ namespace peloton { namespace catalog { class DatabaseMetricsCatalog : public AbstractCatalog { + friend class logging::TimestampCheckpointManager; + public: ~DatabaseMetricsCatalog(); diff --git a/src/include/catalog/index_catalog.h b/src/include/catalog/index_catalog.h index dcd15ea285a..b098a31495a 100644 --- a/src/include/catalog/index_catalog.h +++ b/src/include/catalog/index_catalog.h @@ -71,6 +71,7 @@ class IndexCatalog : public AbstractCatalog { friend class IndexCatalogEntry; friend class TableCatalogEntry; friend class Catalog; + friend class logging::TimestampCheckpointManager; public: IndexCatalog(concurrency::TransactionContext *txn, diff --git a/src/include/catalog/index_metrics_catalog.h b/src/include/catalog/index_metrics_catalog.h index 6047bfe063f..994cfdea1d6 100644 --- a/src/include/catalog/index_metrics_catalog.h +++ b/src/include/catalog/index_metrics_catalog.h @@ -37,6 +37,8 @@ namespace peloton { namespace catalog { class IndexMetricsCatalog : public AbstractCatalog { + friend class logging::TimestampCheckpointManager; + public: IndexMetricsCatalog(concurrency::TransactionContext *txn, const std::string &database_name); diff --git a/src/include/catalog/language_catalog.h b/src/include/catalog/language_catalog.h index 7a6e906d422..b04fab35e45 100644 --- a/src/include/catalog/language_catalog.h +++ b/src/include/catalog/language_catalog.h @@ -50,6 +50,8 @@ class LanguageCatalogEntry { }; class LanguageCatalog : public AbstractCatalog { + friend class logging::TimestampCheckpointManager; + public: ~LanguageCatalog(); diff --git a/src/include/catalog/proc_catalog.h b/src/include/catalog/proc_catalog.h index 19fc39fbd43..8c757243f14 100644 --- a/src/include/catalog/proc_catalog.h +++ b/src/include/catalog/proc_catalog.h @@ -77,6 +77,8 @@ class ProcCatalogEntry { // The pg_proc catalog table. //===----------------------------------------------------------------------===// class ProcCatalog : public AbstractCatalog { + friend class logging::TimestampCheckpointManager; + public: ~ProcCatalog(); diff --git a/src/include/catalog/query_metrics_catalog.h b/src/include/catalog/query_metrics_catalog.h index 30e44ccbc43..25e706b0323 100644 --- a/src/include/catalog/query_metrics_catalog.h +++ b/src/include/catalog/query_metrics_catalog.h @@ -42,6 +42,8 @@ namespace peloton { namespace catalog { class QueryMetricsCatalog : public AbstractCatalog { + friend class logging::TimestampCheckpointManager; + public: QueryMetricsCatalog(concurrency::TransactionContext *txn, const std::string &database_name); diff --git a/src/include/catalog/schema_catalog.h b/src/include/catalog/schema_catalog.h index 357afd911e6..015a71b7467 100644 --- a/src/include/catalog/schema_catalog.h +++ b/src/include/catalog/schema_catalog.h @@ -54,6 +54,7 @@ class SchemaCatalogEntry { class SchemaCatalog : public AbstractCatalog { friend class SchemaCatalogEntry; friend class Catalog; + friend class logging::TimestampCheckpointManager; public: SchemaCatalog(concurrency::TransactionContext *txn, diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index f7d373c4d3e..d6cdc77011d 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -181,6 +181,7 @@ class TableCatalog : public AbstractCatalog { friend class LayoutCatalog; friend class ConstraintCatalog; friend class Catalog; + friend class logging::TimestampCheckpointManager; public: TableCatalog(concurrency::TransactionContext *txn, diff --git a/src/include/catalog/table_metrics_catalog.h b/src/include/catalog/table_metrics_catalog.h index ba2e8d1c5b1..d22c6e453a6 100644 --- a/src/include/catalog/table_metrics_catalog.h +++ b/src/include/catalog/table_metrics_catalog.h @@ -37,6 +37,8 @@ namespace peloton { namespace catalog { class TableMetricsCatalog : public AbstractCatalog { + friend class logging::TimestampCheckpointManager; + public: TableMetricsCatalog(concurrency::TransactionContext *txn, const std::string &database_name); diff --git a/src/include/catalog/trigger_catalog.h b/src/include/catalog/trigger_catalog.h index 83088727aa8..28486bbd714 100644 --- a/src/include/catalog/trigger_catalog.h +++ b/src/include/catalog/trigger_catalog.h @@ -46,6 +46,8 @@ class TriggerList; namespace catalog { class TriggerCatalog : public AbstractCatalog { + friend class logging::TimestampCheckpointManager; + public: TriggerCatalog(concurrency::TransactionContext *txn, const std::string &database_name); diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index b34d2971c70..ed32d0f5feb 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -966,8 +966,9 @@ std::ostream &operator<<(std::ostream &os, const LogRecordType &type); enum class CheckpointingType { INVALID = INVALID_TYPE_ID, - OFF = 1, // turn off GC - ON = 2 // turn on GC + OFF = 1, // turn off checkpoints + LOGICAL = 2, // turn on logical checkpoints + TIMESTAMP = 3 // turn on timestamp checkpoints }; std::string CheckpointingTypeToString(CheckpointingType type); CheckpointingType StringToCheckpointingType(const std::string &str); @@ -1185,6 +1186,7 @@ extern size_t CONNECTION_THREAD_COUNT; extern size_t LOGGING_THREAD_COUNT; extern size_t GC_THREAD_COUNT; extern size_t EPOCH_THREAD_COUNT; +extern size_t CHECKPOINTING_THREAD_COUNT; //===--------------------------------------------------------------------===// // TupleMetadata diff --git a/src/include/logging/checkpoint_manager.h b/src/include/logging/checkpoint_manager.h index 78d8bc3c120..c9f387ecc51 100644 --- a/src/include/logging/checkpoint_manager.h +++ b/src/include/logging/checkpoint_manager.h @@ -55,6 +55,8 @@ class CheckpointManager { virtual void StopCheckpointing() {} + virtual bool DoCheckpointRecovery() { return false;} + virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} diff --git a/src/include/logging/checkpoint_manager_factory.h b/src/include/logging/checkpoint_manager_factory.h index 44492147ba3..fed0c45a8e7 100644 --- a/src/include/logging/checkpoint_manager_factory.h +++ b/src/include/logging/checkpoint_manager_factory.h @@ -14,41 +14,53 @@ #include "logging/checkpoint_manager.h" #include "logging/logical_checkpoint_manager.h" +#include "logging/timestamp_checkpoint_manager.h" namespace peloton { namespace logging { class CheckpointManagerFactory { public: - - static CheckpointManager& GetInstance() { + static CheckpointManager &GetInstance() { switch (checkpointing_type_) { + case CheckpointingType::LOGICAL: + return LogicalCheckpointManager::GetInstance( + checkpointing_thread_count_); + + case CheckpointingType::TIMESTAMP: + return TimestampCheckpointManager::GetInstance( + checkpointing_thread_count_); - case CheckpointingType::ON: - return LogicalCheckpointManager::GetInstance(checkpointing_thread_count_); - default: return CheckpointManager::GetInstance(); } } - static void Configure(const int thread_count = 1) { + static void Configure( + const int thread_count = 1, + const CheckpointingType type = CheckpointingType::TIMESTAMP) { if (thread_count == 0) { checkpointing_type_ = CheckpointingType::OFF; } else { - checkpointing_type_ = CheckpointingType::ON; + checkpointing_type_ = type; checkpointing_thread_count_ = thread_count; } } - inline static CheckpointingType GetCheckpointingType() { return checkpointing_type_; } + inline static CheckpointingType GetCheckpointingType() { + return checkpointing_type_; + } + + inline static int GetCheckpointingThreadCount() { + return checkpointing_thread_count_; + } -private: + private: // checkpointing type static CheckpointingType checkpointing_type_; static int checkpointing_thread_count_; }; -} // namespace logging -} // namespace peloton +} // namespace logging +} // namespace peloton diff --git a/src/include/logging/logging_util.h b/src/include/logging/logging_util.h new file mode 100644 index 00000000000..1a47b671938 --- /dev/null +++ b/src/include/logging/logging_util.h @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logging_util.h +// +// Identification: src/include/logging/logging_util.h +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "type/type.h" +#include "common/logger.h" +#include "storage/data_table.h" +#include "type/serializer.h" + +namespace peloton { +namespace logging { + +//===--------------------------------------------------------------------===// +// LoggingUtil +//===--------------------------------------------------------------------===// + +class LoggingUtil { + public: + // FILE SYSTEM RELATED OPERATIONS + static bool CheckDirectoryExistence(const char *dir_name); + + static bool CreateDirectory(const char *dir_name, int mode); + + static bool RemoveDirectory(const char *dir_name, bool only_remove_file); + + static bool GetDirectoryList(const char *dir_name, + std::vector &dir_name_list); + + static bool GetFileList(const char *dir_name, + std::vector &file_name_list); + + static void FFlushFsync(FileHandle &file_handle); + + static bool OpenFile(const char *name, const char *mode, + FileHandle &file_handle); + + static bool MoveFile(const char *oldname, const char *newname); + + static bool CloseFile(FileHandle &file_handle); + + static bool IsFileTruncated(FileHandle &file_handle, size_t size_to_read); + + static size_t GetFileSize(FileHandle &file_handle); + + static bool ReadNBytesFromFile(FileHandle &file_handle, void *bytes_read, + size_t n); +}; + +} // namespace logging +} // namespace peloton diff --git a/src/include/logging/logical_checkpoint_manager.h b/src/include/logging/logical_checkpoint_manager.h index b93a89e680f..5a2884dac75 100644 --- a/src/include/logging/logical_checkpoint_manager.h +++ b/src/include/logging/logical_checkpoint_manager.h @@ -39,11 +39,13 @@ class LogicalCheckpointManager : public CheckpointManager { virtual void Reset() { is_running_ = false; } - virtual void StartLogging(std::vector> & UNUSED_ATTRIBUTE) {} + virtual void StartCheckpointing(std::vector> & UNUSED_ATTRIBUTE) {} - virtual void StartLogging() {} + virtual void StartCheckpointing() {} - virtual void StopLogging() {} + virtual void StopCheckpointing() {} + + virtual bool DoCheckpointRecovery() { return false;} virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} diff --git a/src/include/logging/timestamp_checkpoint_manager.h b/src/include/logging/timestamp_checkpoint_manager.h new file mode 100644 index 00000000000..77361903da2 --- /dev/null +++ b/src/include/logging/timestamp_checkpoint_manager.h @@ -0,0 +1,320 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// timestamp_checkpoint_manager.h +// +// Identification: src/include/logging/timestamp_checkpoint_manager.h +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "logging/checkpoint_manager.h" + +#include "catalog/database_catalog.h" +#include "catalog/table_catalog.h" +#include "catalog/schema.h" +#include "logging/logging_util.h" +#include "settings/settings_manager.h" +#include "storage/database.h" +#include "storage/data_table.h" +#include "storage/tile_group_header.h" +#include "util/string_util.h" + +namespace peloton { +namespace logging { + +//===--------------------------------------------------------------------===// +// Timestamp checkpoint Manager +//===--------------------------------------------------------------------===// + +class TimestampCheckpointManager : public CheckpointManager { + public: + TimestampCheckpointManager(const TimestampCheckpointManager &) = delete; + TimestampCheckpointManager &operator=(const TimestampCheckpointManager &) = + delete; + TimestampCheckpointManager(TimestampCheckpointManager &&) = delete; + TimestampCheckpointManager &operator=(TimestampCheckpointManager &&) = delete; + + TimestampCheckpointManager(const int thread_count) + : CheckpointManager(), checkpointer_thread_count_(thread_count) { + SetCheckpointInterval(settings::SettingsManager::GetInt( + settings::SettingId::checkpoint_interval)); + SetCheckpointBaseDirectory(settings::SettingsManager::GetString( + settings::SettingId::checkpoint_dir)); + } + + virtual ~TimestampCheckpointManager() {} + + static TimestampCheckpointManager &GetInstance(const int thread_count = 1) { + static TimestampCheckpointManager checkpoint_manager(thread_count); + return checkpoint_manager; + } + + virtual void Reset() { is_running_ = false; } + + virtual void StartCheckpointing(); + + virtual void StopCheckpointing(); + + virtual bool DoCheckpointRecovery(); + + virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} + + virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} + + virtual size_t GetTableCount() { return 0; } + + // setter for checkpoint interval + void SetCheckpointInterval(const int interval) { + checkpoint_interval_ = interval; + } + + // setter for checkpoint base directory + // checkpoint files are made in a below directory structure: + // base_directory / [epoch_id | checkpointing] / checkpoint_files + void SetCheckpointBaseDirectory(const std::string &dir_name) { + // check the existence of checkpoint base directory. + // if not exist, then create the directory. + if (LoggingUtil::CheckDirectoryExistence(dir_name.c_str()) == false) { + LOG_INFO("Create base checkpoint directory %s", dir_name.c_str()); + CreateDirectory(dir_name); + } + checkpoint_base_dir_ = dir_name; + } + + // get a recovered epoch id, or a latest checkpoint epoch for recovery + eid_t GetRecoveryCheckpointEpoch(); + + protected: + //===--------------------------------------------------------------------===// + // Checkpointing Functions + //===--------------------------------------------------------------------===// + + // execute checkpointing in a designated interval + void PerformCheckpointing(); + + // create checkpoints for all tables + void CreateCheckpoints(concurrency::TransactionContext *txn, + const cid_t begin_cid); + + // read table data and write it down to checkpoint data file + void CheckpointingTableData(const storage::DataTable *table, + const cid_t &begin_cid, + FileHandle &file_handle); + + // read table data without tile group and write it down to checkpoint data + // file for catalog table checkpointing + void CheckpointingTableDataWithoutTileGroup(const storage::DataTable *table, + const cid_t &begin_cid, + FileHandle &file_handle); + + // Read table data without tile group and write it down to checkpoint data + // file if its persistent flag is true. + // This function works for only settings_catalog now. + void CheckpointingTableDataWithPersistentCheck(const storage::DataTable *table, + const cid_t &begin_cid, + oid_t flag_column_id, + FileHandle &file_handle); + + // check the value is committed before the checkpointing begins + bool IsVisible(const storage::TileGroupHeader *header, + const oid_t &tuple_id, + const cid_t &begin_cid); + + //===--------------------------------------------------------------------===// + // Checkpoint Recovery Functions + //===--------------------------------------------------------------------===// + + // load catalog table checkpoints + bool LoadCatalogTableCheckpoints(concurrency::TransactionContext *txn, + const eid_t &epoch_id); + + // load a specific catalog table checkpoint + bool LoadCatalogTableCheckpoint(concurrency::TransactionContext *txn, + const eid_t &epoch_id, + const oid_t db_oid, + const oid_t table_oid); + + // load user table checkpoints + bool LoadUserTableCheckpoints(concurrency::TransactionContext *txn, + const eid_t &epoch_id); + + // recover an user table storage object by using recovered catalog + // Note: Foreign key constraint is recovered just for the source table. + // Caller has to register its constraint in the sink table. + bool RecoverTableStorageObject(concurrency::TransactionContext *txn, + const oid_t db_oid, + const oid_t table_oid); + + // Read a checkpoint data file and recover the table + // This function is provided for checkpointed user table + void RecoverTableData(concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle); + + // Read a checkpoint data file without tile group and recover the table + // This function is provided for initialized catalog table not including + // default value + // return: Tuple count inserted into the table + oid_t RecoverTableDataWithoutTileGroup(concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle); + + // Read a checkpoint data file with duplicate check without tile group. + // This function keeps default values of catalogs. + // return: inserted tuple count into table without default values + oid_t RecoverTableDataWithDuplicateCheck(concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle); + + // Read a checkpoint data file without tile group. + // If a same key is existed and its persistent flag is true, + // then overwrite it by recovered value. + // This function works for only settings_catalog now. + // return: inserted/updated tuple count into table without default value + oid_t RecoverTableDataWithPersistentCheck(concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle, + std::vector key_colmun_ids, + oid_t flag_column_id); + + //===--------------------------------------------------------------------===// + // Utility Functions for Checkpoint Directory + //===--------------------------------------------------------------------===// + + // create hierarchical directory + void CreateDirectory(const std::string &dir_name) { + std::string sub_dir_path = ""; + for (auto sub_dir_name : StringUtil::Split(dir_name, '/')) { + sub_dir_path += sub_dir_name + "/"; + if (LoggingUtil::CheckDirectoryExistence(sub_dir_path.c_str()) == false) { + LOG_TRACE("Create sub directory %s", sub_dir_path.c_str()); + if (LoggingUtil::CreateDirectory(sub_dir_path.c_str(), 0700) == false) { + LOG_ERROR("Cannot create directory in this checkpoints: %s", + dir_name.c_str()); + break; + } + } + } + } + + // create checkpoint directory in base directory (default: ./data/checkpoint/) + void CreateCheckpointDirectory(const std::string &dir_name) { + std::string checkpoint_dir = checkpoint_base_dir_ + "/" + dir_name; + + // check the existence of checkpoint directory. + // if exists, then remove all files in the directory + // else then create the directory. + if (LoggingUtil::CheckDirectoryExistence(checkpoint_dir.c_str()) == false) { + LOG_TRACE("Create checkpoint directory %s", checkpoint_dir.c_str()); + CreateDirectory(checkpoint_dir); + } else { + LOG_TRACE("Found checkpoint directory %s, and delete all old files", + checkpoint_dir.c_str()); + if (LoggingUtil::RemoveDirectory(checkpoint_dir.c_str(), true) == false) { + LOG_ERROR("Cannot delete files in directory: %s", + checkpoint_dir.c_str()); + } + } + } + + // create working checkpoint directory in base directory. + // (default: ./data//checkpoint/checkpointing) + // this avoids to recover the failed (broken) checkpointing files. + // e.g. system crash during checkpointing + void CreateWorkingCheckpointDirectory() { + CreateCheckpointDirectory(checkpoint_working_dir_name_); + } + + // move the working checkpoint directory to completed checkpoint directory + // named epoch id that the checkpointing is started. + void MoveWorkingToCheckpointDirectory(const std::string dir_name) { + std::string working_dir_path = + checkpoint_base_dir_ + "/" + checkpoint_working_dir_name_; + std::string checkpoint_dir_path = checkpoint_base_dir_ + "/" + dir_name; + + CreateCheckpointDirectory(dir_name); + if (LoggingUtil::MoveFile(working_dir_path.c_str(), + checkpoint_dir_path.c_str()) == false) { + LOG_ERROR("Cannot move checkpoint file from %s to %s", + working_dir_path.c_str(), checkpoint_dir_path.c_str()); + } + } + + std::string GetCheckpointFileFullPath(const std::string &database_name, + const std::string &schema_name, + const std::string &table_name, + const eid_t &epoch_id) { + return checkpoint_base_dir_ + "/" + std::to_string(epoch_id) + "/" + + checkpoint_filename_prefix_ + "_" + database_name + "_" + + schema_name + "_" + table_name; + } + std::string GetWorkingCheckpointFileFullPath(const std::string &database_name, + const std::string &schema_name, + const std::string &table_name) { + return checkpoint_base_dir_ + "/" + checkpoint_working_dir_name_ + "/" + + checkpoint_filename_prefix_ + "_" + database_name + "_" + + schema_name + "_" + table_name;; + } + std::string GetMetadataFileFullPath(const eid_t &epoch_id) { + return checkpoint_base_dir_ + "/" + std::to_string(epoch_id) + "/" + + metadata_filename_prefix_; + } + std::string GetWorkingMetadataFileFullPath() { + return checkpoint_base_dir_ + "/" + checkpoint_working_dir_name_ + "/" + + metadata_filename_prefix_; + } + + // remove old checkpoints except for the designated epoch id (latest + // checkpointing time) + // TODO: Consider whether some old checkpoints will remain + void RemoveOldCheckpoints(const eid_t &begin_epoch_id) { + std::vector dir_name_list; + + // Get list of checkpoint directories + if (LoggingUtil::GetDirectoryList(checkpoint_base_dir_.c_str(), + dir_name_list) == false) { + LOG_ERROR("Failed to get directory list in %s", + checkpoint_base_dir_.c_str()); + } + + // Remove old checkpoint directory + for (const auto dir_name : dir_name_list) { + if (strcmp(dir_name.c_str(), checkpoint_working_dir_name_.c_str()) != 0) { + eid_t epoch_id = std::strtoul(dir_name.c_str(), NULL, 10); + if (epoch_id == 0) { + LOG_ERROR("Unexpected epoch value in checkpoints directory: %s", + dir_name.c_str()); + } else if (epoch_id == begin_epoch_id) { + // not be old + continue; + } + } + + std::string remove_dir = checkpoint_base_dir_ + '/' + dir_name; + if (LoggingUtil::RemoveDirectory(remove_dir.c_str(), false) == false) { + LOG_ERROR("Failure to remove checkpoint dir: %s", remove_dir.c_str()); + } + } + } + + private: + int checkpointer_thread_count_; + std::unique_ptr central_checkpoint_thread_; + + int checkpoint_interval_ = 30; + + std::string checkpoint_base_dir_; + std::string checkpoint_working_dir_name_ = "checkpointing"; + std::string checkpoint_filename_prefix_ = "checkpoint"; + std::string metadata_filename_prefix_ = "checkpoint_metadata"; + + eid_t recovered_epoch_id_ = INVALID_EID; +}; + +} // namespace logging +} // namespace peloton diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index a442fd151cf..6cc4050242d 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -131,6 +131,22 @@ SETTING_int(min_parallel_table_scan_size, // WRITE AHEAD LOG //===----------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// +// CHECKPOINTS +//===----------------------------------------------------------------------===// + +SETTING_bool(checkpointing, + "Enable Checkpointing and recovery (default: false)", false, + false, false) + +SETTING_int(checkpoint_interval, + "Checkpoint interval in seconds (default: 30 second)", 30, + 0, 31536000, false, false) + +SETTING_string(checkpoint_dir, + "Direcotry for checkpoints (default: ./checkpoints)", + "./data/checkpoints", false, false) + //===----------------------------------------------------------------------===// // ERROR REPORTING AND LOGGING //===----------------------------------------------------------------------===// @@ -229,11 +245,9 @@ SETTING_bool(hash_join_bloom_filter, SETTING_int(task_execution_timeout, "Maximum allowed length of time (in ms) for task " - "execution step of optimizer, " - "assuming one plan has been found (default 5000)", - 5000, - 1000, 60000, - true, true) + "execution step of optimizer, " + "assuming one plan has been found (default 5000)", + 5000, 1000, 60000, true, true) //===----------------------------------------------------------------------===// // GENERAL diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 48708b9edb5..29611804fdd 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -55,12 +55,9 @@ class Index; namespace logging { class LogManager; +class TimestampCheckpointManager; } // namespace logging -namespace concurrency { -class TransactionContext; -} // namespace concurrency - namespace storage { class Tuple; @@ -86,6 +83,7 @@ class DataTable : public AbstractTable { friend class TableFactory; friend class catalog::Catalog; friend class logging::LogManager; + friend class logging::TimestampCheckpointManager; DataTable() = delete; DataTable(DataTable const &) = delete; @@ -141,11 +139,13 @@ class DataTable : public AbstractTable { // TILE GROUP //===--------------------------------------------------------------------===// - // coerce into adding a new tile group with a tile group id - void AddTileGroupWithOidForRecovery(const oid_t &tile_group_id); - + // for test void AddTileGroup(const std::shared_ptr &tile_group); + // for checkpoint recovery + void AddTileGroup(const std::shared_ptr &tile_group, + const size_t &active_tile_group_id); + // Offset is a 0-based number local to the table std::shared_ptr GetTileGroup( const std::size_t &tile_group_offset) const; diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 8ba397f118c..acf5363d132 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -16,6 +16,7 @@ #include "common/internal_types.h" #include "common/printable.h" +#include "type/serializeio.h" namespace peloton { @@ -71,6 +72,9 @@ class Layout : public Printable { /** @brief Check whether this layout is a hybrid store. */ bool IsHybridStore() const { return (layout_type_ == LayoutType::HYBRID); } + /** @brief Return the layout_type_ of this object. */ + LayoutType GetLayoutType() const { return layout_type_; } + /** @brief Return the layout_oid_ of this object. */ oid_t GetOid() const { return layout_oid_; } diff --git a/src/include/type/serializeio.h b/src/include/type/serializeio.h index e845b3db482..373edbfc23f 100644 --- a/src/include/type/serializeio.h +++ b/src/include/type/serializeio.h @@ -94,6 +94,11 @@ class SerializeInput { current_ -= bytes; } + // Return the rest size of data + size_t RestSize() { + return end_ - current_; + } + private: template T ReadPrimitive() { diff --git a/src/logging/checkpoint_manager_factory.cpp b/src/logging/checkpoint_manager_factory.cpp index 4f025729a7d..83aeb1cd7da 100644 --- a/src/logging/checkpoint_manager_factory.cpp +++ b/src/logging/checkpoint_manager_factory.cpp @@ -16,7 +16,7 @@ namespace peloton { namespace logging { -CheckpointingType CheckpointManagerFactory::checkpointing_type_ = CheckpointingType::ON; +CheckpointingType CheckpointManagerFactory::checkpointing_type_ = CheckpointingType::TIMESTAMP; int CheckpointManagerFactory::checkpointing_thread_count_ = 1; } // namespace gc diff --git a/src/logging/logging_util.cpp b/src/logging/logging_util.cpp new file mode 100644 index 00000000000..873350c9658 --- /dev/null +++ b/src/logging/logging_util.cpp @@ -0,0 +1,245 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logging_util.cpp +// +// Identification: src/logging/logging_util.cpp +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +#include "catalog/manager.h" +#include "logging/logging_util.h" +#include "storage/database.h" + +namespace peloton { +namespace logging { + +//===--------------------------------------------------------------------===// +// LoggingUtil +//===--------------------------------------------------------------------===// + +bool LoggingUtil::CreateDirectory(const char *dir_name, int mode) { + int return_val = mkdir(dir_name, mode); + if (return_val == 0) { + LOG_TRACE("Created directory %s successfully", dir_name); + } else if (errno == EEXIST) { + LOG_ERROR("Directory %s already exists", dir_name); + } else { + LOG_ERROR("Creating directory failed: %s", strerror(errno)); + return false; + } + return true; +} + +bool LoggingUtil::CheckDirectoryExistence(const char *dir_name) { + struct stat info; + int return_val = stat(dir_name, &info); + return return_val == 0 && S_ISDIR(info.st_mode); +} + +/** + * @return false if fail to remove directory + */ +bool LoggingUtil::RemoveDirectory(const char *dir_name, bool only_remove_file) { + struct dirent *file; + DIR *dir; + + dir = opendir(dir_name); + if (dir == nullptr) { + LOG_ERROR("Failed to open directory: %s, error: %s", dir_name, + strerror(errno)); + return false; + } + + // XXX readdir is not thread safe??? + while ((file = readdir(dir)) != nullptr) { + if (strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) { + continue; + } + std::ostringstream complete_path; + complete_path << dir_name << "/" << file->d_name; + auto ret_val = remove(complete_path.str().c_str()); + if (ret_val != 0) { + LOG_ERROR("Failed to delete file: %s, error: %s", + complete_path.str().c_str(), strerror(errno)); + } + } + closedir(dir); + if (!only_remove_file) { + auto ret_val = remove(dir_name); + if (ret_val != 0) { + LOG_ERROR("Failed to delete dir: %s, error: %s", file->d_name, + strerror(errno)); + } + } + return true; +} + +void LoggingUtil::FFlushFsync(FileHandle &file_handle) { + // First, flush + PELOTON_ASSERT(file_handle.fd != INVALID_FILE_DESCRIPTOR); + if (file_handle.fd == INVALID_FILE_DESCRIPTOR) return; + int ret = fflush(file_handle.file); + if (ret != 0) { + LOG_ERROR("Error occured in fflush(%d)", ret); + } + // Finally, sync + ret = fsync(file_handle.fd); + if (ret != 0) { + LOG_ERROR("Error occured in fsync(%d)", ret); + } +} + +bool LoggingUtil::GetDirectoryList(const char *dir_name, + std::vector &dir_name_list) { + DIR *dir; + struct dirent *element; + + // get a list of elements in the directory + dir = opendir(dir_name); + if (dir == nullptr) { + LOG_ERROR("Failed to open directory: %s, error: %s", dir_name, + strerror(errno)); + return false; + } + + // read directories containing in the directory + while ((element = readdir(dir)) != nullptr) { + char *element_name = element->d_name; + + // ignore '.' and '..' + if (strcmp(element_name, ".") == 0 || + strcmp(element_name, "..") == 0) { + continue; + } + + // check directory or not + std::string target_dir = std::string(dir_name) + '/' + element_name; + if (CheckDirectoryExistence(target_dir.c_str()) == true) { + dir_name_list.push_back(element_name); + } + } + + closedir(dir); + return true; +} + +bool LoggingUtil::GetFileList(const char *dir_name, + std::vector &file_name_list) { + DIR *dir; + struct dirent *element; + + // get a list of elements in the directory + dir = opendir(dir_name); + if (dir == nullptr) { + LOG_ERROR("Failed to open directory: %s, error: %s", dir_name, + strerror(errno)); + return false; + } + + // read directories containing in the directory + while ((element = readdir(dir)) != nullptr) { + char *element_name = element->d_name; + + // ignore '.' and '..' + if (strcmp(element_name, ".") == 0 || + strcmp(element_name, "..") == 0) { + continue; + } + + std::string target_dir = std::string(dir_name) + '/' + element_name; + if (CheckDirectoryExistence(target_dir.c_str()) != false) { + file_name_list.push_back(element_name); + } + } + + closedir(dir); + return true; +} + +bool LoggingUtil::OpenFile(const char *name, const char *mode, + FileHandle &file_handle) { + auto file = fopen(name, mode); + if (file == NULL) { + LOG_ERROR("Checkpoint File is NULL"); + return false; + } else { + file_handle.file = file; + } + + // also, get the descriptor + auto fd = fileno(file); + if (fd == INVALID_FILE_DESCRIPTOR) { + LOG_ERROR("checkpoint_file_fd_ is -1"); + return false; + } else { + file_handle.fd = fd; + } + + file_handle.size = GetFileSize(file_handle); + return true; +} + +bool LoggingUtil::MoveFile(const char *oldname, const char *newname) { + int ret; + ret = rename(oldname, newname); + if (ret != 0) { + LOG_ERROR("Error occured in rename"); + return false; + } + return true; +} + +bool LoggingUtil::CloseFile(FileHandle &file_handle) { + PELOTON_ASSERT(file_handle.file != nullptr && + file_handle.fd != INVALID_FILE_DESCRIPTOR); + int ret = fclose(file_handle.file); + + if (ret == 0) { + file_handle.file = nullptr; + file_handle.fd = INVALID_FILE_DESCRIPTOR; + } else { + LOG_ERROR("Error when closing log file"); + } + + return ret == 0; +} + +bool LoggingUtil::IsFileTruncated(FileHandle &file_handle, + size_t size_to_read) { + // Cache current position + size_t current_position = ftell(file_handle.file); + + // Check if the actual file size is less than the expected file size + // Current position + frame length + if (current_position + size_to_read <= file_handle.size) { + return false; + } else { + fseek(file_handle.file, 0, SEEK_END); + return true; + } +} + +size_t LoggingUtil::GetFileSize(FileHandle &file_handle) { + struct stat file_stats; + fstat(file_handle.fd, &file_stats); + return file_stats.st_size; +} + +bool LoggingUtil::ReadNBytesFromFile(FileHandle &file_handle, void *bytes_read, + size_t n) { + PELOTON_ASSERT(file_handle.fd != INVALID_FILE_DESCRIPTOR && + file_handle.file != nullptr); + int res = fread(bytes_read, n, 1, file_handle.file); + return res == 1; +} + +} // namespace logging +} // namespace peloton diff --git a/src/logging/timestamp_checkpoint_manager.cpp b/src/logging/timestamp_checkpoint_manager.cpp new file mode 100644 index 00000000000..7f0db845b75 --- /dev/null +++ b/src/logging/timestamp_checkpoint_manager.cpp @@ -0,0 +1,1373 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// timestamp_checkpoint_manager.cpp +// +// Identification: src/logging/timestamp_checkpoint_manager.cpp +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "logging/timestamp_checkpoint_manager.h" + +#include "catalog/catalog.h" +#include "catalog/database_catalog.h" +#include "catalog/database_metrics_catalog.h" +#include "catalog/column.h" +#include "catalog/column_catalog.h" +#include "catalog/index_catalog.h" +#include "catalog/index_metrics_catalog.h" +#include "catalog/manager.h" +#include "catalog/language_catalog.h" +#include "catalog/proc_catalog.h" +#include "catalog/query_metrics_catalog.h" +#include "catalog/query_history_catalog.h" +#include "catalog/schema.h" +#include "catalog/settings_catalog.h" +#include "catalog/system_catalogs.h" +#include "catalog/table_catalog.h" +#include "catalog/table_metrics_catalog.h" +#include "catalog/trigger_catalog.h" +#include "common/container_tuple.h" +#include "common/timer.h" +#include "concurrency/timestamp_ordering_transaction_manager.h" +#include "concurrency/transaction_manager_factory.h" +#include "executor/executor_context.h" +#include "executor/insert_executor.h" +#include "index/index.h" +#include "index/index_factory.h" +#include "planner/insert_plan.h" +#include "settings/settings_manager.h" +#include "storage/database.h" +#include "storage/data_table.h" +#include "storage/storage_manager.h" +#include "storage/table_factory.h" +#include "storage/tile_group.h" +#include "storage/tile_group_factory.h" +#include "type/serializeio.h" +#include "type/type.h" + +namespace peloton { +namespace logging { + +/** + * @brief Start checkpointing by making a background thread + */ +void TimestampCheckpointManager::StartCheckpointing() { + is_running_ = true; + central_checkpoint_thread_.reset( + new std::thread(&TimestampCheckpointManager::PerformCheckpointing, this)); +} + +/** + * @brief Stop checkpointing and wait for a background thread to quit + */ +void TimestampCheckpointManager::StopCheckpointing() { + is_running_ = false; + central_checkpoint_thread_->join(); +} + +/** + * @brief Do checkpoint recovery for all tables + * @return bool whether success to recover checkpoints or not + */ +bool TimestampCheckpointManager::DoCheckpointRecovery() { + eid_t epoch_id = GetRecoveryCheckpointEpoch(); + if (epoch_id == INVALID_EID) { + LOG_INFO("No checkpoint for recovery"); + return false; + } else { + LOG_INFO("Start checkpoint recovery"); + Timer recovery_timer; + recovery_timer.Start(); + + // recover catalog table checkpoint + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + if (!LoadCatalogTableCheckpoints(txn, epoch_id)) { + LOG_ERROR("Catalog table checkpoint recovery was failed"); + txn_manager.AbortTransaction(txn); + return false; + } + txn_manager.CommitTransaction(txn); + + // recover user table checkpoint + txn = txn_manager.BeginTransaction(); + if (!LoadUserTableCheckpoints(txn, epoch_id)) { + LOG_ERROR("User table checkpoint recovery was failed"); + txn_manager.AbortTransaction(txn); + return false; + } + txn_manager.CommitTransaction(txn); + + // set recovered epoch id + recovered_epoch_id_ = epoch_id; + LOG_INFO("Complete checkpoint recovery for epoch %" PRIu64, epoch_id); + + recovery_timer.Stop(); + LOG_INFO("Checkpoint recovery time: %lf ms", recovery_timer.GetDuration()); + + return true; + } +} + +/** + * @brief Get a recovered epoch id. If not recovered yet, then get a latest + * checkpoint epoch id for recovery from checkpoint directories. + * @return eid_t, or INVALID_EID if failed + */ +eid_t TimestampCheckpointManager::GetRecoveryCheckpointEpoch() { + // already recovered + if (recovered_epoch_id_ != INVALID_EID) { + return recovered_epoch_id_; + } + // for checkpoint recovery + else { + eid_t max_epoch = INVALID_EID; + std::vector dir_name_list; + + // Get list of checkpoint directories + if (LoggingUtil::GetDirectoryList(checkpoint_base_dir_.c_str(), + dir_name_list) == false) { + LOG_ERROR("Failed to get directory list %s", + checkpoint_base_dir_.c_str()); + return INVALID_EID; + } + + // Get the newest epoch from checkpoint directories + for (const auto dir_name : dir_name_list) { + eid_t epoch_id; + + // get the directory name as epoch id + // if it includes character, go next + if ((epoch_id = std::strtoul(dir_name.c_str(), NULL, 10)) == 0) { + continue; + } + + if (epoch_id == INVALID_EID) { + LOG_ERROR("Unexpected epoch value in checkpoints directory: %s", + dir_name.c_str()); + } + + // the largest epoch is a recovered epoch id + if (epoch_id > max_epoch) { + max_epoch = epoch_id; + } + } + LOG_DEBUG("max epoch : %" PRIu64, max_epoch); + return max_epoch; + } +} + +/** + * @brief Do checkpointing and manage checkpoint files/directories + */ +void TimestampCheckpointManager::PerformCheckpointing() { + int count = checkpoint_interval_ - 1; + while (1) { + if (is_running_ == false) { + LOG_INFO("Finish checkpoint thread"); + break; + } + // wait for interval + std::this_thread::sleep_for(std::chrono::seconds(1)); + count++; + if (count == checkpoint_interval_) { + LOG_INFO("Start checkpointing"); + Timer checkpoint_timer; + checkpoint_timer.Start(); + + // create working checkpoint directory + CreateWorkingCheckpointDirectory(); + + // begin transaction and get epoch id as this checkpointing beginning + auto &txn_manager = + concurrency::TimestampOrderingTransactionManager::GetInstance( + ProtocolType::TIMESTAMP_ORDERING, + IsolationLevelType::SERIALIZABLE, ConflictAvoidanceType::WAIT); + auto txn = txn_manager.BeginTransaction(); + cid_t begin_cid = txn->GetReadId(); + eid_t begin_epoch_id = txn->GetEpochId(); + + // create checkpoint + CreateCheckpoints(txn, begin_cid); + + // end transaction + txn_manager.EndTransaction(txn); + + // finalize checkpoint directory: + // 1) move working directory to epoch directory + // 2) remove all of old checkpoints + MoveWorkingToCheckpointDirectory(std::to_string(begin_epoch_id)); + RemoveOldCheckpoints(begin_epoch_id); + + checkpoint_timer.Stop(); + LOG_INFO( + "Complete Checkpointing for epoch %" PRIu64 " (cid = %" PRIu64 ")", + begin_epoch_id, begin_cid); + LOG_INFO("Checkpointing time: %lf ms", checkpoint_timer.GetDuration()); + + count = 0; + } + } +} + +/** + * @brief Create checkpoints for all tables + * @param txn TransactionContext + * @param begin_cid Commit id as time stamp to get checkpoints + */ +void TimestampCheckpointManager::CreateCheckpoints( + concurrency::TransactionContext *txn, + const cid_t begin_cid) { + // prepare for data loading + auto catalog = catalog::Catalog::GetInstance(); + auto storage_manager = storage::StorageManager::GetInstance(); + + // do checkpointing to take tables into each file + for (auto &db_catalog_entry_pair : catalog->GetDatabaseCatalogEntries(txn)) { + auto &db_oid = db_catalog_entry_pair.first; + auto &db_catalog_entry = db_catalog_entry_pair.second; + auto database = storage_manager->GetDatabaseWithOid(db_oid); + + for (auto &table_catalog_entry_pair : db_catalog_entry->GetTableCatalogEntries()) { + auto &table_oid = table_catalog_entry_pair.first; + auto &table_catalog_entry = table_catalog_entry_pair.second; + + // make sure the table exists in this epoch + // and system catalogs in catalog database are out of checkpoint + if (table_catalog_entry == nullptr || + (db_oid == CATALOG_DATABASE_OID && ( + table_catalog_entry->GetTableOid() == SCHEMA_CATALOG_OID || + table_catalog_entry->GetTableOid() == TABLE_CATALOG_OID || + table_catalog_entry->GetTableOid() == COLUMN_CATALOG_OID || + table_catalog_entry->GetTableOid() == INDEX_CATALOG_OID || + table_catalog_entry->GetTableOid() == LAYOUT_CATALOG_OID))) { + continue; + } + + // create a checkpoint file for the table + FileHandle file_handle; + std::string file_name = GetWorkingCheckpointFileFullPath( + db_catalog_entry->GetDatabaseName(), table_catalog_entry->GetSchemaName(), + table_catalog_entry->GetTableName()); + if (LoggingUtil::OpenFile(file_name.c_str(), "wb", file_handle) != true) { + LOG_ERROR("file open error: %s", file_name.c_str()); + return; + } + + LOG_DEBUG("Checkpoint table %d '%s.%s' in database %d '%s'", table_oid, + table_catalog_entry->GetSchemaName().c_str(), + table_catalog_entry->GetTableName().c_str(), + db_catalog_entry->GetDatabaseOid(), + db_catalog_entry->GetDatabaseName().c_str()); + + // insert data to checkpoint file + // if table is catalog, then insert data without tile group info + auto table = database->GetTableWithOid(table_oid); + if (table_catalog_entry->GetSchemaName() == CATALOG_SCHEMA_NAME) { + // settings_catalog_entry takes only persistent values + if (table_catalog_entry->GetTableName() == "pg_settings") { + CheckpointingTableDataWithPersistentCheck(table, begin_cid, + table->GetSchema()->GetColumnID("is_persistent"), + file_handle); + } else { + CheckpointingTableDataWithoutTileGroup(table, begin_cid, file_handle); + } + } else { + CheckpointingTableData(table, begin_cid, file_handle); + } + + LoggingUtil::CloseFile(file_handle); + } // end table loop + + } // end database loop + +} + +/** + * @brief Read a table data and write it down to a checkpoint file + * @param txn TransactionContext + * @param table Table data written to the checkpoint file + * @param begin_cid Commit id as time stamp to get checkpoints + * @param file_handle File handler for the checkpoint file + */ +void TimestampCheckpointManager::CheckpointingTableData( + const storage::DataTable *table, + const cid_t &begin_cid, + FileHandle &file_handle) { + CopySerializeOutput output_buffer; + + LOG_TRACE("Do checkpointing to table %d in database %d", table->GetOid(), + table->GetDatabaseOid()); + + // load all table data + size_t tile_group_count = table->GetTileGroupCount(); + output_buffer.WriteLong(tile_group_count); + LOG_TRACE("Tile group count: %lu", tile_group_count); + for (oid_t tg_offset = START_OID; tg_offset < tile_group_count; tg_offset++) { + auto tile_group = table->GetTileGroup(tg_offset); + auto tile_group_header = tile_group->GetHeader(); + + // write the tile group information + output_buffer.WriteInt(tile_group->GetLayout().GetOid()); + output_buffer.WriteInt(tile_group->GetAllocatedTupleCount()); + + LOG_TRACE("Tile group %u in %d \n%s", tile_group->GetTileGroupId(), + table->GetOid(), tile_group->GetLayout().GetInfo().c_str()); + + // collect and count visible tuples + std::vector visible_tuples; + oid_t max_tuple_count = tile_group->GetNextTupleSlot(); + oid_t column_count = table->GetSchema()->GetColumnCount(); + for (oid_t tuple_id = START_OID; tuple_id < max_tuple_count; tuple_id++) { + if (IsVisible(tile_group_header, tuple_id, begin_cid)) { + // load all field data of each column in the tuple + output_buffer.WriteBool(true); + for (oid_t column_id = START_OID; column_id < column_count; column_id++) { + type::Value value = tile_group->GetValue(tuple_id, column_id); + value.SerializeTo(output_buffer); + LOG_TRACE("%s(column %d, tuple %d):%s\n", table->GetName().c_str(), + column_id, tuple_id, value.ToString().c_str()); + } + } else { + LOG_TRACE("%s's tuple %d is invisible\n", table->GetName().c_str(), + tuple_id); + } + } + output_buffer.WriteBool(false); + + // write down tuple data to file + int ret = fwrite((void *)output_buffer.Data(), output_buffer.Size(), 1, + file_handle.file); + if (ret != 1) { + LOG_ERROR("Write error"); + return; + } + + output_buffer.Reset(); + } + + LoggingUtil::FFlushFsync(file_handle); +} + +/** + * @brief Read a table data without tile group and write it down to + * a checkpoint file. This is used for catalog tables. + * @param txn TransactionContext + * @param table Table data written to the checkpoint file + * @param begin_cid Commit id as time stamp to get checkpoints + * @param file_handle File handler for the checkpoint file + */ +void TimestampCheckpointManager::CheckpointingTableDataWithoutTileGroup( + const storage::DataTable *table, + const cid_t &begin_cid, + FileHandle &file_handle) { + CopySerializeOutput output_buffer; + + LOG_TRACE("Do checkpointing without tile group to table %d in database %d", + table->GetOid(), table->GetDatabaseOid()); + + // load all table data without tile group information + size_t tile_group_count = table->GetTileGroupCount(); + for (oid_t tg_offset = START_OID; tg_offset < tile_group_count; tg_offset++) { + auto tile_group = table->GetTileGroup(tg_offset); + auto tile_group_header = tile_group->GetHeader(); + + // load visible tuples data in the table + auto max_tuple_count = tile_group->GetNextTupleSlot(); + auto column_count = table->GetSchema()->GetColumnCount(); + for (oid_t tuple_id = START_OID; tuple_id < max_tuple_count; tuple_id++) { + if (IsVisible(tile_group_header, tuple_id, begin_cid)) { + // load all field data of each column in the tuple + for (oid_t column_id = START_OID; column_id < column_count; + column_id++) { + type::Value value = tile_group->GetValue(tuple_id, column_id); + value.SerializeTo(output_buffer); + LOG_TRACE("%s(column %d, tuple %d):%s\n", table->GetName().c_str(), + column_id, tuple_id, value.ToString().c_str()); + } + } else { + LOG_TRACE("%s's tuple %d is invisible\n", table->GetName().c_str(), + tuple_id); + } + } + + // write down tuple data to file + int ret = fwrite((void *)output_buffer.Data(), output_buffer.Size(), 1, + file_handle.file); + if (ret != 1 && ret != 0) { + LOG_ERROR("Write error: %d", ret); + return; + } + + output_buffer.Reset(); + } + + LoggingUtil::FFlushFsync(file_handle); +} + +/** + * @brief Read a table data only if persistent flag is true, and write it + * down to a checkpoint file. This is used for catalog tables that + * checkpointing needs to choose whether writing each tuple down or + * not, like pg_settings. + * @param txn TransactionContext + * @param table Table data written to the checkpoint file + * @param begin_cid Commit id as time stamp to get checkpoints + * @param file_handle File handler for the checkpoint file + */ +void TimestampCheckpointManager::CheckpointingTableDataWithPersistentCheck( + const storage::DataTable *table, const cid_t &begin_cid, + oid_t flag_column_id, FileHandle &file_handle) { + CopySerializeOutput output_buffer; + + LOG_TRACE("Do checkpointing without tile group to table %d in database %d", + table->GetOid(), table->GetDatabaseOid()); + + // load all table data without tile group information + size_t tile_group_count = table->GetTileGroupCount(); + for (oid_t tg_offset = START_OID; tg_offset < tile_group_count; tg_offset++) { + auto tile_group = table->GetTileGroup(tg_offset); + auto tile_group_header = tile_group->GetHeader(); + + // load visible tuples data in the table + auto max_tuple_count = tile_group->GetNextTupleSlot(); + auto column_count = table->GetSchema()->GetColumnCount(); + for (oid_t tuple_id = START_OID; tuple_id < max_tuple_count; tuple_id++) { + // check visible and persistent flag. + if (IsVisible(tile_group_header, tuple_id, begin_cid) && + tile_group->GetValue(tuple_id, flag_column_id).IsTrue()) { + // load all field data of each column in the tuple + for (oid_t column_id = START_OID; column_id < column_count; + column_id++) { + type::Value value = tile_group->GetValue(tuple_id, column_id); + value.SerializeTo(output_buffer); + LOG_TRACE("%s(column %d, tuple %d):%s\n", table->GetName().c_str(), + column_id, tuple_id, value.ToString().c_str()); + } + } else { + LOG_TRACE("%s's tuple %d is invisible\n", table->GetName().c_str(), + tuple_id); + } + } + + // write down tuple data to file + int ret = fwrite((void *)output_buffer.Data(), output_buffer.Size(), 1, + file_handle.file); + if (ret != 1 && ret != 0) { + LOG_ERROR("Write error: %d", ret); + return; + } + + output_buffer.Reset(); + } + + LoggingUtil::FFlushFsync(file_handle); +} + +/** + * @brief Whether a tuple is visible in terms of a time stamp + * @param header Header of a tile group that the tuple belongs to + * @param tuple_id Tuple to be checked its visibility. + * @param begin_cid Commit id as time stamp to check the tuple + * @return true if the tuple's commit time is before begin_cid + */ +bool TimestampCheckpointManager::IsVisible( + const storage::TileGroupHeader *header, const oid_t &tuple_id, + const cid_t &begin_cid) { + txn_id_t tuple_txn_id = header->GetTransactionId(tuple_id); + cid_t tuple_begin_cid = header->GetBeginCommitId(tuple_id); + cid_t tuple_end_cid = header->GetEndCommitId(tuple_id); + + // the tuple has already been committed + bool activated = (begin_cid >= tuple_begin_cid); + // the tuple is not visible + bool invalidated = (begin_cid >= tuple_end_cid); + + if (tuple_txn_id == INVALID_TXN_ID) { + // this tuple is not available. + return false; + } + + if (tuple_txn_id == INITIAL_TXN_ID) { + // this tuple is not owned by any other transaction. + return activated && !invalidated; + } else { + // this tuple is owned by other transactions. + if (tuple_begin_cid == MAX_CID) { + // this tuple is an uncommitted version. + return false; + } else { + return activated && !invalidated; + } + } +} + +/** + * @brief Load all catalog table checkpoints and recover them. + * @param txn TransactionContext + * @param epoch_cid Epoch id as time stamp when checkpoints is recovered + */ +bool TimestampCheckpointManager::LoadCatalogTableCheckpoints( + concurrency::TransactionContext *txn, + const eid_t &epoch_id) { + // prepare for catalog data file loading + auto storage_manager = storage::StorageManager::GetInstance(); + auto catalog = catalog::Catalog::GetInstance(); + + // first, recover all catalogs within catalog database + auto catalog_db_catalog_entry = catalog->GetDatabaseCatalogEntry(txn, CATALOG_DATABASE_OID); + PELOTON_ASSERT(catalog_db_catalog_entry != nullptr); + for (auto &table_catalog_entry : + catalog_db_catalog_entry->GetTableCatalogEntries((std::string)CATALOG_SCHEMA_NAME)) { + if (!LoadCatalogTableCheckpoint(txn, + epoch_id, + catalog_db_catalog_entry->GetDatabaseOid(), + table_catalog_entry->GetTableOid())) { + return false; + } + } + + // recover all catalog tables within each database + for (auto &db_catalog_entry_pair : catalog->GetDatabaseCatalogEntries(txn)) { + auto &db_oid = db_catalog_entry_pair.first; + auto &db_catalog_entry = db_catalog_entry_pair.second; + + // catalog database has already been recovered + if (db_oid == CATALOG_DATABASE_OID) continue; + + // load system catalogs in the database + // if not exist, then create the database storage object and catalogs + if (storage_manager->HasDatabase(db_oid) == false) { + LOG_DEBUG("Create database storage object %d '%s'", db_oid, + db_catalog_entry->GetDatabaseName().c_str()); + LOG_DEBUG("And create system catalog objects for this database"); + + // create database storage object + auto database = new storage::Database(db_oid); + // TODO: This should be deprecated, dbname should only exists in pg_db + database->setDBName(db_catalog_entry->GetDatabaseName()); + storage_manager->AddDatabaseToStorageManager(database); + + // put database object into rw_object_set + txn->RecordCreate(db_oid, INVALID_OID, INVALID_OID); + + // add core & non-core system catalog tables into database + catalog->BootstrapSystemCatalogs(txn, database); + catalog->catalog_map_[db_oid]->Bootstrap(txn, + db_catalog_entry->GetDatabaseName()); + } else { + LOG_DEBUG("Use existed database storage object %d '%s'", db_oid, + db_catalog_entry->GetDatabaseName().c_str()); + LOG_DEBUG("And use its system catalog objects"); + } + + for (auto &table_catalog_entry : + db_catalog_entry->GetTableCatalogEntries((std::string)CATALOG_SCHEMA_NAME)) { + if (!LoadCatalogTableCheckpoint(txn, + epoch_id, + db_oid, + table_catalog_entry->GetTableOid())) { + return false; + } + } // table loop end + + } // database loop end + + return true; +} + +/** + * @brief Load a catalog table checkpoint and recover it. + * @param txn TransactionContext + * @param epoch_cid Epoch id as time stamp when checkpoints is recovered + * @param db_oid Database in which the recovered table is + * @param table_oid Catalog table recovered + */ +bool TimestampCheckpointManager::LoadCatalogTableCheckpoint( + concurrency::TransactionContext *txn, + const eid_t &epoch_id, + const oid_t db_oid, + const oid_t table_oid) { + auto catalog = catalog::Catalog::GetInstance(); + auto db_catalog_entry = catalog->GetDatabaseCatalogEntry(txn, db_oid); + auto table_catalog_entry = db_catalog_entry->GetTableCatalogEntry(table_oid); + auto &table_name = table_catalog_entry->GetTableName(); + auto system_catalogs = catalog->GetSystemCatalogs(db_oid); + PELOTON_ASSERT(table_catalog_entry->GetSchemaName() == CATALOG_SCHEMA_NAME); + + auto storage_manager = storage::StorageManager::GetInstance(); + auto table = storage_manager->GetTableWithOid(db_oid, table_oid); + + // load checkpoint files for catalog data. + // except for catalogs that initialized in unsupported area: + // ColumnStatsCatalog, ZoneMapCatalog + // TODO: Move these catalogs' initialization to catalog bootstrap + // except for catalogs requiring to select recovered values: + // SettingsCatalog. + // TODO: Implement a logic for selecting recovered values + + // catalogs out of recovery + if (table_name == "pg_column_stats" || table_name == "zone_map" || + (db_oid == CATALOG_DATABASE_OID && ( + table_oid == SCHEMA_CATALOG_OID || table_oid == TABLE_CATALOG_OID || + table_oid == COLUMN_CATALOG_OID || table_oid == INDEX_CATALOG_OID || + table_oid == LAYOUT_CATALOG_OID || table_oid == CONSTRAINT_CATALOG_OID))) { + // nothing to do (keep the default values, and not recover other data) + } else { + // read a checkpoint file for the catalog + oid_t oid_align; + FileHandle table_file; + std::string table_filename = GetCheckpointFileFullPath( + db_catalog_entry->GetDatabaseName(), table_catalog_entry->GetSchemaName(), + table_name, epoch_id); + if (!LoggingUtil::OpenFile(table_filename.c_str(), "rb", table_file)) { + // Not existed here means that there is not the table in last checkpoint + // because last checkpointing for the table was failed or + // related settings like 'brain' was disabled in the checkpointing + LOG_ERROR("Checkpoint file for catalog table %d '%s' is not existed", + table_catalog_entry->GetTableOid(), table_name.c_str()); + return true; + } + + LOG_DEBUG("Recover catalog table %d '%s.%s' in database %d '%s'", + table_catalog_entry->GetTableOid(), + table_catalog_entry->GetSchemaName().c_str(), table_name.c_str(), + db_catalog_entry->GetDatabaseOid(), + db_catalog_entry->GetDatabaseName().c_str()); + + // catalogs with duplicate check + // keep the embedded tuples which are stored in Peloton initialization, + // but user tuples is recovered + if (table_oid == DATABASE_CATALOG_OID || table_oid == SCHEMA_CATALOG_OID || + table_oid == TABLE_CATALOG_OID || table_oid == COLUMN_CATALOG_OID || + table_oid == INDEX_CATALOG_OID || table_oid == LAYOUT_CATALOG_OID || + table_oid == CONSTRAINT_CATALOG_OID || table_name == "pg_language" || + table_name == "pg_proc") { + oid_align = + RecoverTableDataWithDuplicateCheck(txn, table, table_file); + } + // catalogs with persistent check + // recover settings which need to update embedded tuples with user set values. + else if (table_name == "pg_settings") { + std::vector key_column_ids = {table->GetSchema()->GetColumnID("name")}; + oid_t flag_column_id = table->GetSchema()->GetColumnID("is_persistent"); + oid_align = + RecoverTableDataWithPersistentCheck(txn, + table, + table_file, + key_column_ids, + flag_column_id); + // Reload settings data + // TODO: settings::SettingsManager::GetInstance()->UpdateSettingListFromCatalog(txn) + } + // catalogs to be recovered without duplicate check + // these catalog tables have no embedded tuples + // Targets: pg_trigger, pg_*_metrics, pg_query_history, pg_column_stats, zone_map + else { + oid_align = RecoverTableDataWithoutTileGroup(txn, table, table_file); + } + + LoggingUtil::CloseFile(table_file); + + // modify next OID of each catalog + if (table_oid == DATABASE_CATALOG_OID) { + catalog::DatabaseCatalog::GetInstance()->UpdateOid(oid_align); + } else if (table_oid == SCHEMA_CATALOG_OID) { + system_catalogs->GetSchemaCatalog()->UpdateOid(oid_align); + } else if (table_oid == TABLE_CATALOG_OID) { + system_catalogs->GetTableCatalog()->UpdateOid(oid_align); + } else if (table_oid == COLUMN_CATALOG_OID) { + system_catalogs->GetColumnCatalog()->UpdateOid(oid_align); + } else if (table_oid == INDEX_CATALOG_OID) { + system_catalogs->GetIndexCatalog()->UpdateOid(oid_align); + } else if (table_oid == LAYOUT_CATALOG_OID) { + // Layout OID is controlled within each DataTable object + } else if (table_oid == CONSTRAINT_CATALOG_OID) { + system_catalogs->GetConstraintCatalog()->UpdateOid(oid_align); + } else if (table_name == "pg_proc") { + catalog::ProcCatalog::GetInstance().UpdateOid(oid_align); + } else if (table_name == "pg_trigger") { + system_catalogs->GetTriggerCatalog()->UpdateOid(oid_align); + } else if (table_name == "pg_language") { + catalog::LanguageCatalog::GetInstance().UpdateOid(oid_align); + } else if (table_name == "pg_settings" || table_name == "pg_database_metrics" || + table_name == "pg_table_metrics" || table_name == "pg_index_metrics" || + table_name == "pg_query_metrics" || table_name == "pg_query_history" || + table_name == "pg_column_stats" || table_name == "zone_map") { + // OID is not used + } else { + LOG_ERROR("Unexpected catalog table appears: %s", table_name.c_str()); + PELOTON_ASSERT(false); + } + } + + return true; +} + +/** + * @brief Load all user table checkpoints and recover them. + * @param txn TransactionContext + * @param epoch_cid Epoch id as time stamp when checkpoints is recovered + */ +bool TimestampCheckpointManager::LoadUserTableCheckpoints( + concurrency::TransactionContext *txn, + const eid_t &epoch_id) { + // Recover table + auto storage_manager = storage::StorageManager::GetInstance(); + auto catalog = catalog::Catalog::GetInstance(); + for (auto &db_catalog_entry_pair : catalog->GetDatabaseCatalogEntries(txn)) { + auto &db_oid = db_catalog_entry_pair.first; + auto database = storage_manager->GetDatabaseWithOid(db_oid); + auto &db_catalog_entry = db_catalog_entry_pair.second; + + // the catalog database has already been recovered. + if (db_oid == CATALOG_DATABASE_OID) continue; + + for (auto &table_catalog_entry_pair : db_catalog_entry->GetTableCatalogEntries()) { + auto &table_oid = table_catalog_entry_pair.first; + auto &table_catalog_entry = table_catalog_entry_pair.second; + + // catalog tables in each database have already been recovered + if (table_catalog_entry->GetSchemaName() == CATALOG_SCHEMA_NAME) continue; + + // Recover storage object of the table + if (!RecoverTableStorageObject(txn, db_oid, table_oid)) { + LOG_ERROR("Storage object recovery for table %s failed", + table_catalog_entry->GetTableName().c_str()); + continue; + } + + // read a checkpoint file for the catalog + FileHandle table_file; + std::string table_filename = GetCheckpointFileFullPath( + db_catalog_entry->GetDatabaseName(), table_catalog_entry->GetSchemaName(), + table_catalog_entry->GetTableName(), epoch_id); + if (!LoggingUtil::OpenFile(table_filename.c_str(), "rb", table_file)) { + LOG_ERROR("Checkpoint file for table %d '%s' is not existed", table_oid, + table_catalog_entry->GetTableName().c_str()); + continue; + } + + LOG_DEBUG("Recover user table %d '%s.%s' in database %d '%s'", table_oid, + table_catalog_entry->GetSchemaName().c_str(), + table_catalog_entry->GetTableName().c_str(), + db_catalog_entry->GetDatabaseOid(), + db_catalog_entry->GetDatabaseName().c_str()); + + // recover the table from the checkpoint file + RecoverTableData(txn, database->GetTableWithOid(table_oid), table_file); + + LoggingUtil::CloseFile(table_file); + + } // table loop end + + // register foreign key to sink table + for (auto &table_catalog_entry_pair : db_catalog_entry->GetTableCatalogEntries()) { + auto source_table_schema = + database->GetTableWithOid(table_catalog_entry_pair.first)->GetSchema(); + if (source_table_schema->HasForeignKeys()) { + for(auto constraint : source_table_schema->GetForeignKeyConstraints()) { + auto sink_table = database->GetTableWithOid(constraint->GetFKSinkTableOid()); + sink_table->GetSchema()->RegisterForeignKeySource(constraint); + } + } + } + + } // database loop end + + return true; +} + +/** + * @brief Recover a table storage object from related catalogs. + * @param txn TransactionContext + * @param db_oid Database in which the recovered table is + * @param table_oid Table which storage object is recovered + * Note: Foreign key constraint is recovered just for the source table. + * Caller has to register its constraint in the sink table. + */ +bool TimestampCheckpointManager::RecoverTableStorageObject( + concurrency::TransactionContext *txn, + const oid_t db_oid, + const oid_t table_oid) { + auto database = + storage::StorageManager::GetInstance()->GetDatabaseWithOid(db_oid); + auto table_catalog_entry = + catalog::Catalog::GetInstance()->GetTableCatalogEntry(txn, db_oid, table_oid); + + // This function targets user tables rather than catalogs + PELOTON_ASSERT(db_oid != CATALOG_DATABASE_OID); + PELOTON_ASSERT(table_catalog_entry->GetSchemaName() != CATALOG_SCHEMA_NAME); + + LOG_DEBUG("Create table storage object %d '%s.%s'", table_oid, + table_catalog_entry->GetSchemaName().c_str(), + table_catalog_entry->GetTableName().c_str()); + + // recover column information + std::vector columns; + for (auto column_catalog_entry_pair : table_catalog_entry->GetColumnCatalogEntries()) { + auto column_oid = column_catalog_entry_pair.first; + auto column_catalog_entry = column_catalog_entry_pair.second; + + // create column storage object + auto column = catalog::Column(column_catalog_entry->GetColumnType(), + column_catalog_entry->GetColumnLength(), + column_catalog_entry->GetColumnName(), + column_catalog_entry->IsInlined(), + column_catalog_entry->GetColumnOffset()); + + // recover column constraints: NOT NULL + if (column_catalog_entry->IsNotNull()) { + column.SetNotNull(); + } + + // recover column constraints: DEFAULT + if (column_catalog_entry->HasDefault()) { + column.SetDefaultValue(column_catalog_entry->GetDefaultValue()); + } + + // Set a column into the vector in order of the column_oid. + // Cannot use push_back of vector because column_catalog_entries doesn't acquire + // the column info in the order from pg_attribute. + auto column_itr = columns.begin(); + for (oid_t idx_count = START_OID; idx_count < column_oid; idx_count++) { + if (column_itr == columns.end() || + column_itr->GetOffset() > column.GetOffset()) { + break; + } else { + column_itr++; + } + } + columns.insert(column_itr, column);; + + } // column loop end + + // create schema for the table + std::unique_ptr schema(new catalog::Schema(columns)); + + // recover default layout + auto default_layout = + table_catalog_entry->GetLayout(table_catalog_entry->GetDefaultLayoutOid()); + + // create table storage object. + // if the default layout type is hybrid, set default layout separately + bool own_schema = true; + bool adapt_table = false; + bool is_catalog = false; + storage::DataTable *table; + if (default_layout->IsHybridStore()) { + table = + storage::TableFactory::GetDataTable(db_oid, + table_oid, + schema.release(), + table_catalog_entry->GetTableName(), + DEFAULT_TUPLES_PER_TILEGROUP, + own_schema, + adapt_table, + is_catalog); + table->SetDefaultLayout(default_layout); + // adjust layout oid value + table->GetNextLayoutOid(); + } else { + table = + storage::TableFactory::GetDataTable(db_oid, + table_oid, + schema.release(), + table_catalog_entry->GetTableName(), + DEFAULT_TUPLES_PER_TILEGROUP, + own_schema, + adapt_table, + is_catalog, + default_layout->GetLayoutType()); + } + database->AddTable(table, is_catalog); + + // put data table object into rw_object_set + txn->RecordCreate(db_oid, table_oid, INVALID_OID); + + // recover triggers of the storage table + table->UpdateTriggerListFromCatalog(txn); + + // recover index storage objects + for (auto &index_catalog_entry_pair : + table_catalog_entry->GetIndexCatalogEntries()) { + auto index_oid = index_catalog_entry_pair.first; + auto index_catalog_entry = index_catalog_entry_pair.second; + + LOG_TRACE( + "|- Index %d '%s': Index type %s, Index constraint %s, unique " + "keys %d", + index_oid, index_catalog_entry->GetIndexName().c_str(), + IndexTypeToString(index_catalog_entry->GetIndexType()).c_str(), + IndexConstraintTypeToString(index_catalog_entry->GetIndexConstraint()) + .c_str(), + index_catalog_entry->HasUniqueKeys()); + + auto &key_attrs = index_catalog_entry->GetKeyAttrs(); + auto key_schema = + catalog::Schema::CopySchema(table->GetSchema(), key_attrs); + key_schema->SetIndexedColumns(key_attrs); + + // Set index metadata + auto index_metadata = + new index::IndexMetadata(index_catalog_entry->GetIndexName(), + index_oid, + table_oid, + db_oid, + index_catalog_entry->GetIndexType(), + index_catalog_entry->GetIndexConstraint(), + table->GetSchema(), + key_schema, + key_attrs, + index_catalog_entry->HasUniqueKeys()); + + // create index storage objects and add it to the table + std::shared_ptr index( + index::IndexFactory::GetIndex(index_metadata)); + table->AddIndex(index); + + // Put index object into rw_object_set + txn->RecordCreate(db_oid, table_oid, index_oid); + + } // index loop end + + // recover table constraints + for (auto constraint_catalog_entry_pair : + table_catalog_entry->GetConstraintCatalogEntries()) { + auto constraint_oid = constraint_catalog_entry_pair.first; + auto constraint_catalog_entry = constraint_catalog_entry_pair.second; + + // create constraint + std::shared_ptr constraint; + switch (constraint_catalog_entry->GetConstraintType()) { + case ConstraintType::PRIMARY: + case ConstraintType::UNIQUE: { + constraint = + std::make_shared(constraint_oid, + constraint_catalog_entry->GetConstraintType(), + constraint_catalog_entry->GetConstraintName(), + constraint_catalog_entry->GetTableOid(), + constraint_catalog_entry->GetColumnIds(), + constraint_catalog_entry->GetIndexOid()); + break; + } + case ConstraintType::FOREIGN: { + constraint = + std::make_shared(constraint_oid, + constraint_catalog_entry->GetConstraintType(), + constraint_catalog_entry->GetConstraintName(), + constraint_catalog_entry->GetTableOid(), + constraint_catalog_entry->GetColumnIds(), + constraint_catalog_entry->GetIndexOid(), + constraint_catalog_entry->GetFKSinkTableOid(), + constraint_catalog_entry->GetFKSinkColumnIds(), + constraint_catalog_entry->GetFKUpdateAction(), + constraint_catalog_entry->GetFKDeleteAction()); + // cannot register foreign key into a sink table here + // because the sink table might not be recovered yet. + break; + } + case ConstraintType::CHECK: { + constraint = + std::make_shared(constraint_oid, + constraint_catalog_entry->GetConstraintType(), + constraint_catalog_entry->GetConstraintName(), + constraint_catalog_entry->GetTableOid(), + constraint_catalog_entry->GetColumnIds(), + constraint_catalog_entry->GetIndexOid(), + constraint_catalog_entry->GetCheckExp()); + break; + } + default: + LOG_ERROR("Unexpected constraint type is appeared: %s", + ConstraintTypeToString(constraint_catalog_entry->GetConstraintType()).c_str()); + break; + } + + LOG_TRACE("|- %s", constraint->GetInfo().c_str()); + + // set the constraint into the table + table->GetSchema()->AddConstraint(constraint); + + } // constraint loop end + + + return true; +} + +/** + * @brief Recover a table data from its checkpoint file. + * @param txn TransactionContext + * @param table Table storage object which data is recovered + * @param file_handle File handler for the checkpoint file + */ +void TimestampCheckpointManager::RecoverTableData( + concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle) { + size_t table_size = LoggingUtil::GetFileSize(file_handle); + if (table_size == 0) return; + std::unique_ptr data(new char[table_size]); + if (!LoggingUtil::ReadNBytesFromFile(file_handle, data.get(), table_size)) { + LOG_ERROR("Checkpoint table file read error"); + return; + } + CopySerializeInput input_buffer(data.get(), table_size); + + LOG_TRACE("Recover table %d data (%lu byte)", table->GetOid(), table_size); + + // Drop all default tile groups created by table catalog recovery + table->DropTileGroups(); + + // Create tile group + std::unique_ptr pool(new type::EphemeralPool()); + auto schema = table->GetSchema(); + auto default_layout = table->GetDefaultLayout(); + auto column_count = schema->GetColumnCount(); + oid_t tile_group_count = input_buffer.ReadLong(); + for (oid_t tg_idx = START_OID; tg_idx < tile_group_count; tg_idx++) { + // recover layout for this tile group + oid_t layout_oid = input_buffer.ReadInt(); + std::shared_ptr layout; + if (default_layout->GetOid() != layout_oid) { + layout = catalog::Catalog::GetInstance() + ->GetTableCatalogEntry(txn, table->GetDatabaseOid(), table->GetOid()) + ->GetLayout(layout_oid); + } else { + layout = default_layout; + } + + // The tile_group_id can't be recovered. + // Because if active_tile_group_count_ in DataTable class is changed after + // restart (e.g. in case of change of connection_thread_count setting), + // then a recovered tile_group_id might get collision with a tile_group_id + // which set for the default tile group of a table. + oid_t tile_group_id = + storage::StorageManager::GetInstance()->GetNextTileGroupId(); + oid_t allocated_tuple_count = input_buffer.ReadInt(); + + // recover tile group + auto layout_schemas = layout->GetLayoutSchemas(schema); + std::shared_ptr tile_group( + storage::TileGroupFactory::GetTileGroup(table->GetDatabaseOid(), + table->GetOid(), + tile_group_id, + table, + layout_schemas, + layout, + allocated_tuple_count)); + + LOG_TRACE("Recover tile group %u in %d \n%s", tile_group->GetTileGroupId(), + table->GetOid(), tile_group->GetLayout().GetInfo().c_str()); + + // add the tile group to table + oid_t active_tile_group_id = tg_idx % table->GetActiveTileGroupCount(); + table->AddTileGroup(tile_group, active_tile_group_id); + + // recover tuples located in the tile group + while (input_buffer.ReadBool()) { + // recover values on each column + std::unique_ptr tuple(new storage::Tuple(schema, true)); + for (oid_t column_id = 0; column_id < column_count; column_id++) { + auto value = type::Value::DeserializeFrom(input_buffer, + schema->GetType(column_id), + NULL); + tuple->SetValue(column_id, value, pool.get()); + } + + // insert the tuple into the tile group + oid_t tuple_slot = tile_group->InsertTuple(tuple.get()); + ItemPointer location(tile_group->GetTileGroupId(), tuple_slot); + if (location.block != INVALID_OID) { + // register the location of the inserted tuple to the table without + // foreign key check to avoid an error which occurs in tables with + // the mutual foreign keys each other + ItemPointer *index_entry_ptr = nullptr; + if (table->InsertTuple(tuple.get(), + location, + txn, + &index_entry_ptr, + false)) { + concurrency::TransactionManagerFactory::GetInstance().PerformInsert( + txn, location, index_entry_ptr); + } else { + LOG_ERROR("Tuple insert error for table %d", table->GetOid()); + } + } else { + LOG_ERROR("Tuple insert error for tile group %d of table %d", + tile_group->GetTileGroupId(), table->GetOid()); + } + + } // tuple loop end + + } // tile group loop end + + // if # of recovered tile_groups is smaller than # of active tile + // groups, then create default tile group for rest of the slots of + // active tile groups + for (auto active_tile_group_id = tile_group_count; + active_tile_group_id < table->GetActiveTileGroupCount(); + active_tile_group_id++) { + table->AddDefaultTileGroup(active_tile_group_id); + } +} + +/** + * @brief Recover a table data without tile group from its checkpoint file. + * This function is provided for catalog tables initialized without + * default values, like pg_trigger. + * @param txn TransactionContext + * @param table Table storage object which data is recovered + * @param file_handle File handler for the checkpoint file + * @return Tuple count inserted into the table + */ +oid_t TimestampCheckpointManager::RecoverTableDataWithoutTileGroup( + concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle) { + size_t table_size = LoggingUtil::GetFileSize(file_handle); + if (table_size == 0) return 0; + std::unique_ptr data(new char[table_size]); + if (!LoggingUtil::ReadNBytesFromFile(file_handle, data.get(), table_size)) { + LOG_ERROR("Checkpoint table file read error"); + return 0; + } + CopySerializeInput input_buffer(data.get(), table_size); + + LOG_TRACE("Recover table %d data without tile group (%lu byte)", + table->GetOid(), table_size); + + // recover table tuples + std::unique_ptr pool(new type::EphemeralPool()); + oid_t insert_tuple_count = 0; + auto schema = table->GetSchema(); + oid_t column_count = schema->GetColumnCount(); + while (input_buffer.RestSize() > 0) { + // recover values on each column + std::unique_ptr tuple(new storage::Tuple(schema, true)); + ItemPointer *index_entry_ptr = nullptr; + for (oid_t column_id = 0; column_id < column_count; column_id++) { + auto value = type::Value::DeserializeFrom(input_buffer, + schema->GetType(column_id), + NULL); + tuple->SetValue(column_id, value, pool.get()); + } + + // insert tuple into the table without foreign key check to avoid an error + // which occurs in tables with the mutual foreign keys each other + ItemPointer location = + table->InsertTuple(tuple.get(), txn, &index_entry_ptr, false); + if (location.block != INVALID_OID) { + concurrency::TransactionManagerFactory::GetInstance().PerformInsert( + txn, location, index_entry_ptr); + insert_tuple_count++; + } else { + LOG_ERROR("Tuple insert error for table %d", table->GetOid()); + } + } + + return insert_tuple_count; +} + +/** + * @brief Recover a table data without tile group from its checkpoint file, + * and keep values already inserted. This is provided for catalogs + * which has default values inserted in Peloton initialization. + * @param txn TransactionContext + * @param table Table storage object which data is recovered + * @param file_handle File handler for the checkpoint file + * @return Tuple count inserted into the table + */ +oid_t TimestampCheckpointManager::RecoverTableDataWithDuplicateCheck( + concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle) { + size_t table_size = LoggingUtil::GetFileSize(file_handle); + if (table_size == 0) return 0; + std::unique_ptr data(new char[table_size]); + if (!LoggingUtil::ReadNBytesFromFile(file_handle, data.get(), table_size)) { + LOG_ERROR("Checkpoint table file read error"); + return 0; + } + CopySerializeInput input_buffer(data.get(), table_size); + + LOG_TRACE("Recover table %d data with duplicate check (%lu byte)", + table->GetOid(), table_size); + + // look for all primary key columns + std::vector pk_columns; + auto schema = table->GetSchema(); + PELOTON_ASSERT(schema->HasPrimary()); + oid_t column_count = schema->GetColumnCount(); + for (auto constraint : schema->GetConstraints()) { + if (constraint.second->GetType() == ConstraintType::PRIMARY) { + pk_columns = constraint.second->GetColumnIds(); + } + } + + // recover table tuples + std::unique_ptr pool(new type::EphemeralPool()); + oid_t insert_tuple_count = 0; + while (input_buffer.RestSize() > 0) { + // recover values on each column + std::unique_ptr tuple(new storage::Tuple(schema, true)); + ItemPointer *index_entry_ptr = nullptr; + for (oid_t column_id = 0; column_id < column_count; column_id++) { + auto value = type::Value::DeserializeFrom(input_buffer, + schema->GetType(column_id), + NULL); + tuple->SetValue(column_id, value, pool.get()); + } + + // duplicate check + // if all primary key values are existed, the tuple is not stored in the + // table + bool duplicated = false; + for (oid_t tg_offset = 0; tg_offset < table->GetTileGroupCount(); + tg_offset++) { + auto tile_group = table->GetTileGroup(tg_offset); + auto max_tuple_count = tile_group->GetNextTupleSlot(); + for (oid_t tuple_id = 0; tuple_id < max_tuple_count; tuple_id++) { + // check all primary key columns + bool check_all_pk_values_same = true; + for (auto pk_column : pk_columns) { + if (tile_group->GetValue(tuple_id, pk_column) + .CompareNotEquals(tuple->GetValue(pk_column)) + == CmpBool::CmpTrue) { + check_all_pk_values_same = false; + break; + } + } + if (check_all_pk_values_same) { + duplicated = true; + break; + } + } + } + + // if not duplicated, insert the tuple + if (!duplicated) { + // insert tuple into the table without foreign key check to avoid an error + // which occurs in tables with the mutual foreign keys each other + ItemPointer location = + table->InsertTuple(tuple.get(), txn, &index_entry_ptr, false); + if (location.block != INVALID_OID) { + concurrency::TransactionManagerFactory::GetInstance().PerformInsert( + txn, location, index_entry_ptr); + insert_tuple_count++; + } else { + LOG_ERROR("Tuple insert error for table %d", table->GetOid()); + } + } + } + + return insert_tuple_count; +} + +/** + * @brief Recover a table data without tile group from its checkpoint file. + * If a recovered tuple's key equals to already inserted tuple's key + * and its tuple's persistent flag is true, then overwrite the inserted + * tuple by the recovered tuple. This is now provided for pg_settings. + * @param txn TransactionContext + * @param table Table storage object which data is recovered + * @param file_handle File handler for the checkpoint file + * @param key_column_ids Column for key matching to overwrite the tuple + * @param flag_column_id Column for checking persistency of the tuple + * @return Tuple count inserted into the table + */ +oid_t TimestampCheckpointManager::RecoverTableDataWithPersistentCheck( + concurrency::TransactionContext *txn, + storage::DataTable *table, + FileHandle &file_handle, + std::vector key_colmun_ids, + oid_t flag_column_id) { + size_t table_size = LoggingUtil::GetFileSize(file_handle); + if (table_size == 0) return 0; + std::unique_ptr data(new char[table_size]); + if (!LoggingUtil::ReadNBytesFromFile(file_handle, data.get(), table_size)) { + LOG_ERROR("Checkpoint table file read error"); + return 0; + } + CopySerializeInput input_buffer(data.get(), table_size); + + LOG_TRACE("Recover table %d data with persistent check (%lu byte)", + table->GetOid(), table_size); + + // recover table tuples + std::unique_ptr pool(new type::EphemeralPool()); + oid_t insert_tuple_count = 0; + auto schema = table->GetSchema(); + oid_t column_count = schema->GetColumnCount(); + while (input_buffer.RestSize() > 0) { + // recover values on each column + std::unique_ptr tuple(new storage::Tuple(schema, true)); + ItemPointer *index_entry_ptr = nullptr; + for (oid_t column_id = 0; column_id < column_count; column_id++) { + auto value = type::Value::DeserializeFrom(input_buffer, + schema->GetType(column_id), + NULL); + tuple->SetValue(column_id, value, pool.get()); + } + + // Persistent check + // If same name tuple exists in the table already, delete its tuple. + // If the tuple's persistent is changed to false, then recovered tuple + // is discarded + storage::Tuple target_tuple; + bool do_insert = true; + for (oid_t tg_offset = 0; tg_offset < table->GetTileGroupCount(); + tg_offset++) { + auto tile_group = table->GetTileGroup(tg_offset); + auto max_tuple_count = tile_group->GetNextTupleSlot(); + for (oid_t tuple_id = 0; tuple_id < max_tuple_count; tuple_id++) { + // check all key columns which identify the tuple + bool check_all_values_same = true; + for (auto key_column : key_colmun_ids) { + if (tile_group->GetValue(tuple_id, key_column) + .CompareNotEquals(tuple->GetValue(key_column)) + == CmpBool::CmpTrue) { + check_all_values_same = false; + break; + } + } + + // If found a same tuple and its parameter is persistent, + // then update it, + if (check_all_values_same && + tile_group->GetValue(tuple_id, flag_column_id).IsTrue()) { + for (oid_t column_id = 0; column_id < column_count; column_id++) { + auto new_value = tuple->GetValue(column_id); + tile_group->SetValue(new_value, tuple_id, column_id); + } + do_insert = false; + break; + } + } + } + + // If there is no same tuple in the table, insert it. + if (do_insert) { + // insert tuple into the table without foreign key check to avoid an error + // which occurs in tables with the mutual foreign keys each other + ItemPointer location = + table->InsertTuple(tuple.get(), txn, &index_entry_ptr, false); + if (location.block != INVALID_OID) { + concurrency::TransactionManagerFactory::GetInstance().PerformInsert( + txn, location, index_entry_ptr); + insert_tuple_count++; + } else { + LOG_ERROR("Tuple insert error for table %d", table->GetOid()); + } + } + } + + return insert_tuple_count; +} + +} // namespace logging +} // namespace peloton diff --git a/src/planner/plan_util.cpp b/src/planner/plan_util.cpp index aecedb80486..a0e43e90d91 100644 --- a/src/planner/plan_util.cpp +++ b/src/planner/plan_util.cpp @@ -56,7 +56,7 @@ const std::set PlanUtil::GetAffectedIndexes( table_name = delete_stmt.GetTableName(); schema_name = delete_stmt.GetSchemaName(); } - auto indexes_map = catalog_cache.GetDatabaseObject(db_name) + auto indexes_map = catalog_cache.GetDatabaseCatalogEntry(db_name) ->GetTableCatalogEntry(table_name, schema_name) ->GetIndexCatalogEntries(); for (auto &index : indexes_map) { @@ -69,7 +69,7 @@ const std::set PlanUtil::GetAffectedIndexes( db_name = update_stmt.table->GetDatabaseName(); table_name = update_stmt.table->GetTableName(); schema_name = update_stmt.table->GetSchemaName(); - auto table_object = catalog_cache.GetDatabaseObject(db_name) + auto table_object = catalog_cache.GetDatabaseCatalogEntry(db_name) ->GetTableCatalogEntry(table_name, schema_name); auto &update_clauses = update_stmt.updates; @@ -133,7 +133,7 @@ const std::vector PlanUtil::GetIndexableColumns( try { auto plan = optimizer->BuildPelotonPlanTree(sql_stmt_list, txn); - auto db_object = catalog_cache.GetDatabaseObject(db_name); + auto db_object = catalog_cache.GetDatabaseCatalogEntry(db_name); database_id = db_object->GetDatabaseOid(); // Perform a breadth first search on plan tree diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 85240e196e1..e6cbcba1438 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -905,52 +905,15 @@ oid_t DataTable::AddDefaultTileGroup(const size_t &active_tile_group_id) { return tile_group_id; } -void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { - PELOTON_ASSERT(tile_group_id); - - std::vector schemas; - schemas.push_back(*schema); - std::shared_ptr layout = nullptr; - - // The TileGroup for recovery is always added in ROW layout, - // This was a part of the previous design. If you are planning - // to change this, make sure the layout is added to the catalog - if (default_layout_->IsRowStore()) { - layout = default_layout_; - } else { - layout = std::shared_ptr( - new const Layout(schema->GetColumnCount())); - } - - std::shared_ptr tile_group(TileGroupFactory::GetTileGroup( - database_oid, table_oid, tile_group_id, this, schemas, layout, - tuples_per_tilegroup_)); - - auto tile_groups_exists = tile_groups_.Contains(tile_group_id); - - if (tile_groups_exists == false) { - tile_groups_.Append(tile_group_id); - - LOG_TRACE("Added a tile group "); - - // add tile group metadata in locator - storage::StorageManager::GetInstance()->AddTileGroup(tile_group_id, - tile_group); - - // we must guarantee that the compiler always add tile group before adding - // tile_group_count_. - COMPILER_MEMORY_FENCE; - - tile_group_count_++; - - LOG_TRACE("Recording tile group : %u ", tile_group_id); - } -} - // NOTE: This function is only used in test cases. void DataTable::AddTileGroup(const std::shared_ptr &tile_group) { size_t active_tile_group_id = number_of_tuples_ % active_tilegroup_count_; + AddTileGroup(tile_group, active_tile_group_id); +} +void DataTable::AddTileGroup( + const std::shared_ptr &tile_group, + const size_t &active_tile_group_id) { active_tile_groups_[active_tile_group_id] = tile_group; oid_t tile_group_id = tile_group->GetTileGroupId(); @@ -1004,8 +967,10 @@ void DataTable::DropTileGroups() { } // Clear array + active_tile_groups_.clear(); tile_groups_.Clear(); + active_tile_groups_.resize(active_tilegroup_count_); tile_group_count_ = 0; } diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 8458d167dd0..296f13a6d87 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -14,6 +14,8 @@ #include +#include "catalog/manager.h" +#include "catalog/schema.h" #include "storage/storage_manager.h" #include "common/container_tuple.h" #include "common/internal_types.h" @@ -77,9 +79,7 @@ type::AbstractPool *TileGroup::GetTilePool(const oid_t tile_id) const { return nullptr; } -oid_t TileGroup::GetTileGroupId() const { - return tile_group_id; -} +oid_t TileGroup::GetTileGroupId() const { return tile_group_id; } // TODO: check when this function is called. --Yingjun oid_t TileGroup::GetNextTupleSlot() const { @@ -92,7 +92,6 @@ oid_t TileGroup::GetActiveTupleCount() const { return tile_group_header->GetActiveTupleCount(); } - //===--------------------------------------------------------------------===// // Operations //===--------------------------------------------------------------------===// @@ -156,7 +155,7 @@ oid_t TileGroup::InsertTuple(const Tuple *tuple) { // Set MVCC info PELOTON_ASSERT(tile_group_header->GetTransactionId(tuple_slot_id) == - INVALID_TXN_ID); + INVALID_TXN_ID); PELOTON_ASSERT(tile_group_header->GetBeginCommitId(tuple_slot_id) == MAX_CID); PELOTON_ASSERT(tile_group_header->GetEndCommitId(tuple_slot_id) == MAX_CID); return tuple_slot_id; @@ -323,8 +322,7 @@ type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id); } -void TileGroup::SetValue(type::Value &value, oid_t tuple_id, - oid_t column_id) { +void TileGroup::SetValue(type::Value &value, oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; tile_group_layout_->LocateTileAndColumn(column_id, tile_offset, @@ -332,7 +330,6 @@ void TileGroup::SetValue(type::Value &value, oid_t tuple_id, GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id); } - std::shared_ptr TileGroup::GetTileReference( const oid_t tile_offset) const { PELOTON_ASSERT(tile_offset < tile_count_); @@ -363,7 +360,8 @@ const std::string TileGroup::GetInfo() const { for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { Tile *tile = GetTile(tile_itr); if (tile != nullptr) { - os << std::endl << (*tile); + os << std::endl + << (*tile); } } diff --git a/test/common/internal_types_test.cpp b/test/common/internal_types_test.cpp index a8ff6b0881c..cc7b9b21d91 100644 --- a/test/common/internal_types_test.cpp +++ b/test/common/internal_types_test.cpp @@ -457,7 +457,8 @@ TEST_F(InternalTypesTests, LoggingTypeTest) { TEST_F(InternalTypesTests, CheckpointingTypeTest) { std::vector list = { - CheckpointingType::INVALID, CheckpointingType::OFF, CheckpointingType::ON + CheckpointingType::INVALID, CheckpointingType::OFF, CheckpointingType::LOGICAL, + CheckpointingType::TIMESTAMP }; // Make sure that ToString and FromString work diff --git a/test/logging/timestamp_checkpoint_recovery_test.cpp b/test/logging/timestamp_checkpoint_recovery_test.cpp new file mode 100644 index 00000000000..838d9226214 --- /dev/null +++ b/test/logging/timestamp_checkpoint_recovery_test.cpp @@ -0,0 +1,517 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// timestamp_checkpoint_recovery_test.cpp +// +// Identification: test/logging/timestamp_checkpoint_recovery_test.cpp +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/catalog.h" +#include "catalog/column_catalog.h" +#include "catalog/constraint_catalog.h" +#include "catalog/index_catalog.h" +#include "common/init.h" +#include "common/harness.h" +#include "concurrency/transaction_manager_factory.h" +#include "logging/timestamp_checkpoint_manager.h" +#include "settings/settings_manager.h" +#include "storage/storage_manager.h" +#include "sql/testing_sql_util.h" + +namespace peloton { +namespace test { + +//===--------------------------------------------------------------------===// +// Checkpoint Recovery Tests +//===--------------------------------------------------------------------===// + +class TimestampCheckpointRecoveryTests : public PelotonTest {}; + +TEST_F(TimestampCheckpointRecoveryTests, CheckpointRecoveryTest) { + PelotonInit::Initialize(); + + // do checkpoint recovery + auto &checkpoint_manager = logging::TimestampCheckpointManager::GetInstance(); + auto result = checkpoint_manager.DoCheckpointRecovery(); + if (!result) { + LOG_ERROR("Recovery failed. Has to do timestamp_checkpointing_test" + " in advance."); + EXPECT_TRUE(false); + return; + } + + //-------------------------------------------------------------------------- + // LOW LEVEL TEST + //-------------------------------------------------------------------------- + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto catalog = catalog::Catalog::GetInstance(); + auto storage = storage::StorageManager::GetInstance(); + + // check an specific catalog: SettingCatalog + auto &setting_manager = settings::SettingsManager::GetInstance(); + (void)setting_manager; + LOG_DEBUG("Setting info\n%s", setting_manager.GetInfo().c_str()); + + // make sure data structure created in checkpointing test + + // check all tables in the default database + auto default_db_catalog_entry = catalog->GetDatabaseCatalogEntry(txn, DEFAULT_DB_NAME); + for (auto table_catalog_entry : + default_db_catalog_entry->GetTableCatalogEntries((std::string)DEFAULT_SCHEMA_NAME)) { + auto table_name = table_catalog_entry->GetTableName(); + auto table = storage->GetTableWithOid(table_catalog_entry->GetDatabaseOid(), + table_catalog_entry->GetTableOid()); + auto schema = table->GetSchema(); + + LOG_INFO("Check the table %d %s", table_catalog_entry->GetTableOid(), + table_name.c_str()); + + // check the basic table information + if (table_name == "checkpoint_table_test") { + // column info + for (auto column_pair : table_catalog_entry->GetColumnCatalogEntries()) { + auto column_catalog_entry = column_pair.second; + auto column_name = column_catalog_entry->GetColumnName(); + auto column = schema->GetColumn(column_pair.first); + + LOG_INFO("Check the column %d %s\n%s", column_catalog_entry->GetColumnId(), + column_name.c_str(), column.GetInfo().c_str()); + + if (column_name == "id") { + EXPECT_EQ(column_name, column.GetName()); + EXPECT_EQ(type::TypeId::INTEGER, column_catalog_entry->GetColumnType()); + EXPECT_EQ(0, column.GetOffset()); + EXPECT_EQ(4, column.GetLength()); + EXPECT_TRUE(column.IsInlined()); + EXPECT_FALSE(column.IsNotNull()); + EXPECT_FALSE(column.HasDefault()); + } else if (column_name == "value1") { + EXPECT_EQ(column_name, column.GetName()); + EXPECT_EQ(type::TypeId::DECIMAL, column_catalog_entry->GetColumnType()); + EXPECT_EQ(4, column.GetOffset()); + EXPECT_EQ(8, column.GetLength()); + EXPECT_TRUE(column.IsInlined()); + EXPECT_FALSE(column.IsNotNull()); + EXPECT_FALSE(column.HasDefault()); + } else if (column_name == "value2") { + EXPECT_EQ(column_name, column.GetName()); + EXPECT_EQ(type::TypeId::VARCHAR, column_catalog_entry->GetColumnType()); + EXPECT_EQ(12, column.GetOffset()); + EXPECT_EQ(32, column.GetLength()); + EXPECT_FALSE(column.IsInlined()); + EXPECT_FALSE(column.IsNotNull()); + EXPECT_FALSE(column.HasDefault()); + } else { + LOG_ERROR("Unexpected column is found: %s", column_name.c_str()); + EXPECT_TRUE(false); + } + } + + // constraints + EXPECT_EQ(0, schema->GetNotNullColumns().size()); + EXPECT_TRUE(schema->HasPrimary()); + EXPECT_FALSE(schema->HasUniqueConstraints()); + EXPECT_FALSE(schema->HasForeignKeys()); + EXPECT_EQ(0, schema->GetForeignKeyConstraints().size()); + EXPECT_TRUE(schema->HasForeignKeySources()); + EXPECT_EQ(1, schema->GetForeignKeySources().size()); + } + // end: check the basic information of columns + + // check the index recovery + else if (table_name == "checkpoint_index_test") { + for (auto index_pair : table_catalog_entry->GetIndexCatalogEntries()) { + auto index_catalog_entry = index_pair.second; + auto index_name = index_catalog_entry->GetIndexName(); + auto index = table->GetIndexWithOid(index_pair.first); + + LOG_INFO("Check the index %s", index_name.c_str()); + + // unique primary key for attribute "pid" (primary key) + if (index_name == "checkpoint_index_test_pkey") { + EXPECT_EQ(index_name, index->GetName()); + EXPECT_EQ(IndexType::BWTREE, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::PRIMARY_KEY, index->GetIndexType()); + EXPECT_TRUE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(2, key_columns.size()); + EXPECT_EQ("upid1", key_columns.at(0).GetName()); + EXPECT_EQ("upid2", key_columns.at(1).GetName()); + } + // unique primary key for attribute "upid" (unique) + else if (index_name == "checkpoint_index_test_upid1_UNIQ") { + EXPECT_EQ(index_name, index->GetName()); + EXPECT_EQ(IndexType::BWTREE, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::UNIQUE, index->GetIndexType()); + EXPECT_TRUE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(1, key_columns.size()); + EXPECT_EQ("upid1", key_columns.at(0).GetName()); + } + // ART index for attribute "value1" + else if (index_name == "index_test1") { + EXPECT_EQ(index_name, index->GetName()); + EXPECT_EQ(IndexType::ART, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::DEFAULT, index->GetIndexType()); + EXPECT_FALSE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(1, key_columns.size()); + EXPECT_EQ("value1", key_columns.at(0).GetName()); + } + // SKIPLIST index for attributes "value2" and "value3" + else if (index_name == "index_test2") { + EXPECT_EQ(index_name, index->GetName()); + EXPECT_EQ(IndexType::SKIPLIST, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::DEFAULT, index->GetIndexType()); + EXPECT_FALSE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(2, key_columns.size()); + EXPECT_EQ("value2", key_columns.at(0).GetName()); + EXPECT_EQ("value3", key_columns.at(1).GetName()); + } + // unique index for attribute "value2" + else if (index_name == "unique_index_test") { + EXPECT_EQ(index_name, index->GetName()); + EXPECT_EQ(IndexType::BWTREE, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::UNIQUE, index->GetIndexType()); + EXPECT_TRUE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(1, key_columns.size()); + EXPECT_EQ("value2", key_columns.at(0).GetName()); + } else { + LOG_ERROR("Unexpected index is found: %s", index_name.c_str()); + EXPECT_TRUE(false); + } + } + + // constraints + EXPECT_EQ(0, schema->GetNotNullColumns().size()); + EXPECT_TRUE(schema->HasPrimary()); + EXPECT_TRUE(schema->HasUniqueConstraints()); + EXPECT_FALSE(schema->HasForeignKeys()); + EXPECT_EQ(0, schema->GetForeignKeyConstraints().size()); + EXPECT_TRUE(schema->HasForeignKeySources()); + EXPECT_EQ(1, schema->GetForeignKeySources().size()); + } + // end: check the index recovery + + // check the constraints recovery + else if (table_name == "checkpoint_constraint_test") { + // column constraints + EXPECT_EQ(1, schema->GetNotNullColumns().size()); + for (auto column_pair : table_catalog_entry->GetColumnCatalogEntries()) { + auto column_catalog_entry = column_pair.second; + auto column_name = column_catalog_entry->GetColumnName(); + auto column = schema->GetColumn(column_pair.first); + LOG_INFO("Check constraint within the column %d %s\n%s", + column_catalog_entry->GetColumnId(), column_name.c_str(), + column.GetInfo().c_str()); + + // No column constraints + if (column_name == "pid1" || column_name == "pid2" || + column_name == "value3" || column_name == "value4" || + column_name == "value5") { + EXPECT_FALSE(column.IsNotNull()); + EXPECT_FALSE(column.HasDefault()); + } + // DEFAULT constraint (value: 0) for column 'value1' + else if (column_name == "value1") { + EXPECT_FALSE(column.IsNotNull()); + EXPECT_TRUE(column.HasDefault()); + EXPECT_EQ(0, column.GetDefaultValue()->GetAs()); + } + // NOT NULL constraint for column 'value2' + else if (column_name == "value2") { + EXPECT_TRUE(column.IsNotNull()); + EXPECT_FALSE(column.HasDefault()); + } else { + LOG_ERROR("Unexpected column is found: %s", column_name.c_str()); + EXPECT_TRUE(false); + } + } // loop end: column constraints + + // table constraints + EXPECT_TRUE(schema->HasPrimary()); + EXPECT_TRUE(schema->HasUniqueConstraints()); + EXPECT_TRUE(schema->HasForeignKeys()); + EXPECT_EQ(2, schema->GetForeignKeyConstraints().size()); + EXPECT_FALSE(schema->HasForeignKeySources()); + EXPECT_EQ(0, schema->GetForeignKeySources().size()); + for (auto constraint_catalog_entry_pair : + table_catalog_entry->GetConstraintCatalogEntries()) { + auto constraint_oid = constraint_catalog_entry_pair.first; + auto constraint_catalog_entry = constraint_catalog_entry_pair.second; + auto &constraint_name = constraint_catalog_entry->GetConstraintName(); + auto constraint = schema->GetConstraint(constraint_oid); + LOG_INFO("Check table constraint: %d %s", + constraint_catalog_entry->GetConstraintOid(), + constraint_name.c_str()); + + // PRIMARY KEY for a set of columns 'pid1' and 'pid2' + if (constraint_name == "con_primary") { + EXPECT_EQ(constraint_name, constraint->GetName()); + EXPECT_EQ(ConstraintType::PRIMARY, constraint->GetType()); + EXPECT_EQ(table->GetOid(), constraint->GetTableOid()); + auto &key_oids = constraint->GetColumnIds(); + EXPECT_EQ(2, key_oids.size()); + EXPECT_EQ("pid1", schema->GetColumn(key_oids.at(0)).GetName()); + EXPECT_EQ("pid2", schema->GetColumn(key_oids.at(1)).GetName()); + + // index for the constraint + auto index = table->GetIndexWithOid(constraint->GetIndexOid()); + EXPECT_EQ("checkpoint_constraint_test_pkey", index->GetName()); + EXPECT_EQ(IndexType::BWTREE, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::PRIMARY_KEY, index->GetIndexType()); + EXPECT_TRUE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(2, key_columns.size()); + EXPECT_EQ("pid1", key_columns.at(0).GetName()); + EXPECT_EQ("pid2", key_columns.at(1).GetName()); + } + // UNIQUE constraint for a column 'value1' + else if (constraint_name == "con_unique") { + EXPECT_EQ(constraint_name, constraint->GetName()); + EXPECT_EQ(ConstraintType::UNIQUE, constraint->GetType()); + EXPECT_EQ(table->GetOid(), constraint->GetTableOid()); + auto &key_oids = constraint->GetColumnIds(); + EXPECT_EQ(1, key_oids.size()); + EXPECT_EQ("value1", schema->GetColumn(key_oids.at(0)).GetName()); + + // index for the constraint + auto index = table->GetIndexWithOid(constraint->GetIndexOid()); + EXPECT_EQ("checkpoint_constraint_test_value1_UNIQ", index->GetName()); + EXPECT_EQ(IndexType::BWTREE, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::UNIQUE, index->GetIndexType()); + EXPECT_TRUE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(1, key_columns.size()); + EXPECT_EQ("value1", key_columns.at(0).GetName()); + } + // CHECK constraint for a column 'value2' (value2 > 2) + else if (constraint_name == "con_check") { + EXPECT_EQ(constraint_name, constraint->GetName()); + EXPECT_EQ(ConstraintType::CHECK, constraint->GetType()); + EXPECT_EQ(table->GetOid(), constraint->GetTableOid()); + auto &key_oids = constraint->GetColumnIds(); + EXPECT_EQ(1, key_oids.size()); + EXPECT_EQ("value2", schema->GetColumn(key_oids.at(0)).GetName()); + EXPECT_EQ(ExpressionType::COMPARE_GREATERTHAN, + constraint->GetCheckExpression().first); + EXPECT_EQ(2, constraint->GetCheckExpression().second.GetAs()); + + // index for the constraint (No index) + EXPECT_EQ(INVALID_OID, constraint->GetIndexOid()); + } + // FOREIGN KEY constraint for a column 'value3' to checkpoint_table_test.pid + else if (constraint_name == + "FK_checkpoint_constraint_test->checkpoint_table_test") { + EXPECT_EQ(constraint_name, constraint->GetName()); + EXPECT_EQ(ConstraintType::FOREIGN, constraint->GetType()); + EXPECT_EQ(table->GetOid(), constraint->GetTableOid()); + auto &source_column_ids = constraint->GetColumnIds(); + EXPECT_EQ(1, source_column_ids.size()); + EXPECT_EQ("value3", schema->GetColumn(source_column_ids.at(0)).GetName()); + + // sink table info + auto sink_table = storage->GetTableWithOid(table->GetDatabaseOid(), + constraint->GetFKSinkTableOid()); + auto sink_schema = sink_table->GetSchema(); + EXPECT_TRUE(sink_schema->HasForeignKeySources()); + auto fk_sources = sink_schema->GetForeignKeySources(); + EXPECT_EQ(1, fk_sources.size()); + EXPECT_EQ(constraint->GetConstraintOid(), fk_sources.at(0)->GetConstraintOid()); + auto sink_column_ids = constraint->GetFKSinkColumnIds(); + EXPECT_EQ(1, sink_column_ids.size()); + EXPECT_EQ("id", sink_schema->GetColumn(sink_column_ids.at(0)).GetName()); + EXPECT_EQ(FKConstrActionType::NOACTION, constraint->GetFKUpdateAction()); + EXPECT_EQ(FKConstrActionType::NOACTION, constraint->GetFKDeleteAction()); + + // index for the constraint + auto index = table->GetIndexWithOid(constraint->GetIndexOid()); + EXPECT_EQ("checkpoint_constraint_test_value3_fkey", index->GetName()); + EXPECT_EQ(IndexType::BWTREE, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::DEFAULT, index->GetIndexType()); + EXPECT_FALSE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(1, key_columns.size()); + EXPECT_EQ("value3", key_columns.at(0).GetName()); + } + // FOREIGN KEY constraint for a set of columns 'value4' and 'value5' + // to (checkpoint_index_test.upid1, checkpoint_index_test.upid2) + else if (constraint_name == + "FK_checkpoint_constraint_test->checkpoint_index_test") { + EXPECT_EQ(constraint_name, constraint->GetName()); + EXPECT_EQ(ConstraintType::FOREIGN, constraint->GetType()); + EXPECT_EQ(table->GetOid(), constraint->GetTableOid()); + auto &source_column_ids = constraint->GetColumnIds(); + EXPECT_EQ(2, source_column_ids.size()); + EXPECT_EQ("value4", schema->GetColumn(source_column_ids.at(0)).GetName()); + EXPECT_EQ("value5", schema->GetColumn(source_column_ids.at(1)).GetName()); + + // sink table info + auto sink_table = storage->GetTableWithOid(table->GetDatabaseOid(), + constraint->GetFKSinkTableOid()); + auto sink_schema = sink_table->GetSchema(); + EXPECT_TRUE(sink_schema->HasForeignKeySources()); + auto fk_sources = sink_schema->GetForeignKeySources(); + EXPECT_EQ(1, fk_sources.size()); + EXPECT_EQ(constraint->GetConstraintOid(), fk_sources.at(0)->GetConstraintOid()); + auto sink_column_ids = constraint->GetFKSinkColumnIds(); + EXPECT_EQ(2, sink_column_ids.size()); + EXPECT_EQ("upid1", sink_schema->GetColumn(sink_column_ids.at(0)).GetName()); + EXPECT_EQ("upid2", sink_schema->GetColumn(sink_column_ids.at(1)).GetName()); + EXPECT_EQ(FKConstrActionType::NOACTION, constraint->GetFKUpdateAction()); + EXPECT_EQ(FKConstrActionType::NOACTION, constraint->GetFKDeleteAction()); + + // index for the constraint + auto index = table->GetIndexWithOid(constraint->GetIndexOid()); + EXPECT_EQ("checkpoint_constraint_test_value4_value5_fkey", index->GetName()); + EXPECT_EQ(IndexType::BWTREE, index->GetIndexMethodType()); + EXPECT_EQ(IndexConstraintType::DEFAULT, index->GetIndexType()); + EXPECT_FALSE(index->HasUniqueKeys()); + auto &key_columns = index->GetKeySchema()->GetColumns(); + EXPECT_EQ(2, key_columns.size()); + EXPECT_EQ("value4", key_columns.at(0).GetName()); + EXPECT_EQ("value5", key_columns.at(1).GetName()); + } else { + LOG_ERROR("Unexpected constraint is found: %s", constraint_name.c_str()); + EXPECT_TRUE(false); + } + } // loop end: table constraints + } + // end: check the constraints recovery + + else { + LOG_ERROR("Unexpected table is found: %s", table_name.c_str()); + EXPECT_TRUE(false); + } + } // table loop end + + // finish the low level test + txn_manager.CommitTransaction(txn); + + + //-------------------------------------------------------------------------- + // HIGH LEVEL TEST + //-------------------------------------------------------------------------- + // make sure the records of 3 user tables created checkpointing test + // are correct and their constraints are working correctly through + // SQL execution. + + // make sure the data in each user table is correct + std::string sql1 = "SELECT * FROM checkpoint_table_test;"; + std::vector expected1 = {"0|1.2|aaa", "1|12.34|bbbbbb", + "2|12345.7|ccccccccc", "3|0|xxxx"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql1, expected1, false); + + std::string sql2 = "SELECT * FROM checkpoint_index_test;"; + std::vector expected2 = {"1|2|3|4|5", "6|7|8|9|10", + "11|12|13|14|15"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql2, expected2, false); + + std::string sql3 = "SELECT * FROM checkpoint_constraint_test;"; + std::vector expected3 = {"1|2|3|4|0|1|2", "5|6|7|8|1|6|7", + "9|10|11|12|2|11|12"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql3, expected3, false); + + // make sure the constraints are working + // PRIMARY KEY (1 column: pid) + LOG_INFO("PRIMARY KEY (1 column) check"); + std::string primary_key_dml1 = + "INSERT INTO checkpoint_table_test VALUES (0, 5.5, 'eee');"; + ResultType primary_key_result1 = + TestingSQLUtil::ExecuteSQLQuery(primary_key_dml1); + EXPECT_EQ(ResultType::ABORTED, primary_key_result1); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // PRIMARY KEY (2 column: pid1, pid2) + LOG_INFO("PRIMARY KEY (2 columns) check"); + std::string primary_key_dml2 = + "INSERT INTO checkpoint_constraint_test VALUES (1, 2, 15, 16, 0, 1 ,2);"; + ResultType primary_key_result2 = + TestingSQLUtil::ExecuteSQLQuery(primary_key_dml2); + EXPECT_EQ(ResultType::ABORTED, primary_key_result2); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // DEFAULT (value1 = 0) + LOG_INFO("DEFAULT check"); + std::string default_dml = + "INSERT INTO checkpoint_constraint_test" + " (pid1, pid2, value2, value3, value4, value5)" + " VALUES (13, 14, 16, 0, 1 ,2);"; + ResultType default_result1 = TestingSQLUtil::ExecuteSQLQuery(default_dml); + EXPECT_EQ(ResultType::SUCCESS, default_result1); + + std::string default_sql = + "SELECT value1 FROM checkpoint_constraint_test" + " WHERE pid1 = 13 AND pid2 = 14;"; + std::vector result_value; + ResultType default_result2 = + TestingSQLUtil::ExecuteSQLQuery(default_sql, result_value); + EXPECT_EQ(ResultType::SUCCESS, default_result2); + EXPECT_EQ("0", result_value.at(0)); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // UNIQUE (value1) + LOG_INFO("UNIQUE check"); + std::string unique_dml = + "INSERT INTO checkpoint_constraint_test VALUES (17, 18, 3, 20, 1, 6 ,7);"; + ResultType unique_result = TestingSQLUtil::ExecuteSQLQuery(unique_dml); + EXPECT_EQ(ResultType::ABORTED, unique_result); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // NOT NULL (value2) + LOG_INFO("NOT NULL check"); + std::string not_null_dml = + "INSERT INTO checkpoint_constraint_test VALUES (17, 18, 19, NULL, 1, 6 " + ",7);"; + ResultType not_null_result = TestingSQLUtil::ExecuteSQLQuery(not_null_dml); + // EXPECT_EQ(ResultType::ABORTED, not_null_result); + EXPECT_EQ(ResultType::FAILURE, not_null_result); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // CHECK (value2 > 2) + LOG_INFO("CHECK check"); + std::string check_dml = + "INSERT INTO checkpoint_constraint_test VALUES (17, 18, 19, 1, 1, 6 ,7);"; + ResultType check_result = TestingSQLUtil::ExecuteSQLQuery(check_dml); + // EXPECT_EQ(ResultType::FAILURE, check_result); + EXPECT_EQ(ResultType::SUCCESS, check_result); // check doesn't work correctly + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // FOREIGN KEY (1 column: value3 => pid) + LOG_INFO("FOREIGN KEY (1 column) check"); + std::string foreign_key_dml1 = + "INSERT INTO checkpoint_constraint_test VALUES (21, 22, 23, 24, 10, 6 " + ",7);"; + ResultType foreign_key_result1 = + TestingSQLUtil::ExecuteSQLQuery(foreign_key_dml1); + EXPECT_EQ(ResultType::ABORTED, foreign_key_result1); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // FOREIGN KEY (2 column: (value4, value5) => (upid1, upid2)) + LOG_INFO("FOREIGN KEY (2 columns) check"); + std::string foreign_key_dml2 = + "INSERT INTO checkpoint_constraint_test VALUES (21, 22, 23, 24, 1, 20 " + ",20);"; + ResultType foreign_key_result2 = + TestingSQLUtil::ExecuteSQLQuery(foreign_key_dml2); + EXPECT_EQ(ResultType::ABORTED, foreign_key_result2); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // Final check + std::string sql4 = "SELECT * FROM checkpoint_constraint_test;"; + std::vector expected4 = {"1|2|3|4|0|1|2", "5|6|7|8|1|6|7", + "9|10|11|12|2|11|12", + "13|14|0|16|0|1|2", "17|18|19|1|1|6|7"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql4, expected4, false); + + PelotonInit::Shutdown(); +} +} +} diff --git a/test/logging/timestamp_checkpointing_test.cpp b/test/logging/timestamp_checkpointing_test.cpp new file mode 100644 index 00000000000..7e43267dd4d --- /dev/null +++ b/test/logging/timestamp_checkpointing_test.cpp @@ -0,0 +1,381 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// timestamp_checkpointing_test.cpp +// +// Identification: test/logging/timestamp_checkpointing_test.cpp +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/catalog.h" +#include "catalog/column_catalog.h" +#include "catalog/layout_catalog.h" +#include "catalog/system_catalogs.h" +#include "common/init.h" +#include "common/harness.h" +#include "concurrency/transaction_manager_factory.h" +#include "logging/timestamp_checkpoint_manager.h" +#include "logging/logging_util.h" +#include "storage/storage_manager.h" +#include "storage/tile_group_factory.h" +#include "sql/testing_sql_util.h" +#include "type/ephemeral_pool.h" +#include "type/value_factory.h" + +namespace peloton { +namespace test { + +//===--------------------------------------------------------------------===// +// Checkpointing Tests +//===--------------------------------------------------------------------===// + +class TimestampCheckpointingTests : public PelotonTest {}; + +bool RecoverTileGroupFromFile( + std::vector> &tile_groups, + storage::DataTable *table, FileHandle table_file, type::AbstractPool *pool, + concurrency::TransactionContext *txn) { + size_t table_size = logging::LoggingUtil::GetFileSize(table_file); + if (table_size == 0) return false; + std::unique_ptr data(new char[table_size]); + if (!logging::LoggingUtil::ReadNBytesFromFile(table_file, data.get(), table_size)) { + LOG_ERROR("Checkpoint table file read error: table %d", table->GetOid()); + return false; + } + CopySerializeInput input_buffer(data.get(), table_size); + + auto schema = table->GetSchema(); + auto default_layout = table->GetDefaultLayout(); + auto column_count = schema->GetColumnCount(); + oid_t tile_group_count = input_buffer.ReadLong(); + for (oid_t tg_idx = START_OID; tg_idx < tile_group_count; tg_idx++) { + // recover layout for this tile group + oid_t layout_oid = input_buffer.ReadInt(); + std::shared_ptr layout; + if (default_layout->GetOid() != layout_oid) { + layout = catalog::Catalog::GetInstance() + ->GetTableCatalogEntry(txn, table->GetDatabaseOid(), table->GetOid()) + ->GetLayout(layout_oid); + } else { + layout = default_layout; + } + // recover tile group + oid_t tile_group_id = + storage::StorageManager::GetInstance()->GetNextTileGroupId(); + oid_t allocated_tuple_count = input_buffer.ReadInt(); + auto layout_schemas = layout->GetLayoutSchemas(schema); + std::shared_ptr tile_group( + storage::TileGroupFactory::GetTileGroup( + table->GetDatabaseOid(), table->GetOid(), tile_group_id, table, + layout_schemas, layout, allocated_tuple_count)); + + LOG_TRACE("Deserialized tile group %u in %s \n%s", tile_group->GetTileGroupId(), + table->GetName().c_str(), tile_group->GetLayout().GetInfo().c_str()); + + // recover tuples located in the tile group + while (input_buffer.ReadBool()) { + // recover values on each column + std::unique_ptr tuple(new storage::Tuple(schema, true)); + for (oid_t column_id = 0; column_id < column_count; column_id++) { + auto value = type::Value::DeserializeFrom( + input_buffer, schema->GetType(column_id), pool); + tuple->SetValue(column_id, value, pool); + } + + // insert the tuple into the tile group + oid_t tuple_slot = tile_group->InsertTuple(tuple.get()); + EXPECT_NE(tuple_slot, INVALID_OID); + if (tuple_slot == INVALID_OID) { + LOG_ERROR("Tuple insert error for tile group %d of table %d", + tile_group->GetTileGroupId(), table->GetOid()); + return false; + } + } + + tile_groups.push_back(tile_group); + } + + return true; +} + + +TEST_F(TimestampCheckpointingTests, CheckpointingTest) { + PelotonInit::Initialize(); + + auto &checkpoint_manager = logging::TimestampCheckpointManager::GetInstance(); + + // checkpoint_manager.SetCheckpointBaseDirectory("/var/tmp/peloton/checkpoints") + + // generate table and data taken into storage. + // basic table test + TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE checkpoint_table_test (id INTEGER PRIMARY KEY, value1 " + "REAL, value2 VARCHAR(32));"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_table_test VALUES (0, 1.2, 'aaa');"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_table_test VALUES (1, 12.34, 'bbbbbb');"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_table_test VALUES (2, 12345.678912345, " + "'ccccccccc');"); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // primary key and index test + TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE checkpoint_index_test (" + "upid1 INTEGER UNIQUE PRIMARY KEY, " + "upid2 INTEGER PRIMARY KEY, " + "value1 INTEGER, value2 INTEGER, value3 INTEGER);"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE INDEX index_test1 ON checkpoint_index_test USING art (value1);"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE INDEX index_test2 ON checkpoint_index_test USING skiplist " + "(value2, value3);"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE UNIQUE INDEX unique_index_test ON checkpoint_index_test " + "(value2);"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_index_test VALUES (1, 2, 3, 4, 5);"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_index_test VALUES (6, 7, 8, 9, 10);"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_index_test VALUES (11, 12, 13, 14, 15);"); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // column constraint test + TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + std::string constraint_test_sql = + "CREATE TABLE checkpoint_constraint_test (" + "pid1 INTEGER, pid2 INTEGER, " + "value1 INTEGER DEFAULT 0 UNIQUE, " + "value2 INTEGER NOT NULL CHECK (value2 > 2), " // check doesn't work + // correctly + "value3 INTEGER REFERENCES checkpoint_table_test (id), " + "value4 INTEGER, value5 INTEGER, " + "FOREIGN KEY (value4, value5) REFERENCES checkpoint_index_test (upid1, " + "upid2), " + // not supported yet "UNIQUE (value4, value5), " + "PRIMARY KEY (pid1, pid2));"; + TestingSQLUtil::ExecuteSQLQuery(constraint_test_sql); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_constraint_test VALUES (1, 2, 3, 4, 0, 1, 2);"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_constraint_test VALUES (5, 6, 7, 8, 1, 6, 7);"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_constraint_test VALUES (9, 10, 11, 12, 2, 11, " + "12);"); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // insert test + TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_table_test VALUES (3, 0.0, 'xxxx');"); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // generate table and data that will be out of checkpointing. + TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_table_test VALUES (4, -1.0, 'out of the " + "checkpoint');"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO checkpoint_table_test VALUES (5, -2.0, 'out of the " + "checkpoint');"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE out_of_checkpoint_test (pid INTEGER PRIMARY KEY);"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO out_of_checkpoint_test VALUES (1);"); + // TestingSQLUtil::ExecuteSQLQuery("CREATE DATABASE out_of_checkpoint;"); + + // do checkpointing + checkpoint_manager.StartCheckpointing(); + + EXPECT_TRUE(checkpoint_manager.GetStatus()); + + std::this_thread::sleep_for(std::chrono::seconds(3)); + checkpoint_manager.StopCheckpointing(); + + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + EXPECT_FALSE(checkpoint_manager.GetStatus()); + + // test files created by this checkpointing + // prepare file check + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto catalog = catalog::Catalog::GetInstance(); + auto storage = storage::StorageManager::GetInstance(); + std::unique_ptr pool(new type::EphemeralPool()); + + // check the created directory of the checkpoint + eid_t checkpointed_epoch = checkpoint_manager.GetRecoveryCheckpointEpoch(); + std::string checkpoint_dir = "./data/checkpoints/" + std::to_string(checkpointed_epoch); + EXPECT_TRUE(logging::LoggingUtil::CheckDirectoryExistence(checkpoint_dir.c_str())); + + // check user table file + auto default_db_catalog_entry = catalog->GetDatabaseCatalogEntry(txn, DEFAULT_DB_NAME); + for (auto table_catalog_entry : + default_db_catalog_entry->GetTableCatalogEntries((std::string)DEFAULT_SCHEMA_NAME)) { + auto table = storage->GetTableWithOid(table_catalog_entry->GetDatabaseOid(), + table_catalog_entry->GetTableOid()); + FileHandle table_file; + std::string file = checkpoint_dir + "/" + "checkpoint_" + + default_db_catalog_entry->GetDatabaseName() + "_" + + table_catalog_entry->GetSchemaName() + "_" + table_catalog_entry->GetTableName(); + + LOG_INFO("Check the user table %s.%s\n%s", table_catalog_entry->GetSchemaName().c_str(), + table_catalog_entry->GetTableName().c_str(), table->GetInfo().c_str()); + + // open table file + // table 'out_of_checkpoint_test' is not targeted for the checkpoint + if (table_catalog_entry->GetTableName() == "out_of_checkpoint_test") { + EXPECT_FALSE(logging::LoggingUtil::OpenFile(file.c_str(), "rb", table_file)); + continue; + } else { + bool open_file_result = logging::LoggingUtil::OpenFile(file.c_str(), "rb", table_file); + EXPECT_TRUE(open_file_result); + if(open_file_result == false) { + LOG_ERROR("Unexpected table is found: %s", table_catalog_entry->GetTableName().c_str()); + continue; + } + } + + // read data (tile groups and records) + std::vector> tile_groups; + RecoverTileGroupFromFile(tile_groups, table, table_file, pool.get(), txn); + + logging::LoggingUtil::CloseFile(table_file); + + // check the tile group + auto schema = table->GetSchema(); + auto column_count = schema->GetColumnCount(); + for (auto tile_group : tile_groups) { + // check the layout of columns in the tile group + EXPECT_EQ(column_count, tile_group->GetLayout().GetColumnCount()); + + // check the records + oid_t max_tuple_count = tile_group->GetNextTupleSlot(); + for (oid_t tuple_id = START_OID; tuple_id < max_tuple_count; tuple_id++) { + for (oid_t column_id = START_OID; column_id < column_count; column_id++) { + type::Value value = tile_group->GetValue(tuple_id, column_id); + // for checkpoint_table_test + if (table_catalog_entry->GetTableName() == "checkpoint_table_test") { + switch (column_id) { + case 0: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(0)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(1)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(2)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(3)) == CmpBool::CmpTrue); + break; + case 1: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetDecimalValue(1.2)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetDecimalValue(12.34)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetDecimalValue(12345.678912345)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetDecimalValue(0.0)) == CmpBool::CmpTrue); + break; + case 2: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetVarcharValue(std::string("aaa"), pool.get())) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetVarcharValue(std::string("bbbbbb"), pool.get())) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetVarcharValue(std::string("ccccccccc"), pool.get())) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetVarcharValue(std::string("xxxx"), pool.get())) == CmpBool::CmpTrue); + break; + default: + LOG_ERROR("Unexpected column is found in %s: %d", table_catalog_entry->GetTableName().c_str(), column_id); + EXPECT_TRUE(false); + } + } + // for checkpoint_index_test + else if (table_catalog_entry->GetTableName() == "checkpoint_index_test") { + switch (column_id) { + case 0: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(1)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(6)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(11)) == CmpBool::CmpTrue); + break; + case 1: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(2)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(7)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(12)) == CmpBool::CmpTrue); + break; + case 2: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(3)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(8)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(13)) == CmpBool::CmpTrue); + break; + case 3: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(4)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(9)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(14)) == CmpBool::CmpTrue); + break; + case 4: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(5)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(10)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(15)) == CmpBool::CmpTrue); + break; + default: + LOG_ERROR("Unexpected column is found in %s: %d", table_catalog_entry->GetTableName().c_str(), column_id); + EXPECT_TRUE(false); + } + } + // for checkpoint_index_test + else if (table_catalog_entry->GetTableName() == "checkpoint_constraint_test") { + switch (column_id) { + case 0: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(1)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(5)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(9)) == CmpBool::CmpTrue); + break; + case 1: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(2)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(6)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(10)) == CmpBool::CmpTrue); + break; + case 2: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(3)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(7)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(11)) == CmpBool::CmpTrue); + break; + case 3: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(4)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(8)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(12)) == CmpBool::CmpTrue); + break; + case 4: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(0)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(1)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(2)) == CmpBool::CmpTrue); + break; + case 5: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(1)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(6)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(11)) == CmpBool::CmpTrue); + break; + case 6: + EXPECT_TRUE(value.CompareEquals(type::ValueFactory::GetIntegerValue(2)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(7)) == CmpBool::CmpTrue || + value.CompareEquals(type::ValueFactory::GetIntegerValue(12)) == CmpBool::CmpTrue); + break; + default: + LOG_ERROR("Unexpected column is found in %s: %d", table_catalog_entry->GetTableName().c_str(), column_id); + EXPECT_TRUE(false); + } + } + else { + LOG_ERROR("Unexpected table is found: %s", table_catalog_entry->GetTableName().c_str()); + PELOTON_ASSERT(false); + } + } + } + } + } + + txn_manager.CommitTransaction(txn); + + PelotonInit::Shutdown(); +} +} +}