Create DB properties for internal table cache

graetzer · graetzer · commit 54a9cb464dca · 2021-04-16T11:53:23.000+02:00
diff --git a/db/db_properties_test.cc b/db/db_properties_test.cc
@@ -245,7 +245,8 @@ void GetExpectedTableProperties(
   const int kDeletionCount = kTableCount * kDeletionsPerTable;
   const int kMergeCount = kTableCount * kMergeOperandsPerTable;
   const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
-  const int kKeyCount = kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
+  const int kKeyCount =
+      kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
   const int kAvgSuccessorSize = kKeySize / 5;
   const int kEncodingSavePerKey = kKeySize / 4;
   expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
@@ -256,7 +257,8 @@ void GetExpectedTableProperties(
   expected_tp->num_merge_operands = kMergeCount;
   expected_tp->num_range_deletions = kRangeDeletionCount;
   expected_tp->num_data_blocks =
-      kTableCount * (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
+      kTableCount *
+      (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
       kBlockSize;
   expected_tp->data_size =
       kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
@@ -1090,7 +1092,8 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
     std::string encoded;
     PutVarint32(&encoded, count_);
     *properties = UserCollectedProperties{
-        {"CountingUserTblPropCollector", message_}, {"Count", encoded},
+        {"CountingUserTblPropCollector", message_},
+        {"Count", encoded},
     };
     return Status::OK();
   }
@@ -1713,6 +1716,46 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) {
   ASSERT_EQ(0, value);
 }
 
+TEST_F(DBPropertiesTest, TableCacheProperties) {
+  Options options;
+  uint64_t value, new_value;
+
+  options.env = CurrentOptions().env;
+
+  Reopen(options);
+
+  //
+  // test table_cache access is "live"
+  //  TableCacheCapacity originally comes from DBOptions::max_open_files
+  //  and can vary by system.  Get its current value.
+  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheCapacity, &value));
+
+  // now, change max_open_files to prove we are really accessing the value of
+  // interest
+  new_value = value / 2;
+  std::unordered_map<std::string, std::string> new_options;
+  new_options.insert(std::pair<std::string, std::string>(
+      "max_open_files", std::to_string(new_value)));
+  ASSERT_OK(db_->SetDBOptions(new_options));
+
+  // did the value we are reading update. NOTE: rocksdb internally reduces
+  //  the value we pass by 10.
+  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheCapacity, &value));
+  ASSERT_EQ(new_value - 10, value);
+
+  //
+  //  TableCacheUsage is a count of open .sst files.  Force the creation of a
+  //   a new table file.  First add a record via Put().  Then force that
+  //   record from write buffer to new .sst via Flush().  New .sst
+  //   automatically opens and gets position in table cache ... raising usage
+  //   count
+  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kTableCacheUsage, &value));
+  ASSERT_OK(Put("foo", "v1"));
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  ASSERT_TRUE(
+      db_->GetIntProperty(DB::Properties::kTableCacheUsage, &new_value));
+  ASSERT_EQ(new_value, value + 1);
+}
 #endif  // ROCKSDB_LITE
 }  // namespace ROCKSDB_NAMESPACE
 
diff --git a/db/internal_stats.cc b/db/internal_stats.cc
@@ -260,6 +260,8 @@ static const std::string estimate_oldest_key_time = "estimate-oldest-key-time";
 static const std::string block_cache_capacity = "block-cache-capacity";
 static const std::string block_cache_usage = "block-cache-usage";
 static const std::string block_cache_pinned_usage = "block-cache-pinned-usage";
+static const std::string tablecache_capacity = "table-cache-capacity";
+static const std::string tablecache_usage = "table-cache-usage";
 static const std::string options_statistics = "options-statistics";
 
 const std::string DB::Properties::kNumFilesAtLevelPrefix =
@@ -348,6 +350,10 @@ const std::string DB::Properties::kBlockCacheUsage =
     rocksdb_prefix + block_cache_usage;
 const std::string DB::Properties::kBlockCachePinnedUsage =
     rocksdb_prefix + block_cache_pinned_usage;
+const std::string DB::Properties::kTableCacheCapacity =
+    rocksdb_prefix + tablecache_capacity;
+const std::string DB::Properties::kTableCacheUsage =
+    rocksdb_prefix + tablecache_usage;
 const std::string DB::Properties::kOptionsStatistics =
     rocksdb_prefix + options_statistics;
 
@@ -487,6 +493,12 @@ const std::unordered_map<std::string, DBPropertyInfo>
         {DB::Properties::kBlockCachePinnedUsage,
          {false, nullptr, &InternalStats::HandleBlockCachePinnedUsage, nullptr,
           nullptr}},
+        {DB::Properties::kTableCacheCapacity,
+         {false, nullptr, &InternalStats::HandleTableCacheCapacity, nullptr,
+          nullptr}},
+        {DB::Properties::kTableCacheUsage,
+         {false, nullptr, &InternalStats::HandleTableCacheUsage, nullptr,
+          nullptr}},
         {DB::Properties::kOptionsStatistics,
          {false, nullptr, nullptr, nullptr,
           &DBImpl::GetPropertyHandleOptionsStatistics}},
@@ -987,6 +999,18 @@ bool InternalStats::HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* /*db*/,
   return true;
 }
 
+bool InternalStats::HandleTableCacheCapacity(uint64_t* value, DBImpl* db,
+                                             Version* /*version*/) {
+  *value = static_cast<uint64_t>(db->table_cache_->GetCapacity());
+  return true;
+}
+
+bool InternalStats::HandleTableCacheUsage(uint64_t* value, DBImpl* db,
+                                          Version* /*version*/) {
+  *value = static_cast<uint64_t>(db->table_cache_->GetUsage());
+  return true;
+}
+
 void InternalStats::DumpDBStats(std::string* value) {
   char buf[1000];
   // DB-level stats, only available from default column family
diff --git a/db/internal_stats.h b/db/internal_stats.h
@@ -296,7 +296,7 @@ class InternalStats {
       this->num_dropped_records += c.num_dropped_records;
       this->count += c.count;
       int num_of_reasons = static_cast<int>(CompactionReason::kNumOfReasons);
-      for (int i = 0; i< num_of_reasons; i++) {
+      for (int i = 0; i < num_of_reasons; i++) {
         counts[i] += c.counts[i];
       }
     }
@@ -437,8 +437,8 @@ class InternalStats {
   struct CFStatsSnapshot {
     // ColumnFamily-level stats
     CompactionStats comp_stats;
-    uint64_t ingest_bytes_flush;      // Bytes written to L0 (Flush)
-    uint64_t stall_count;             // Stall count
+    uint64_t ingest_bytes_flush;  // Bytes written to L0 (Flush)
+    uint64_t stall_count;         // Stall count
     // Stats from compaction jobs - bytes written, bytes read, duration.
     uint64_t compact_bytes_write;
     uint64_t compact_bytes_read;
@@ -480,10 +480,10 @@ class InternalStats {
 
   struct DBStatsSnapshot {
     // DB-level stats
-    uint64_t ingest_bytes;            // Bytes written by user
-    uint64_t wal_bytes;               // Bytes written to WAL
-    uint64_t wal_synced;              // Number of times WAL is synced
-    uint64_t write_with_wal;          // Number of writes that request WAL
+    uint64_t ingest_bytes;    // Bytes written by user
+    uint64_t wal_bytes;       // Bytes written to WAL
+    uint64_t wal_synced;      // Number of times WAL is synced
+    uint64_t write_with_wal;  // Number of writes that request WAL
     // These count the number of writes processed by the calling thread or
     // another thread.
     uint64_t write_other;
@@ -594,6 +594,8 @@ class InternalStats {
   bool HandleBlockCacheUsage(uint64_t* value, DBImpl* db, Version* version);
   bool HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* db,
                                    Version* version);
+  bool HandleTableCacheCapacity(uint64_t* value, DBImpl* db, Version* version);
+  bool HandleTableCacheUsage(uint64_t* value, DBImpl* db, Version* version);
   // Total number of background errors encountered. Every time a flush task
   // or compaction task fails, this counter is incremented. The failure can
   // be caused by any possible reason, including file system errors, out of
@@ -697,13 +699,14 @@ class InternalStats {
     return false;
   }
 
-  bool GetIntProperty(const DBPropertyInfo& /*property_info*/, uint64_t* /*value*/,
-                      DBImpl* /*db*/) const {
+  bool GetIntProperty(const DBPropertyInfo& /*property_info*/,
+                      uint64_t* /*value*/, DBImpl* /*db*/) const {
     return false;
   }
 
   bool GetIntPropertyOutOfMutex(const DBPropertyInfo& /*property_info*/,
-                                Version* /*version*/, uint64_t* /*value*/) const {
+                                Version* /*version*/,
+                                uint64_t* /*value*/) const {
     return false;
   }
 };
diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h
@@ -923,6 +923,13 @@ class DB {
     //      entries being pinned.
     static const std::string kBlockCachePinnedUsage;
 
+    //  "rocksdb.table-cache-capacity" - returns table cache capacity.
+    static const std::string kTableCacheCapacity;
+
+    //  "rocksdb.table-cache-usage" - returns the memory size for the entries
+    //      residing in table cache.
+    static const std::string kTableCacheUsage;
+
     // "rocksdb.options-statistics" - returns multi-line string
     //      of options.statistics
     static const std::string kOptionsStatistics;
@@ -986,6 +993,8 @@ class DB {
   //  "rocksdb.block-cache-capacity"
   //  "rocksdb.block-cache-usage"
   //  "rocksdb.block-cache-pinned-usage"
+  //  "rocksdb.table-cache-capacity"
+  //  "rocksdb.table-cache-usage"
   virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                               const Slice& property, uint64_t* value) = 0;
   virtual bool GetIntProperty(const Slice& property, uint64_t* value) {
