Avoid a spurious assert in tests (#21543)

This avoids the following crash:

```
thread '<unnamed>' panicked at crates/sui-core/src/execution_cache/cache_types.rs:307:17:
entry is newer than value 0x86aacc42979583a6797f4bb0cab20d436a1f41e215b2c87cb6e99ffdda5ec7db
stack backtrace:
   0: rust_begin_unwind
             at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:692:5
   1: core::panicking::panic_fmt
             at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:75:14
   2: sui_core::execution_cache::cache_types::MonotonicCache<K,V>::insert
             at /Users/marklogan/dev/sui-wb/crates/sui-core/src/execution_cache/cache_types.rs:307:17
   3: sui_core::execution_cache::writeback_cache::WritebackCache::cache_latest_object_by_id
             at /Users/marklogan/dev/sui-wb/crates/sui-core/src/execution_cache/writeback_cache.rs:1210:12
   4: sui_core::execution_cache::writeback_cache::WritebackCache::cache_object_not_found
             at /Users/marklogan/dev/sui-wb/crates/sui-core/src/execution_cache/writeback_cache.rs:1224:9
   5: <sui_core::execution_cache::writeback_cache::WritebackCache as sui_core::execution_cache::ObjectCacheRead>::find_object_lt_or_eq_version::{{closure}}
   ...
```

The fix is to not overwrite an existing latest cache entry in this
scenario, since it must already be in the correct state at the point at
which the crash was triggered before.

Author: mystenmark

crates/sui-core/src/execution_cache/cache_types.rs

@@ -188,7 +188,7 @@ const KEY_GENERATION_SIZE: usize = 1024 * 16;
 
 impl<K, V> MonotonicCache<K, V>
 where
-    K: Hash + Eq + Send + Sync + Copy + 'static,
+    K: Hash + Eq + Send + Sync + Copy + std::fmt::Debug + 'static,
     V: IsNewer + Clone + Send + Sync + 'static,
 {
     pub fn new(cache_size: u64) -> Self {
@@ -304,7 +304,7 @@ where
 
             // Ticket expiry should make this assert impossible.
             if entry.is_newer_than(&value) {
-                debug_fatal!("entry is newer than value");
+                debug_fatal!("entry is newer than value {:?}", key);
             } else {
                 *entry = value;
             }

crates/sui-core/src/execution_cache/writeback_cache.rs

@@ -173,6 +173,13 @@ impl LatestObjectCacheEntry {
             LatestObjectCacheEntry::NonExistent => None,
         }
     }
+
+    fn is_alive(&self) -> bool {
+        match self {
+            LatestObjectCacheEntry::Object(_, entry) => !entry.is_tombstone(),
+            LatestObjectCacheEntry::NonExistent => false,
+        }
+    }
 }
 
 impl IsNewer for LatestObjectCacheEntry {
@@ -1522,7 +1529,8 @@ impl ObjectCacheRead for WritebackCache {
         // if we have the latest version cached, and it is within the bound, we are done
         self.metrics
             .record_cache_request("object_lt_or_eq_version", "object_by_id");
-        if let Some(latest) = self.cached.object_by_id_cache.get(&object_id) {
+        let latest_cache_entry = self.cached.object_by_id_cache.get(&object_id);
+        if let Some(latest) = &latest_cache_entry {
             let latest = latest.lock();
             match &*latest {
                 LatestObjectCacheEntry::Object(latest_version, object) => {
@@ -1628,12 +1636,18 @@ impl ObjectCacheRead for WritebackCache {
                             .find_object_lt_or_eq_version(object_id, version_bound)
                             .expect("db error")
                     }
+
+                // no object found in dirty set or db, object does not exist
+                // When this is called from a read api (i.e. not the execution path) it is
+                // possible that the object has been deleted and pruned. In this case,
+                // there would be no entry at all on disk, but we may have a tombstone in the
+                // cache
+                } else if let Some(latest_cache_entry) = latest_cache_entry {
+                    // If there is a latest cache entry, it had better not be a live object!
+                    assert!(!latest_cache_entry.lock().is_alive());
+                    None
                 } else {
-                    // no object found in dirty set or db, object does not exist
-                    // When this is called from a read api (i.e. not the execution path) it is
-                    // possible that the object has been deleted and pruned. In this case,
-                    // there would be no entry at all on disk, but we may have a tombstone in the
-                    // cache
+                    // If there is no latest cache entry, we can insert one.
                     let highest = cached_entry.and_then(|c| c.get_highest());
                     assert!(highest.is_none() || highest.unwrap().1.is_tombstone());
                     self.cache_object_not_found(