diff --git a/bench-cargo-miri/string-replace/Cargo.lock b/bench-cargo-miri/string-replace/Cargo.lock new file mode 100644 index 0000000000..443115c126 --- /dev/null +++ b/bench-cargo-miri/string-replace/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "string-replace" +version = "0.1.0" diff --git a/bench-cargo-miri/string-replace/Cargo.toml b/bench-cargo-miri/string-replace/Cargo.toml new file mode 100644 index 0000000000..f0785cd693 --- /dev/null +++ b/bench-cargo-miri/string-replace/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "string-replace" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/bench-cargo-miri/string-replace/data.json b/bench-cargo-miri/string-replace/data.json new file mode 100644 index 0000000000..7e074cd695 --- /dev/null +++ b/bench-cargo-miri/string-replace/data.json @@ -0,0 +1 @@ +[{"_id":"6724e6fc58417687afba2b85","index":0,"guid":"5c1bd108-2ee2-40bd-bce8-895c206409df","isActive":true,"balance":"$2,927.88","picture":"http://placehold.it/32x32","age":40,"eyeColor":"green","name":"Wynn Bradshaw","gender":"male","company":"ROTODYNE","email":"wynnbradshaw@rotodyne.com","phone":"+1 (904) 559-3130","address":"287 Bergen Avenue, Sperryville, Alaska, 5392","about":"Adipisicing fugiat aute adipisicing qui esse cillum. Lorem consequat consectetur voluptate id pariatur nostrud incididunt aliquip incididunt laboris aliqua. Magna nulla adipisicing cupidatat ea velit aliquip magna duis duis sunt ipsum. Cillum labore mollit fugiat tempor dolor sit.\r\n","registered":"2017-01-26T01:28:10 -01:00","latitude":46.089504,"longitude":51.763723,"greeting":"Hello, Wynn Bradshaw! You have 6 unread messages.","favoriteFruit":"banana"},{"_id":"6724e6fce8619d86c0389ccf","index":1,"guid":"ced7fbb7-3b1a-419b-9fc7-d47582bbb3ea","isActive":true,"balance":"$1,856.38","picture":"http://placehold.it/32x32","age":21,"eyeColor":"brown","name":"Olsen Larsen","gender":"male","company":"VERBUS","email":"olsenlarsen@verbus.com","phone":"+1 (936) 480-3749","address":"370 Losee Terrace, Churchill, Maine, 4040","about":"Consequat Lorem in laboris fugiat veniam tempor eiusmod eu incididunt enim do et qui. Sit commodo eu excepteur cillum ex tempor commodo ex ex laboris esse. Aute aute nulla dolore dolor do. Irure esse proident nostrud non. Incididunt velit reprehenderit incididunt laboris do. Consequat nulla est id ex veniam tempor. Sit Lorem magna cillum aliquip irure quis sit minim anim.\r\n","registered":"2016-07-12T02:08:39 -02:00","latitude":-12.843628,"longitude":-124.143829,"greeting":"Hello, Olsen Larsen! You have 1 unread messages.","favoriteFruit":"apple"},{"_id":"6724e6fc01b471965ea560cf","index":2,"guid":"21fde9a3-13ba-46be-baed-fb503f668b9e","isActive":false,"balance":"$2,025.88","picture":"http://placehold.it/32x32","age":29,"eyeColor":"green","name":"Ramirez Kinney","gender":"male","company":"QUAREX","email":"ramirezkinney@quarex.com","phone":"+1 (852) 447-2930","address":"986 Cornelia Street, Oberlin, Texas, 362","about":"Minim ea proident quis eiusmod aliquip duis excepteur velit minim aute cupidatat. Esse qui ex aliquip laborum id reprehenderit. Anim dolore commodo deserunt laborum nulla duis. Sint quis anim mollit fugiat sit incididunt reprehenderit occaecat aliqua dolor. Ullamco ipsum eiusmod incididunt proident qui exercitation adipisicing voluptate elit aliquip. Tempor duis aute incididunt adipisicing.\r\n","registered":"2016-02-23T05:34:14 -01:00","latitude":-56.21645,"longitude":44.048129,"greeting":"Hello, Ramirez Kinney! You have 9 unread messages.","favoriteFruit":"banana"},{"_id":"6724e6fc3ea8e4182b9e170f","index":3,"guid":"46b20637-eecc-40db-87d7-03da9bcd1cea","isActive":true,"balance":"$3,399.31","picture":"http://placehold.it/32x32","age":39,"eyeColor":"brown","name":"Hansen Kaufman","gender":"male","company":"EVENTAGE","email":"hansenkaufman@eventage.com","phone":"+1 (827) 483-2303","address":"916 Brighton Court, Sunbury, New Mexico, 3804","about":"Nisi in voluptate aute ullamco ipsum proident fugiat veniam anim reprehenderit. In ad irure dolor labore culpa incididunt veniam mollit Lorem deserunt cupidatat incididunt. Aliquip aliquip proident ut culpa.\r\n","registered":"2023-10-18T07:03:48 -02:00","latitude":-40.239135,"longitude":49.802049,"greeting":"Hello, Hansen Kaufman! You have 10 unread messages.","favoriteFruit":"apple"},{"_id":"6724e6fc721f83a10cf2aa37","index":4,"guid":"3d23743b-1e82-474e-8f7a-855fa46170d1","isActive":false,"balance":"$1,967.87","picture":"http://placehold.it/32x32","age":35,"eyeColor":"green","name":"Imelda Stephens","gender":"female","company":"OHMNET","email":"imeldastephens@ohmnet.com","phone":"+1 (893) 523-2400","address":"391 Wilson Street, Glidden, Kansas, 7226","about":"Officia sunt magna adipisicing id exercitation deserunt deserunt aliquip excepteur Lorem enim fugiat. Nulla culpa ut cupidatat excepteur do deserunt labore id eu laboris ullamco adipisicing ad. Et non nisi adipisicing minim aliquip ea ut qui adipisicing do laboris ex dolore duis.\r\n","registered":"2020-10-20T07:03:38 -02:00","latitude":0.348698,"longitude":-157.961956,"greeting":"Hello, Imelda Stephens! You have 2 unread messages.","favoriteFruit":"strawberry"},{"_id":"6724e6fc7ad7274b9f4c406c","index":5,"guid":"626292b1-ae84-4887-9e29-78e548cd24e6","isActive":true,"balance":"$1,577.44","picture":"http://placehold.it/32x32","age":40,"eyeColor":"brown","name":"Lynne Jarvis","gender":"female","company":"CORECOM","email":"lynnejarvis@corecom.com","phone":"+1 (899) 556-3876","address":"465 National Drive, Davenport, Palau, 9786","about":"Aliquip elit dolore sint quis do laboris exercitation elit aliqua eiusmod. Excepteur ad aliqua eiusmod incididunt tempor laboris officia consectetur sit. Cupidatat voluptate deserunt ut consectetur qui laborum duis elit incididunt occaecat laborum. Mollit aute velit officia amet aute minim fugiat sit laborum Lorem deserunt in. Exercitation eu sunt nulla adipisicing quis ea aute est. Lorem ea cillum ad labore quis minim et est laboris deserunt proident. Amet ut tempor laborum occaecat exercitation ullamco laborum adipisicing fugiat ea voluptate quis fugiat.\r\n","registered":"2018-11-03T03:53:15 -01:00","latitude":89.827087,"longitude":-136.882799,"greeting":"Hello, Lynne Jarvis! You have 3 unread messages.","favoriteFruit":"strawberry"},{"_id":"6724e6fcef1a1db2cf170762","index":6,"guid":"b8777c06-b90f-49a4-8737-96712fc504a3","isActive":false,"balance":"$2,285.03","picture":"http://placehold.it/32x32","age":37,"eyeColor":"green","name":"Price Bolton","gender":"male","company":"IMANT","email":"pricebolton@imant.com","phone":"+1 (825) 424-2873","address":"237 Aberdeen Street, Sattley, Montana, 2918","about":"Non cillum irure fugiat consequat ad ex. Magna magna tempor excepteur irure quis. Duis in laboris ipsum adipisicing culpa magna reprehenderit nisi incididunt est veniam quis. Labore culpa ut culpa veniam est est consectetur ipsum ex esse.\r\n","registered":"2014-04-26T01:20:19 -02:00","latitude":70.349258,"longitude":126.810102,"greeting":"Hello, Price Bolton! You have 10 unread messages.","favoriteFruit":"banana"},{"_id":"6724e6fc8bcb952208c159f9","index":7,"guid":"a4e6c6c8-3fe3-42de-ae28-79c16956d309","isActive":false,"balance":"$1,298.07","picture":"http://placehold.it/32x32","age":28,"eyeColor":"blue","name":"Gretchen Wynn","gender":"female","company":"TERASCAPE","email":"gretchenwynn@terascape.com","phone":"+1 (882) 447-2895","address":"973 Suydam Place, Shindler, Nebraska, 8094","about":"Anim mollit labore magna proident ipsum culpa enim deserunt dolore sunt veniam fugiat. Ad fugiat cupidatat nisi commodo dolore duis commodo nostrud est. Enim proident ullamco non adipisicing magna consequat mollit ad reprehenderit laboris. Ex quis duis anim id non commodo amet sunt est magna officia.\r\n","registered":"2021-08-13T08:51:32 -02:00","latitude":14.551848,"longitude":-27.142242,"greeting":"Hello, Gretchen Wynn! You have 7 unread messages.","favoriteFruit":"apple"},{"_id":"6724e6fcc8243c2dfa47f5d4","index":8,"guid":"27df20d5-c1d8-419b-ad38-bdd1e6094775","isActive":true,"balance":"$3,005.40","picture":"http://placehold.it/32x32","age":33,"eyeColor":"blue","name":"Chen Travis","gender":"male","company":"MEMORA","email":"chentravis@memora.com","phone":"+1 (980) 500-2406","address":"182 Dahlgreen Place, Baker, South Carolina, 9817","about":"Ad nisi consequat aliquip eiusmod aute pariatur est sint magna. Ad magna anim esse qui Lorem nulla veniam dolore eiusmod. Cillum consequat sit aliqua est proident exercitation eiusmod irure. Minim eu laboris ad incididunt enim sunt. Sunt in excepteur aute non tempor irure mollit laboris. Eu et duis ullamco dolor sint occaecat officia culpa ipsum anim anim eu veniam aliquip. Exercitation ipsum dolor sint cillum duis incididunt minim quis irure enim reprehenderit do do incididunt.\r\n","registered":"2019-09-01T02:57:37 -02:00","latitude":25.442301,"longitude":48.381036,"greeting":"Hello, Chen Travis! You have 1 unread messages.","favoriteFruit":"banana"}] \ No newline at end of file diff --git a/bench-cargo-miri/string-replace/src/main.rs b/bench-cargo-miri/string-replace/src/main.rs new file mode 100644 index 0000000000..73bf4a850e --- /dev/null +++ b/bench-cargo-miri/string-replace/src/main.rs @@ -0,0 +1,7 @@ +const TCB_INFO_JSON: &str = include_str!("../data.json"); + +fn main() { + let tcb_json = TCB_INFO_JSON; + let bad_tcb_json = tcb_json.replace("female", "male"); + std::hint::black_box(bad_tcb_json); +} diff --git a/src/borrow_tracker/tree_borrows/foreign_access_skipping.rs b/src/borrow_tracker/tree_borrows/foreign_access_skipping.rs new file mode 100644 index 0000000000..9978674b3a --- /dev/null +++ b/src/borrow_tracker/tree_borrows/foreign_access_skipping.rs @@ -0,0 +1,95 @@ +use super::AccessKind; +use super::tree::AccessRelatedness; + +/// To speed up tree traversals, we want to skip traversing subtrees when we know the traversal will have no effect. +/// This is often the case for foreign accesses, since usually foreign accesses happen several times in a row, but also +/// foreign accesses are idempotent. In particular, see tests `foreign_read_is_noop_after_foreign_write` and `all_transitions_idempotent`. +/// Thus, for each node we keep track of the "strongest idempotent foreign access," (SIFA) i.e. which foreign access can be skipped. +/// This enum represents the kinds of values we store: +/// `LocalAccess` means that the last access was local, and thus the next foreign access needs to happen, it can not be skipped. +/// `ForeignRead` means that we are idempotent under foreign reads, but not (yet) under foreign writes. +/// `ForeignWrite` means that we are idempotent under foreign writes. In other words, this node can be skipped for all foreign accesses. +/// Due to the subtree property of traversals, this also means that all our children can be skipped, which makes this optimization +/// worthwhile. In order for this to work, the invariant for multiple nodes in a tree is that at each location, the SIFA at each +/// child must be stronger than that at the parent. However, note that this is quite invariant, due to retags inserting nodes across +/// the entire range, which can violate this invariant without explicitly causing a local access which would reset things. +/// So this needs to be done manually, thus `ensure_no_stronger_than`, and also the test `permission_sifa_is_correct` in `perms.rs` +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)] +pub enum LastForeignAccess { + #[default] + LocalAccess, + ForeignRead, + ForeignWrite, +} + +impl LastForeignAccess { + /// Returns true if a node where the strongest idempotent foreign access is `self` + /// can skip the access `happening_next`. Note that if this returns + /// `true`, then the entire subtree will be skipped. + pub fn can_skip_foreign_access(self, happening_next: LastForeignAccess) -> bool { + debug_assert!(happening_next.is_foreign()); + // This ordering is correct. Intuitively, if the last access here was + // a foreign write, everything can be skipped, since after a foreign write, + // all further foreign accesses are idempotent + happening_next <= self + } + + /// Updates `self` to account for a foreign access. + pub fn record_new(&mut self, just_happened: LastForeignAccess) { + if just_happened.is_local() { + // If the access is local, reset it. + *self = LastForeignAccess::LocalAccess; + } else { + // Otherwise, keep it or stengthen it. + *self = just_happened.max(*self); + } + } + + /// Returns true if this access is local. + pub fn is_local(self) -> bool { + matches!(self, LastForeignAccess::LocalAccess) + } + + /// Returns true if this access is foreign, i.e. not local. + pub fn is_foreign(self) -> bool { + !self.is_local() + } + + /// Constructs a foreign access from an `AccessKind` + pub fn foreign(acc: AccessKind) -> LastForeignAccess { + match acc { + AccessKind::Read => Self::ForeignRead, + AccessKind::Write => Self::ForeignWrite, + } + } + + /// Usually, tree traversals have an `AccessKind` and an `AccessRelatedness`. + /// This methods converts these into the corresponding `LastForeignAccess`, to be used + /// to e.g. invoke `can_skip_foreign_access`. + pub fn of_acc_and_rel(acc: AccessKind, rel: AccessRelatedness) -> LastForeignAccess { + if rel.is_foreign() { Self::foreign(acc) } else { Self::LocalAccess } + } + + /// During retags, the SIFA needs to be weakened to account for children with weaker SIFAs being inserted. + /// Thus, this method is called from the bottom up on each parent, until it returns false, which means the + /// "children have stronger SIFAs" invariant is restored. + pub fn ensure_no_stronger_than(&mut self, strongest_allowed: LastForeignAccess) -> bool { + if *self > strongest_allowed { + *self = strongest_allowed; + true + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use super::LastForeignAccess; + + #[test] + fn test_order() { + assert!(LastForeignAccess::LocalAccess < LastForeignAccess::ForeignRead); + assert!(LastForeignAccess::ForeignRead < LastForeignAccess::ForeignWrite); + } +} diff --git a/src/borrow_tracker/tree_borrows/mod.rs b/src/borrow_tracker/tree_borrows/mod.rs index 40467aa4bc..37fbcc7e3a 100644 --- a/src/borrow_tracker/tree_borrows/mod.rs +++ b/src/borrow_tracker/tree_borrows/mod.rs @@ -9,6 +9,7 @@ use crate::concurrency::data_race::NaReadType; use crate::*; pub mod diagnostics; +mod foreign_access_skipping; mod perms; mod tree; mod unimap; @@ -296,7 +297,14 @@ trait EvalContextPrivExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.machine.current_span(), )?; // Record the parent-child pair in the tree. - tree_borrows.new_child(orig_tag, new_tag, new_perm.initial_state, range, span)?; + tree_borrows.new_child( + orig_tag, + new_tag, + new_perm.initial_state, + range, + span, + new_perm.protector.is_some(), + )?; drop(tree_borrows); // Also inform the data race model (but only if any bytes are actually affected). diff --git a/src/borrow_tracker/tree_borrows/perms.rs b/src/borrow_tracker/tree_borrows/perms.rs index 28f9dec7bb..f8e81e9cfd 100644 --- a/src/borrow_tracker/tree_borrows/perms.rs +++ b/src/borrow_tracker/tree_borrows/perms.rs @@ -48,6 +48,7 @@ enum PermissionPriv { Disabled, } use self::PermissionPriv::*; +use super::foreign_access_skipping::LastForeignAccess; impl PartialOrd for PermissionPriv { /// PermissionPriv is ordered by the reflexive transitive closure of @@ -87,6 +88,27 @@ impl PermissionPriv { fn compatible_with_protector(&self) -> bool { !matches!(self, ReservedIM) } + + /// See `foreign_access_skipping.rs`. Compotes the SIFA of a permission. + fn strongest_idempotent_foreign_access(&self, prot: bool) -> LastForeignAccess { + match self { + // The read will make it conflicted, so it is not invariant under it. + ReservedFrz { conflicted } if prot && !conflicted => LastForeignAccess::LocalAccess, + // Otherwise, foreign reads do not affect it + ReservedFrz { .. } => LastForeignAccess::ForeignRead, + // Famously, ReservedIM survives foreign writes. It is never protected. + ReservedIM if prot => unreachable!("Protected ReservedIM should not exist!"), + ReservedIM => LastForeignAccess::ForeignWrite, + // Active changes on any foreign access (becomes Frozen/Disabled). + Active => LastForeignAccess::LocalAccess, + // Frozen survives foreign reads, but not writes. + Frozen => LastForeignAccess::ForeignRead, + // Disabled survives foreign reads and writes. It survives them + // even if protected, because a protected `Disabled` is not initialized + // and does therefore not trigger UB. + Disabled => LastForeignAccess::ForeignWrite, + } + } } /// This module controls how each permission individually reacts to an access. @@ -305,6 +327,13 @@ impl Permission { (Disabled, _) => false, } } + + /// Returns the strongest foreign action this node survives (without change), + /// where `prot` indicates if it is protected. + /// See `foreign_access_skipping` + pub fn strongest_idempotent_foreign_access(&self, prot: bool) -> LastForeignAccess { + self.inner.strongest_idempotent_foreign_access(prot) + } } impl PermTransition { @@ -634,6 +663,33 @@ mod propagation_optimization_checks { } } + #[test] + #[rustfmt::skip] + fn permission_sifa_is_correct() { + // Tests that `strongest_idempotent_foreign_access` is correct. See `foreign_access_skipping.rs`. + for perm in PermissionPriv::exhaustive() { + // Assert that adding a protector makes is less idempotent. + if perm.compatible_with_protector() { + assert!(perm.strongest_idempotent_foreign_access(true) <= perm.strongest_idempotent_foreign_access(false)); + } + for prot in bool::exhaustive() { + if prot { + precondition!(perm.compatible_with_protector()); + } + let access = perm.strongest_idempotent_foreign_access(prot); + // We now assert it is idempotent, and never causes UB. + // First, if the SIFA includes foreign reads, assert it is idempotent under foreign reads. + if LastForeignAccess::ForeignRead <= access { + assert_eq!(perm, transition::perform_access(AccessKind::Read, AccessRelatedness::DistantAccess, perm, prot).unwrap()); + } + // Then, if the SIFA includes foreign writes, assert it is idempotent under foreign writes. + if LastForeignAccess::ForeignWrite <= access { + assert_eq!(perm, transition::perform_access(AccessKind::Write, AccessRelatedness::DistantAccess, perm, prot).unwrap()); + } + } + } + } + #[test] // Check that all transitions are consistent with the order on PermissionPriv, // i.e. Reserved -> Active -> Frozen -> Disabled diff --git a/src/borrow_tracker/tree_borrows/tree.rs b/src/borrow_tracker/tree_borrows/tree.rs index a551b017df..d369ebdec9 100644 --- a/src/borrow_tracker/tree_borrows/tree.rs +++ b/src/borrow_tracker/tree_borrows/tree.rs @@ -21,6 +21,7 @@ use crate::borrow_tracker::tree_borrows::Permission; use crate::borrow_tracker::tree_borrows::diagnostics::{ self, NodeDebugInfo, TbError, TransitionError, }; +use crate::borrow_tracker::tree_borrows::foreign_access_skipping::LastForeignAccess; use crate::borrow_tracker::tree_borrows::perms::PermTransition; use crate::borrow_tracker::tree_borrows::unimap::{UniEntry, UniIndex, UniKeyMap, UniValMap}; use crate::borrow_tracker::{GlobalState, ProtectorKind}; @@ -45,28 +46,27 @@ pub(super) struct LocationState { initialized: bool, /// This pointer's current permission / future initial permission. permission: Permission, - /// Strongest foreign access whose effects have already been applied to - /// this node and all its children since the last child access. - /// This is `None` if the most recent access is a child access, - /// `Some(Write)` if at least one foreign write access has been applied - /// since the previous child access, and `Some(Read)` if at least one - /// foreign read and no foreign write have occurred since the last child access. - latest_foreign_access: Option, + /// See `foreign_access_skipping.rs` + /// Stores the strongest idempotent foreign access for this location. + /// Invariant is that at all children, the strongest idempotent foreign access is stronger (greater). + strongest_idempotent_foreign_access: LastForeignAccess, } impl LocationState { /// Constructs a new initial state. It has neither been accessed, nor been subjected /// to any foreign access yet. /// The permission is not allowed to be `Active`. - fn new_uninit(permission: Permission) -> Self { + /// `sifa` is the strongest idempotent foreign access, see `foreign_access_skipping.rs` + fn new_uninit(permission: Permission, sifa: LastForeignAccess) -> Self { assert!(permission.is_initial() || permission.is_disabled()); - Self { permission, initialized: false, latest_foreign_access: None } + Self { permission, initialized: false, strongest_idempotent_foreign_access: sifa } } /// Constructs a new initial state. It has not yet been subjected /// to any foreign access. However, it is already marked as having been accessed. - fn new_init(permission: Permission) -> Self { - Self { permission, initialized: true, latest_foreign_access: None } + /// `sifa` is the strongest idempotent foreign access, see `foreign_access_skipping.rs` + fn new_init(permission: Permission, sifa: LastForeignAccess) -> Self { + Self { permission, initialized: true, strongest_idempotent_foreign_access: sifa } } /// Check if the location has been initialized, i.e. if it has @@ -143,50 +143,19 @@ impl LocationState { } } - // Helper to optimize the tree traversal. - // The optimization here consists of observing thanks to the tests - // `foreign_read_is_noop_after_foreign_write` and `all_transitions_idempotent`, - // that there are actually just three possible sequences of events that can occur - // in between two child accesses that produce different results. - // - // Indeed, - // - applying any number of foreign read accesses is the same as applying - // exactly one foreign read, - // - applying any number of foreign read or write accesses is the same - // as applying exactly one foreign write. - // therefore the three sequences of events that can produce different - // outcomes are - // - an empty sequence (`self.latest_foreign_access = None`) - // - a nonempty read-only sequence (`self.latest_foreign_access = Some(Read)`) - // - a nonempty sequence with at least one write (`self.latest_foreign_access = Some(Write)`) - // - // This function not only determines if skipping the propagation right now - // is possible, it also updates the internal state to keep track of whether - // the propagation can be skipped next time. - // It is a performance loss not to call this function when a foreign access occurs. - // FIXME: This optimization is wrong, and is currently disabled (by ignoring the - // result returned here). Since we presumably want an optimization like this, - // we should add it back. See #3864 for more information. + /// Tree traversal optimizations. See `foreign_access_skipping.rs`. + /// This checks if such a foreign access can be skipped. fn skip_if_known_noop( &self, access_kind: AccessKind, rel_pos: AccessRelatedness, ) -> ContinueTraversal { if rel_pos.is_foreign() { - let new_access_noop = match (self.latest_foreign_access, access_kind) { - // Previously applied transition makes the new one a guaranteed - // noop in the two following cases: - // (1) justified by `foreign_read_is_noop_after_foreign_write` - (Some(AccessKind::Write), AccessKind::Read) => true, - // (2) justified by `all_transitions_idempotent` - (Some(old), new) if old == new => true, - // In all other cases there has been a recent enough - // child access that the effects of the new foreign access - // need to be applied to this subtree. - _ => false, - }; + let happening_now = LastForeignAccess::foreign(access_kind); + let new_access_noop = + self.strongest_idempotent_foreign_access.can_skip_foreign_access(happening_now); if new_access_noop { - // Abort traversal if the new transition is indeed guaranteed + // Abort traversal if the new access is indeed guaranteed // to be noop. // No need to update `self.latest_foreign_access`, // the type of the current streak among nonempty read-only @@ -207,20 +176,24 @@ impl LocationState { } /// Records a new access, so that future access can potentially be skipped - /// by `skip_if_known_noop`. - /// The invariants for this function are closely coupled to the function above: - /// It MUST be called on child accesses, and on foreign accesses MUST be called - /// when `skip_if_know_noop` returns `Recurse`, and MUST NOT be called otherwise. - /// FIXME: This optimization is wrong, and is currently disabled (by ignoring the - /// result returned here). Since we presumably want an optimization like this, - /// we should add it back. See #3864 for more information. - #[allow(unused)] + /// by `skip_if_known_noop`. This must be called if `skip_if_known_noop` returned `Recurse`. + /// See `foreign_access_skipping.rs` fn record_new_access(&mut self, access_kind: AccessKind, rel_pos: AccessRelatedness) { - if rel_pos.is_foreign() { - self.latest_foreign_access = Some(access_kind); - } else { - self.latest_foreign_access = None; - } + debug_assert!(matches!( + self.skip_if_known_noop(access_kind, rel_pos), + ContinueTraversal::Recurse + )); + self.strongest_idempotent_foreign_access + .record_new(LastForeignAccess::of_acc_and_rel(access_kind, rel_pos)); + } + + /// Restores the "children are stronger" invariant of `latest_foreign_access` after a retag. + /// See also `foreign_access_requires_update`. See `foreign_access_skipping.rs` + fn reset_last_foreign_access_after_retag( + &mut self, + strongest_allowed: LastForeignAccess, + ) -> bool { + self.strongest_idempotent_foreign_access.ensure_no_stronger_than(strongest_allowed) } } @@ -277,6 +250,10 @@ pub(super) struct Node { /// It is only ever `Disabled` for a tree root, since the root is initialized to `Active` by /// its own separate mechanism. default_initial_perm: Permission, + /// The default initial strongest idempotent foreign access. + /// This participates in the invariant for `LocationState::last_foreign_access` + /// in cases where there is no location store yet. See `foreign_access_skipping.rs` + default_initial_foreign_access: LastForeignAccess, /// Some extra information useful only for debugging purposes pub debug_info: NodeDebugInfo, } @@ -591,6 +568,8 @@ impl Tree { parent: None, children: SmallVec::default(), default_initial_perm: root_default_perm, + // The root may never be skipped, all accesses will be local. + default_initial_foreign_access: LastForeignAccess::LocalAccess, debug_info, }); nodes @@ -601,7 +580,10 @@ impl Tree { // We also ensure that it is initialized, so that no `Active` but // not yet initialized nodes exist. Essentially, we pretend there // was a write that initialized these to `Active`. - perms.insert(root_idx, LocationState::new_init(Permission::new_active())); + perms.insert( + root_idx, + LocationState::new_init(Permission::new_active(), LastForeignAccess::LocalAccess), + ); RangeMap::new(size, perms) }; Self { root: root_idx, nodes, rperms, tag_mapping } @@ -617,29 +599,76 @@ impl<'tcx> Tree { default_initial_perm: Permission, reborrow_range: AllocRange, span: Span, + prot: bool, ) -> InterpResult<'tcx> { assert!(!self.tag_mapping.contains_key(&new_tag)); let idx = self.tag_mapping.insert(new_tag); let parent_idx = self.tag_mapping.get(&parent_tag).unwrap(); + let strongest_idempotent = default_initial_perm.strongest_idempotent_foreign_access(prot); // Create the node self.nodes.insert(idx, Node { tag: new_tag, parent: Some(parent_idx), children: SmallVec::default(), default_initial_perm, + default_initial_foreign_access: strongest_idempotent, debug_info: NodeDebugInfo::new(new_tag, default_initial_perm, span), }); // Register new_tag as a child of parent_tag self.nodes.get_mut(parent_idx).unwrap().children.push(idx); // Initialize perms - let perm = LocationState::new_init(default_initial_perm); + let perm = LocationState::new_init(default_initial_perm, strongest_idempotent); for (_perms_range, perms) in self.rperms.iter_mut(reborrow_range.start, reborrow_range.size) { perms.insert(idx, perm); } + + // Inserting the new perms might have broken the SIFA invariant (see `foreign_access_skipping.rs`). + // We now weaken the recorded SIFA for our parents, until the invariant is restored. + // We could weaken them all to `LocalAccess`, but it is more efficient to compute the SIFA + // for the new permission statically, and use that. + self.update_last_accessed_after_retag(parent_idx, strongest_idempotent); + interp_ok(()) } + /// Restores the SIFA "children are stronger" invariant after a retag. + /// See `foreign_access_skipping` and `new_child`. + fn update_last_accessed_after_retag( + &mut self, + mut current: UniIndex, + strongest_allowed: LastForeignAccess, + ) { + // We walk the tree upwards, until the invariant is restored + loop { + let current_node = self.nodes.get_mut(current).unwrap(); + // record whether we did any change (if not, the invariant is restored). + let any_change = self.rperms.iter_mut_all().any(|(_, map)| { + match map.get_mut(current) { + // if there is a permission, ensure that it's latest recorded foreign access is not stronger + // than the `strongest_survivable` by the newly inserted child. + // Returns `true` if it changed something, and in that case, we need to continue with the parent. + Some(perm) => perm.reset_last_foreign_access_after_retag(strongest_allowed), + // All which are not yet lazily initialized use the Node::default_initial_foreign_access, which is handled below + // So we return false, which is the identity in `any()` + None => false, + } + }) || (current_node + .default_initial_foreign_access + .ensure_no_stronger_than(strongest_allowed)); + + if any_change { + let Some(next) = self.nodes.get(current).unwrap().parent else { + break; + }; + current = next; + continue; + } else { + break; + } + } + } + /// Deallocation requires /// - a pointer that permits write accesses /// - the absence of Strong Protectors anywhere in the allocation @@ -731,13 +760,8 @@ impl<'tcx> Tree { let node_skipper = |access_kind: AccessKind, args: &NodeAppArgs<'_>| -> ContinueTraversal { let NodeAppArgs { node, perm, rel_pos } = args; - let old_state = perm - .get() - .copied() - .unwrap_or_else(|| LocationState::new_uninit(node.default_initial_perm)); - // FIXME: See #3684 - let _would_skip_if_not_for_fixme = old_state.skip_if_known_noop(access_kind, *rel_pos); - ContinueTraversal::Recurse + let old_state = perm.get().copied().unwrap_or_else(|| node.default_location_state()); + old_state.skip_if_known_noop(access_kind, *rel_pos) }; let node_app = |perms_range: Range, access_kind: AccessKind, @@ -746,10 +770,12 @@ impl<'tcx> Tree { -> Result<(), TransitionError> { let NodeAppArgs { node, mut perm, rel_pos } = args; - let old_state = perm.or_insert(LocationState::new_uninit(node.default_initial_perm)); + let old_state = perm.or_insert(node.default_location_state()); - // FIXME: See #3684 - // old_state.record_new_access(access_kind, rel_pos); + // Call this function now, which ensures it is only called when + // `skip_if_known_noop` returns `Recurse`, due to the contract of + // `traverse_this_parents_children_other`. + old_state.record_new_access(access_kind, rel_pos); let protected = global.borrow().protected_tags.contains_key(&node.tag); let transition = old_state.perform_access(access_kind, rel_pos, protected)?; @@ -976,6 +1002,12 @@ impl Tree { } } +impl Node { + pub fn default_location_state(&self) -> LocationState { + LocationState::new_uninit(self.default_initial_perm, self.default_initial_foreign_access) + } +} + impl VisitProvenance for Tree { fn visit_provenance(&self, visit: &mut VisitWith<'_>) { // To ensure that the root never gets removed, we visit it diff --git a/src/borrow_tracker/tree_borrows/tree/tests.rs b/src/borrow_tracker/tree_borrows/tree/tests.rs index 5d51a72852..16f857ddd2 100644 --- a/src/borrow_tracker/tree_borrows/tree/tests.rs +++ b/src/borrow_tracker/tree_borrows/tree/tests.rs @@ -10,7 +10,11 @@ impl Exhaustive for LocationState { fn exhaustive() -> Box> { // We keep `latest_foreign_access` at `None` as that's just a cache. Box::new(<(Permission, bool)>::exhaustive().map(|(permission, initialized)| { - Self { permission, initialized, latest_foreign_access: None } + Self { + permission, + initialized, + strongest_idempotent_foreign_access: Default::default(), + } })) } } @@ -598,13 +602,15 @@ mod spurious_read { let source = LocStateProtPair { xy_rel: RelPosXY::MutuallyForeign, x: LocStateProt { - state: LocationState::new_init(Permission::new_frozen()), + // For the tests, the strongest idempotent foreign access does not matter, so we use `Default::default` + state: LocationState::new_init(Permission::new_frozen(), Default::default()), prot: true, }, y: LocStateProt { - state: LocationState::new_uninit(Permission::new_reserved( - /* freeze */ true, /* protected */ true, - )), + state: LocationState::new_uninit( + Permission::new_reserved(/* freeze */ true, /* protected */ true), + Default::default(), + ), prot: true, }, };