[move-model-2] Rewrite normalized module for broader use (MystenLabs#21751)

## Description 

- Improved efficiency of normalized module through use of Rcs
- Generalized Identifiers to allow for different "interned" symbols

## Test plan 

- Ran tests

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] gRPC:
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:

Author: tnowacki

Cargo.lock

@@ -329,7 +329,11 @@ async fn test_upgrade_package_happy_path() {
         .unwrap();
     let config = ProtocolConfig::get_for_max_version_UNSAFE();
     let binary_config = to_binary_config(&config);
-    let normalized_modules = package.move_package().normalize(&binary_config).unwrap();
+    let pool = &mut move_binary_format::normalized::RcPool::new();
+    let normalized_modules = package
+        .move_package()
+        .normalize(pool, &binary_config, /* include code */ true)
+        .unwrap();
     assert!(normalized_modules.contains_key("new_module"));
     assert!(normalized_modules["new_module"]
         .functions

crates/sui-core/src/unit_tests/move_package_upgrade_tests.rs

@@ -1,6 +1,7 @@
 // Copyright (c) Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
+use move_binary_format::normalized;
 use move_binary_format::{
     binary_config::BinaryConfig, compatibility::Compatibility, CompiledModule,
 };
@@ -248,14 +249,17 @@ pub async fn compare_system_package<S: ObjectStore>(
         .try_as_package_mut()
         .expect("Created as package");
 
-    let cur_normalized = match cur_pkg.normalize(binary_config) {
+    let pool = &mut normalized::RcPool::new();
+    let cur_normalized = match cur_pkg.normalize(pool, binary_config, /* include code */ false) {
         Ok(v) => v,
         Err(e) => {
             error!("Could not normalize existing package: {e:?}");
             return None;
         }
     };
-    let mut new_normalized = new_pkg.normalize(binary_config).ok()?;
+    let mut new_normalized = new_pkg
+        .normalize(pool, binary_config, /* include code */ false)
+        .ok()?;
 
     for (name, cur_module) in cur_normalized {
         let new_module = new_normalized.remove(&name)?;

crates/sui-framework/src/lib.rs

@@ -111,12 +111,12 @@ fn normalized_type(sig: &OpenSignatureBody) -> SuiMoveNormalizedType {
         S::U128 => T::U128,
         S::U256 => T::U256,
         S::Vector(sig) => T::Vector(Box::new(normalized_type(sig))),
-        S::Datatype(t, params) => T::Struct {
-            address: t.package.to_canonical_string(/* with_prefix */ true),
-            module: t.module.to_string(),
-            name: t.name.to_string(),
-            type_arguments: params.iter().map(normalized_type).collect(),
-        },
+        S::Datatype(t, params) => T::new_struct(
+            t.package.to_canonical_string(/* with_prefix */ true),
+            t.module.to_string(),
+            t.name.to_string(),
+            params.iter().map(normalized_type).collect(),
+        ),
         S::TypeParameter(ix) => T::TypeParameter(*ix),
     }
 }

crates/sui-indexer-alt-jsonrpc/src/api/move_utils/response.rs

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use async_trait::async_trait;
 use jsonrpsee::core::RpcResult;
 use jsonrpsee::RpcModule;
-use move_binary_format::normalized::Module as NormalizedModule;
+use move_binary_format::normalized;
 
 use sui_json_rpc::error::SuiRpcInputError;
 use sui_json_rpc::SuiRpcModule;
@@ -39,9 +39,13 @@ impl MoveUtilsServer for MoveUtilsApi {
         package_id: ObjectID,
     ) -> RpcResult<BTreeMap<String, SuiMoveNormalizedModule>> {
         let resolver_modules = self.inner.get_package(package_id).await?.modules().clone();
+        let pool = &mut normalized::RcPool::new();
         let sui_normalized_modules = resolver_modules
             .into_iter()
-            .map(|(k, v)| (k, NormalizedModule::new(v.bytecode()).into()))
+            .map(|(k, v)| {
+                let m = &normalized::Module::new(pool, v.bytecode(), /* include code */ false);
+                (k, m.into())
+            })
             .collect::<BTreeMap<String, SuiMoveNormalizedModule>>();
         Ok(sui_normalized_modules)
     }

crates/sui-indexer/src/apis/move_utils.rs

@@ -5,7 +5,7 @@ use colored::Colorize;
 use itertools::Itertools;
 use move_binary_format::file_format::{Ability, AbilitySet, DatatypeTyParameter, Visibility};
 use move_binary_format::normalized::{
-    Enum as NormalizedEnum, Field as NormalizedField, Function as SuiNormalizedFunction,
+    self, Enum as NormalizedEnum, Field as NormalizedField, Function as NormalizedFunction,
     Module as NormalizedModule, Struct as NormalizedStruct, Type as NormalizedType,
 };
 use move_core_types::annotated_value::{MoveStruct, MoveValue, MoveVariant};
@@ -91,19 +91,25 @@ pub enum SuiMoveNormalizedType {
     U256,
     Address,
     Signer,
-    #[serde(rename_all = "camelCase")]
     Struct {
-        address: String,
-        module: String,
-        name: String,
-        type_arguments: Vec<SuiMoveNormalizedType>,
+        #[serde(flatten)]
+        inner: Box<SuiMoveNormalizedStructType>,
     },
     Vector(Box<SuiMoveNormalizedType>),
     TypeParameter(SuiMoveTypeParameterIndex),
     Reference(Box<SuiMoveNormalizedType>),
     MutableReference(Box<SuiMoveNormalizedType>),
 }
 
+#[derive(Serialize, Deserialize, Debug, JsonSchema, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SuiMoveNormalizedStructType {
+    pub address: String,
+    pub module: String,
+    pub name: String,
+    pub type_arguments: Vec<SuiMoveNormalizedType>,
+}
+
 #[derive(Serialize, Deserialize, Debug, JsonSchema, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct SuiMoveNormalizedFunction {
@@ -141,45 +147,53 @@ impl PartialEq for SuiMoveNormalizedModule {
     }
 }
 
-impl From<NormalizedModule> for SuiMoveNormalizedModule {
-    fn from(module: NormalizedModule) -> Self {
+impl<S: std::hash::Hash + Eq + ToString> From<&NormalizedModule<S>> for SuiMoveNormalizedModule {
+    fn from(module: &NormalizedModule<S>) -> Self {
         Self {
             file_format_version: module.file_format_version,
-            address: module.address.to_hex_literal(),
-            name: module.name.to_string(),
+            address: module.address().to_hex_literal(),
+            name: module.name().to_string(),
             friends: module
                 .friends
-                .into_iter()
+                .iter()
                 .map(|module_id| SuiMoveModuleId {
-                    address: module_id.address().to_hex_literal(),
-                    name: module_id.name().to_string(),
+                    address: module_id.address.to_hex_literal(),
+                    name: module_id.name.to_string(),
                 })
                 .collect::<Vec<SuiMoveModuleId>>(),
             structs: module
                 .structs
-                .into_iter()
-                .map(|(name, struct_)| (name.to_string(), SuiMoveNormalizedStruct::from(struct_)))
+                .iter()
+                .map(|(name, struct_)| {
+                    (name.to_string(), SuiMoveNormalizedStruct::from(&**struct_))
+                })
                 .collect::<BTreeMap<String, SuiMoveNormalizedStruct>>(),
             enums: module
                 .enums
-                .into_iter()
-                .map(|(name, enum_)| (name.to_string(), SuiMoveNormalizedEnum::from(enum_)))
+                .iter()
+                .map(|(name, enum_)| (name.to_string(), SuiMoveNormalizedEnum::from(&**enum_)))
                 .collect(),
             exposed_functions: module
                 .functions
-                .into_iter()
-                .filter_map(|(name, function)| {
+                .iter()
+                .filter(|(_name, function)| {
+                    function.is_entry || function.visibility != Visibility::Private
+                })
+                .map(|(name, function)| {
                     // TODO: Do we want to expose the private functions as well?
-                    (function.is_entry || function.visibility != Visibility::Private)
-                        .then(|| (name.to_string(), SuiMoveNormalizedFunction::from(function)))
+
+                    (
+                        name.to_string(),
+                        SuiMoveNormalizedFunction::from(&**function),
+                    )
                 })
                 .collect::<BTreeMap<String, SuiMoveNormalizedFunction>>(),
         }
     }
 }
 
-impl From<SuiNormalizedFunction> for SuiMoveNormalizedFunction {
-    fn from(function: SuiNormalizedFunction) -> Self {
+impl<S: ToString> From<&NormalizedFunction<S>> for SuiMoveNormalizedFunction {
+    fn from(function: &NormalizedFunction<S>) -> Self {
         Self {
             visibility: match function.visibility {
                 Visibility::Private => SuiMoveVisibility::Private,
@@ -189,59 +203,62 @@ impl From<SuiNormalizedFunction> for SuiMoveNormalizedFunction {
             is_entry: function.is_entry,
             type_parameters: function
                 .type_parameters
-                .into_iter()
+                .iter()
+                .copied()
                 .map(|a| a.into())
                 .collect::<Vec<SuiMoveAbilitySet>>(),
             parameters: function
                 .parameters
-                .into_iter()
-                .map(SuiMoveNormalizedType::from)
+                .iter()
+                .map(|t| SuiMoveNormalizedType::from(&**t))
                 .collect::<Vec<SuiMoveNormalizedType>>(),
             return_: function
                 .return_
-                .into_iter()
-                .map(SuiMoveNormalizedType::from)
+                .iter()
+                .map(|t| SuiMoveNormalizedType::from(&**t))
                 .collect::<Vec<SuiMoveNormalizedType>>(),
         }
     }
 }
 
-impl From<NormalizedStruct> for SuiMoveNormalizedStruct {
-    fn from(struct_: NormalizedStruct) -> Self {
+impl<S: ToString> From<&NormalizedStruct<S>> for SuiMoveNormalizedStruct {
+    fn from(struct_: &NormalizedStruct<S>) -> Self {
         Self {
             abilities: struct_.abilities.into(),
             type_parameters: struct_
                 .type_parameters
-                .into_iter()
+                .iter()
+                .copied()
                 .map(SuiMoveStructTypeParameter::from)
                 .collect::<Vec<SuiMoveStructTypeParameter>>(),
             fields: struct_
                 .fields
-                .into_iter()
-                .map(SuiMoveNormalizedField::from)
+                .iter()
+                .map(|f| SuiMoveNormalizedField::from(&**f))
                 .collect::<Vec<SuiMoveNormalizedField>>(),
         }
     }
 }
 
-impl From<NormalizedEnum> for SuiMoveNormalizedEnum {
-    fn from(value: NormalizedEnum) -> Self {
+impl<S: ToString> From<&NormalizedEnum<S>> for SuiMoveNormalizedEnum {
+    fn from(value: &NormalizedEnum<S>) -> Self {
         Self {
             abilities: value.abilities.into(),
             type_parameters: value
                 .type_parameters
-                .into_iter()
+                .iter()
+                .copied()
                 .map(SuiMoveStructTypeParameter::from)
                 .collect::<Vec<SuiMoveStructTypeParameter>>(),
             variants: value
                 .variants
-                .into_iter()
+                .iter()
                 .map(|variant| {
                     (
                         variant.name.to_string(),
                         variant
                             .fields
-                            .into_iter()
+                            .iter()
                             .map(SuiMoveNormalizedField::from)
                             .collect::<Vec<SuiMoveNormalizedField>>(),
                     )
@@ -260,17 +277,17 @@ impl From<DatatypeTyParameter> for SuiMoveStructTypeParameter {
     }
 }
 
-impl From<NormalizedField> for SuiMoveNormalizedField {
-    fn from(normalized_field: NormalizedField) -> Self {
+impl<S: ToString> From<&NormalizedField<S>> for SuiMoveNormalizedField {
+    fn from(normalized_field: &NormalizedField<S>) -> Self {
         Self {
             name: normalized_field.name.to_string(),
-            type_: SuiMoveNormalizedType::from(normalized_field.type_),
+            type_: SuiMoveNormalizedType::from(&normalized_field.type_),
         }
     }
 }
 
-impl From<NormalizedType> for SuiMoveNormalizedType {
-    fn from(type_: NormalizedType) -> Self {
+impl<S: ToString> From<&NormalizedType<S>> for SuiMoveNormalizedType {
+    fn from(type_: &NormalizedType<S>) -> Self {
         match type_ {
             NormalizedType::Bool => SuiMoveNormalizedType::Bool,
             NormalizedType::U8 => SuiMoveNormalizedType::U8,
@@ -281,30 +298,32 @@ impl From<NormalizedType> for SuiMoveNormalizedType {
             NormalizedType::U256 => SuiMoveNormalizedType::U256,
             NormalizedType::Address => SuiMoveNormalizedType::Address,
             NormalizedType::Signer => SuiMoveNormalizedType::Signer,
-            NormalizedType::Struct {
-                address,
-                module,
-                name,
-                type_arguments,
-            } => SuiMoveNormalizedType::Struct {
-                address: address.to_hex_literal(),
-                module: module.to_string(),
-                name: name.to_string(),
-                type_arguments: type_arguments
-                    .into_iter()
-                    .map(SuiMoveNormalizedType::from)
-                    .collect::<Vec<SuiMoveNormalizedType>>(),
-            },
-            NormalizedType::Vector(v) => {
-                SuiMoveNormalizedType::Vector(Box::new(SuiMoveNormalizedType::from(*v)))
+            NormalizedType::Datatype(dt) => {
+                let normalized::Datatype {
+                    module,
+                    name,
+                    type_arguments,
+                } = &**dt;
+                SuiMoveNormalizedType::new_struct(
+                    module.address.to_hex_literal(),
+                    module.name.to_string(),
+                    name.to_string(),
+                    type_arguments
+                        .iter()
+                        .map(SuiMoveNormalizedType::from)
+                        .collect::<Vec<SuiMoveNormalizedType>>(),
+                )
             }
-            NormalizedType::TypeParameter(t) => SuiMoveNormalizedType::TypeParameter(t),
-            NormalizedType::Reference(r) => {
-                SuiMoveNormalizedType::Reference(Box::new(SuiMoveNormalizedType::from(*r)))
+            NormalizedType::Vector(v) => {
+                SuiMoveNormalizedType::Vector(Box::new(SuiMoveNormalizedType::from(&**v)))
             }
-            NormalizedType::MutableReference(mr) => {
-                SuiMoveNormalizedType::MutableReference(Box::new(SuiMoveNormalizedType::from(*mr)))
+            NormalizedType::TypeParameter(t) => SuiMoveNormalizedType::TypeParameter(*t),
+            NormalizedType::Reference(false, r) => {
+                SuiMoveNormalizedType::Reference(Box::new(SuiMoveNormalizedType::from(&**r)))
             }
+            NormalizedType::Reference(true, mr) => SuiMoveNormalizedType::MutableReference(
+                Box::new(SuiMoveNormalizedType::from(&**mr)),
+            ),
         }
     }
 }
@@ -325,6 +344,24 @@ impl From<AbilitySet> for SuiMoveAbilitySet {
     }
 }
 
+impl SuiMoveNormalizedType {
+    pub fn new_struct(
+        address: String,
+        module: String,
+        name: String,
+        type_arguments: Vec<SuiMoveNormalizedType>,
+    ) -> Self {
+        SuiMoveNormalizedType::Struct {
+            inner: Box::new(SuiMoveNormalizedStructType {
+                address,
+                module,
+                name,
+                type_arguments,
+            }),
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug, JsonSchema, Clone)]
 pub enum ObjectValueKind {
     ByImmutableReference,
@@ -653,3 +690,8 @@ impl From<MoveStruct> for SuiMoveStruct {
         }
     }
 }
+
+#[test]
+fn enum_size() {
+    assert_eq!(std::mem::size_of::<SuiMoveNormalizedType>(), 16);
+}