feat(encoding): add max length check for bytes (#1399)

* feat(encoding): add constants MaxLength and ByteArrayLength in encoding

* feat(encoding): add encoding for byte array and generic array

* feat(beacon): refactor BeaconEntry with ByteArray

* feat(serde): add module checked_serde_bytes

* chore(lint): fix lint

* feat(serde): apply serde_byte_array to ProofBytes and VRFBytes

* chore(serde): clean files

* feat(encoding): add tests for serde_byte_array

* chore(lint): make clippy happy

* doc(encoding): fix typo

Author: clearloop

blockchain/beacon/src/beacon_entries.rs

@@ -1,16 +1,16 @@
 // Copyright 2019-2022 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
-use encoding::{serde_bytes, tuple::*};
+use encoding::{serde_byte_array, tuple::*};
 
 /// The result from getting an entry from Drand.
 /// The entry contains the round, or epoch as well as the BLS signature for that round of
 /// randomness.
 /// This beacon entry is stored on chain in the block header.
-#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize_tuple, Deserialize_tuple)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize_tuple, Serialize_tuple)]
 pub struct BeaconEntry {
     round: u64,
-    #[serde(with = "serde_bytes")]
+    #[serde(with = "serde_byte_array")]
     data: Vec<u8>,
 }
 

crypto/src/vrf.rs

@@ -3,13 +3,13 @@
 
 use crate::signature::verify_bls_sig;
 use address::Address;
-use encoding::{blake2b_256, serde_bytes};
+use encoding::{blake2b_256, serde_byte_array};
 use serde::{Deserialize, Serialize};
 
 /// The output from running a VRF proof.
 #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Default, Serialize, Deserialize)]
 #[serde(transparent)]
-pub struct VRFProof(#[serde(with = "serde_bytes")] pub Vec<u8>);
+pub struct VRFProof(#[serde(with = "serde_byte_array")] pub Vec<u8>);
 
 impl VRFProof {
     /// Creates a VRFProof from a raw vector.

encoding/src/checked_serde_bytes.rs

@@ -0,0 +1,118 @@
+// Copyright 2019-2022 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use crate::BYTE_ARRAY_MAX_LEN;
+use serde::{de, ser, Deserializer, Serializer};
+use serde_bytes::{Deserialize, Serialize};
+
+/// serde_bytes with max length check
+pub mod serde_byte_array {
+    use super::*;
+
+    /// checked if input > `crate::BYTE_ARRAY_MAX_LEN`
+    pub fn serialize<T, S>(bytes: &T, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        T: ?Sized + Serialize + AsRef<[u8]>,
+        S: Serializer,
+    {
+        let len = bytes.as_ref().len();
+        if len > BYTE_ARRAY_MAX_LEN {
+            return Err(ser::Error::custom::<String>(
+                "Array exceed max length".into(),
+            ));
+        }
+
+        Serialize::serialize(bytes, serializer)
+    }
+
+    /// checked if output > `crate::ByteArrayMaxLen`
+    pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
+    where
+        T: Deserialize<'de> + AsRef<[u8]>,
+        D: Deserializer<'de>,
+    {
+        Deserialize::deserialize(deserializer).and_then(|bytes: T| {
+            if bytes.as_ref().len() > BYTE_ARRAY_MAX_LEN {
+                Err(de::Error::custom::<String>(
+                    "Array exceed max length".into(),
+                ))
+            } else {
+                Ok(bytes)
+            }
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::serde_byte_array;
+    use crate::BYTE_ARRAY_MAX_LEN;
+    use serde::{Deserialize, Serialize};
+
+    #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+    struct ByteArray {
+        #[serde(with = "serde_byte_array")]
+        pub inner: Vec<u8>,
+    }
+
+    #[test]
+    fn can_serialize_byte_array() {
+        for len in [0, 1, BYTE_ARRAY_MAX_LEN] {
+            let bytes = ByteArray {
+                inner: vec![0; len],
+            };
+
+            assert!(serde_cbor::to_vec(&bytes).is_ok());
+        }
+    }
+
+    #[test]
+    fn cannot_serialize_byte_array_overflow() {
+        let bytes = ByteArray {
+            inner: vec![0; BYTE_ARRAY_MAX_LEN + 1],
+        };
+
+        assert_eq!(
+            format!("{}", serde_cbor::to_vec(&bytes).err().unwrap()),
+            "Array exceed max length".to_string()
+        );
+    }
+
+    #[test]
+    fn can_deserialize_byte_array() {
+        for len in [0, 1, BYTE_ARRAY_MAX_LEN] {
+            let bytes = ByteArray {
+                inner: vec![0; len],
+            };
+
+            let encoding = serde_cbor::to_vec(&bytes).unwrap();
+            assert_eq!(
+                serde_cbor::from_slice::<ByteArray>(&encoding).unwrap(),
+                bytes
+            );
+        }
+    }
+
+    #[test]
+    fn cannot_deserialize_byte_array_overflow() {
+        let max_length_bytes = ByteArray {
+            inner: vec![0; BYTE_ARRAY_MAX_LEN],
+        };
+
+        // prefix: 2 ^ 21 -> 2 ^ 21 + 1
+        let mut overflow_encoding = serde_cbor::to_vec(&max_length_bytes).unwrap();
+        let encoding_len = overflow_encoding.len();
+        overflow_encoding[encoding_len - BYTE_ARRAY_MAX_LEN - 1] = 1;
+        overflow_encoding.push(0);
+
+        assert_eq!(
+            format!(
+                "{}",
+                serde_cbor::from_slice::<ByteArray>(&overflow_encoding)
+                    .err()
+                    .unwrap()
+            ),
+            "Array exceed max length".to_string()
+        );
+    }
+}

encoding/src/lib.rs

@@ -3,6 +3,7 @@
 
 mod bytes;
 mod cbor;
+mod checked_serde_bytes;
 mod errors;
 mod hash;
 
@@ -12,6 +13,7 @@ pub use serde_cbor::{error, from_reader, from_slice, tags, to_vec, to_writer};
 
 pub use self::bytes::*;
 pub use self::cbor::*;
+pub use self::checked_serde_bytes::serde_byte_array;
 pub use self::errors::*;
 pub use self::hash::*;
 
@@ -22,3 +24,15 @@ pub mod tuple {
 pub mod repr {
     pub use serde_repr::{Deserialize_repr, Serialize_repr};
 }
+
+/// lotus use cbor-gen for generating codec for types, it has a length limit of generic array
+/// for `8192`
+///
+/// https://github.com/whyrusleeping/cbor-gen/blob/f57984553008dd4285df16d4ec2760f97977d713/gen.go#L14
+pub const GENERIC_ARRAY_MAX_LEN: usize = 8192;
+
+/// lotus use cbor-gen for generating codec for types, it has a length limit for byte
+/// array as `2 << 20`
+///
+/// https://github.com/whyrusleeping/cbor-gen/blob/f57984553008dd4285df16d4ec2760f97977d713/gen.go#L16
+pub const BYTE_ARRAY_MAX_LEN: usize = 2097152;

types/src/sector/post.rs

@@ -3,7 +3,7 @@
 
 use crate::{ActorID, Randomness, RegisteredPoStProof, RegisteredSealProof, SectorNumber};
 use cid::Cid;
-use encoding::{serde_bytes, tuple::*};
+use encoding::{serde_byte_array, tuple::*};
 
 /// Randomness type used for generating PoSt proof randomness.
 pub type PoStRandomness = Randomness;
@@ -21,7 +21,7 @@ pub struct SectorInfo {
 #[derive(Debug, PartialEq, Clone, Eq, Serialize_tuple, Deserialize_tuple)]
 pub struct PoStProof {
     pub post_proof: RegisteredPoStProof,
-    #[serde(with = "serde_bytes")]
+    #[serde(with = "serde_byte_array")]
     pub proof_bytes: Vec<u8>,
 }
 

utils/bitfield/src/rleplus/writer.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0, MIT
 
 /// The maximum varint that can be serialized on 9 bytes.
-const MAX_VARINT: u64 = 2u64.pow(63)-1;
+const MAX_VARINT: u64 = 2u64.pow(63) - 1;
 
 #[derive(Default, Clone, Debug)]
 /// A `BitWriter` allows for efficiently writing bits to a byte buffer, up to a byte at a time.