Separate raw documents from the database.

This CR implements a layered-approach for the database: the first
layer is a meta database that stores all the information except the raw
document content. Instead, the meta database stores ids of the raw
documents. The second layer is a simple key value storage, which stores
the raw documents and the meta database as a encrypted data blob.

Assuming a pair memory id and a blob id is about 20 bytes, this allows users store about 500000 documents, each below 10M.

BUG: 412698203
Change-Id: Ia40fd696547d3279bc13ed58f4374682ec3a6a41

Author: Changochen

oak_private_memory/BUILD

@@ -24,10 +24,10 @@ package(
 )
 
 rust_library(
-    name = "private_memory_database_server_lib",
+    name = "private_memory_test_database_server_lib",
     srcs = [
-        "src/database/lib.rs",
-        "src/database/service.rs",
+        "test/database/lib.rs",
+        "test/database/service.rs",
     ],
     proc_macro_deps = ["@oak_crates_index//:async-trait"],
     deps = [
@@ -52,6 +52,7 @@ rust_library(
         "src/app.rs",
         "src/app_config.rs",
         "src/app_service.rs",
+        "src/database.rs",
         "src/encryption.rs",
         "src/lib.rs",
     ],
@@ -66,9 +67,11 @@ rust_library(
         "@oak_crates_index//:aes-gcm",
         "@oak_crates_index//:anyhow",
         "@oak_crates_index//:async-stream",
+        "@oak_crates_index//:futures",
         "@oak_crates_index//:log",
         "@oak_crates_index//:prost",
         "@oak_crates_index//:prost-types",
+        "@oak_crates_index//:rand",
         "@oak_crates_index//:serde",
         "@oak_crates_index//:serde_json",
         "@oak_crates_index//:tokio",
@@ -106,8 +109,8 @@ rust_test(
     srcs = ["test/standalone_test.rs"],
     proc_macro_deps = ["@oak_crates_index//:async-trait"],
     deps = [
-        ":private_memory_database_server_lib",
         ":private_memory_server_lib",
+        ":private_memory_test_database_server_lib",
         "//proto:sealed_memory_rust_proto",
         "//proto/grpc:sealed_memory_grpc_proto",
         "@oak//oak_client",

oak_private_memory/Cargo.bazel.lock

@@ -929,7 +929,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
  "generic-array",
- "rand_core",
+ "rand_core 0.6.4",
  "subtle",
  "zeroize",
 ]
@@ -941,7 +941,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array",
- "rand_core",
+ "rand_core 0.6.4",
  "typenum",
 ]
 
@@ -1252,9 +1252,9 @@ dependencies = [
  "prost-derive",
  "prost-types",
  "quote",
- "rand",
- "rand_chacha",
- "rand_core",
+ "rand 0.9.1",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
  "regex",
  "regex-lite",
  "rlsf",
@@ -1377,7 +1377,7 @@ dependencies = [
  "group",
  "pem-rfc7468",
  "pkcs8",
- "rand_core",
+ "rand_core 0.6.4",
  "sec1",
  "subtle",
  "zeroize",
@@ -1481,7 +1481,7 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393"
 dependencies = [
- "rand_core",
+ "rand_core 0.6.4",
  "subtle",
 ]
 
@@ -1760,7 +1760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63"
 dependencies = [
  "ff",
- "rand_core",
+ "rand_core 0.6.4",
  "subtle",
 ]
 
@@ -1913,7 +1913,7 @@ dependencies = [
  "generic-array",
  "hkdf",
  "hmac",
- "rand_core",
+ "rand_core 0.6.4",
  "sha2",
  "subtle",
  "x25519-dalek",
@@ -2627,7 +2627,7 @@ dependencies = [
  "num-integer",
  "num-iter",
  "num-traits",
- "rand",
+ "rand 0.8.5",
  "smallvec",
  "zeroize",
 ]
@@ -2802,7 +2802,7 @@ dependencies = [
  "opentelemetry",
  "ordered-float 4.6.0",
  "percent-encoding",
- "rand",
+ "rand 0.8.5",
  "serde_json",
  "thiserror 1.0.69",
  "tokio",
@@ -3082,7 +3082,7 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9"
 dependencies = [
- "rand",
+ "rand 0.8.5",
 ]
 
 [[package]]
@@ -3363,8 +3363,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
+dependencies = [
+ "rand_chacha 0.9.0",
+ "rand_core 0.9.3",
 ]
 
 [[package]]
@@ -3374,7 +3384,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.3",
 ]
 
 [[package]]
@@ -3386,6 +3406,15 @@ dependencies = [
  "getrandom 0.2.15",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+dependencies = [
+ "getrandom 0.3.2",
+]
+
 [[package]]
 name = "raw-cpuid"
 version = "10.7.0"
@@ -3541,7 +3570,7 @@ dependencies = [
  "num-traits",
  "pkcs1",
  "pkcs8",
- "rand_core",
+ "rand_core 0.6.4",
  "signature",
  "spki",
  "subtle",
@@ -3863,7 +3892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
 dependencies = [
  "digest",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -4452,7 +4481,7 @@ dependencies = [
  "indexmap 1.9.3",
  "pin-project",
  "pin-project-lite",
- "rand",
+ "rand 0.8.5",
  "slab",
  "tokio",
  "tokio-util",
@@ -5399,7 +5428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277"
 dependencies = [
  "curve25519-dalek",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]

oak_private_memory/bazel/crates/oak_crates.bzl

@@ -431,7 +431,7 @@ OAK_STD_CRATES = _common_crates(std = True) | {
     "prost-derive": crate.spec(version = "0.12.4"),
     "prost-types": crate.spec(version = "0.12.3"),
     "quote": crate.spec(version = "1.0.35"),
-    "rand": crate.spec(version = "0.8.5"),
+    "rand": crate.spec(version = "0.9.1"),
     "regex": crate.spec(
         default_features = False,
         version = "1.10.2",