Skip to content

Commit

Permalink
Allow persisting entities as documents
Browse files Browse the repository at this point in the history
Introduce a document-based repository that allows persisting the entire entity payload to a single column.

Regardless of the serialization strategy used, we persist the type full name and assembly version of the persisted entity, which can be invaluable in data migration scenarios.

In addition to the built-in JSON text serialization, we provide three binary serializers too as separate packages: Bson, MessagePack and Protobuf. The last two require annotating the entity as required by the underlying libraries.

Fixes #24.
  • Loading branch information
kzu committed May 31, 2021
1 parent db525bd commit 3756104
Show file tree
Hide file tree
Showing 39 changed files with 1,284 additions and 16 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ _site
.jekyll-metadata
.jekyll-cache
Gemfile.lock
package-lock.json
package-lock.json
__azurite_db_table__.json
44 changes: 44 additions & 0 deletions TableStorage.sln
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,20 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "src\Tests\Tests.cs
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Source", "src\TableStorage.Source\TableStorage.Source.csproj", "{B58183AB-38E7-42FA-BE98-5D7B58C06266}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Bson", "src\TableStorage.Bson\TableStorage.Bson.csproj", "{D0C6041C-D796-483A-8470-F78A4C659BD5}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Bson.Source", "src\TableStorage.Bson.Source\TableStorage.Bson.Source.csproj", "{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Source", "Source", "{7412B98E-AA65-4B6E-AD83-6F9960C1C452}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.MessagePack", "src\TableStorage.MessagePack\TableStorage.MessagePack.csproj", "{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.MessagePack.Source", "src\TableStorage.MessagePack.Source\TableStorage.MessagePack.Source.csproj", "{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Protobuf", "src\TableStorage.Protobuf\TableStorage.Protobuf.csproj", "{3DEA170D-5637-4A01-AA53-B727013FA850}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TableStorage.Protobuf.Source", "src\TableStorage.Protobuf.Source\TableStorage.Protobuf.Source.csproj", "{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -36,10 +50,40 @@ Global
{B58183AB-38E7-42FA-BE98-5D7B58C06266}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B58183AB-38E7-42FA-BE98-5D7B58C06266}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B58183AB-38E7-42FA-BE98-5D7B58C06266}.Release|Any CPU.Build.0 = Release|Any CPU
{D0C6041C-D796-483A-8470-F78A4C659BD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D0C6041C-D796-483A-8470-F78A4C659BD5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D0C6041C-D796-483A-8470-F78A4C659BD5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D0C6041C-D796-483A-8470-F78A4C659BD5}.Release|Any CPU.Build.0 = Release|Any CPU
{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B}.Release|Any CPU.Build.0 = Release|Any CPU
{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C5857B21-FDB6-49FB-8FDE-A1450B0C7EC6}.Release|Any CPU.Build.0 = Release|Any CPU
{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C}.Release|Any CPU.Build.0 = Release|Any CPU
{3DEA170D-5637-4A01-AA53-B727013FA850}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3DEA170D-5637-4A01-AA53-B727013FA850}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3DEA170D-5637-4A01-AA53-B727013FA850}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3DEA170D-5637-4A01-AA53-B727013FA850}.Release|Any CPU.Build.0 = Release|Any CPU
{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{B58183AB-38E7-42FA-BE98-5D7B58C06266} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
{3CDC8EBB-39EB-4195-8A65-2CAEBF6C250B} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
{09DC6B3D-2950-4C2A-9D53-CFFCBBBA612C} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
{7801FE93-4A6C-4C49-82B0-F32DEFE3A2E2} = {7412B98E-AA65-4B6E-AD83-6F9960C1C452}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {9E26BFEF-9184-4EA2-8A64-BCD61E247C33}
EndGlobalSection
Expand Down
Binary file added assets/img/document.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added assets/img/entity.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
71 changes: 67 additions & 4 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,62 @@ This is quite convenient for handling reference data, for example. Enumerating a
in the partition wouldn't be something you'd typically do for your "real" data, but for
reference data, it could be useful.

Stored entities will use individual columns for properties, which makes it easy to browse
the data. If you don't need the individual columns, and would like a document-like storage
mechanism instead, you can use the `DocumentRepository.Create` and `DocumentPartition.Create`
factory methods instead. The API is otherwise the same, but you can see the effect of using
one or the other in the following screenshots of the [Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/)
for the same `Product` entity shown in the first example above:

![Screenshot of entity persisted with separate columns for properties](assets/img/entity.png)

![Screenshot of entity persisted as a document](assets/img/document.png)

The code that persisted both entities is:

```csharp
var repo = TableRepository.Create<Product>(
CloudStorageAccount.DevelopmentStorageAccount,
tableName: "Products",
partitionKey: p => p.Category,
rowKey: p => p.Id);

await repo.PutAsync(new Product("book", "9781473217386")
{
Title = "Neuromancer",
Price = 7.32
});

var docs = DocumentRepository.Create<Product>(
CloudStorageAccount.DevelopmentStorageAccount,
tableName: "Documents",
partitionKey: p => p.Category,
rowKey: p => p.Id);

await docs.PutAsync(new Product("book", "9781473217386")
{
Title = "Neuromancer",
Price = 7.32
});
```

The `DocumentType` is the `Type.FullName` of the entity type, and the `DocumentVersion` is
the `Major.Minor` of its assembly, which could be used for advanced data migration scenarios.

In addition to the default built-in JSON plain-text based serializer, you can choose from
various binary serializers which will instead persist the document as a byte array:

[![Bson](https://img.shields.io/nuget/v/Devlooped.TableStorage.svg?color=royalblue&label=Bson)](https://www.nuget.org/packages/Devlooped.TableStorage)
[![MessagePack](https://img.shields.io/nuget/v/Devlooped.TableStorage.svg?color=royalblue&label=MessagePack)](https://www.nuget.org/packages/Devlooped.TableStorage)
[![Protobuf](https://img.shields.io/nuget/v/Devlooped.TableStorage.svg?color=royalblue&label=Protobuf)](https://www.nuget.org/packages/Devlooped.TableStorage)

You can pass the serializer to use to the factory method as follows:

```csharp
var repo = TableRepository.Create<Product>(...,
serializer: [BsonDocumentSerializer|MessagePackDocumentSerializer|ProtobufDocumentSerializer].Default);
```

### Attributes

If you want to avoid using strings with the factory methods, you can also annotate the
Expand All @@ -128,12 +184,11 @@ entity type to modify the default values used:
* `[PartitionKey]`: annotates the property that should be used as the partition key
* `[RowKey]`: annotates the property that should be used as the row key.

Values passed to the `TableRepository.Create<T>` or `TablePartition.Create<T>` override
declarative attributes.
Values passed to the factory methods override declarative attributes.

### TableEntity Support

Since these repository APIs are quite a bit more intuitive than working against a direct
Since these repository APIs are quite a bit more intuitive than working directly against a
`TableClient`, you might want to retrieve/enumerate entities just by their built-in `ITableEntity`
properties, like `PartitionKey`, `RowKey`, `Timestamp` and `ETag`. For this scenario, we
also support creating `ITableRepository<TableEntity>` and `ITablePartition<TableEntity>`
Expand Down Expand Up @@ -161,7 +216,7 @@ await foreach (TableEntity region in repo.EnumerateAsync())
> Install-Package Devlooped.TableStorage
```

There is also a source-only version, if you want to avoid an additional assembly:
There is also a source-only version, if you want to avoid an additional assembly dependency:

```
> Install-Package Devlooped.TableStorage.Source
Expand All @@ -179,8 +234,16 @@ namespace Devlooped
public partial interface ITablePartition<T> { }
public partial class TableRepository { }
public partial class TableRepository<T> { }
public partial class AttributedTableRepository<T> { }
public partial class DocumentRepository { }
public partial class DocumentRepository<T> { }
public partial class AttributedDocumentRepository<T> { }
public partial interface IDocumentSerializer { }
public partial interface IBinaryDocumentSerializer { }
public partial interface IStringDocumentSerializer { }
public partial class TablePartition { }
public partial class TablePartition<T> { }
public partial class DocumentPartition { }

// Perhaps make the attributes visible too if you use them?
public partial class TableAttribute { }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<Project>

<ItemGroup>
<Compile Update="@(Compile -> WithMetadataValue('NuGetPackageId', 'Devlooped.TableStorage.Bson.Source'))">
<Visible>false</Visible>
<Link>Devlooped\TableStorage.Bson\%(Filename)%(Extension)</Link>
</Compile>
</ItemGroup>

</Project>
32 changes: 32 additions & 0 deletions src/TableStorage.Bson.Source/TableStorage.Bson.Source.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<AssemblyName>Devlooped.TableStorage.Bson.Source</AssemblyName>
<TargetFramework>netstandard2.0</TargetFramework>
<IsPackable>true</IsPackable>
<PackBuildOutput>false</PackBuildOutput>
<PackCompile>true</PackCompile>
<Description>A source-only BSON binary serializer for use with document-based repositories.

Usage:

var repo = DocumentRepository.Create&lt;Product&gt;(storageAccount, serializer: BsonDocumentSerializer.Default);
</Description>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="NuGetizer" Version="0.7.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="all" />
<PackageReference Include="Newtonsoft.Json.Bson" Version="1.0.2" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\TableStorage.Source\TableStorage.Source.csproj" />
</ItemGroup>

<ItemGroup>
<Compile Include="..\TableStorage.Bson\**\*.cs" Exclude="..\TableStorage.Bson\Visibility.cs;..\TableStorage.Bson\obj\**\*.cs;" />
<None Update="Devlooped.TableStorage.Bson.Source.targets" PackFolder="build" />
</ItemGroup>

</Project>
44 changes: 44 additions & 0 deletions src/TableStorage.Bson/BsonDocumentSerializer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
//<auto-generated/>
#nullable enable
using System.IO;
using Newtonsoft.Json;

namespace Devlooped
{
/// <summary>
/// Default implementation of <see cref="IBinaryDocumentSerializer"/> which
/// uses Newtonsoft.Json implementation of BSON for serialization.
/// </summary>
partial class BsonDocumentSerializer : IBinaryDocumentSerializer
{
static readonly JsonSerializer serializer = new JsonSerializer();

/// <summary>
/// Default instance of the serializer.
/// </summary>
public static IDocumentSerializer Default { get; } = new BsonDocumentSerializer();

/// <inheritdoc />
public T? Deserialize<T>(byte[] data)
{
if (data.Length == 0)
return default;

using var mem = new MemoryStream(data);
using var reader = new Newtonsoft.Json.Bson.BsonDataReader(mem);
return (T?)serializer.Deserialize<T>(reader);
}

/// <inheritdoc />
public byte[] Serialize<T>(T value)
{
if (value == null)
return new byte[0];

using var mem = new MemoryStream();
using var writer = new Newtonsoft.Json.Bson.BsonDataWriter(mem);
serializer.Serialize(writer, value);
return mem.ToArray();
}
}
}
25 changes: 25 additions & 0 deletions src/TableStorage.Bson/TableStorage.Bson.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<AssemblyName>Devlooped.TableStorage.Bson</AssemblyName>
<TargetFramework>netstandard2.0</TargetFramework>
<IsPackable>true</IsPackable>
<Description>A BSON binary serializer for use with document-based repositories.

Usage:

var repo = DocumentRepository.Create&lt;Product&gt;(storageAccount, serializer: BsonDocumentSerializer.Default);
</Description>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="NuGetizer" Version="0.7.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="all" />
<PackageReference Include="Newtonsoft.Json.Bson" Version="1.0.2" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\TableStorage\TableStorage.csproj" />
</ItemGroup>

</Project>
6 changes: 6 additions & 0 deletions src/TableStorage.Bson/Visibility.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//<auto-generated/>
namespace Devlooped
{
// Sets default visibility when using compiled version, where everything is public
public partial class BsonDocumentSerializer { }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<Project>

<ItemGroup>
<Compile Update="@(Compile -> WithMetadataValue('NuGetPackageId', 'Devlooped.TableStorage.MessagePack.Source'))">
<Visible>false</Visible>
<Link>Devlooped\TableStorage.MessagePack\%(Filename)%(Extension)</Link>
</Compile>
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<AssemblyName>Devlooped.TableStorage.MessagePack.Source</AssemblyName>
<TargetFramework>netstandard2.0</TargetFramework>
<IsPackable>true</IsPackable>
<PackBuildOutput>false</PackBuildOutput>
<PackCompile>true</PackCompile>
<Description>A source-only MessagePack binary serializer for use with document-based repositories.

Usage:

var repo = DocumentRepository.Create&lt;Product&gt;(storageAccount, serializer: MessagePackDocumentSerializer.Default);
</Description>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="NuGetizer" Version="0.7.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="all" />
<PackageReference Include="MessagePack" Version="2.2.85" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\TableStorage.Source\TableStorage.Source.csproj" />
</ItemGroup>

<ItemGroup>
<Compile Include="..\TableStorage.MessagePack\**\*.cs" Exclude="..\TableStorage.MessagePack\Visibility.cs;..\TableStorage.MessagePack\obj\**\*.cs;" />
<None Update="Devlooped.TableStorage.MessagePack.Source.targets" PackFolder="build" />
</ItemGroup>

</Project>
24 changes: 24 additions & 0 deletions src/TableStorage.MessagePack/MessagePackDocumentSerializer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
//<auto-generated/>
#nullable enable
using MessagePack;

namespace Devlooped
{
/// <summary>
/// Default implementation of <see cref="IBinaryDocumentSerializer"/> which
/// uses Newtonsoft.Json implementation of BSON for serialization.
/// </summary>
partial class MessagePackDocumentSerializer : IBinaryDocumentSerializer
{
/// <summary>
/// Default instance of the serializer.
/// </summary>
public static IDocumentSerializer Default { get; } = new MessagePackDocumentSerializer();

/// <inheritdoc />
public T? Deserialize<T>(byte[] data) => data.Length == 0 ? default : MessagePackSerializer.Deserialize<T>(data);

/// <inheritdoc />
public byte[] Serialize<T>(T value) => value == null ? new byte[0] : MessagePackSerializer.Serialize(value.GetType(), value);
}
}
25 changes: 25 additions & 0 deletions src/TableStorage.MessagePack/TableStorage.MessagePack.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<AssemblyName>Devlooped.TableStorage.MessagePack</AssemblyName>
<TargetFramework>netstandard2.0</TargetFramework>
<IsPackable>true</IsPackable>
<Description>A MessagePack binary serializer for use with document-based repositories.

Usage:

var repo = DocumentRepository.Create&lt;Product&gt;(storageAccount, serializer: BsonDocumentSerializer.Default);
</Description>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="NuGetizer" Version="0.7.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="all" />
<PackageReference Include="MessagePack" Version="2.2.85" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\TableStorage\TableStorage.csproj" />
</ItemGroup>

</Project>
6 changes: 6 additions & 0 deletions src/TableStorage.MessagePack/Visibility.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//<auto-generated/>
namespace Devlooped
{
// Sets default visibility when using compiled version, where everything is public
public partial class MessagePackDocumentSerializer { }
}
Loading

0 comments on commit 3756104

Please sign in to comment.