Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generalize excel writing with a common write adapter and implement writing IAsyncEnumerable #712

Merged
merged 14 commits into from
Jan 20, 2025
+860 −1,009
Merged

Generalize excel writing with a common write adapter and implement writing IAsyncEnumerable #712

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
d764f1b
Generalize excel writing with a common write adapter
Discolai Dec 10, 2024
b9e1d7a
Remove testcase for single row in empty table
Discolai Dec 10, 2024
82b1aac
Implement sync IExcelWriteAdapter
Discolai Jan 11, 2025
239fc5d
Remove testcase for single row in empty table
Discolai Jan 11, 2025
b814ebf
Add back support for writing IMiniExcelDataReader
Discolai Jan 12, 2025
7e1ea12
Add support for writing AsyncEnumerable
Discolai Jan 12, 2025
02752a4
Add cancellation support
Discolai Jan 12, 2025
6a43c47
Add MiniExcelWriteAdapterFactory
Discolai Jan 15, 2025
5067add
Implement write adapters for csv writing
Discolai Jan 15, 2025
52ffa75
cancel break change
izanhzh Jan 20, 2025
c888cf9
fix build bug
izanhzh Jan 20, 2025
156b8e7
fix get props bug
izanhzh Jan 20, 2025
286db62
cancel some adjust
izanhzh Jan 20, 2025
9293f73
clean code
izanhzh Jan 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
257 changes: 102 additions & 155 deletions src/MiniExcel/Csv/CsvWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
using MiniExcelLibs.Utils;
using MiniExcelLibs.WriteAdapter;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

Expand All @@ -31,205 +31,152 @@ public CsvWriter(Stream stream, object value, IConfiguration configuration, bool

public void SaveAs()
{
var seperator = _configuration.Seperator.ToString();
var newLine = _configuration.NewLine;
if (_value == null)
{
if (_value == null)
{
_writer.Write("");
this._writer.Flush();
return;
}

var type = _value.GetType();

if (_value is IDataReader dataReader)
{
GenerateSheetByIDataReader(dataReader, seperator, newLine, _writer);
}
else if (_value is IEnumerable enumerable)
{
GenerateSheetByIEnumerable(enumerable, seperator, newLine, _writer);
}
else if (_value is DataTable dataTable)
{
GenerateSheetByDataTable(_writer, dataTable, seperator, newLine);
}
else
{
throw new NotImplementedException($"Type {type?.Name} not Implemented. please issue for me.");
}

this._writer.Flush();
_writer.Write("");
_writer.Flush();
return;
}
}

public async Task SaveAsAsync(CancellationToken cancellationToken = default)
{
await Task.Run(() => SaveAs(), cancellationToken).ConfigureAwait(false);
WriteValues(_writer, _value);
_writer.Flush();
}

public void Insert(bool overwriteSheet = false)
{
SaveAs();
}

public async Task InsertAsync(bool overwriteSheet = false, CancellationToken cancellationToken = default)
private void AppendColumn(StringBuilder rowBuilder, CellWriteInfo column)
{
await Task.Run(() => SaveAs(), cancellationToken).ConfigureAwait(false);
rowBuilder.Append(CsvHelpers.ConvertToCsvValue(ToCsvString(column.Value, column.Prop), _configuration.AlwaysQuote, _configuration.Seperator));
rowBuilder.Append(_configuration.Seperator);
}

private void GenerateSheetByIEnumerable(IEnumerable values, string seperator, string newLine, StreamWriter writer)
private void RemoveTrailingSeparator(StringBuilder rowBuilder)
{
Type genericType = null;
List<ExcelColumnInfo> props = null;
string mode = null;

var enumerator = values.GetEnumerator();
var empty = !enumerator.MoveNext();
if (empty)
{
// only when empty IEnumerable need to check this issue #133 https://github.com/shps951023/MiniExcel/issues/133
genericType = TypeHelper.GetGenericIEnumerables(values).FirstOrDefault();
if (genericType == null || genericType == typeof(object) // sometime generic type will be object, e.g: https://user-images.githubusercontent.com/12729184/132812859-52984314-44d1-4ee8-9487-2d1da159f1f0.png
|| typeof(IDictionary<string, object>).IsAssignableFrom(genericType)
|| typeof(IDictionary).IsAssignableFrom(genericType)
|| typeof(KeyValuePair<string, object>).IsAssignableFrom(genericType))
{
_writer.Write(newLine);
this._writer.Flush();
return;
}

mode = "Properties";
props = CustomPropertyHelper.GetSaveAsProperties(genericType, _configuration);
}
else
if (rowBuilder.Length == 0)
{
var firstItem = enumerator.Current;
if (firstItem is IDictionary<string, object> genericDic)
{
mode = "IDictionary<string, object>";
props = CustomPropertyHelper.GetDictionaryColumnInfo(genericDic, null, _configuration);
}
else if (firstItem is IDictionary dic)
{
mode = "IDictionary";
props = CustomPropertyHelper.GetDictionaryColumnInfo(null, dic, _configuration);
mode = "IDictionary";
}
else
{
mode = "Properties";
genericType = firstItem.GetType();
props = CustomPropertyHelper.GetSaveAsProperties(genericType, _configuration);
}
}

if (this._printHeader)
{
_writer.Write(string.Join(seperator, props.Select(s => CsvHelpers.ConvertToCsvValue(s?.ExcelColumnName, _configuration.AlwaysQuote, _configuration.Seperator))));
_writer.Write(newLine);
}

if (!empty)
{
if (mode == "IDictionary<string, object>") //Dapper Row
GenerateSheetByDapperRow(_writer, enumerator, props.Select(x => x.Key.ToString()).ToList(), seperator, newLine);
else if (mode == "IDictionary") //IDictionary
GenerateSheetByIDictionary(_writer, enumerator, props.Select(x => x.Key).ToList(), seperator, newLine);
else if (mode == "Properties")
GenerateSheetByProperties(_writer, enumerator, props, seperator, newLine);
else
throw new NotImplementedException($"Mode for genericType {genericType?.Name} not Implemented. please issue for me.");
return;
}
rowBuilder.Remove(rowBuilder.Length - 1, 1);
}

private void GenerateSheetByIDataReader(IDataReader reader, string seperator, string newLine, StreamWriter writer)
private string GetHeader(List<ExcelColumnInfo> props) => string.Join(
_configuration.Seperator.ToString(),
props.Select(s => CsvHelpers.ConvertToCsvValue(s?.ExcelColumnName, _configuration.AlwaysQuote, _configuration.Seperator)));

private void WriteValues(StreamWriter writer, object values)
{
int fieldCount = reader.FieldCount;
if (fieldCount == 0)
throw new InvalidDataException("fieldCount is 0");
IMiniExcelWriteAdapter writeAdapter = MiniExcelWriteAdapterFactory.GetWriteAdapter(values, _configuration);

if (this._printHeader)
var props = writeAdapter.GetColumns();
if (props == null)
{
for (int i = 0; i < fieldCount; i++)
{
var columnName = reader.GetName(i);
_writer.Write(_configuration.NewLine);
_writer.Flush();
return;
}

if (i != 0)
writer.Write(seperator);
writer.Write(CsvHelpers.ConvertToCsvValue(ToCsvString(columnName, null), _configuration.AlwaysQuote, _configuration.Seperator));
}
writer.Write(newLine);
if (_printHeader)
{
_writer.Write(GetHeader(props));
_writer.Write(_configuration.NewLine);
}

while (reader.Read())
var rowBuilder = new StringBuilder();
if (writeAdapter != null)
{
for (int i = 0; i < fieldCount; i++)
foreach (var row in writeAdapter.GetRows(props))
{
var cellValue = reader.GetValue(i);
if (i != 0)
writer.Write(seperator);
writer.Write(CsvHelpers.ConvertToCsvValue(ToCsvString(cellValue, null), _configuration.AlwaysQuote, _configuration.Seperator));
rowBuilder.Clear();
foreach (var column in row)
{
AppendColumn(rowBuilder, column);
}
RemoveTrailingSeparator(rowBuilder);
_writer.Write(rowBuilder.ToString());
_writer.Write(_configuration.NewLine);
}
writer.Write(newLine);
}
}

private void GenerateSheetByDataTable(StreamWriter writer, DataTable dt, string seperator, string newLine)
private async Task WriteValuesAsync(StreamWriter writer, object values, string seperator, string newLine, CancellationToken cancellationToken)
{
#if NETSTANDARD2_0_OR_GREATER || NET
IMiniExcelWriteAdapter writeAdapter = null;
if (!MiniExcelWriteAdapterFactory.TryGetAsyncWriteAdapter(values, _configuration, out var asyncWriteAdapter))
{
writeAdapter = MiniExcelWriteAdapterFactory.GetWriteAdapter(values, _configuration);
}
var props = writeAdapter != null ? writeAdapter.GetColumns() : await asyncWriteAdapter.GetColumnsAsync();
#else
IMiniExcelWriteAdapter writeAdapter = MiniExcelWriteAdapterFactory.GetWriteAdapter(values, _configuration);
var props = writeAdapter.GetColumns();
#endif
if (props == null)
{
await _writer.WriteAsync(_configuration.NewLine);
await _writer.FlushAsync();
return;
}
if (_printHeader)
{
writer.Write(string.Join(seperator, dt.Columns.Cast<DataColumn>().Select(s => CsvHelpers.ConvertToCsvValue(s.Caption ?? s.ColumnName, _configuration.AlwaysQuote, _configuration.Seperator))));
writer.Write(newLine);
await _writer.WriteAsync(GetHeader(props));
await _writer.WriteAsync(newLine);
}
for (int i = 0; i < dt.Rows.Count; i++)
var rowBuilder = new StringBuilder();
if (writeAdapter != null)
{
var first = true;
for (int j = 0; j < dt.Columns.Count; j++)
foreach (var row in writeAdapter.GetRows(props, cancellationToken))
{
var cellValue = CsvHelpers.ConvertToCsvValue(ToCsvString(dt.Rows[i][j], null), _configuration.AlwaysQuote, _configuration.Seperator);
if (!first)
writer.Write(seperator);
writer.Write(cellValue);
first = false;
rowBuilder.Clear();
foreach (var column in row)
{
AppendColumn(rowBuilder, column);
}
RemoveTrailingSeparator(rowBuilder);
await _writer.WriteAsync(rowBuilder.ToString());
await _writer.WriteAsync(newLine);
}
writer.Write(newLine);
}
}

private void GenerateSheetByProperties(StreamWriter writer, IEnumerator value, List<ExcelColumnInfo> props, string seperator, string newLine)
{
do
#if NETSTANDARD2_0_OR_GREATER || NET
else
{
var v = value.Current;
var values = props.Select(s => CsvHelpers.ConvertToCsvValue(ToCsvString(s?.Property.GetValue(v), s), _configuration.AlwaysQuote, _configuration.Seperator));
writer.Write(string.Join(seperator, values));
writer.Write(newLine);
} while (value.MoveNext());
await foreach (var row in asyncWriteAdapter.GetRowsAsync(props, cancellationToken))
{
rowBuilder.Clear();
await foreach (var column in row)
{
AppendColumn(rowBuilder, column);
}
RemoveTrailingSeparator(rowBuilder);
await _writer.WriteAsync(rowBuilder.ToString());
await _writer.WriteAsync(newLine);
}
}
#endif
}

private void GenerateSheetByIDictionary(StreamWriter writer, IEnumerator value, List<object> keys, string seperator, string newLine)
public async Task SaveAsAsync(CancellationToken cancellationToken = default)
{
do
var seperator = _configuration.Seperator.ToString();
var newLine = _configuration.NewLine;

if (_value == null)
{
var v = (IDictionary)value.Current;
var values = keys.Select(key => CsvHelpers.ConvertToCsvValue(ToCsvString(v[key], null), _configuration.AlwaysQuote, _configuration.Seperator));
writer.Write(string.Join(seperator, values));
writer.Write(newLine);
} while (value.MoveNext());
await _writer.WriteAsync("");
await _writer.FlushAsync();
return;
}

await WriteValuesAsync(_writer, _value, seperator, newLine, cancellationToken);
await _writer.FlushAsync();
}

private void GenerateSheetByDapperRow(StreamWriter writer, IEnumerator value, List<string> keys, string seperator, string newLine)
public async Task InsertAsync(bool overwriteSheet = false, CancellationToken cancellationToken = default)
{
do
{
var v = (IDictionary<string, object>)value.Current;
var values = keys.Select(key => CsvHelpers.ConvertToCsvValue(ToCsvString(v[key], null), _configuration.AlwaysQuote, _configuration.Seperator));
writer.Write(string.Join(seperator, values));
writer.Write(newLine);
} while (value.MoveNext());
await SaveAsAsync(cancellationToken);
}

public string ToCsvString(object value, ExcelColumnInfo p)
Expand Down
11 changes: 8 additions & 3 deletions src/MiniExcel/MiniExcelLibs.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
<TargetFrameworks>net45;netstandard2.0;net8.0;</TargetFrameworks>
<Version>1.36.1</Version>
</PropertyGroup>
<PropertyGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
<LangVersion>8</LangVersion>
</PropertyGroup>
<PropertyGroup>
<AssemblyName>MiniExcel</AssemblyName>
<Company>Mini-Software</Company>
Expand Down Expand Up @@ -35,9 +38,6 @@ Todo : https://github.com/mini-software/MiniExcel/projects/1?fullscreen=true</De
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
<PackageReadmeFile>README.md</PackageReadmeFile>
</PropertyGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'net461'">
<Reference Include="System.IO.Compression" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'net45'">
<Reference Include="System.IO.Compression" />
</ItemGroup>
Expand All @@ -56,4 +56,9 @@ Todo : https://github.com/mini-software/MiniExcel/projects/1?fullscreen=true</De
<ItemGroup Condition="'$(GITHUB_ACTIONS)' == 'true'">
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces">
<Version>9.0.0</Version>
</PackageReference>
</ItemGroup>
</Project>
Loading
Loading