diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f652b45 --- /dev/null +++ b/.gitignore @@ -0,0 +1,299 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ +**/Properties/launchSettings.json + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Typescript v1 declaration files +typings/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs diff --git a/BenchmarkDotNet.ResultDiff.sln b/BenchmarkDotNet.ResultDiff.sln new file mode 100644 index 0000000..09cf61a --- /dev/null +++ b/BenchmarkDotNet.ResultDiff.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.ResultDiff", "src\BenchmarkDotNet.ResultDiff\BenchmarkDotNet.ResultDiff.csproj", "{73E99FED-64DD-4BDC-B70E-D88CD7498B8F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {73E99FED-64DD-4BDC-B70E-D88CD7498B8F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {73E99FED-64DD-4BDC-B70E-D88CD7498B8F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {73E99FED-64DD-4BDC-B70E-D88CD7498B8F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {73E99FED-64DD-4BDC-B70E-D88CD7498B8F}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7af5a19 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +### The MIT License + +Copyright (c) 2018 Marko Lahma and contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..fe225d1 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +BenchmarkDotNet.ResultDiff is a simple command line program that takes two directories as parameters +and outputs a diff view for the BenchmarkDotNet results. + +The input used is the CSV result format and output is the GitHub flavored version of Markdown. + +Each result file comparison is added to output with heading containing the name of benchmark (file name). + +See [this PR for Jint](https://github.com/sebastienros/jint/pull/495) for an example how output file contents can be easily pasted to PRs. + +## Usage + +General workflow that works at least for me + +* Run your benchmark on original branch +* Rename the result directory BenchmarkDotNet.Artifacts to for example BenchmarkDotNet.Artifacts_original +* Switch to your optimization branch +* Run benchmarks again +* Run this tool and paths as parameters (BenchmarkDotNet.Artifacts_original BenchmarkDotNet.Artifacts) + +` +cd C:\Sources\BenchmarkDotNet.ResultDiff\src\BenchmarkDotNet.ResultDiff +` + +` +dotnet run ..\jint\Jint.Benchmark\BenchmarkDotNet.Artifacts_dev ..\jint\Jint.Benchmark\BenchmarkDotNet.Artifacts_my_feature +` + +` +Analyzing pair ArrayBenchmark-report.csv +Wrote results to C:\BenchmarkDotNet.ResultDiff\BenchmarkDotNet.Artifacts_dev_vs_BenchmarkDotNet.Artifacts-github.md +` + +## BenchmarkDotNet input + +The tool turns these two results tables (taken from markdown output, tool actually uses CSV): + +*BenchmarkDotNet.Artifacts_dev\results* + +| Method | N | Mean | Error | StdDev | Gen 0 | Allocated | +|------------------- |---- |------------:|-----------:|-----------:|----------:|------------:| +| Slice | 100 | 436.2 us | 0.7824 us | 0.6936 us | 161.1328 | 660.16 KB | +| Concat | 100 | 468.4 us | 1.0230 us | 0.9569 us | 175.7813 | 720.31 KB | +| Unshift | 100 | 17,912.2 us | 29.0475 us | 24.2560 us | 3562.5000 | 14672.66 KB | +| Push | 100 | 10,861.8 us | 17.9719 us | 16.8109 us | 343.7500 | 1438.28 KB | +| Index | 100 | 12,106.5 us | 7.0282 us | 6.5742 us | 390.6250 | 1637.5 KB | +| Map | 100 | 3,382.7 us | 13.8354 us | 12.9416 us | 765.6250 | 3149.22 KB | +| Apply | 100 | 569.2 us | 1.1511 us | 1.0767 us | 188.4766 | 774.22 KB | +| JsonStringifyParse | 100 | 4,523.7 us | 6.5277 us | 6.1060 us | 1273.4375 | 5225 KB | + +*BenchmarkDotNet.Artifacts_my_feature\results* + + +| Method | N | Mean | Error | StdDev | Gen 0 | Allocated | +|------------------- |---- |------------:|-----------:|----------:|----------:|------------:| +| Slice | 100 | 455.8 us | 3.482 us | 3.086 us | 161.1328 | 660.16 KB | +| Concat | 100 | 496.6 us | 9.547 us | 10.611 us | 175.7813 | 720.31 KB | +| Unshift | 100 | 19,023.0 us | 103.525 us | 96.838 us | 3562.5000 | 14672.66 KB | +| Push | 100 | 11,274.1 us | 31.569 us | 29.530 us | 343.7500 | 1438.28 KB | +| Index | 100 | 12,471.8 us | 33.521 us | 29.716 us | 390.6250 | 1643.75 KB | +| Map | 100 | 3,624.8 us | 31.269 us | 29.249 us | 691.4063 | 2833.59 KB | +| Apply | 100 | 600.3 us | 6.965 us | 6.515 us | 188.4766 | 774.22 KB | +| JsonStringifyParse | 100 | 4,602.2 us | 49.303 us | 43.706 us | 1273.4375 | 5225.78 KB | + + +## Output + +Output for each file single result table that allows easier examination of differences between the results: + +*BenchmarkDotNet.Artifacts_dev_vs_BenchmarkDotNet.Artifacts_my_feature-github.md* + +## ArrayBenchmark + +| **Diff**|Method|N|Mean|Gen 0|Allocated| +|------- |-------|-------|-------:|-------:|-------:| +| Old |Slice|100|436.2 us|161.1328|660.16 KB| +| **New** | | | **455.8 us (+4%)** | **161.1328 (0%)** | **660.16 KB (0%)** | +| Old |Concat|100|468.4 us|175.7813|720.31 KB| +| **New** | | | **496.6 us (+6%)** | **175.7813 (0%)** | **720.31 KB (0%)** | +| Old |Unshift|100|17,912.2 us|3562.5000|14672.66 KB| +| **New** | | | **19,023.0 us (+6%)** | **3562.5000 (0%)** | **14672.66 KB (0%)** | +| Old |Push|100|10,861.8 us|343.7500|1438.28 KB| +| **New** | | | **11,274.1 us (+4%)** | **343.7500 (0%)** | **1438.28 KB (0%)** | +| Old |Index|100|12,106.5 us|390.6250|1637.5 KB| +| **New** | | | **12,471.8 us (+3%)** | **390.6250 (0%)** | **1643.75 KB (0%)** | +| Old |Map|100|3,382.7 us|765.6250|3149.22 KB| +| **New** | | | **3,624.8 us (+7%)** | **691.4063 (-10%)** | **2833.59 KB (-10%)** | +| Old |Apply|100|569.2 us|188.4766|774.22 KB| +| **New** | | | **600.3 us (+5%)** | **188.4766 (0%)** | **774.22 KB (0%)** | +| Old |JsonStringifyParse|100|4,523.7 us|1273.4375|5225 KB| +| **New** | | | **4,602.2 us (+2%)** | **1273.4375 (0%)** | **5225.78 KB (0%)** | \ No newline at end of file diff --git a/src/BenchmarkDotNet.ResultDiff/BenchmarkDotNet.ResultDiff.csproj b/src/BenchmarkDotNet.ResultDiff/BenchmarkDotNet.ResultDiff.csproj new file mode 100644 index 0000000..6b2fbad --- /dev/null +++ b/src/BenchmarkDotNet.ResultDiff/BenchmarkDotNet.ResultDiff.csproj @@ -0,0 +1,8 @@ + + + + Exe + netcoreapp2.0 + + + diff --git a/src/BenchmarkDotNet.ResultDiff/Program.cs b/src/BenchmarkDotNet.ResultDiff/Program.cs new file mode 100644 index 0000000..08d8886 --- /dev/null +++ b/src/BenchmarkDotNet.ResultDiff/Program.cs @@ -0,0 +1,264 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace BenchmarkDotNet.ResultDiff +{ + /// + /// Reads BenchmarkDotNet CSV format results from two directories and creates a rudimentary diff view. + /// + public static class Program + { + static void Main(string[] args) + { + if (args.Length < 2) + { + Console.Error.WriteLine("syntax: [target dir to save files to]"); + return; + } + + string targetDir = Directory.GetCurrentDirectory(); + if (args.Length == 3) + { + targetDir = args[2]; + } + + var oldDir = FindDirectory(args[0]); + var newDir = FindDirectory(args[1]); + + var pairs = CreateFilePairs(oldDir, newDir); + + var columns = new List + { + "Method", + "FileName", + "N", + "Mean", + "Gen 0", + "Gen 1", + "Gen 2", + "Allocated" + }; + + var oldDirName = oldDir.Name != newDir.Name ? oldDir.Name : oldDir.Parent.Name; + var newDirName = newDir.Name != oldDir.Name ? newDir.Name : newDir.Parent.Name; + var targetFile = Path.Combine(targetDir, oldDirName + "_vs_" + newDirName + "-github.md"); + using (var writer = new StreamWriter(targetFile)) + { + foreach (var pair in pairs) + { + writer.WriteLine("## " + pair.OldFile.Name.Replace("-report.csv", "")); + writer.WriteLine(); + + Console.WriteLine("Analyzing pair " + pair.OldFile.Name); + + using (var oldReader = new StreamReader(pair.OldFile.FullName)) + using (var newReader = new StreamReader(pair.NewFile.FullName)) + { + ReadLinePair(oldReader, newReader, out var headers); + var oldHeaders = headers.Old.Split(";").Select((x, i) => (x, i)).ToDictionary(x => x.Item1, x => x.Item2); + var newHeaders = headers.New.Split(";").Select((x, i) => (x, i)).ToDictionary(x => x.Item1, x => x.Item2); + + var readLines = new List<(string Old, string New)>(); + while (ReadLinePair(oldReader, newReader, out var lines)) + { + if (string.IsNullOrEmpty(lines.Old)) + { + break; + } + + readLines.Add(lines); + } + + var effectiveHeaders = columns + .Where(x => oldHeaders.ContainsKey(x) || newHeaders.ContainsKey(x)) + .ToList(); + + writer.WriteLine("| **Diff**|" + string.Join("|", effectiveHeaders) + "|"); + + writer.Write("|------- "); + foreach (var effectiveHeader in effectiveHeaders) + { + writer.Write("|-------"); + if (effectiveHeader.IndexOf("Gen ") > -1 || effectiveHeader == "Allocated" || effectiveHeader == "Mean") + { + writer.Write(":"); + } + } + + writer.WriteLine("|"); + + foreach (var lines in readLines) + { + if (string.IsNullOrEmpty(lines.Old)) + { + break; + } + + var oldie = lines.Old.Split(";"); + var newbie = lines.New.Split(";"); + + var oldColumnValues = new Dictionary(); + + writer.Write("| Old |"); + foreach (var effectiveHeader in effectiveHeaders) + { + string value = "-"; + if (oldHeaders.TryGetValue(effectiveHeader, out var oldHeader)) + { + value = oldie[oldHeader].TrimStart('"').TrimEnd('"'); + } + oldColumnValues[effectiveHeader] = value; + writer.Write(value + "|"); + } + + writer.WriteLine(); + + writer.Write("| **New** |"); + foreach (var effectiveHeader in effectiveHeaders) + { + if (effectiveHeader == "Method" || effectiveHeader == "N" || effectiveHeader == "FileName") + { + writer.Write("\t|"); + } + else + { + string value = "-"; + if (newHeaders.ContainsKey(effectiveHeader)) + { + value = newbie[newHeaders[effectiveHeader]].TrimStart('"').TrimEnd('"'); + } + + if (oldColumnValues.TryGetValue(effectiveHeader, out var oldString)) + { + var oldTokens = oldString.Split(" "); + var newTokens = value.Split(" "); + + if (oldTokens.Length == newTokens.Length) + { + bool canCalculateDiff = oldTokens[0] != "-" && newTokens[0] != "-"; + + decimal newMultiplier = 1; + + if (canCalculateDiff && oldTokens.Length > 1) + { + var oldUnit = oldTokens[1]; + var newUnit = newTokens[1]; + if (oldUnit == newUnit) + { + // ok + } + else if (oldUnit == "MB" && newUnit == "KB" + || oldUnit == "GB" && newUnit == "MB" + || oldUnit == "s" && newUnit == "ms" + || oldUnit == "ms" && newUnit == "us") + { + newMultiplier = 0.001M; + } + else + { + canCalculateDiff = false; + } + } + + if (canCalculateDiff) + { + var old = decimal.Parse(oldTokens[0], CultureInfo.InvariantCulture); + var newValue = decimal.Parse(newTokens[0], CultureInfo.InvariantCulture); + + var diff = ((newValue * newMultiplier) / old - 1) * 100; + value += $" ({diff:+#;-#;0}%)"; + } + else if (oldTokens[0] == "-" || newTokens[0] == "-") + { + // OK + } + else + { + Console.Error.WriteLine("Cannot calculate difff for " + oldString + " vs " + value); + } + } + } + + writer.Write(" **" + value + "** |"); + } + } + + writer.WriteLine(); + } + } + + writer.WriteLine(); + writer.WriteLine(); + } + + Console.WriteLine("Wrote results to " + targetFile); + } + } + + private static List<(FileInfo OldFile, FileInfo NewFile)> CreateFilePairs(DirectoryInfo oldDir, DirectoryInfo newDir) + { + var pairs = new List<(FileInfo OldFile, FileInfo NewFile)>(); + foreach (var oldReportFile in oldDir.GetFiles("*-report.csv")) + { + var fileName = oldReportFile.Name; + var newReportFile = new FileInfo(Path.Combine(newDir.FullName, fileName)); + if (newReportFile.Exists) + { + pairs.Add((oldReportFile, newReportFile)); + } + else + { + // check if new file name format without namespace + var tokens = fileName.Split('.'); + if (tokens.Length > 1) + { + fileName = tokens[tokens.Length - 2] + "." + tokens[tokens.Length - 1]; + newReportFile = new FileInfo(Path.Combine(newDir.FullName, fileName)); + if (newReportFile.Exists) + { + pairs.Add((oldReportFile, newReportFile)); + } + } + } + } + + return pairs; + } + + private static DirectoryInfo FindDirectory(string path) + { + var dir = new DirectoryInfo(path); + if (!dir.Exists) + { + Console.Error.WriteLine("directory does not exist: " + path); + } + + if (dir.GetFiles("*.csv").Length == 0) + { + var resultsDirectory = dir.GetDirectories().FirstOrDefault(x => x.Name == "results"); + if (resultsDirectory != null) + { + dir = resultsDirectory; + } + } + + return dir; + } + + private static bool ReadLinePair( + TextReader oldReader, + TextReader newReader, + out (string Old, string New) pair) + { + string oldLine; + string newLine; + var ok = (oldLine = oldReader.ReadLine()) != null; + ok &= (newLine = newReader.ReadLine()) != null; + pair = (oldLine, newLine); + return ok; + } + } +} \ No newline at end of file