-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathPerformanceMeasurement.cs
280 lines (259 loc) · 13.5 KB
/
PerformanceMeasurement.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
/* Copyright (c) Microsoft Corporation. All rights reserved. */
/* AUTHOR: Vance Morrison Date : 10/20/2007 */
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Web; // for HttpUtility.HtmlEncode
using System.Reflection; // for Assembly
using System.Diagnostics; // for Process, DebuggableAttribute
/* files are best browsed in 'outline form'. You outline code with Ctrl-M Ctrl-O. */
/* see code:PerformanceMeasurement.StatsLogger to get started */
namespace PerformanceMeasurement
{
/// <summary>
/// A code:StatsLogger is something that remembers a set of common
/// atrributes (verison of the code, Machine used, NGENed or JITed ...) as
/// well as a set of performance results from mulitple benchmarks.
/// (represented by a code:StatsCollection)
///
/// The primary value of a StatsLogger is the
/// code:StatsLogger.DisplayHtmlReport which displayes the data in a
/// user-friendly way.
/// </summary>
public class StatsLogger
{
public StatsLogger(StatsCollection dataSet)
{
this.dataSet = dataSet;
attributes = new Dictionary<string, object>();
}
StatsCollection DataSet { get { return dataSet; } }
public object this[string key]
{
get { return attributes[key]; }
set { attributes[key] = value; }
}
public void Add(string name, Stats sample) { dataSet.Add(name, sample); }
public string Category
{
get { return category; }
set { category = value; }
}
public void AddWithCount(string name, int iterationCount, float scale, Stats sample)
{
if (!string.IsNullOrEmpty(category))
name = category + ": " + name;
if (iterationCount != 1 || scale != 1)
{
name += " [";
if (iterationCount != 1)
{
name = name + "count=" + iterationCount.ToString();
if (scale != 1)
name += " ";
}
if (scale != 1)
name = name + " scale=" + scale.ToString("f1");
name += "]";
}
Add(name, sample);
}
public void DisplayHtmlReport(string reportFileName)
{
TextWriter writer = File.CreateText(reportFileName);
writer.WriteLine("<html>");
writer.WriteLine("<h1> MeasureIt Performance Results </h1>");
object optimizationValue;
if (attributes.TryGetValue("CodeOptimization", out optimizationValue) && ((string)optimizationValue) == "Unoptimized")
{
writer.WriteLine("<font color=red><p>");
writer.WriteLine("Warning: the MeasureIt code was not optimized. The results are likely invalid.");
writer.WriteLine("</p></font>");
}
if (Environment.OSVersion.Version.Major < 6)
{
writer.WriteLine("<b><p>");
writer.WriteLine("Data was collected on a Pre-Vista machine. MeasureIt does NOT automatically");
writer.WriteLine("set the CPU to a high performance power policy. This means the CPU might");
writer.WriteLine("be throttled to save power and can lead to");
writer.WriteLine("incorrect and inconsistant benchmark measurements.");
writer.WriteLine("</p></b>");
}
else if (PowerManagment.CurrentPolicy != PowerManagment.HighPerformance)
{
writer.WriteLine("<font color=red><p>");
writer.WriteLine("Warning: The power policy settings were not set at 'High Performance' during this run.");
writer.WriteLine("This means that the CPU could be throttled to lower frequency resulting in");
writer.WriteLine("incorrect and inconsistant benchmark measurements.");
writer.WriteLine("To correct go to Start Menu -> Contol Panel -> System and Maintance -> Power Options");
writer.WriteLine("and set the power policy to 'High Performance' for the duration of the tests.");
writer.WriteLine("</p></font>");
}
writer.WriteLine("<p>");
writer.WriteLine("Below are the results of running a series of benchmarks. Use the");
writer.WriteLine("<b>MeasureIt /usersGuide</b> for more details on exactly what the benchmarks do.");
writer.WriteLine("</p><p>");
writer.WriteLine("It is very easy for benchmark results to be wrong or misleading. You should read the guidance");
writer.WriteLine("in the <b>MeasureIt /usersGuide</b> before making important decisions based on this data.");
writer.WriteLine("</p><p>");
writer.WriteLine("To improve the stability of the measurements, a may be cloned several times");
writer.WriteLine("and this cloned code is then run in a loop.");
writer.WriteLine("If the benchmark was cloned the 'scale' attribute represents the number of times");
writer.WriteLine("it was cloned, and the count represents the number of times the cloned code was run in a loop");
writer.WriteLine("before the measurement was made. The reported number divides by both");
writer.WriteLine("of these values, so it represents a single instance of the operation being measured.");
writer.WriteLine("</p>");
writer.WriteLine("<p>");
writer.WriteLine("The benchmarks data can vary from run to run, so the benchmark is run several times and");
writer.WriteLine("the statistics are displayed. If we assume a normal distribution, you can expect 68% of all measureuments");
writer.WriteLine("to fall within 1 StdDev of the Mean. You can expect over 95% of all measurements");
writer.WriteLine("to fall witin 2 StdDev of the Mean. Thus 2 StdDev is a good error bound.");
writer.WriteLine("Keep in mind, however that it is not uncommon for the statistics to be quite stable");
writer.WriteLine("during a run and yet very widely across different runs. See the users guide for more.");
writer.WriteLine("</p>");
writer.WriteLine("<p>");
writer.WriteLine("Generally the mean is a better measurment if you use the number to compute an");
writer.WriteLine("aggregate throughput for a large number of items. The median is a better");
writer.WriteLine("guess if you want to best guess of a typical sample. The median is also");
writer.WriteLine("more stable if the sample is noisy (eg has outliers).");
writer.WriteLine("</p>");
writer.WriteLine("<h3>Data collected</h3>");
{
writer.WriteLine("<p>");
writer.WriteLine(UnitsDescription);
writer.WriteLine("</p>");
dataSet.WriteReportTable(writer, Scale);
}
writer.WriteLine("<p>");
{
writer.WriteLine("<h2>Attributes of the machine used to collect the data</h2>");
writer.WriteLine("<table border>");
writer.WriteLine("<tr><th>Attribute</th><th>Value</th></tr>");
foreach (string key in attributes.Keys)
{
object valueObj = this[key];
writer.Write("<tr>");
writer.Write("<td>" + HttpUtility.HtmlEncode(key) + "</td>");
string valueStr = HttpUtility.HtmlEncode(valueObj.ToString());
writer.Write("<td>" + valueStr + "</td>");
writer.WriteLine("<tr>");
}
writer.WriteLine("</table>");
}
writer.WriteLine("</p>");
writer.WriteLine("</html>");
writer.Close();
}
static public void LaunchIE(string fileName)
{
Process process = new Process();
process.StartInfo = new ProcessStartInfo(fileName);
process.Start();
}
public float Scale = 1.0F;
public string UnitsDescription = "Scale in usec";
// TODO: probabably does not belong in this class.
public void CaptureCurrentMachineInfo(Assembly assemblyWithCode, bool skipMachineStats)
{
this["Computer Name"] = Environment.MachineName;
if (!skipMachineStats)
{
ComputerSpecs specs = new ComputerSpecs();
this["Number of Processors"] = specs.NumberOfProcessors;
this["Processor Name "] = specs.ProcessorName;
this["Processor Mhz"] = specs.ProcessorClockSpeedMhz;
this["Memory MBytes"] = specs.MemoryMBytes;
this["L1 Cache KBytes"] = specs.L1KBytes;
this["L2 Cache KBytes"] = specs.L2KBytes;
this["Operating System"] = specs.OperatingSystem;
this["Operating System Version"] = specs.OperatingSystemVersion;
this["Stopwatch resolution (nsec)"] = (CodeTimer.ResolutionUsec * 1000.0).ToString("f3");
}
// Are we NGENed or JITTed?
if (IsNGenedCodeLoaded(assemblyWithCode))
this["CompileType"] = "NGEN";
else
this["CompileType"] = "JIT";
// Are we Appdomain Shared, or not?
MethodInfo currentMethod = Assembly.GetEntryAssembly().EntryPoint;
LoaderOptimizationAttribute loaderAttribute = (LoaderOptimizationAttribute)System.Attribute.GetCustomAttribute(currentMethod, typeof(LoaderOptimizationAttribute));
if (loaderAttribute != null && loaderAttribute.Value == LoaderOptimization.MultiDomain)
this["CodeSharing"] = "AppDomainShared";
else
this["CodeSharing"] = "AppDomainSpecific";
DebuggableAttribute debugAttribute = (DebuggableAttribute)System.Attribute.GetCustomAttribute(assemblyWithCode, typeof(System.Diagnostics.DebuggableAttribute));
if (debugAttribute != null && debugAttribute.IsJITOptimizerDisabled)
this["CodeOptimization"] = "Unoptimized";
else
this["CodeOptimization"] = "Optimized";
}
static bool IsNGenedCodeLoaded(Assembly assembly)
{
// This is a bit of a hack, basically I find the assemblies file name, then
// look for a module loaded called '<filename>.ni.<ext>'. It is not foolproof,
// but it more than good enough for most purposes.
string assemblyFileName = Path.GetFileName(assembly.ManifestModule.FullyQualifiedName);
string nativeImageExt = "ni" + Path.GetExtension(assemblyFileName);
string nativeImageSuffix = @"\" + Path.ChangeExtension(assemblyFileName, nativeImageExt);
System.Diagnostics.Process myProcess = System.Diagnostics.Process.GetCurrentProcess();
foreach (System.Diagnostics.ProcessModule module in myProcess.Modules)
{
if (module.FileName.EndsWith(nativeImageSuffix, StringComparison.OrdinalIgnoreCase))
return true;
}
return false;
}
#region privates
Dictionary<string, object> attributes;
StatsCollection dataSet;
string category;
#endregion
}
/// <summary>
/// StatsCollection represents a collecton of named of samples (class
/// Stats) that have have been given string names. The data can be
/// looked up by name, but is also collection also remembers the order in
/// which the samples were added, and the names can be enumerated in that
/// order.
/// </summary>
public class StatsCollection
{
public StatsCollection()
{
dict = new Dictionary<string, Stats>();
order = new List<string>();
}
public Stats this[string key] { get { return dict[key]; } }
public bool ContainsKey(string key) { return dict.ContainsKey(key); }
public IEnumerable<string> Keys { get { return order; } }
public void Add(string key, Stats value)
{
dict.Add(key, value);
order.Add(key);
}
public void WriteReportTable(TextWriter writer, float scale)
{
writer.WriteLine("<table border>");
writer.WriteLine("<tr><th>Name</th><th>Median</th><th>Mean</th><th>StdDev</th><th>Min</th><th>Max</th><th>Samples</th></tr>");
foreach (string key in this.Keys)
{
Stats value = this[key];
writer.Write("<tr>");
writer.Write("<td>" + HttpUtility.HtmlEncode(key) + "</td>");
writer.Write("<td>" + (value.Median / scale).ToString("f3") + "</td>");
writer.Write("<td>" + (value.Mean / scale).ToString("f3") + "</td>");
writer.Write("<td>" + (value.StandardDeviation / scale).ToString("f3") + "</td>");
writer.Write("<td>" + (value.Minimum / scale).ToString("f3") + "</td>");
writer.Write("<td>" + (value.Maximum / scale).ToString("f3") + "</td>");
writer.Write("<td>" + value.Count + "</td>");
writer.WriteLine("</tr>");
}
writer.WriteLine("</table>");
}
#region privates
Dictionary<string, Stats> dict;
List<string> order;
#endregion
}
}