Skip to content

Commit f62a7a4

Browse files
timcasselltimcassell
committed
Inline engine stages.
Apply AggressiveOptimization to engine methods.
1 parent 804482d commit f62a7a4

File tree

3 files changed

+204
-26
lines changed

3 files changed

+204
-26
lines changed

‎src/BenchmarkDotNet/Engines/Engine.cs

+188-10
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Diagnostics;
34
using System.Globalization;
45
using System.Linq;
56
using System.Runtime.CompilerServices;
67
using BenchmarkDotNet.Characteristics;
8+
using BenchmarkDotNet.Environments;
79
using BenchmarkDotNet.Jobs;
10+
using BenchmarkDotNet.Mathematics;
811
using BenchmarkDotNet.Portability;
912
using BenchmarkDotNet.Reports;
1013
using JetBrains.Annotations;
@@ -102,8 +105,14 @@ public void Dispose()
102105
}
103106
}
104107

108+
// AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted,
109+
// eliminating tiered JIT as a potential variable in measurements.
110+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
105111
public RunResults Run()
106112
{
113+
// This method is huge, because all stages are inlined. This ensures the stack size
114+
// remains constant for each benchmark invocation, eliminating stack sizes as a potential variable in measurements.
115+
// #1120
107116
var measurements = new List<Measurement>();
108117
measurements.AddRange(jittingMeasurements);
109118

@@ -116,23 +125,187 @@ public RunResults Run()
116125
{
117126
if (Strategy != RunStrategy.Monitoring)
118127
{
119-
var pilotStageResult = pilotStage.Run();
120-
invokeCount = pilotStageResult.PerfectInvocationCount;
121-
measurements.AddRange(pilotStageResult.Measurements);
128+
// Pilot Stage
129+
{
130+
long Autocorrect(long count) => (count + UnrollFactor - 1) / UnrollFactor * UnrollFactor;
131+
132+
// If InvocationCount is specified, pilot stage should be skipped
133+
if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic))
134+
{
135+
}
136+
// Here we want to guess "perfect" amount of invocation
137+
else if (TargetJob.HasValue(RunMode.IterationTimeCharacteristic))
138+
{
139+
// Perfect invocation count
140+
invokeCount = Autocorrect(MinInvokeCount);
141+
142+
int iterationCounter = 0;
143+
144+
int downCount = 0; // Amount of iterations where newInvokeCount < invokeCount
145+
while (true)
146+
{
147+
iterationCounter++;
148+
var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor));
149+
measurements.Add(measurement);
150+
double actualIterationTime = measurement.Nanoseconds;
151+
long newInvokeCount = Autocorrect(Math.Max(pilotStage.minInvokeCount, (long) Math.Round(invokeCount * pilotStage.targetIterationTime / actualIterationTime)));
152+
153+
if (newInvokeCount < invokeCount)
154+
downCount++;
155+
156+
if (Math.Abs(newInvokeCount - invokeCount) <= 1 || downCount >= 3)
157+
break;
158+
159+
invokeCount = newInvokeCount;
160+
}
161+
WriteLine();
162+
}
163+
else
164+
{
165+
// A case where we don't have specific iteration time.
166+
invokeCount = Autocorrect(pilotStage.minInvokeCount);
167+
168+
int iterationCounter = 0;
169+
while (true)
170+
{
171+
iterationCounter++;
172+
var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, UnrollFactor));
173+
measurements.Add(measurement);
174+
double iterationTime = measurement.Nanoseconds;
175+
double operationError = 2.0 * pilotStage.resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision
176+
177+
// Max acceptable operation error
178+
double operationMaxError1 = iterationTime / invokeCount * pilotStage.maxRelativeError;
179+
double operationMaxError2 = pilotStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
180+
double operationMaxError = Math.Min(operationMaxError1, operationMaxError2);
181+
182+
bool isFinished = operationError < operationMaxError && iterationTime >= pilotStage.minIterationTime.Nanoseconds;
183+
if (isFinished)
184+
break;
185+
if (invokeCount >= EnginePilotStage.MaxInvokeCount)
186+
break;
187+
188+
if (UnrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput)
189+
invokeCount += 1;
190+
else
191+
invokeCount *= 2;
192+
}
193+
WriteLine();
194+
}
195+
}
196+
// End Pilot Stage
122197

123198
if (EvaluateOverhead)
124199
{
125-
measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor));
126-
measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor));
200+
// Warmup Overhead
201+
{
202+
var warmupMeasurements = new List<Measurement>();
203+
204+
var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Overhead, RunStrategy.Throughput);
205+
int iterationCounter = 0;
206+
while (!criteria.Evaluate(warmupMeasurements).IsFinished)
207+
{
208+
iterationCounter++;
209+
warmupMeasurements.Add(RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor)));
210+
}
211+
WriteLine();
212+
213+
measurements.AddRange(warmupMeasurements);
214+
}
215+
// End Warmup Overhead
216+
217+
// Actual Overhead
218+
{
219+
var measurementsForStatistics = new List<Measurement>(actualStage.maxIterationCount);
220+
221+
int iterationCounter = 0;
222+
double effectiveMaxRelativeError = EngineActualStage.MaxOverheadRelativeError;
223+
while (true)
224+
{
225+
iterationCounter++;
226+
var measurement = RunIteration(new IterationData(IterationMode.Overhead, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor));
227+
measurements.Add(measurement);
228+
measurementsForStatistics.Add(measurement);
229+
230+
var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode);
231+
double actualError = statistics.LegacyConfidenceInterval.Margin;
232+
233+
double maxError1 = effectiveMaxRelativeError * statistics.Mean;
234+
double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
235+
double maxError = Math.Min(maxError1, maxError2);
236+
237+
if (iterationCounter >= actualStage.minIterationCount && actualError < maxError)
238+
break;
239+
240+
if (iterationCounter >= actualStage.maxIterationCount || iterationCounter >= EngineActualStage.MaxOverheadIterationCount)
241+
break;
242+
}
243+
WriteLine();
244+
}
245+
// End Actual Overhead
127246
}
128247
}
129248

130-
measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy));
249+
// Warmup Workload
250+
{
251+
var workloadMeasurements = new List<Measurement>();
252+
253+
var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(TargetJob, Resolver, IterationMode.Workload, Strategy);
254+
int iterationCounter = 0;
255+
while (!criteria.Evaluate(workloadMeasurements).IsFinished)
256+
{
257+
iterationCounter++;
258+
workloadMeasurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Warmup, iterationCounter, invokeCount, UnrollFactor)));
259+
}
260+
WriteLine();
261+
262+
measurements.AddRange(workloadMeasurements);
263+
}
264+
// End Warmup Workload
131265
}
132266

133267
Host.BeforeMainRun();
134268

135-
measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring));
269+
// Actual Workload
270+
{
271+
if (actualStage.iterationCount == null && Strategy != RunStrategy.Monitoring)
272+
{
273+
// RunAuto
274+
var measurementsForStatistics = new List<Measurement>(actualStage.maxIterationCount);
275+
276+
int iterationCounter = 0;
277+
double effectiveMaxRelativeError = actualStage.maxRelativeError;
278+
while (true)
279+
{
280+
iterationCounter++;
281+
var measurement = RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, iterationCounter, invokeCount, UnrollFactor));
282+
measurements.Add(measurement);
283+
measurementsForStatistics.Add(measurement);
284+
285+
var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, actualStage.outlierMode);
286+
double actualError = statistics.LegacyConfidenceInterval.Margin;
287+
288+
double maxError1 = effectiveMaxRelativeError * statistics.Mean;
289+
double maxError2 = actualStage.maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
290+
double maxError = Math.Min(maxError1, maxError2);
291+
292+
if (iterationCounter >= actualStage.minIterationCount && actualError < maxError)
293+
break;
294+
295+
if (iterationCounter >= actualStage.maxIterationCount)
296+
break;
297+
}
298+
}
299+
else
300+
{
301+
// RunSpecific
302+
var iterationCount = actualStage.iterationCount ?? EngineActualStage.DefaultWorkloadCount;
303+
for (int i = 0; i < iterationCount; i++)
304+
measurements.Add(RunIteration(new IterationData(IterationMode.Workload, IterationStage.Actual, i + 1, invokeCount, UnrollFactor)));
305+
}
306+
WriteLine();
307+
}
308+
// End Actual Workload
136309

137310
Host.AfterMainRun();
138311

@@ -148,11 +321,15 @@ public RunResults Run()
148321
return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
149322
}
150323

324+
[MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
151325
public Measurement RunIteration(IterationData data)
152326
{
153327
// Initialization
154328
long invokeCount = data.InvokeCount;
155329
int unrollFactor = data.UnrollFactor;
330+
if (invokeCount % unrollFactor != 0)
331+
throw new ArgumentOutOfRangeException($"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor}).");
332+
156333
long totalOperations = invokeCount * OperationsPerInvoke;
157334
bool isOverhead = data.IterationMode == IterationMode.Overhead;
158335
bool randomizeMemory = !isOverhead && MemoryRandomization;
@@ -167,7 +344,7 @@ public Measurement RunIteration(IterationData data)
167344
EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations);
168345

169346
var clockSpan = randomizeMemory
170-
? MeasureWithRandomMemory(action, invokeCount / unrollFactor)
347+
? MeasureWithRandomStack(action, invokeCount / unrollFactor)
171348
: Measure(action, invokeCount / unrollFactor);
172349

173350
if (EngineEventSource.Log.IsEnabled())
@@ -193,8 +370,8 @@ public Measurement RunIteration(IterationData data)
193370
// This is in a separate method, because stackalloc can affect code alignment,
194371
// resulting in unexpected measurements on some AMD cpus,
195372
// even if the stackalloc branch isn't executed. (#2366)
196-
[MethodImpl(MethodImplOptions.NoInlining)]
197-
private unsafe ClockSpan MeasureWithRandomMemory(Action<long> action, long invokeCount)
373+
[MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)]
374+
private unsafe ClockSpan MeasureWithRandomStack(Action<long> action, long invokeCount)
198375
{
199376
byte* stackMemory = stackalloc byte[random.Next(32)];
200377
var clockSpan = Measure(action, invokeCount);
@@ -205,6 +382,7 @@ private unsafe ClockSpan MeasureWithRandomMemory(Action<long> action, long invok
205382
[MethodImpl(MethodImplOptions.NoInlining)]
206383
private unsafe void Consume(byte* _) { }
207384

385+
[MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)]
208386
private ClockSpan Measure(Action<long> action, long invokeCount)
209387
{
210388
var clock = Clock.Start();

‎src/BenchmarkDotNet/Engines/EngineGeneralStage.cs

+9-9
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,15 @@ namespace BenchmarkDotNet.Engines
1111
public class EngineActualStage : EngineStage
1212
{
1313
internal const int MaxOverheadIterationCount = 20;
14-
private const double MaxOverheadRelativeError = 0.05;
15-
private const int DefaultWorkloadCount = 10;
16-
17-
private readonly int? iterationCount;
18-
private readonly double maxRelativeError;
19-
private readonly TimeInterval? maxAbsoluteError;
20-
private readonly OutlierMode outlierMode;
21-
private readonly int minIterationCount;
22-
private readonly int maxIterationCount;
14+
internal const double MaxOverheadRelativeError = 0.05;
15+
internal const int DefaultWorkloadCount = 10;
16+
17+
internal readonly int? iterationCount;
18+
internal readonly double maxRelativeError;
19+
internal readonly TimeInterval? maxAbsoluteError;
20+
internal readonly OutlierMode outlierMode;
21+
internal readonly int minIterationCount;
22+
internal readonly int maxIterationCount;
2323

2424
public EngineActualStage(IEngine engine) : base(engine)
2525
{

‎src/BenchmarkDotNet/Engines/EnginePilotStage.cs

+7-7
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ public PilotStageResult(long perfectInvocationCount)
3030

3131
internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2;
3232

33-
private readonly int unrollFactor;
34-
private readonly TimeInterval minIterationTime;
35-
private readonly int minInvokeCount;
36-
private readonly double maxRelativeError;
37-
private readonly TimeInterval? maxAbsoluteError;
38-
private readonly double targetIterationTime;
39-
private readonly double resolution;
33+
internal readonly int unrollFactor;
34+
internal readonly TimeInterval minIterationTime;
35+
internal readonly int minInvokeCount;
36+
internal readonly double maxRelativeError;
37+
internal readonly TimeInterval? maxAbsoluteError;
38+
internal readonly double targetIterationTime;
39+
internal readonly double resolution;
4040

4141
public EnginePilotStage(IEngine engine) : base(engine)
4242
{

0 commit comments

Comments
 (0)