Skip to content

Commit fffd53a

Browse files
lostmsuAlexCatarino
lostmsu
authored and
AlexCatarino
committed
CI for performance tests (pythonnet#992)
* attempted to add performance tests to CI * attempt to fix PerformanceTests xplat CI build * enabling building PerformanceTests for Mono * fixed AppVeyor path to Python.PerformanceTests.dll * fixed Mono deb sources to bionic * slightly relaxed perf target for WriteInt64Property * PerformanceTests: explicitly specify platform * use framework-specific build of perf tests in xplat and generic otherwise * added perf tests run to Travis CI * better error message for a failure to run benchmarks * appveyor: don't run perf tests in unsupported configurations * fixed performance test Python version condition in AppVeyor * explicitly notify when performance tests are skipped in AppVeyor * relax performance targets to ~10%, improve perf failure message * switch to the release of Microsoft.NETFramework.ReferenceAssemblies package
1 parent 687552f commit fffd53a

5 files changed

Lines changed: 63 additions & 9 deletions

File tree

‎.travis.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ python:
99

1010
env:
1111
matrix:
12-
- BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/
13-
- BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH=""
12+
- BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/
13+
- BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH=""
1414

1515
global:
1616
- LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
@@ -45,6 +45,8 @@ install:
4545
script:
4646
- python -m pytest
4747
- $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll
48+
# does not work on Linux, because NuGet package for 2.3 is Windows only
49+
# - "if [[ $TRAVIS_PYTHON_VERSION == '3.5' && $PERF_TESTS_PATH != '' ]]; then mono $NUNIT_PATH src/perf_tests/bin/$PERF_TESTS_PATH/Python.PerformanceTests.dll; fi"
4850

4951
after_script:
5052
# Waiting on mono-coverage, SharpCover or xr.Baboon

‎ci/appveyor_run_tests.ps1‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
# Test Runner framework being used for embedded tests
44
$CS_RUNNER = "nunit3-console"
55

6+
$XPLAT = $env:BUILD_OPTS -eq "--xplat"
7+
68
# Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3
79
if ($FALSE -and $env:PLATFORM -eq "x86"){
810
$CS_RUNNER = $CS_RUNNER + "-x86"
@@ -11,7 +13,7 @@ if ($FALSE -and $env:PLATFORM -eq "x86"){
1113
# Executable paths for OpenCover
1214
# Note if OpenCover fails, it won't affect the exit codes.
1315
$OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe
14-
if ($env:BUILD_OPTS -eq "--xplat"){
16+
if ($XPLAT){
1517
$CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe
1618
}
1719
else{
@@ -42,9 +44,31 @@ Write-Host ("Starting embedded tests") -ForegroundColor "Green"
4244
$CS_STATUS = $LastExitCode
4345
if ($CS_STATUS -ne 0) {
4446
Write-Host "Embedded tests failed" -ForegroundColor "Red"
47+
} else {
48+
# NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5
49+
# the test is only built using modern stack
50+
if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) {
51+
# Run C# Performance tests
52+
Write-Host ("Starting performance tests") -ForegroundColor "Green"
53+
if ($XPLAT) {
54+
$CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll"
55+
}
56+
else {
57+
$CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll"
58+
}
59+
&"$CS_RUNNER" "$CS_PERF_TESTS"
60+
$CS_PERF_STATUS = $LastExitCode
61+
if ($CS_PERF_STATUS -ne 0) {
62+
Write-Host "Performance tests (C#) failed" -ForegroundColor "Red"
63+
}
64+
} else {
65+
Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow"
66+
Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow"
67+
$CS_PERF_STATUS = 0
68+
}
4569
}
4670

47-
if ($env:BUILD_OPTS -eq "--xplat"){
71+
if ($XPLAT){
4872
if ($env:PLATFORM -eq "x64") {
4973
$DOTNET_CMD = "dotnet"
5074
}
@@ -54,15 +78,15 @@ if ($env:BUILD_OPTS -eq "--xplat"){
5478

5579
# Run Embedded tests for netcoreapp2.0 (OpenCover currently does not supports dotnet core)
5680
Write-Host ("Starting embedded tests for netcoreapp2.0") -ForegroundColor "Green"
57-
&$DOTNET_CMD .\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll
81+
&$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll"
5882
$CS_STATUS = $LastExitCode
5983
if ($CS_STATUS -ne 0) {
6084
Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red"
6185
}
6286
}
6387

6488
# Set exit code to fail if either Python or Embedded tests failed
65-
if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0) {
89+
if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) {
6690
Write-Host "Tests failed" -ForegroundColor "Red"
6791
$host.SetShouldExit(1)
6892
}

‎setup.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,16 @@ def build_extension(self, ext):
362362
),
363363
shell=use_shell,
364364
)
365+
subprocess.check_call(
366+
" ".join(
367+
cmd
368+
+ [
369+
'"/t:Python_PerformanceTests:publish"',
370+
"/p:TargetFramework=net461",
371+
]
372+
),
373+
shell=use_shell,
374+
)
365375
if DEVTOOLS == "Mono" or DEVTOOLS == "dotnet":
366376
self._build_monoclr()
367377

‎src/perf_tests/BenchmarkTests.cs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,23 @@ public void SetUp()
2121
Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new");
2222
this.summary = BenchmarkRunner.Run<PythonCallingNetBenchmark>();
2323
Assert.IsNotEmpty(this.summary.Reports);
24-
Assert.IsTrue(this.summary.Reports.All(r => r.Success));
24+
Assert.IsTrue(
25+
condition: this.summary.Reports.All(r => r.Success),
26+
message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above.");
2527
}
2628

2729
[Test]
2830
public void ReadInt64Property()
2931
{
3032
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
31-
Assert.LessOrEqual(optimisticPerfRatio, 0.68);
33+
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.66);
3234
}
3335

3436
[Test]
3537
public void WriteInt64Property()
3638
{
3739
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
38-
Assert.LessOrEqual(optimisticPerfRatio, 0.66);
40+
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.64);
3941
}
4042

4143
static double GetOptimisticPerfRatio(
@@ -59,5 +61,14 @@ static double GetOptimisticPerfRatio(
5961
}
6062

6163
public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
64+
65+
public static void AssertPerformanceIsBetterOrSame(
66+
double actual, double target,
67+
double wiggleRoom = 1.1, [CallerMemberName] string testName = null) {
68+
double threshold = target * wiggleRoom;
69+
Assert.LessOrEqual(actual, threshold,
70+
$"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})"
71+
+ ": perf result is higher than the failure threshold.");
72+
}
6273
}
6374
}

‎src/perf_tests/Python.PerformanceTests.csproj‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,19 @@
33
<PropertyGroup>
44
<TargetFramework>net461</TargetFramework>
55
<Configurations>DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3</Configurations>
6+
<OutputPath>bin\</OutputPath>
67

78
<IsPackable>false</IsPackable>
9+
10+
<Platforms>x64;x86</Platforms>
811
</PropertyGroup>
912

1013
<ItemGroup>
1114
<PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
15+
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0">
16+
<PrivateAssets>all</PrivateAssets>
17+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
18+
</PackageReference>
1219
<PackageReference Include="nunit" Version="3.12.0" />
1320
<PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />
1421
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />

0 commit comments

Comments
 (0)