diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4e37eabcd..ea6b2c99a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -107,10 +107,4 @@ jobs: if: ${{ matrix.python != '3.13' }} run: dotnet test --runtime any-${{ matrix.os.platform }} src/python_tests_runner/ - - name: Perf tests - if: ${{ (matrix.python == '3.8') && (matrix.os.platform == 'x64') }} - run: | - pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2 - dotnet test --configuration Release --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/perf_tests/ - # TODO: Run mono tests on Windows? diff --git a/pythonnet.sln b/pythonnet.sln index cf684c0e1..9dfeb44b1 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -8,8 +8,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest", "src EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\Python.Test.csproj", "{14EF9518-5BB7-4F83-8686-015BD2CC788E}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "tests\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" diff --git a/src/perf_tests/BaselineComparisonBenchmarkBase.cs b/src/perf_tests/BaselineComparisonBenchmarkBase.cs deleted file mode 100644 index 06adcbc67..000000000 --- a/src/perf_tests/BaselineComparisonBenchmarkBase.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Reflection; - -using Python.Runtime; - -namespace Python.PerformanceTests -{ - public class BaselineComparisonBenchmarkBase - { - public BaselineComparisonBenchmarkBase() - { - Console.WriteLine($"CWD: {Environment.CurrentDirectory}"); - Console.WriteLine($"Using Python.Runtime from {typeof(PythonEngine).Assembly.Location} {typeof(PythonEngine).Assembly.GetName()}"); - - try { - PythonEngine.Initialize(); - Console.WriteLine("Python Initialized"); - Trace.Assert(PythonEngine.BeginAllowThreads() != IntPtr.Zero); - Console.WriteLine("Threading enabled"); - } - catch (Exception e) { - Console.WriteLine(e); - throw; - } - } - - static BaselineComparisonBenchmarkBase() - { - SetupRuntimeResolve(); - } - - public static void SetupRuntimeResolve() - { - string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); - if (string.IsNullOrEmpty(pythonRuntimeDll)) - { - throw new ArgumentException( - "Required environment variable is missing", - BaselineComparisonConfig.EnvironmentVariableName); - } - - Console.WriteLine("Preloading " + pythonRuntimeDll); - Assembly.LoadFrom(pythonRuntimeDll); - foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) { - if (assembly.FullName.StartsWith("Python.Runtime")) - Console.WriteLine(assembly.Location); - foreach(var dependency in assembly.GetReferencedAssemblies()) - if (dependency.FullName.Contains("Python.Runtime")) { - Console.WriteLine($"{assembly} -> {dependency}"); - } - } - - AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; - } - - static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { - if (!args.Name.StartsWith("Python.Runtime")) - return null; - - var preloaded = AppDomain.CurrentDomain.GetAssemblies() - .FirstOrDefault(a => a.GetName().Name == "Python.Runtime"); - if (preloaded != null) return preloaded; - - string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); - if (string.IsNullOrEmpty(pythonRuntimeDll)) - return null; - - return Assembly.LoadFrom(pythonRuntimeDll); - } - } -} diff --git a/src/perf_tests/BaselineComparisonConfig.cs b/src/perf_tests/BaselineComparisonConfig.cs deleted file mode 100644 index 3f6766554..000000000 --- a/src/perf_tests/BaselineComparisonConfig.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; - -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Jobs; - -using Perfolizer.Horology; - -namespace Python.PerformanceTests -{ - public class BaselineComparisonConfig : ManualConfig - { - public const string EnvironmentVariableName = "PythonRuntimeDLL"; - - public BaselineComparisonConfig() - { - this.Options |= ConfigOptions.DisableOptimizationsValidator; - - string deploymentRoot = BenchmarkTests.DeploymentRoot; - - var baseJob = Job.Default - .WithLaunchCount(1) - .WithWarmupCount(3) - .WithMaxIterationCount(100) - .WithIterationTime(TimeInterval.FromMilliseconds(100)); - this.Add(baseJob - .WithId("baseline") - .WithEnvironmentVariable(EnvironmentVariableName, - Path.Combine(deploymentRoot, "baseline", "Python.Runtime.dll")) - .WithBaseline(true)); - this.Add(baseJob - .WithId("new") - .WithEnvironmentVariable(EnvironmentVariableName, - Path.Combine(deploymentRoot, "new", "Python.Runtime.dll"))); - } - - static BaselineComparisonConfig() { - AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; - } - - static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { - Console.WriteLine(args.Name); - if (!args.Name.StartsWith("Python.Runtime")) - return null; - string pythonRuntimeDll = Environment.GetEnvironmentVariable(EnvironmentVariableName); - if (string.IsNullOrEmpty(pythonRuntimeDll)) - pythonRuntimeDll = Path.Combine(BenchmarkTests.DeploymentRoot, "baseline", "Python.Runtime.dll"); - return Assembly.LoadFrom(pythonRuntimeDll); - } - } -} diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs deleted file mode 100644 index 9e033d11f..000000000 --- a/src/perf_tests/BenchmarkTests.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Reflection; - -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Running; -using NUnit.Framework; - -namespace Python.PerformanceTests -{ - public class BenchmarkTests - { - Summary summary; - - [OneTimeSetUp] - public void SetUp() - { - Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new"); - this.summary = BenchmarkRunner.Run(); - Assert.IsNotEmpty(this.summary.Reports); - Assert.IsTrue( - condition: this.summary.Reports.All(r => r.Success), - message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above."); - } - - [Test] - public void ReadInt64Property() - { - double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); - AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 1.35); - } - - [Test] - public void WriteInt64Property() - { - double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); - AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 1.25); - } - - static double GetOptimisticPerfRatio( - IReadOnlyList reports, - [CallerMemberName] string methodName = null) - { - reports = reports.Where(r => r.BenchmarkCase.Descriptor.WorkloadMethod.Name == methodName).ToArray(); - if (reports.Count == 0) - throw new ArgumentException( - message: $"No reports found for {methodName}. " - + "You have to match test method name to benchmark method name or " - + "pass benchmark method name explicitly", - paramName: nameof(methodName)); - - var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics; - var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics; - - double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5; - - return newTimeOptimistic / baseline.Mean; - } - - public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - - public static void AssertPerformanceIsBetterOrSame( - double actual, double target, - double wiggleRoom = 1.1, [CallerMemberName] string testName = null) { - double threshold = target * wiggleRoom; - Assert.LessOrEqual(actual, threshold, - $"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})" - + ": perf result is higher than the failure threshold."); - } - } -} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj deleted file mode 100644 index bde07ecab..000000000 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ /dev/null @@ -1,48 +0,0 @@ - - - - net472 - false - x64 - x64 - - - - - - PreserveNewest - - - - - - false - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - diff --git a/src/perf_tests/PythonCallingNetBenchmark.cs b/src/perf_tests/PythonCallingNetBenchmark.cs deleted file mode 100644 index d7edd4583..000000000 --- a/src/perf_tests/PythonCallingNetBenchmark.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Reflection; -using System.Text; - -using BenchmarkDotNet.Attributes; -using Python.Runtime; - -namespace Python.PerformanceTests -{ - [Config(typeof(BaselineComparisonConfig))] - public class PythonCallingNetBenchmark: BaselineComparisonBenchmarkBase - { - [Benchmark] - public void ReadInt64Property() - { - using (Py.GIL()) - { - var locals = new PyDict(); - locals.SetItem("a", new NetObject().ToPython()); - Exec($@" -s = 0 -for i in range(50000): - s += a.{nameof(NetObject.LongProperty)} -", locals: locals); - } - } - - [Benchmark] - public void WriteInt64Property() { - using (Py.GIL()) { - var locals = new PyDict(); - locals.SetItem("a", new NetObject().ToPython()); - Exec($@" -s = 0 -for i in range(50000): - a.{nameof(NetObject.LongProperty)} += i -", locals: locals); - } - } - - static void Exec(string code, PyDict locals) - { - MethodInfo exec = typeof(PythonEngine).GetMethod(nameof(PythonEngine.Exec)); - object localsArg = typeof(PyObject).Assembly.GetName().Version.Major >= 3 - ? locals : locals.Handle; - exec.Invoke(null, new[] - { - code, localsArg, null - }); - } - } - - class NetObject - { - public long LongProperty { get; set; } = 42; - } -} diff --git a/src/perf_tests/baseline/.gitkeep b/src/perf_tests/baseline/.gitkeep deleted file mode 100644 index e69de29bb..000000000