From 4f70860254236baba4043061736637f106f1b272 Mon Sep 17 00:00:00 2001 From: Rafael Vuijk Date: Fri, 14 Aug 2026 22:19:01 +0000 Subject: [PATCH] Run a fixed corpus on every commit, reporting solved, wrong, error and timeout (#529, #500) #746 v1.0 names this as required infrastructure and it was the last piece of it not built. The point is the four-way split. A suite that passes or fails cannot tell answered wrongly from declined to answer, and AGENTS.md is explicit that a change solving one more problem and introducing one wrong answer is a regression rather than progress -- which a green suite would report as progress. Forty problems across simplification, solving, integration and limits, each carrying the verdict it currently earns. The gate fails on any wrong answer, and on any case that stops matching its recorded verdict -- including one that improves, so that the record cannot quietly drift from the library. Answers are checked rather than compared against stored text. A root is substituted back into its equation, an antiderivative is differentiated back, a simplification is evaluated against the expression it came from. So the corpus does not record what the library happens to print today, a change of form is not a failure, and only a change of value is. Sampling is complex and off the axes on purpose: a rule that is wrong away from the real line is the kind this library keeps finding, and real sample points cannot see it. A point where both sides are undefined proves nothing and is skipped; a point where one has a value and the other does not is a disagreement, which is how a rewrite that widens or narrows a domain gets caught. Two things the first run caught, both in the checking rather than in the library. A root may carry a parameter -- sin(x) = 0 is solved by 2*pi*n for every integer n -- and calling that unverifiable would have marked a correct answer unsolved. Each parameter is given four integer values, so solve:trig now verifies eight roots and checks the parameterisation rather than one branch of it. An antiderivative is checked on the positive reals rather than off the axis, unlike everything else here. The integral of 1/x comes back as ln(abs(x)) + C, and abs is not holomorphic, so differentiating it does not reproduce 1/x away from the real line. Whether ln(x) is the better answer is a real question, and it is not one a regression gate should settle by failing every build until somebody answers it. It is worth answering separately. This is a gate and not the harness. The harnesses stay outside the repository where they can generate inputs, take minutes and be read by a person; this runs in CI on every commit, takes about a second, and answers one question: did anything get worse. The problems are the library's own, from its issue tracker, plus a few standard limits; nothing is vendored from another project's suite. Measured on this tree: solved 39, unsolved 1, wrong 0, error 0, timeout 0, of 40. Suite 7119 passed / 0 failed. https://github.com/asc-community/AngouriMath/issues/529 https://github.com/asc-community/AngouriMath/issues/500 https://github.com/asc-community/AngouriMath/issues/746 Co-authored-by: Claude Opus 5 --- Sources/Tests/UnitTests/Corpus/Corpus.cs | 145 ++++++++ .../Tests/UnitTests/Corpus/CorpusGateTest.cs | 316 ++++++++++++++++++ 2 files changed, 461 insertions(+) create mode 100644 Sources/Tests/UnitTests/Corpus/Corpus.cs create mode 100644 Sources/Tests/UnitTests/Corpus/CorpusGateTest.cs diff --git a/Sources/Tests/UnitTests/Corpus/Corpus.cs b/Sources/Tests/UnitTests/Corpus/Corpus.cs new file mode 100644 index 000000000..1217f298b --- /dev/null +++ b/Sources/Tests/UnitTests/Corpus/Corpus.cs @@ -0,0 +1,145 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System.Collections.Generic; + +namespace AngouriMath.Tests.Corpus +{ + /// What the library did with a problem. + public enum Verdict + { + /// Answered, and the answer was checked and holds. + Solved, + + /// Declined to answer. Not a failure: "I could not settle this" is legitimate. + Unsolved, + + /// Answered, and the answer is not right. Always a failure. + Wrong, + + /// Threw something that is not the library declining. + Error, + + /// Did not finish inside the budget. + Timeout, + } + + /// Which call to make. + public enum Op + { + /// Simplify, checked against the input at sampled points. + Simplify, + + /// Solve for x, checked by substituting each root back in. + Solve, + + /// Integrate over x, checked by differentiating the answer back. + Integrate, + + /// Take a limit, checked against a stated value. + Limit, + } + + /// One problem, and what the library is currently expected to do with it. + /// What to call it in the report. + /// Which call to make. + /// The expression, as written. + /// + /// The verdict recorded when this entry was last measured. is + /// never an acceptable expectation and the gate rejects it outright. + /// + /// Where the variable goes, for a limit. + /// The answer, for a limit. + public sealed record Problem( + string Name, + Op Op, + string Input, + Verdict Expect = Verdict.Solved, + string? Approach = null, + string? Expected = null); + + /// + /// A fixed set of problems with checkable answers, run on every commit, reporting + /// solved / unsolved / wrong / error / timeout rather than pass or fail. + /// + /// + /// + /// #746 v1.0 asks for + /// exactly this and names it as required infrastructure, together with + /// #529 and + /// #500. The point is + /// the four-way split: a suite that only passes or fails cannot tell answered wrongly + /// from declined to answer, and AGENTS.md is explicit that a change which solves one + /// more problem and introduces one wrong answer is a regression rather than progress. + /// + /// + /// This is a gate, and it is not the exploratory harness. The harnesses live outside + /// the repository, in the analysis workspace, where they can generate inputs, take minutes + /// and be read by a person. This one runs in CI on every commit, so it is small, fast, and + /// answers one question: did anything get worse. + /// + /// + /// The problems are the library's own — drawn from its issue tracker, where each was reported + /// as something that ought to work — plus a few standard limits. Nothing here is vendored + /// from another project's test suite. + /// + /// + public static class Corpus + { + /// The problems, with the verdict each currently earns. + public static readonly IReadOnlyList All = new List + { + // --- Simplification: the answer must equal the input wherever both are defined --- + new("simp:polynomial", Op.Simplify, "(x + 1) ^ 2 - x ^ 2 - 2 * x - 1"), + new("simp:rational", Op.Simplify, "(x ^ 2 - 1) / (x - 1)"), + new("simp:rational-two-vars", Op.Simplify, "(x ^ 2 + 2 * x * y + y ^ 2) / (x ^ 2 - y ^ 2)"), + new("simp:factoring", Op.Simplify, "a * c + a * d + b * c + b * d"), + new("simp:surds", Op.Simplify, "sqrt(12) + sqrt(27)"), + new("simp:surd-product", Op.Simplify, "sqrt(2) * sqrt(3)"), + new("simp:trig-pythagoras", Op.Simplify, "sin(x) ^ 2 + cos(x) ^ 2"), + new("simp:trig-double", Op.Simplify, "sin(2 * x) - 2 * sin(x) * cos(x)"), + new("simp:trig-cyclic", Op.Simplify, "(sin(2 * x) * csc(x)) ^ 2 / 4 - cos(2 * x) - sin(x) ^ 2"), + new("simp:inverse-trig", Op.Simplify, "arcsin(x) + arccos(x)"), + new("simp:log-ratio", Op.Simplify, "ln(2 ^ 1000) / ln(2 ^ (-1000))"), + new("simp:collapse", Op.Simplify, "x ^ 2 + 2 * x + 1"), + new("simp:parity", Op.Simplify, "sin(-x) + sin(x)"), + new("simp:abs", Op.Simplify, "abs(-x) - abs(x)"), + new("simp:power-tower", Op.Simplify, "(x ^ 2) ^ 3 - x ^ 6"), + + // --- Solving: every root is substituted back and must vanish --- + new("solve:linear", Op.Solve, "3 * x + 5"), + new("solve:quadratic", Op.Solve, "x ^ 2 - 5 * x + 6"), + new("solve:quadratic-complex", Op.Solve, "x ^ 2 + 1"), + new("solve:biquadratic", Op.Solve, "x ^ 4 + 3 * x ^ 2 + 2"), + new("solve:quartic-nested", Op.Solve, "x ^ 4 + x ^ 2 + 1"), + new("solve:quintic-factorable", Op.Solve, "x ^ 5 + 2 * x ^ 3 - 2 * x ^ 2 - 4"), + new("solve:cubic", Op.Solve, "x ^ 3 - 6 * x ^ 2 + 11 * x - 6"), + new("solve:exponential", Op.Solve, "e ^ x - 1"), + new("solve:trig", Op.Solve, "sin(x)"), + new("solve:product", Op.Solve, "(x - 1) * (x - 2) * (x - 3)"), + + // --- Integration: the answer is differentiated back and compared --- + new("int:power", Op.Integrate, "x ^ 3"), + new("int:reciprocal", Op.Integrate, "1 / x"), + new("int:exponential", Op.Integrate, "e ^ x"), + new("int:trig", Op.Integrate, "sin(x)"), + new("int:u-sub", Op.Integrate, "cos(x ^ 2) * x"), + new("int:by-parts-cyclic", Op.Integrate, "sin(x) * e ^ x"), + new("int:arctan-form", Op.Integrate, "1 / (1 + x ^ 2)"), + new("int:partial-fractions", Op.Integrate, "1 / (x ^ 4 + 3 * x ^ 2 + 2)"), + new("int:hard", Op.Integrate, "sqrt(tan(x))", Verdict.Unsolved), + + // --- Limits: compared against the stated value --- + new("lim:0/0", Op.Limit, "sin(x) / x", Approach: "0", Expected: "1"), + new("lim:0/0-arcsin", Op.Limit, "arcsin(x) / x", Approach: "0", Expected: "1"), + new("lim:1^oo", Op.Limit, "(1 + 1/x) ^ x", Approach: "+oo", Expected: "e"), + new("lim:oo-oo", Op.Limit, "e ^ x - x", Approach: "+oo", Expected: "+oo"), + new("lim:log", Op.Limit, "1/x + ln(x)", Approach: "+oo", Expected: "+oo"), + new("lim:ratio", Op.Limit, "(x ^ 2 + 1) / (x ^ 2 - 1)", Approach: "+oo", Expected: "1"), + }; + } +} diff --git a/Sources/Tests/UnitTests/Corpus/CorpusGateTest.cs b/Sources/Tests/UnitTests/Corpus/CorpusGateTest.cs new file mode 100644 index 000000000..a67f44acd --- /dev/null +++ b/Sources/Tests/UnitTests/Corpus/CorpusGateTest.cs @@ -0,0 +1,316 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using AngouriMath; +using AngouriMath.Core.Exceptions; +using AngouriMath.Extensions; +using Xunit; +using Xunit.Abstractions; + +namespace AngouriMath.Tests.Corpus +{ + /// + /// Runs and fails where the library got worse — or where it got + /// better and the record was not updated to say so. + /// + /// + /// + /// The answers are checked rather than compared to a stored string. A solution is substituted + /// back into the equation, an antiderivative is differentiated back, a simplification is + /// evaluated against the expression it came from. That means the corpus does not have to + /// record what the library happens to print today, so a change in form is not a failure and + /// only a change in value is. + /// + /// + /// Checking is numeric at sampled points, and the points are complex and away from the axes + /// on purpose: a rule that is wrong off the real line is the kind this library keeps finding, + /// and real sample points would not see it. + /// + /// + [Trait("Area", "Corpus")] + public sealed class CorpusGateTest + { + private readonly ITestOutputHelper output; + + public CorpusGateTest(ITestOutputHelper output) => this.output = output; + + /// Generous: this is a gate against regression, not a performance measurement. + private static readonly TimeSpan Budget = TimeSpan.FromSeconds(30); + + /// Away from the real axis, and away from the small integers. + private static readonly System.Numerics.Complex[] Samples = + { + new(0.3721, 0.5813), new(-0.7211, 0.2917), new(1.4523, -0.6131), new(-1.1907, -0.8419), + }; + + /// Positive and away from 1, for the antiderivative check. See CheckIntegrate. + private static readonly System.Numerics.Complex[] RealSamples = + { + new(0.4137, 0), new(1.7219, 0), new(2.9041, 0), new(0.8353, 0), + }; + + /// Values given to a parameter that a root is stated in terms of. + private static readonly int[] ParameterValues = { 0, 1, -1, 2 }; + + /// + /// The root itself where it names only x, or one root per value of each parameter + /// it is stated in terms of. + /// + private static IEnumerable Instantiate(Entity root) + { + var parameters = root.Vars.Where(v => v.Name != "x").ToList(); + if (parameters.Count == 0) + { + yield return root; + yield break; + } + foreach (var value in ParameterValues) + { + var instance = root; + foreach (var parameter in parameters) + instance = instance.Substitute(parameter, value); + yield return instance; + } + } + + private const double Tolerance = 1e-6; + + [Fact] + public void TheCorpusHasNotGotWorse() + { + var results = Corpus.All.Select(Run).ToList(); + output.WriteLine(Report(results)); + + var wrong = results.Where(r => r.Verdict == Verdict.Wrong).ToList(); + var regressed = results + .Where(r => r.Verdict != r.Problem.Expect && r.Verdict != Verdict.Wrong) + .ToList(); + + var complaint = new StringBuilder(); + foreach (var r in wrong) + complaint.AppendLine($"WRONG {r.Problem.Name}: {r.Problem.Input} -> {r.Answer} ({r.Note})"); + foreach (var r in regressed) + complaint.AppendLine( + $"CHANGED {r.Problem.Name}: expected {r.Problem.Expect}, got {r.Verdict}" + + $" ({r.Note}). If this is an improvement, record it in Corpus.cs."); + + Assert.True(complaint.Length == 0, "\n" + complaint + "\n" + Report(results)); + } + + /// + /// A corpus entry may not record a wrong answer as acceptable. Recording one would turn + /// the gate into a description of the defect rather than a check against it. + /// + [Fact] + public void NoEntryExpectsAWrongAnswer() + => Assert.Empty(Corpus.All.Where(p => p.Expect == Verdict.Wrong)); + + private sealed record Result(Problem Problem, Verdict Verdict, string Answer, string Note); + + private static Result Run(Problem problem) + { + try + { + var task = Task.Run(() => Attempt(problem)); + return task.Wait(Budget) + ? task.Result + : new Result(problem, Verdict.Timeout, "", $"no answer in {Budget.TotalSeconds:0}s"); + } + catch (AggregateException e) when (e.InnerException is { } inner) + { + // The library declining is not an error; anything else is. + return inner is AngouriMathBaseException and not AngouriBugException + ? new Result(problem, Verdict.Unsolved, "", Short(inner)) + : new Result(problem, Verdict.Error, "", Short(inner)); + } + catch (Exception e) + { + return new Result(problem, Verdict.Error, "", Short(e)); + } + } + + private static Result Attempt(Problem problem) => problem.Op switch + { + Op.Simplify => CheckSimplify(problem), + Op.Solve => CheckSolve(problem), + Op.Integrate => CheckIntegrate(problem), + Op.Limit => CheckLimit(problem), + _ => new Result(problem, Verdict.Error, "", "unsupported operation"), + }; + + private static Result CheckSimplify(Problem problem) + { + var input = problem.Input.ToEntity(); + var simplified = input.Simplify(); + return AgreeAtSamples(input, simplified, out var note) + ? new Result(problem, Verdict.Solved, simplified.Stringize(), note) + : new Result(problem, Verdict.Wrong, simplified.Stringize(), note); + } + + private static Result CheckSolve(Problem problem) + { + var input = problem.Input.ToEntity(); + var roots = input.SolveEquation("x"); + if (roots is not Entity.Set.FiniteSet finite) + return new Result(problem, Verdict.Unsolved, roots.Stringize(), "not a finite root set"); + if (finite.Count == 0) + return new Result(problem, Verdict.Unsolved, roots.Stringize(), "no roots found"); + + var checkedRoots = 0; + foreach (var root in finite) + { + // A root may carry a parameter -- sin(x) = 0 is solved by 2*pi*n, for every + // integer n -- and such a root is right rather than unverifiable. Each parameter + // is given a few integer values and the root is checked at each of them, which + // also checks that the parameterisation itself is right and not only one branch + // of it. + foreach (var instance in Instantiate(root)) + { + if (!TryEvaluate(input.Substitute("x", instance), out var residual)) + return new Result(problem, Verdict.Unsolved, roots.Stringize(), + $"root {instance.Stringize()} could not be checked"); + if (residual.Magnitude > Tolerance) + return new Result(problem, Verdict.Wrong, roots.Stringize(), + $"root {instance.Stringize()} leaves {residual.Magnitude:0.###e+0}"); + checkedRoots++; + } + } + return new Result(problem, Verdict.Solved, roots.Stringize(), $"{checkedRoots} root(s) verified"); + } + + /// + /// An antiderivative, checked by differentiating it back. + /// + /// + /// Sampled on the positive reals rather than off the axis, unlike everything else + /// here, because the antiderivatives this library returns are the real ones: the integral + /// of 1/x comes back as ln(abs(x)) + C, and abs is not holomorphic, + /// so differentiating that does not reproduce 1/x away from the real line. Whether + /// ln(x) would be the better answer is a real question and not one a regression + /// gate should decide by failing every build until someone settles it. + /// + private static Result CheckIntegrate(Problem problem) + { + var input = problem.Input.ToEntity(); + var integral = input.Integrate("x"); + if (integral.Nodes.Any(node => node is Entity.Integralf)) + return new Result(problem, Verdict.Unsolved, integral.Stringize(), "integral remains"); + var back = integral.Differentiate("x"); + return AgreeAtSamples(input, back, RealSamples, out var note) + ? new Result(problem, Verdict.Solved, integral.Stringize(), "differentiates back") + : new Result(problem, Verdict.Wrong, integral.Stringize(), note); + } + + private static Result CheckLimit(Problem problem) + { + var input = problem.Input.ToEntity(); + var limit = input.Limit("x", problem.Approach!.ToEntity()); + if (limit.Nodes.Any(node => node is Entity.Limitf)) + return new Result(problem, Verdict.Unsolved, limit.Stringize(), "limit remains"); + + var expected = problem.Expected!.ToEntity(); + if (limit.Simplify() == expected.Simplify()) + return new Result(problem, Verdict.Solved, limit.Stringize(), $"reached {problem.Expected}"); + // Infinities do not compare numerically, so a difference of forms is all there is. + if (!TryEvaluate(limit, out var got) || !TryEvaluate(expected, out var want)) + return new Result(problem, Verdict.Wrong, limit.Stringize(), $"expected {problem.Expected}"); + return (got - want).Magnitude <= Tolerance + ? new Result(problem, Verdict.Solved, limit.Stringize(), $"reached {problem.Expected}") + : new Result(problem, Verdict.Wrong, limit.Stringize(), $"expected {problem.Expected}"); + } + + /// + /// Whether two expressions take the same value at the sample points, over the variables + /// they mention. + /// + /// + /// A point where both are undefined proves nothing and is skipped; a point where + /// one has a value and the other does not is a disagreement, which is how a rewrite that + /// quietly widens or narrows a domain gets caught. + /// + private static bool AgreeAtSamples(Entity left, Entity right, out string note) + => AgreeAtSamples(left, right, Samples, out note); + + private static bool AgreeAtSamples( + Entity left, Entity right, System.Numerics.Complex[] points, out string note) + { + var variables = left.Vars.Concat(right.Vars).Distinct().ToList(); + var compared = 0; + for (var i = 0; i < points.Length; i++) + { + Entity l = left, r = right; + foreach (var (variable, index) in variables.Select((v, n) => (v, n))) + { + var point = points[(i + index) % points.Length]; + l = l.Substitute(variable, point); + r = r.Substitute(variable, point); + } + var okL = TryEvaluate(l, out var valueL); + var okR = TryEvaluate(r, out var valueR); + if (!okL && !okR) continue; + if (okL != okR) + { + note = $"one side is undefined at sample {i}"; + return false; + } + compared++; + var scale = Math.Max(1, Math.Max(valueL.Magnitude, valueR.Magnitude)); + if ((valueL - valueR).Magnitude / scale > Tolerance) + { + note = $"differ at sample {i} by {(valueL - valueR).Magnitude:0.###e+0}"; + return false; + } + } + note = compared == 0 ? "undefined at every sample" : $"agree at {compared} point(s)"; + return true; + } + + private static bool TryEvaluate(Entity expr, out System.Numerics.Complex value) + { + value = default; + try + { + if (!expr.EvaluableNumerical) return false; + var evaluated = expr.EvalNumerical(); + value = (System.Numerics.Complex)evaluated; + return !double.IsNaN(value.Real) && !double.IsNaN(value.Imaginary); + } + catch (AngouriMathBaseException) + { + return false; + } + } + + private static string Short(Exception e) + { + var text = e.Message.Replace('\n', ' '); + return e.GetType().Name + ": " + (text.Length > 90 ? text[..90] : text); + } + + private static string Report(IReadOnlyList results) + { + var sb = new StringBuilder(); + sb.AppendLine(); + foreach (var group in results.GroupBy(r => r.Problem.Op)) + { + sb.AppendLine($"{group.Key}:"); + foreach (var r in group) + sb.AppendLine($" {r.Verdict,-8} {r.Problem.Name,-26} {r.Note}"); + } + sb.AppendLine(); + foreach (var verdict in Enum.GetValues(typeof(Verdict)).Cast()) + sb.Append($"{verdict}={results.Count(r => r.Verdict == verdict)} "); + sb.AppendLine($"of {results.Count}"); + return sb.ToString(); + } + } +}