diff --git a/Sources/Tests/UnitTests/Corpus/Corpus.cs b/Sources/Tests/UnitTests/Corpus/Corpus.cs new file mode 100644 index 000000000..1217f298b --- /dev/null +++ b/Sources/Tests/UnitTests/Corpus/Corpus.cs @@ -0,0 +1,145 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System.Collections.Generic; + +namespace AngouriMath.Tests.Corpus +{ + /// What the library did with a problem. + public enum Verdict + { + /// Answered, and the answer was checked and holds. + Solved, + + /// Declined to answer. Not a failure: "I could not settle this" is legitimate. + Unsolved, + + /// Answered, and the answer is not right. Always a failure. + Wrong, + + /// Threw something that is not the library declining. + Error, + + /// Did not finish inside the budget. + Timeout, + } + + /// Which call to make. + public enum Op + { + /// Simplify, checked against the input at sampled points. + Simplify, + + /// Solve for x, checked by substituting each root back in. + Solve, + + /// Integrate over x, checked by differentiating the answer back. + Integrate, + + /// Take a limit, checked against a stated value. + Limit, + } + + /// One problem, and what the library is currently expected to do with it. + /// What to call it in the report. + /// Which call to make. + /// The expression, as written. + /// + /// The verdict recorded when this entry was last measured. is + /// never an acceptable expectation and the gate rejects it outright. + /// + /// Where the variable goes, for a limit. + /// The answer, for a limit. + public sealed record Problem( + string Name, + Op Op, + string Input, + Verdict Expect = Verdict.Solved, + string? Approach = null, + string? Expected = null); + + /// + /// A fixed set of problems with checkable answers, run on every commit, reporting + /// solved / unsolved / wrong / error / timeout rather than pass or fail. + /// + /// + /// + /// #746 v1.0 asks for + /// exactly this and names it as required infrastructure, together with + /// #529 and + /// #500. The point is + /// the four-way split: a suite that only passes or fails cannot tell answered wrongly + /// from declined to answer, and AGENTS.md is explicit that a change which solves one + /// more problem and introduces one wrong answer is a regression rather than progress. + /// + /// + /// This is a gate, and it is not the exploratory harness. The harnesses live outside + /// the repository, in the analysis workspace, where they can generate inputs, take minutes + /// and be read by a person. This one runs in CI on every commit, so it is small, fast, and + /// answers one question: did anything get worse. + /// + /// + /// The problems are the library's own — drawn from its issue tracker, where each was reported + /// as something that ought to work — plus a few standard limits. Nothing here is vendored + /// from another project's test suite. + /// + /// + public static class Corpus + { + /// The problems, with the verdict each currently earns. + public static readonly IReadOnlyList All = new List + { + // --- Simplification: the answer must equal the input wherever both are defined --- + new("simp:polynomial", Op.Simplify, "(x + 1) ^ 2 - x ^ 2 - 2 * x - 1"), + new("simp:rational", Op.Simplify, "(x ^ 2 - 1) / (x - 1)"), + new("simp:rational-two-vars", Op.Simplify, "(x ^ 2 + 2 * x * y + y ^ 2) / (x ^ 2 - y ^ 2)"), + new("simp:factoring", Op.Simplify, "a * c + a * d + b * c + b * d"), + new("simp:surds", Op.Simplify, "sqrt(12) + sqrt(27)"), + new("simp:surd-product", Op.Simplify, "sqrt(2) * sqrt(3)"), + new("simp:trig-pythagoras", Op.Simplify, "sin(x) ^ 2 + cos(x) ^ 2"), + new("simp:trig-double", Op.Simplify, "sin(2 * x) - 2 * sin(x) * cos(x)"), + new("simp:trig-cyclic", Op.Simplify, "(sin(2 * x) * csc(x)) ^ 2 / 4 - cos(2 * x) - sin(x) ^ 2"), + new("simp:inverse-trig", Op.Simplify, "arcsin(x) + arccos(x)"), + new("simp:log-ratio", Op.Simplify, "ln(2 ^ 1000) / ln(2 ^ (-1000))"), + new("simp:collapse", Op.Simplify, "x ^ 2 + 2 * x + 1"), + new("simp:parity", Op.Simplify, "sin(-x) + sin(x)"), + new("simp:abs", Op.Simplify, "abs(-x) - abs(x)"), + new("simp:power-tower", Op.Simplify, "(x ^ 2) ^ 3 - x ^ 6"), + + // --- Solving: every root is substituted back and must vanish --- + new("solve:linear", Op.Solve, "3 * x + 5"), + new("solve:quadratic", Op.Solve, "x ^ 2 - 5 * x + 6"), + new("solve:quadratic-complex", Op.Solve, "x ^ 2 + 1"), + new("solve:biquadratic", Op.Solve, "x ^ 4 + 3 * x ^ 2 + 2"), + new("solve:quartic-nested", Op.Solve, "x ^ 4 + x ^ 2 + 1"), + new("solve:quintic-factorable", Op.Solve, "x ^ 5 + 2 * x ^ 3 - 2 * x ^ 2 - 4"), + new("solve:cubic", Op.Solve, "x ^ 3 - 6 * x ^ 2 + 11 * x - 6"), + new("solve:exponential", Op.Solve, "e ^ x - 1"), + new("solve:trig", Op.Solve, "sin(x)"), + new("solve:product", Op.Solve, "(x - 1) * (x - 2) * (x - 3)"), + + // --- Integration: the answer is differentiated back and compared --- + new("int:power", Op.Integrate, "x ^ 3"), + new("int:reciprocal", Op.Integrate, "1 / x"), + new("int:exponential", Op.Integrate, "e ^ x"), + new("int:trig", Op.Integrate, "sin(x)"), + new("int:u-sub", Op.Integrate, "cos(x ^ 2) * x"), + new("int:by-parts-cyclic", Op.Integrate, "sin(x) * e ^ x"), + new("int:arctan-form", Op.Integrate, "1 / (1 + x ^ 2)"), + new("int:partial-fractions", Op.Integrate, "1 / (x ^ 4 + 3 * x ^ 2 + 2)"), + new("int:hard", Op.Integrate, "sqrt(tan(x))", Verdict.Unsolved), + + // --- Limits: compared against the stated value --- + new("lim:0/0", Op.Limit, "sin(x) / x", Approach: "0", Expected: "1"), + new("lim:0/0-arcsin", Op.Limit, "arcsin(x) / x", Approach: "0", Expected: "1"), + new("lim:1^oo", Op.Limit, "(1 + 1/x) ^ x", Approach: "+oo", Expected: "e"), + new("lim:oo-oo", Op.Limit, "e ^ x - x", Approach: "+oo", Expected: "+oo"), + new("lim:log", Op.Limit, "1/x + ln(x)", Approach: "+oo", Expected: "+oo"), + new("lim:ratio", Op.Limit, "(x ^ 2 + 1) / (x ^ 2 - 1)", Approach: "+oo", Expected: "1"), + }; + } +} diff --git a/Sources/Tests/UnitTests/Corpus/CorpusGateTest.cs b/Sources/Tests/UnitTests/Corpus/CorpusGateTest.cs new file mode 100644 index 000000000..a67f44acd --- /dev/null +++ b/Sources/Tests/UnitTests/Corpus/CorpusGateTest.cs @@ -0,0 +1,316 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using AngouriMath; +using AngouriMath.Core.Exceptions; +using AngouriMath.Extensions; +using Xunit; +using Xunit.Abstractions; + +namespace AngouriMath.Tests.Corpus +{ + /// + /// Runs and fails where the library got worse — or where it got + /// better and the record was not updated to say so. + /// + /// + /// + /// The answers are checked rather than compared to a stored string. A solution is substituted + /// back into the equation, an antiderivative is differentiated back, a simplification is + /// evaluated against the expression it came from. That means the corpus does not have to + /// record what the library happens to print today, so a change in form is not a failure and + /// only a change in value is. + /// + /// + /// Checking is numeric at sampled points, and the points are complex and away from the axes + /// on purpose: a rule that is wrong off the real line is the kind this library keeps finding, + /// and real sample points would not see it. + /// + /// + [Trait("Area", "Corpus")] + public sealed class CorpusGateTest + { + private readonly ITestOutputHelper output; + + public CorpusGateTest(ITestOutputHelper output) => this.output = output; + + /// Generous: this is a gate against regression, not a performance measurement. + private static readonly TimeSpan Budget = TimeSpan.FromSeconds(30); + + /// Away from the real axis, and away from the small integers. + private static readonly System.Numerics.Complex[] Samples = + { + new(0.3721, 0.5813), new(-0.7211, 0.2917), new(1.4523, -0.6131), new(-1.1907, -0.8419), + }; + + /// Positive and away from 1, for the antiderivative check. See CheckIntegrate. + private static readonly System.Numerics.Complex[] RealSamples = + { + new(0.4137, 0), new(1.7219, 0), new(2.9041, 0), new(0.8353, 0), + }; + + /// Values given to a parameter that a root is stated in terms of. + private static readonly int[] ParameterValues = { 0, 1, -1, 2 }; + + /// + /// The root itself where it names only x, or one root per value of each parameter + /// it is stated in terms of. + /// + private static IEnumerable Instantiate(Entity root) + { + var parameters = root.Vars.Where(v => v.Name != "x").ToList(); + if (parameters.Count == 0) + { + yield return root; + yield break; + } + foreach (var value in ParameterValues) + { + var instance = root; + foreach (var parameter in parameters) + instance = instance.Substitute(parameter, value); + yield return instance; + } + } + + private const double Tolerance = 1e-6; + + [Fact] + public void TheCorpusHasNotGotWorse() + { + var results = Corpus.All.Select(Run).ToList(); + output.WriteLine(Report(results)); + + var wrong = results.Where(r => r.Verdict == Verdict.Wrong).ToList(); + var regressed = results + .Where(r => r.Verdict != r.Problem.Expect && r.Verdict != Verdict.Wrong) + .ToList(); + + var complaint = new StringBuilder(); + foreach (var r in wrong) + complaint.AppendLine($"WRONG {r.Problem.Name}: {r.Problem.Input} -> {r.Answer} ({r.Note})"); + foreach (var r in regressed) + complaint.AppendLine( + $"CHANGED {r.Problem.Name}: expected {r.Problem.Expect}, got {r.Verdict}" + + $" ({r.Note}). If this is an improvement, record it in Corpus.cs."); + + Assert.True(complaint.Length == 0, "\n" + complaint + "\n" + Report(results)); + } + + /// + /// A corpus entry may not record a wrong answer as acceptable. Recording one would turn + /// the gate into a description of the defect rather than a check against it. + /// + [Fact] + public void NoEntryExpectsAWrongAnswer() + => Assert.Empty(Corpus.All.Where(p => p.Expect == Verdict.Wrong)); + + private sealed record Result(Problem Problem, Verdict Verdict, string Answer, string Note); + + private static Result Run(Problem problem) + { + try + { + var task = Task.Run(() => Attempt(problem)); + return task.Wait(Budget) + ? task.Result + : new Result(problem, Verdict.Timeout, "", $"no answer in {Budget.TotalSeconds:0}s"); + } + catch (AggregateException e) when (e.InnerException is { } inner) + { + // The library declining is not an error; anything else is. + return inner is AngouriMathBaseException and not AngouriBugException + ? new Result(problem, Verdict.Unsolved, "", Short(inner)) + : new Result(problem, Verdict.Error, "", Short(inner)); + } + catch (Exception e) + { + return new Result(problem, Verdict.Error, "", Short(e)); + } + } + + private static Result Attempt(Problem problem) => problem.Op switch + { + Op.Simplify => CheckSimplify(problem), + Op.Solve => CheckSolve(problem), + Op.Integrate => CheckIntegrate(problem), + Op.Limit => CheckLimit(problem), + _ => new Result(problem, Verdict.Error, "", "unsupported operation"), + }; + + private static Result CheckSimplify(Problem problem) + { + var input = problem.Input.ToEntity(); + var simplified = input.Simplify(); + return AgreeAtSamples(input, simplified, out var note) + ? new Result(problem, Verdict.Solved, simplified.Stringize(), note) + : new Result(problem, Verdict.Wrong, simplified.Stringize(), note); + } + + private static Result CheckSolve(Problem problem) + { + var input = problem.Input.ToEntity(); + var roots = input.SolveEquation("x"); + if (roots is not Entity.Set.FiniteSet finite) + return new Result(problem, Verdict.Unsolved, roots.Stringize(), "not a finite root set"); + if (finite.Count == 0) + return new Result(problem, Verdict.Unsolved, roots.Stringize(), "no roots found"); + + var checkedRoots = 0; + foreach (var root in finite) + { + // A root may carry a parameter -- sin(x) = 0 is solved by 2*pi*n, for every + // integer n -- and such a root is right rather than unverifiable. Each parameter + // is given a few integer values and the root is checked at each of them, which + // also checks that the parameterisation itself is right and not only one branch + // of it. + foreach (var instance in Instantiate(root)) + { + if (!TryEvaluate(input.Substitute("x", instance), out var residual)) + return new Result(problem, Verdict.Unsolved, roots.Stringize(), + $"root {instance.Stringize()} could not be checked"); + if (residual.Magnitude > Tolerance) + return new Result(problem, Verdict.Wrong, roots.Stringize(), + $"root {instance.Stringize()} leaves {residual.Magnitude:0.###e+0}"); + checkedRoots++; + } + } + return new Result(problem, Verdict.Solved, roots.Stringize(), $"{checkedRoots} root(s) verified"); + } + + /// + /// An antiderivative, checked by differentiating it back. + /// + /// + /// Sampled on the positive reals rather than off the axis, unlike everything else + /// here, because the antiderivatives this library returns are the real ones: the integral + /// of 1/x comes back as ln(abs(x)) + C, and abs is not holomorphic, + /// so differentiating that does not reproduce 1/x away from the real line. Whether + /// ln(x) would be the better answer is a real question and not one a regression + /// gate should decide by failing every build until someone settles it. + /// + private static Result CheckIntegrate(Problem problem) + { + var input = problem.Input.ToEntity(); + var integral = input.Integrate("x"); + if (integral.Nodes.Any(node => node is Entity.Integralf)) + return new Result(problem, Verdict.Unsolved, integral.Stringize(), "integral remains"); + var back = integral.Differentiate("x"); + return AgreeAtSamples(input, back, RealSamples, out var note) + ? new Result(problem, Verdict.Solved, integral.Stringize(), "differentiates back") + : new Result(problem, Verdict.Wrong, integral.Stringize(), note); + } + + private static Result CheckLimit(Problem problem) + { + var input = problem.Input.ToEntity(); + var limit = input.Limit("x", problem.Approach!.ToEntity()); + if (limit.Nodes.Any(node => node is Entity.Limitf)) + return new Result(problem, Verdict.Unsolved, limit.Stringize(), "limit remains"); + + var expected = problem.Expected!.ToEntity(); + if (limit.Simplify() == expected.Simplify()) + return new Result(problem, Verdict.Solved, limit.Stringize(), $"reached {problem.Expected}"); + // Infinities do not compare numerically, so a difference of forms is all there is. + if (!TryEvaluate(limit, out var got) || !TryEvaluate(expected, out var want)) + return new Result(problem, Verdict.Wrong, limit.Stringize(), $"expected {problem.Expected}"); + return (got - want).Magnitude <= Tolerance + ? new Result(problem, Verdict.Solved, limit.Stringize(), $"reached {problem.Expected}") + : new Result(problem, Verdict.Wrong, limit.Stringize(), $"expected {problem.Expected}"); + } + + /// + /// Whether two expressions take the same value at the sample points, over the variables + /// they mention. + /// + /// + /// A point where both are undefined proves nothing and is skipped; a point where + /// one has a value and the other does not is a disagreement, which is how a rewrite that + /// quietly widens or narrows a domain gets caught. + /// + private static bool AgreeAtSamples(Entity left, Entity right, out string note) + => AgreeAtSamples(left, right, Samples, out note); + + private static bool AgreeAtSamples( + Entity left, Entity right, System.Numerics.Complex[] points, out string note) + { + var variables = left.Vars.Concat(right.Vars).Distinct().ToList(); + var compared = 0; + for (var i = 0; i < points.Length; i++) + { + Entity l = left, r = right; + foreach (var (variable, index) in variables.Select((v, n) => (v, n))) + { + var point = points[(i + index) % points.Length]; + l = l.Substitute(variable, point); + r = r.Substitute(variable, point); + } + var okL = TryEvaluate(l, out var valueL); + var okR = TryEvaluate(r, out var valueR); + if (!okL && !okR) continue; + if (okL != okR) + { + note = $"one side is undefined at sample {i}"; + return false; + } + compared++; + var scale = Math.Max(1, Math.Max(valueL.Magnitude, valueR.Magnitude)); + if ((valueL - valueR).Magnitude / scale > Tolerance) + { + note = $"differ at sample {i} by {(valueL - valueR).Magnitude:0.###e+0}"; + return false; + } + } + note = compared == 0 ? "undefined at every sample" : $"agree at {compared} point(s)"; + return true; + } + + private static bool TryEvaluate(Entity expr, out System.Numerics.Complex value) + { + value = default; + try + { + if (!expr.EvaluableNumerical) return false; + var evaluated = expr.EvalNumerical(); + value = (System.Numerics.Complex)evaluated; + return !double.IsNaN(value.Real) && !double.IsNaN(value.Imaginary); + } + catch (AngouriMathBaseException) + { + return false; + } + } + + private static string Short(Exception e) + { + var text = e.Message.Replace('\n', ' '); + return e.GetType().Name + ": " + (text.Length > 90 ? text[..90] : text); + } + + private static string Report(IReadOnlyList results) + { + var sb = new StringBuilder(); + sb.AppendLine(); + foreach (var group in results.GroupBy(r => r.Problem.Op)) + { + sb.AppendLine($"{group.Key}:"); + foreach (var r in group) + sb.AppendLine($" {r.Verdict,-8} {r.Problem.Name,-26} {r.Note}"); + } + sb.AppendLine(); + foreach (var verdict in Enum.GetValues(typeof(Verdict)).Cast()) + sb.Append($"{verdict}={results.Count(r => r.Verdict == verdict)} "); + sb.AppendLine($"of {results.Count}"); + return sb.ToString(); + } + } +}