Add unstableSince timestamp to tolerated tests

build/scripts/TestTolerance/TestTolerance.Test.ps1

@@ -66,7 +66,7 @@ Describe "TestTolerance" {
   "branch": "main",
   "updatedAt": "2026-04-24T00:00:00Z",
   "tests": [
-    { "extensionId": "ext-1", "codeunitId": 100, "codeunitName": "My Codeunit", "testMethod": "TestA", "reason": "timing", "linkedIssue": "https://example/issues/1" },
+    { "extensionId": "ext-1", "codeunitId": 100, "codeunitName": "My Codeunit", "testMethod": "TestA", "reason": "timing", "linkedIssue": "https://example/issues/1", "unstableSince": "2026-04-24T00:00:00.0000000Z" },
     { "codeunitId": 200, "codeunitName": "Other Codeunit", "testMethod": "TestB" }
   ]
 }
@@ -401,6 +401,52 @@ Describe "TestTolerance" {
             $result['ext-1::300::t1'].FailureMessage | Should -Be 'm1'
             $result['ext-1::300::t1'].FailureDetail | Should -Be 's1'
         }
+
+        It "stamps UnstableSince with the current UTC time for a newly unstable test with no prior entry" {
+            $failed = @{
+                'ext-1::300::t1' = [pscustomobject]@{ ExtensionId = 'ext-1'; CodeunitId = 300; CodeunitName = 'A'; TestMethod = 'T1'; FailureMessage = 'm' }
+            }
+            $before = (Get-Date).ToUniversalTime()
+
+            $result = Update-UnstableTestsList -FailedTests $failed
+            $result['ext-1::300::t1'].UnstableSince | Should -Not -BeNullOrEmpty
+            $parsed = [datetimeoffset]::Parse($result['ext-1::300::t1'].UnstableSince).UtcDateTime
+            $parsed | Should -BeGreaterOrEqual $before.AddSeconds(-5)
+            $parsed | Should -BeLessOrEqual (Get-Date).ToUniversalTime().AddSeconds(5)
+        }
+
+        It "carries forward UnstableSince from the existing artifact for a still-unstable test (survives full recompute)" {
+            $failed = @{
+                'ext-1::300::t1' = [pscustomobject]@{ ExtensionId = 'ext-1'; CodeunitId = 300; CodeunitName = 'A'; TestMethod = 'T1'; FailureMessage = 'm' }
+            }
+            $existing = @(
+                [pscustomobject]@{ extensionId = 'ext-1'; codeunitId = 300; codeunitName = 'A'; testMethod = 'T1'; unstableSince = '2025-12-31T00:00:00.0000000Z' }
+            )
+
+            $result = Update-UnstableTestsList -FailedTests $failed -ExistingTests $existing
+            $result['ext-1::300::t1'].UnstableSince | Should -Be '2025-12-31T00:00:00.0000000Z'
+        }
+    }
+
+    Context "ConvertTo-UnstableTestEntry" {
+        It "stamps UnstableSince with the current UTC time for a test that has none" {
+            $test = [pscustomobject]@{ ExtensionId = 'ext-1'; CodeunitId = 300; CodeunitName = 'A'; TestMethod = 'T1' }
+            $before = (Get-Date).ToUniversalTime()
+
+            $entry = ConvertTo-UnstableTestEntry -Test $test -Repository 'owner/repo'
+
+            $entry.unstableSince | Should -Not -BeNullOrEmpty
+            $parsed = [datetimeoffset]::Parse($entry.unstableSince).UtcDateTime
+            $parsed | Should -BeGreaterOrEqual $before.AddSeconds(-5)
+            $parsed | Should -BeLessOrEqual (Get-Date).ToUniversalTime().AddSeconds(5)
+        }
+
+        It "keeps the test's own UnstableSince when it already has one" {
+            $test = [pscustomobject]@{ ExtensionId = 'ext-1'; CodeunitId = 300; CodeunitName = 'A'; TestMethod = 'T1'; UnstableSince = '2026-02-02T00:00:00.0000000Z' }
+
+            $entry = ConvertTo-UnstableTestEntry -Test $test -Repository 'owner/repo'
+            $entry.unstableSince | Should -Be '2026-02-02T00:00:00.0000000Z'
+        }
     }
 
     Context "Add-FailedTestsToUnstableTests" {
@@ -472,6 +518,23 @@ Describe "TestTolerance" {
             $merged[2].testMethod | Should -Be 'T2'
         }
 
+        It "preserves an existing entry's unstableSince and stamps new entries with the current time" {
+            $existing = @(
+                [pscustomobject]@{ extensionId = 'ext-1'; codeunitId = 300; codeunitName = 'A'; testMethod = 'T1'; reason = 'pre-existing'; unstableSince = '2026-01-01T00:00:00.0000000Z' }
+            )
+            $failed = @{
+                'ext-2::400::t2' = [pscustomobject]@{ ExtensionId = 'ext-2'; CodeunitId = 400; CodeunitName = 'B'; TestMethod = 'T2'; FailureMessage = 'm2'; SourceRunId = '1000' }
+            }
+            $before = (Get-Date).ToUniversalTime()
+
+            $merged = @(Add-FailedTestsToUnstableTests -ExistingTests $existing -FailedTests $failed -Repository 'owner/repo')
+            $merged[0].unstableSince | Should -Be '2026-01-01T00:00:00.0000000Z'
+            $merged[1].unstableSince | Should -Not -BeNullOrEmpty
+            $parsed = [datetimeoffset]::Parse($merged[1].unstableSince).UtcDateTime
+            $parsed | Should -BeGreaterOrEqual $before.AddSeconds(-5)
+            $parsed | Should -BeLessOrEqual (Get-Date).ToUniversalTime().AddSeconds(5)
+        }
+
         It "uses the test's own Reason when the failed test carries one" {
             $failed = @{
                 'ext-1::300::t1' = [pscustomobject]@{ ExtensionId = 'ext-1'; CodeunitId = 300; CodeunitName = 'A'; TestMethod = 'T1'; FailureMessage = 'm1'; SourceRunId = '2001'; Reason = 'Auto-detected: failed on 3 distinct PRs' }

build/scripts/TestTolerance/TestTolerance.psm1

@@ -171,11 +171,11 @@ function Get-UnstableTestKey {
       "branch": "main",
       "updatedAt": "2026-04-24T00:00:00Z",
       "tests": [
-        { "extensionId": "...", "codeunitId": 137404, "codeunitName": "SCM Manufacturing", "testMethod": "TestSomething", "reason": "...", "linkedIssue": "..." }
+        { "extensionId": "...", "codeunitId": 137404, "codeunitName": "SCM Manufacturing", "testMethod": "TestSomething", "reason": "...", "linkedIssue": "...", "unstableSince": "2026-04-24T00:00:00.0000000Z" }
       ]
     }
 
-    The 'reason' and 'linkedIssue' fields are optional. The function returns an empty
+    The 'reason', 'linkedIssue' and 'unstableSince' fields are optional. The function returns an empty
     hashtable when the file does not exist, so callers can treat "no artifact" as
     "no unstable tests yet".
 
@@ -693,18 +693,23 @@ function Receive-UnstableTestsArtifact {
     tests hashtable by marking every failed test key as unstable.
 
     Each returned entry is keyed by 'extensionId::codeunit::testMethod' and contains the test
-    identity fields plus an auto-detected reason of the form:
+    identity fields, an auto-detected reason of the form
       "Auto-detected: failed in at least 1 of the last <RunCount> CI/CD run(s)"
+    and the test's 'unstableSince' timestamp.
 
-    This function does not inspect passed tests or merge with an existing unstable list; it
-    only transforms the supplied failed-test set into the artifact format.
+    Because the sliding window rebuilds the list from scratch each run, a still-unstable test keeps the
+    'unstableSince' it had in the previous artifact ('ExistingTests'); a test that is unstable for the
+    first time is stamped with this run's time.
 
     Returns the updated hashtable keyed by 'extensionId::codeunit::testMethod'.
 
 .Parameter FailedTests
     Hashtable of failed tests keyed by test key to convert into unstable-test entries.
 .Parameter RunCount
     Number of CI/CD runs the window covered; used only to build the auto-detected reason text.
+.Parameter ExistingTests
+    The 'tests' array from the previous artifact (camelCase entries), used to keep each still-unstable
+    test's original 'unstableSince' across the recompute.
 #>
 function Update-UnstableTestsList {
     [CmdletBinding()]
@@ -713,9 +718,22 @@ function Update-UnstableTestsList {
         [Parameter(Mandatory = $true)]
         [hashtable] $FailedTests,
 
-        [int] $RunCount = 0
+        [int] $RunCount = 0,
+
+        [System.Collections.IList] $ExistingTests = @()
     )
 
+    # A newly unstable test is stamped with this run's time; still-unstable tests keep the prior value.
+    $now = (Get-Date).ToUniversalTime().ToString('o')
+    $existingSince = @{}
+    foreach ($entry in $ExistingTests) {
+        $props = $entry.PSObject.Properties
+        if ((-not $props['unstableSince']) -or [string]::IsNullOrWhiteSpace([string]$entry.unstableSince) -or (-not $props['testMethod'])) { continue }
+        $extId = if ($props['extensionId']) { [string]$entry.extensionId } else { '' }
+        $cuId = if ($props['codeunitId']) { [int]$entry.codeunitId } else { 0 }
+        $existingSince[(Get-UnstableTestKey -CodeunitId $cuId -TestMethod ([string]$entry.testMethod) -ExtensionId $extId)] = [string]$entry.unstableSince
+    }
+
     $result = @{}
     foreach ($key in @($FailedTests.Keys)) {
         $ft = $FailedTests[$key]
@@ -730,6 +748,7 @@ function Update-UnstableTestsList {
             SourceRunId    = if ($ft.PSObject.Properties['SourceRunId']) { $ft.SourceRunId } else { '' }
             Reason         = "Auto-detected: failed in at least 1 of the last $RunCount CI/CD run(s)"
             LinkedIssue    = ''
+            UnstableSince  = if ($existingSince.ContainsKey($key)) { $existingSince[$key] } else { $now }
         }
     }
 
@@ -907,6 +926,9 @@ function Get-FailedTestsFromRuns {
     Update-UnstableTestsList). 'Reason' overrides the entry reason; when empty, the test's own Reason
     property (if any) is used. 'Repository' is used to build the sourceRunUrl from the test's SourceRunId.
 
+    Every entry records 'unstableSince', an ISO 8601 UTC timestamp. If the test already carries one it is
+    kept; otherwise the current time is stamped, marking when the test was first added to the list.
+
 .Parameter Test
     A single failed/unstable test object with PascalCase properties.
 .Parameter Reason
@@ -929,6 +951,7 @@ function ConvertTo-UnstableTestEntry {
     $sourceRunId = if ($Test.PSObject.Properties['SourceRunId']) { [string]$Test.SourceRunId } else { '' }
     $reasonValue = if ($Reason) { $Reason } elseif ($Test.PSObject.Properties['Reason']) { [string]$Test.Reason } else { '' }
     $linkedIssue = if ($Test.PSObject.Properties['LinkedIssue']) { [string]$Test.LinkedIssue } else { '' }
+    $unstableSince = if (($Test.PSObject.Properties['UnstableSince']) -and $Test.UnstableSince) { [string]$Test.UnstableSince } else { (Get-Date).ToUniversalTime().ToString('o') }
 
     return [pscustomobject][ordered]@{
         extensionId    = if ($Test.PSObject.Properties['ExtensionId']) { [string]$Test.ExtensionId } else { '' }
@@ -939,6 +962,7 @@ function ConvertTo-UnstableTestEntry {
         failureDetail  = if ($Test.PSObject.Properties['FailureDetail']) { [string]$Test.FailureDetail } else { '' }
         reason         = $reasonValue
         linkedIssue    = $linkedIssue
+        unstableSince  = $unstableSince
         sourceRunUrl   = if ($Repository -and $sourceRunId) { "https://github.com/$Repository/actions/runs/$sourceRunId" } else { '' }
     }
 }

build/scripts/TestTolerance/UpdateUnstableTestsArtifact.ps1

@@ -72,29 +72,31 @@ try {
     Write-Host "::notice::Observed $($failedTests.Count) distinct failed test(s) across $($RunIds.Count) run(s)."
 
     # --- 2. Build the unstable tests list (additive merge or full recompute) ---
-    if ($Additive) {
-        $existingPath = Receive-UnstableTestsArtifact -Branch $Branch -OutputDirectory $downloadDir
-
-        $existingTests = @()
-        if ($existingPath -and (Test-Path $existingPath)) {
-            $existing = Get-Content -Raw -Path $existingPath | ConvertFrom-Json
-            if (($existing.PSObject.Properties['tests']) -and $existing.tests) {
-                $existingTests = @($existing.tests)
-            }
-            Write-Host "Existing unstable tests list for branch '$Branch' has $($existingTests.Count) test(s)."
-        }
-        else {
-            Write-Host "No existing unstable tests artifact found for branch '$Branch'. Starting from an empty list."
+    # Load the existing artifact so each still-unstable test keeps its original 'unstableSince'
+    # (carried forward through both the recompute and the additive merge).
+    $existingPath = Receive-UnstableTestsArtifact -Branch $Branch -OutputDirectory $downloadDir
+    $existingTests = @()
+    if ($existingPath -and (Test-Path $existingPath)) {
+        $existing = Get-Content -Raw -Path $existingPath | ConvertFrom-Json
+        if (($existing.PSObject.Properties['tests']) -and $existing.tests) {
+            $existingTests = @($existing.tests)
         }
+        Write-Host "Existing unstable tests list for branch '$Branch' has $($existingTests.Count) test(s)."
+    }
+    else {
+        Write-Host "No existing unstable tests artifact found for branch '$Branch'."
+    }
 
+    if ($Additive) {
         if ($failedTests.Count -eq 0) {
             Write-Host "::warning::No failed tests found in the supplied run(s). The unstable tests list will be rewritten unchanged."
         }
 
         $tests = @(Add-FailedTestsToUnstableTests -ExistingTests ([System.Collections.IList]$existingTests) -FailedTests $failedTests -Repository $repo)
     }
     else {
-        $updatedTests = Update-UnstableTestsList -FailedTests $failedTests -RunCount $RunIds.Count
+        $updatedTests = Update-UnstableTestsList -FailedTests $failedTests -RunCount $RunIds.Count -ExistingTests ([System.Collections.IList]$existingTests)
+        # Entries already carry their 'unstableSince' (kept or stamped by Update-UnstableTestsList).
         $tests = @($updatedTests.Values | ForEach-Object { ConvertTo-UnstableTestEntry -Test $_ -Repository $repo })
     }
 

build/scripts/TestTolerance/UpdateUnstableTestsCombined.ps1

@@ -85,6 +85,18 @@ New-Item -ItemType Directory -Path $cicdWorkDir -Force | Out-Null
 New-Item -ItemType Directory -Path $prWorkDir -Force | Out-Null
 
 try {
+    # Load the existing artifact once: it carries each test's 'unstableSince' forward across the Path A
+    # recompute, and acts as the fallback base list when there is no CI/CD window to recompute from.
+    $existingTests = @()
+    $existingPath = Receive-UnstableTestsArtifact -Branch $Branch -OutputDirectory $existingDir
+    if ($existingPath -and (Test-Path $existingPath)) {
+        $existing = Get-Content -Raw -Path $existingPath | ConvertFrom-Json
+        if (($existing.PSObject.Properties['tests']) -and $existing.tests) {
+            $existingTests = @($existing.tests)
+        }
+        Write-Host "Existing unstable tests list for '$Branch' has $($existingTests.Count) test(s)."
+    }
+
     # --- Path A: recompute the base list from the recent CI/CD window ---
     Write-Host "::group::Path A · Recompute from recent CI/CD runs (branch '$Branch')"
     $cicdRunIds = @(Find-UnstableTestRunIds `
@@ -97,22 +109,16 @@ try {
     $baseEntries = @()
     if ($cicdRunIds.Count -gt 0) {
         $cicdFailed = Get-FailedTestsFromRuns -RunIds $cicdRunIds -Repository $repo -WorkDirectory $cicdWorkDir
-        $recomputed = Update-UnstableTestsList -FailedTests $cicdFailed -RunCount $cicdRunIds.Count
+        $recomputed = Update-UnstableTestsList -FailedTests $cicdFailed -RunCount $cicdRunIds.Count -ExistingTests ([System.Collections.IList]$existingTests)
+        # Entries already carry their 'unstableSince' (kept or stamped by Update-UnstableTestsList).
         $baseEntries = @($recomputed.Values | ForEach-Object { ConvertTo-UnstableTestEntry -Test $_ -Repository $repo })
         Write-Host "::endgroup::"
         Write-Host "::notice::Path A (CI/CD): recomputed $($baseEntries.Count) unstable test(s) from $($cicdRunIds.Count) run(s) on '$Branch'."
     }
     else {
         # No CI/CD runs to recompute from. Do NOT wipe the list: fall back to the existing artifact as the
-        # base so Path B stays purely additive on top of it.
-        $existingPath = Receive-UnstableTestsArtifact -Branch $Branch -OutputDirectory $existingDir
-        if ($existingPath -and (Test-Path $existingPath)) {
-            $existing = Get-Content -Raw -Path $existingPath | ConvertFrom-Json
-            if (($existing.PSObject.Properties['tests']) -and $existing.tests) {
-                $baseEntries = @($existing.tests)
-            }
-            Write-Host "Existing unstable tests list for '$Branch' has $($baseEntries.Count) test(s)."
-        }
+        # base so Path B stays purely additive on top of it. Existing entries keep their unstableSince.
+        $baseEntries = @($existingTests)
         Write-Host "::endgroup::"
         Write-Host "::warning::Path A (CI/CD): no completed runs with test results for '$Branch'. Preserving the existing list ($($baseEntries.Count) test(s)) as the base."
     }
@@ -134,6 +140,8 @@ try {
     }
 
     # --- Merge Path B additively on top of the Path A base and write once ---
+    # New cross-PR entries are stamped with the current time; existing entries keep their own
+    # 'unstableSince', and Path A recomputed entries kept theirs via Update-UnstableTestsList above.
     $tests = @(Add-FailedTestsToUnstableTests -ExistingTests ([System.Collections.IList]$baseEntries) -FailedTests $crossPrFailed -Repository $repo)
 
     $allRunIds = @()

docs/features/test-tolerance/DESIGN.md

@@ -2,7 +2,7 @@
 
 > Status: Draft — core decisions made, implementation complete
 > Owner: Engineering Systems
-> Last updated: 2026-04-28
+> Last updated: 2026-07-24
 
 ## 1. Summary
 
@@ -95,7 +95,11 @@ The correlation logic lives in `TestTolerance.psm1`, split into a computational,
 
 Each test is identified by a three-part normalized key: `extensionId::codeunit::testMethod` (all lowercase). The `extensionId` is read from the `<properties><property name="extensionid">` element in the test results XML; it is empty-string for suites without that property. Two tests with the same codeunit and testMethod name but different extension IDs are treated as distinct tests.
 
-> **Schema migration note:** Unstable-tests artifacts produced before the `extensionId` field was added use the old two-part key format (`codeunit::testMethod`). When a run downloads such an artifact, no matches will be found and tolerance will silently degrade to zero for that run. The artifact will be overwritten with the new three-part key format after the next UpdateUnstableTests run completes.
+**Unstable-since timestamp**
+
+Each entry records an `unstableSince` field: an ISO 8601 UTC timestamp of when the test first entered the unstable list. The timestamp is assigned when a test is first added. Because the sliding window (Path A) rebuilds the list from scratch on every run, `Update-UnstableTestsList` reads each still-unstable test's `unstableSince` from the previous artifact and carries it forward; only a test with no prior entry is stamped with the current time. The field therefore reflects *since when* the test has been continuously unstable rather than the last recompute time. A test that self-heals (drops off the list) and later returns starts a fresh `unstableSince`.
+
+> **Schema migration note:** Unstable-tests artifacts produced before the `extensionId` field was added use the old two-part key format (`codeunit::testMethod`). When a run downloads such an artifact, no matches will be found and tolerance will silently degrade to zero for that run. The artifact will be overwritten with the new three-part key format after the next UpdateUnstableTests run completes. Likewise, artifacts produced before the `unstableSince` field existed carry no timestamp; the first run after this change stamps `unstableSince` to that run's time for every surviving test, and the value is then preserved on subsequent runs.
 
 **Artifact lifecycle**