From c337d5281a25d2967f3d2b07ac2556061a3fcbd7 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 22:30:47 -0500 Subject: [PATCH 01/24] Add event environments and filtering --- docs/docs/clients/dotnet/configuration.md | 2 + .../javascript/client-configuration.md | 2 + docs/docs/environments.md | 69 ++++++++++++++++ docs/docs/filtering-and-searching.md | 6 ++ docs/docs/versioning.md | 2 + .../Extensions/PersistentEventExtensions.cs | 1 + .../Migrations/010_AddEventEnvironment.cs | 36 +++++++++ src/Exceptionless.Core/Models/Event.cs | 26 +++++- .../Models/EventSummaryModel.cs | 1 + src/Exceptionless.Core/Models/WebHookEvent.cs | 1 + .../Default/70_SessionPlugin.cs | 25 +++--- .../WebHook/Default/020_VersionTwoPlugin.cs | 1 + .../Configuration/Indexes/EventIndex.cs | 2 + .../Queries/EventEnvironmentFilter.cs | 49 +++++++++++ .../PersistentEventQueryValidator.cs | 3 + .../EventEnvironmentConverter.cs | 26 ++++++ .../Api/Handlers/EventHandler.cs | 14 ++-- .../e2e/tests/event-environments.e2e.ts | 81 +++++++++++++++++++ .../events/components/events-overview.svelte | 2 +- .../environment-faceted-filter-builder.svelte | 14 ++++ .../environment-faceted-filter-trigger.svelte | 22 +++++ .../filters/environment-faceted-filter.svelte | 79 ++++++++++++++++++ .../filters/environment-filter.svelte.test.ts | 42 ++++++++++ .../components/filters/helpers.svelte.ts | 4 + .../events/components/filters/index.ts | 5 ++ .../components/filters/models.svelte.ts | 28 +++++++ ...ion-defaults-faceted-filter-builder.svelte | 1 + .../events/components/summary/index.ts | 1 + .../table/event-environment-cell.svelte | 17 ++++ .../events/components/table/options.svelte.ts | 17 +++- .../events/components/views/overview.svelte | 5 ++ .../src/lib/features/events/premium-filter.ts | 1 + .../components/session-table-columns.ts | 11 ++- .../faceted-filter-builder.svelte | 14 +++- .../faceted-filter-multi-select.svelte | 23 +++++- .../components/faceted-filter/filter-scope.ts | 8 ++ .../ClientApp/src/lib/generated/api.ts | 5 ++ .../ClientApp/src/lib/generated/schemas.ts | 5 ++ .../src/routes/(app)/event/+page.svelte | 22 ++++- .../(app)/redirect-to-events.svelte.test.ts | 11 ++- .../routes/(app)/redirect-to-events.svelte.ts | 19 +++++ .../src/routes/(app)/sessions/+page.svelte | 21 ++++- .../src/routes/(app)/stack/+page.svelte | 21 ++++- .../src/routes/(app)/stream/+page.svelte | 33 ++++++-- .../Mcp/ExceptionlessMcpTools.cs | 10 ++- src/Exceptionless.Web/Mcp/McpModels.cs | 3 +- .../Models/SavedView/NewSavedView.cs | 6 +- .../EventEnvironmentSchemaTransformer.cs | 26 ++++++ ...nlessOpenApiServiceCollectionExtensions.cs | 1 + .../Exceptionless.Tests/Api/Data/openapi.json | 9 +++ .../EventEndpointTests.Environment.cs | 77 ++++++++++++++++++ .../Api/Endpoints/SavedViewEndpointTests.cs | 4 +- .../Api/OpenApiSnapshotTests.cs | 5 ++ .../Mcp/ExceptionlessMcpToolsTests.cs | 19 +++++ .../AddEventEnvironmentMigrationTests.cs | 47 +++++++++++ .../Migrations/MigrationRegistrationTests.cs | 8 ++ .../Pipeline/EventPipelineTests.cs | 32 ++++++++ .../Plugins/WebHookDataTests.cs | 9 +++ .../Search/EventEnvironmentFilterTests.cs | 23 ++++++ .../PersistentEventQueryValidatorTests.cs | 5 ++ .../Models/EventEnvironmentTests.cs | 68 ++++++++++++++++ tests/http/events.http | 24 ++++++ 62 files changed, 1108 insertions(+), 46 deletions(-) create mode 100644 docs/docs/environments.md create mode 100644 src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs create mode 100644 src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs create mode 100644 src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs create mode 100644 src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-builder.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-trigger.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/event-environment-cell.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/filter-scope.ts create mode 100644 src/Exceptionless.Web/Utility/OpenApi/EventEnvironmentSchemaTransformer.cs create mode 100644 tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs create mode 100644 tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs create mode 100644 tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs create mode 100644 tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs diff --git a/docs/docs/clients/dotnet/configuration.md b/docs/docs/clients/dotnet/configuration.md index b807877465..7283c9194c 100644 --- a/docs/docs/clients/dotnet/configuration.md +++ b/docs/docs/clients/dotnet/configuration.md @@ -5,6 +5,8 @@ order: 1 # Configuration +Set a deployment environment once at startup, with optional per-event overrides. See [Environments](/docs/environments/) for configuration examples. + There are a few ways to configure Exceptionless in your project. We'll cover them here or you can jump to app-specific examples: [Console App Example](/docs/clients/dotnet/guides/console-apps-example), [Web Server Example](/docs/clients/dotnet/guides/web-server-example). --- diff --git a/docs/docs/clients/javascript/client-configuration.md b/docs/docs/clients/javascript/client-configuration.md index 0db6e31181..06f51e5b3d 100644 --- a/docs/docs/clients/javascript/client-configuration.md +++ b/docs/docs/clients/javascript/client-configuration.md @@ -5,6 +5,8 @@ order: 1 # Configuration +Set a deployment environment once at startup, with optional per-event overrides. See [Environments](/docs/environments/) for configuration examples. + - [Installation](#installation) - [Browser](#browser) - [Node.js](#nodejs) diff --git a/docs/docs/environments.md b/docs/docs/environments.md new file mode 100644 index 0000000000..b73b79a5c5 --- /dev/null +++ b/docs/docs/environments.md @@ -0,0 +1,69 @@ +--- +title: "Environments" +--- + +# Environments + +Use an environment to distinguish production, staging, development, or a custom deployment within one project. Each event can carry one optional top-level `environment` string. The same error still belongs to one stack, with one shared status and fixed version. + +## Configure your client + +Set a default during application startup. In the .NET client: + +```csharp +client.Configuration.SetEnvironment("production"); +client.CreateLog("Deployment complete").SetEnvironment("staging").Submit(); +``` + +You can also set `Exceptionless:Environment` in .NET configuration, or the `Exceptionless__Environment` environment variable. The hosting integration uses `IHostEnvironment.EnvironmentName` when no explicit environment is configured. A per-event value overrides the default. + +In the JavaScript client: + +```javascript +await Exceptionless.startup((config) => { + config.apiKey = "API_KEY_HERE"; + config.environment = "production"; +}); + +await Exceptionless.createLog("Deployment complete") + .setEnvironment("staging") + .submit(); +``` + +For direct API submissions: + +```json +{ + "type": "log", + "message": "Deployment complete", + "environment": "production" +} +``` + +Names are trimmed and lowercased, with a maximum of 64 characters. Custom names such as `qa-west` are supported. Empty, oversized, control-character, and non-string values are treated as unspecified. Historical events and older clients without this field remain unspecified; they are never assumed to be production. + +## Filter your data + +Choose **Manage filters → Environment** on Events, Stacks, Sessions, or Stream. Select one or more names, or **Unspecified** for events without an environment. Clear the selection to include all environments. The picker discovers names for the selected projects and time range; you can also enter a name that has no current events. + +Environment selections persist in URLs and saved views. Events, sessions, and stream tables have an optional Environment column. Event details show Environment in the overview; **Machine & runtime** contains the existing `data.@environment` diagnostics. + +Environment searches and aggregations are available on every plan: + +| Search | Meaning | +| --- | --- | +| `environment:production` | Production events | +| `(environment:production OR environment:staging)` | Either deployment | +| `_missing_:environment` | Historical or unspecified events | +| `_exists_:environment` | Events with an environment | +| `environment:"qa west"` | A custom name containing spaces | + +Stack dashboard counts and charts use events matching the filter. Stack status remains shared, so changing status while viewing production also changes the same stack seen in staging. Automatic session detection separates the same user's activity by environment. + +## Fixing stacks across deployments + +Use [fixed in version](/docs/versioning/) when environments run different releases. For example, if a stack is fixed in `2.4.0`, occurrences from production running `2.3.0` do not represent a regression just because staging already has the fix. Continue sending the application version with events. Plain **Fixed**, without a version, retains its existing behavior across all environments. + +## Rollout + +Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. diff --git a/docs/docs/filtering-and-searching.md b/docs/docs/filtering-and-searching.md index d0ba2f1a50..c0a31d4ff4 100644 --- a/docs/docs/filtering-and-searching.md +++ b/docs/docs/filtering-and-searching.md @@ -6,6 +6,7 @@ title: "Filtering & Searching" - [Filter by Organization \& Project](#filter-by-organization--project) - [Filter by Time Frame](#filter-by-time-frame) +- [Filter by Environment](#filter-by-environment) - [Filter / Search by Specific Criteria](#filter--search-by-specific-criteria) - [Searchable Fields \& Requirements](#searchable-fields--requirements) - [Multiple Queries](#multiple-queries) @@ -30,6 +31,10 @@ Click on the calendar icon in the header to select from multiple preset time fra ![Exceptionless Filter Time Frame](img/filter-by-timeframe.png) +## Filter by Environment + +Choose **Manage filters → Environment** to select production, staging, development, or a custom name. Select **Unspecified** for events without this property, or clear the selection to include all environments. See [Environments](/docs/environments/) for client configuration and shared stack behavior. + ## Filter / Search by Specific Criteria Click the magnifying glass to search by specific criteria. @@ -53,6 +58,7 @@ View a complete list of searchable terms, examples, and FAQs below. | stack | `stack:54d8315ce6bb2d0500bcc7b4` | true | Stack id | | reference | `reference:12345678` | true | Reference id | | session | `session:12345678` | true | Session id | +| environment | `environment:production` or `_missing_:environment` | true | Deployment environment | | type | `type:error` | true | Event type | | source | `source:"my log source"` or `"my log source"` | false | Event source | | level | `level:Error` | true | Log level | diff --git a/docs/docs/versioning.md b/docs/docs/versioning.md index 033b1aaada..8cf36bd12d 100644 --- a/docs/docs/versioning.md +++ b/docs/docs/versioning.md @@ -4,6 +4,8 @@ title: "Versioning" # Versioning +Stacks and their fixed version are shared across [environments](/docs/environments/). Continue sending application versions so older deployments can report occurrences without incorrectly reopening a stack fixed in a newer version. + You can mark error stacks fixed and they won't show up or notify you until they regress! ## How does this work? diff --git a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs index 6a869d0425..98aa5b4f6b 100644 --- a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs +++ b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs @@ -183,6 +183,7 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, I { var startEvent = new PersistentEvent { + Environment = source.Environment, Date = source.Date, Geo = source.Geo, OrganizationId = source.OrganizationId, diff --git a/src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs b/src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs new file mode 100644 index 0000000000..ed0a58b8f4 --- /dev/null +++ b/src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs @@ -0,0 +1,36 @@ +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories.Configuration; +using Foundatio.Parsers.ElasticQueries.Extensions; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Migrations; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Migrations; + +public sealed class AddEventEnvironment : MigrationBase +{ + private readonly ExceptionlessElasticConfiguration _configuration; + + public AddEventEnvironment(ExceptionlessElasticConfiguration configuration, ILoggerFactory loggerFactory) : base(loggerFactory) + { + _configuration = configuration; + MigrationType = MigrationType.VersionedAndResumable; + Version = 10; + } + + public override async Task RunAsync(MigrationContext context) + { + var response = await _configuration.Client.Indices.PutMappingAsync(mapping => mapping + .Indices($"{_configuration.Events.Name}-v*-*") + .AllowNoIndices(true) + .IgnoreUnavailable(true) + .Properties(properties => properties.Text(ev => ev.Environment, + text => text.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField())), context.CancellationToken); + _logger.LogRequest(response); + if (!response.IsValidResponse) + { + throw new InvalidOperationException("Unable to add the event environment mapping: " + response.DebugInformation); + } + } +} diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 84a96c7acd..defefeddd2 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -11,6 +11,26 @@ namespace Exceptionless.Core.Models; [DebuggerDisplay("Type: {Type}, Date: {Date}, Message: {Message}, Value: {Value}, Count: {Count}")] public class Event : IData, IJsonOnDeserialized { + private string? _environment; + + /// + /// The deployment environment, such as production, staging, or development. + /// Missing or invalid names remain unspecified. Machine and runtime information is stored separately in data.@environment. + /// + [StringLength(64)] + [JsonConverter(typeof(EventEnvironmentConverter))] + public string? Environment + { + get => _environment; + set + { + string? name = value?.Trim(); + _environment = String.IsNullOrEmpty(name) || name.Length > 64 || name.Any(Char.IsControl) + ? null + : name.ToLowerInvariant(); + } + } + /// /// The event type (ie. error, log message, feature usage). Check Event.KnownTypes for standard event types. /// Nullable in transit; the pipeline infers a default before save. Validated as required on repository save. @@ -112,7 +132,7 @@ void IJsonOnDeserialized.OnDeserialized() protected bool Equals(Event other) { - return String.Equals(Type, other.Type) && String.Equals(Source, other.Source) && Tags.CollectionEquals(other.Tags) && String.Equals(Message, other.Message) && String.Equals(Geo, other.Geo) && Value == other.Value && Equals(Data, other.Data); + return String.Equals(Environment, other.Environment) && String.Equals(Type, other.Type) && String.Equals(Source, other.Source) && Tags.CollectionEquals(other.Tags) && String.Equals(Message, other.Message) && String.Equals(Geo, other.Geo) && Value == other.Value && Equals(Data, other.Data); } public override bool Equals(object? obj) @@ -138,6 +158,10 @@ public override int GetHashCode() hashCode = (hashCode * 397) ^ (Geo?.GetHashCode() ?? 0); hashCode = (hashCode * 397) ^ Value.GetHashCode(); hashCode = (hashCode * 397) ^ (Data?.GetCollectionHashCode(_exclusions) ?? 0); + if (Environment is not null) + { + hashCode = (hashCode * 397) ^ Environment.GetHashCode(); + } return hashCode; } } diff --git a/src/Exceptionless.Core/Models/EventSummaryModel.cs b/src/Exceptionless.Core/Models/EventSummaryModel.cs index 6bfdbd4782..fd0226d7c5 100644 --- a/src/Exceptionless.Core/Models/EventSummaryModel.cs +++ b/src/Exceptionless.Core/Models/EventSummaryModel.cs @@ -2,6 +2,7 @@ public record EventSummaryModel : SummaryData { + public string? Environment { get; set; } public DateTimeOffset Date { get; set; } public string ProjectId { get; set; } = null!; public string? ProjectName { get; set; } diff --git a/src/Exceptionless.Core/Models/WebHookEvent.cs b/src/Exceptionless.Core/Models/WebHookEvent.cs index e14cc25a7d..dc55e5324a 100644 --- a/src/Exceptionless.Core/Models/WebHookEvent.cs +++ b/src/Exceptionless.Core/Models/WebHookEvent.cs @@ -14,6 +14,7 @@ public WebHookEvent(string baseUrl) public DateTimeOffset? OccurrenceDate { get; init; } public TagSet? Tags { get; init; } public string? Type { get; init; } + public string? Environment { get; init; } public string? Source { get; init; } public string? Message { get; init; } public string ProjectId { get; init; } = null!; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs index 580831e116..b13b920bed 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs @@ -125,11 +125,11 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts) { var identityGroups = contexts .OrderBy(c => c.Event.Date) - .GroupBy(c => c.Event.GetUserIdentity(_serializer, _logger)?.Identity); + .GroupBy(c => (c.Event.ProjectId, c.Event.Environment, Identity: c.Event.GetUserIdentity(_serializer, _logger)?.Identity)); foreach (var identityGroup in identityGroups) { - if (String.IsNullOrEmpty(identityGroup.Key)) + if (String.IsNullOrEmpty(identityGroup.Key.Identity)) continue; string projectId = identityGroup.First().Project.Id; @@ -160,7 +160,7 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts) ctx.IsCancelled = true; }); - string? sessionId = await GetIdentitySessionIdAsync(projectId, identityGroup.Key); + string? sessionId = await GetIdentitySessionIdAsync(projectId, identityGroup.Key.Identity, identityGroup.Key.Environment); // if session end, without any session events, cancel if (String.IsNullOrEmpty(sessionId) && session.Count == 1 && firstSessionEvent.Event.IsSessionEnd()) @@ -192,7 +192,7 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts) } if (!lastSessionEvent.Event.IsSessionEnd()) - await SetIdentitySessionIdAsync(projectId, identityGroup.Key, sessionId); + await SetIdentitySessionIdAsync(projectId, identityGroup.Key.Identity, sessionId, identityGroup.Key.Environment); } else { @@ -217,7 +217,7 @@ public override Task EventProcessedAsync(EventContext context) return Task.CompletedTask; } - private static List> CreateSessionGroups(IGrouping identityGroup) + private static List> CreateSessionGroups(IEnumerable identityGroup) { var sessions = new List>(); var currentSession = new List(); @@ -259,14 +259,19 @@ private Task SetSessionStartEventIdAsync(string projectId, string sessionI return _cache.SetAsync(GetSessionStartEventIdCacheKey(projectId, sessionId), eventId, TimeSpan.FromDays(1)); } - private static string GetIdentitySessionIdCacheKey(string projectId, string identity) + private static string GetIdentitySessionIdCacheKey(string projectId, string identity, string? environment) { + if (environment is not null) + { + return String.Concat(projectId, ":environment:", environment.ToSHA1(), ":identity:", identity.ToSHA1()); + } + return String.Concat(projectId, ":identity:", identity.ToSHA1()); } - private async Task GetIdentitySessionIdAsync(string projectId, string identity) + private async Task GetIdentitySessionIdAsync(string projectId, string identity, string? environment) { - string cacheKey = GetIdentitySessionIdCacheKey(projectId, identity); + string cacheKey = GetIdentitySessionIdCacheKey(projectId, identity, environment); string? sessionId = await _cache.GetAsync(cacheKey, null); if (!String.IsNullOrEmpty(sessionId)) { @@ -279,9 +284,9 @@ await Task.WhenAll( return sessionId; } - private Task SetIdentitySessionIdAsync(string projectId, string identity, string sessionId) + private Task SetIdentitySessionIdAsync(string projectId, string identity, string sessionId, string? environment) { - return _cache.SetAsync(GetIdentitySessionIdCacheKey(projectId, identity), sessionId, _sessionTimeout); + return _cache.SetAsync(GetIdentitySessionIdCacheKey(projectId, identity, environment), sessionId, _sessionTimeout); } private async Task CreateSessionStartEventAsync(EventContext startContext, DateTime? lastActivityUtc, bool? isSessionEnd) diff --git a/src/Exceptionless.Core/Plugins/WebHook/Default/020_VersionTwoPlugin.cs b/src/Exceptionless.Core/Plugins/WebHook/Default/020_VersionTwoPlugin.cs index 81a859181c..ae49c64905 100644 --- a/src/Exceptionless.Core/Plugins/WebHook/Default/020_VersionTwoPlugin.cs +++ b/src/Exceptionless.Core/Plugins/WebHook/Default/020_VersionTwoPlugin.cs @@ -22,6 +22,7 @@ public VersionTwoPlugin(AppOptions options, ILoggerFactory loggerFactory) : base Tags = ev.Tags, Message = ev.Message, Type = ev.Type, + Environment = ev.Environment, Source = ev.Source, ProjectId = ev.ProjectId, ProjectName = ctx.Project.Name, diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs index 9c7b77b4cd..f75c9b0b07 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs @@ -70,6 +70,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor e.ReferenceId) .FieldAlias(Alias.ReferenceId, a => a.Path(f => f.ReferenceId)) .Text(e => e.Type, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Text(e => e.Environment, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) .Text(e => e.Source, t => t.Analyzer(STANDARDPLUS_ANALYZER).SearchAnalyzer(WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) .Date(e => e.Date) .Text(e => e.Message) @@ -260,6 +261,7 @@ private void BuildAnalysis(IndexSettingsAnalysisDescriptor ad) public sealed class Alias { + public const string Environment = "environment"; public const string OrganizationId = "organization"; public const string ProjectId = "project"; public const string StackId = "stack"; diff --git a/src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs b/src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs new file mode 100644 index 0000000000..68a4179655 --- /dev/null +++ b/src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs @@ -0,0 +1,49 @@ +using Foundatio.Parsers.LuceneQueries; +using Foundatio.Parsers.LuceneQueries.Nodes; + +namespace Exceptionless.Core.Repositories.Queries; + +/// +/// Preserves deployment scope when counting all project users, including users without matching errors. +/// +public static class EventEnvironmentFilter +{ + public static async Task GetAsync(string? filter) + { + if (String.IsNullOrWhiteSpace(filter)) + return null; + + return GetConstraint(await new LuceneQueryParser().ParseAsync(filter)); + } + + private static string? GetConstraint(IQueryNode? node, bool negate = false) + { + if (node is not IFieldQueryNode field) + return null; + + if (String.Equals(field.UnescapedField, "environment", StringComparison.OrdinalIgnoreCase)) + return negate ? $"NOT ({node})" : node.ToString(); + + if (field.Field is not null || node is not GroupNode group) + return null; + + negate ^= group.IsNegated == true || group.Prefix == "-"; + if (group.Left is null) + return GetConstraint(group.Right, negate); + if (group.Right is null) + return GetConstraint(group.Left, negate); + + string? left = GetConstraint(group.Left, negate); + string? right = GetConstraint(group.Right, negate); + bool isOr = (group.Operator == GroupOperator.Or) ^ negate; + // An unrelated OR branch permits every environment. Unrelated AND clauses do not restrict it. + if (isOr && (left is null || right is null)) + return null; + if (left is null) + return right; + if (right is null) + return left; + + return $"({left} {(isOr ? "OR" : "AND")} {right})"; + } +} diff --git a/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs b/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs index 5cd11109f5..23f74d0fd9 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs @@ -8,6 +8,7 @@ namespace Exceptionless.Core.Queries.Validation; public sealed class PersistentEventQueryValidator : AppQueryValidator { private static readonly HashSet _freeQueryFields = new(StringComparer.OrdinalIgnoreCase) { + EventIndex.Alias.Environment, "date", "type", EventIndex.Alias.ReferenceId, @@ -22,6 +23,7 @@ public sealed class PersistentEventQueryValidator : AppQueryValidator }; private static readonly HashSet _freeAggregationFields = new(StringComparer.OrdinalIgnoreCase) { + EventIndex.Alias.Environment, "date", "type", "value", @@ -36,6 +38,7 @@ public sealed class PersistentEventQueryValidator : AppQueryValidator }; private static readonly HashSet _allowedAggregationFields = new(StringComparer.OrdinalIgnoreCase) { + EventIndex.Alias.Environment, "date", "source", "tags", diff --git a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs new file mode 100644 index 0000000000..9a6ff454d6 --- /dev/null +++ b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs @@ -0,0 +1,26 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Exceptionless.Core.Serialization; + +/// +/// Ignores malformed optional deployment metadata without rejecting an event or its submission batch. +/// +public sealed class EventEnvironmentConverter : JsonConverter +{ + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + return reader.GetString(); + } + + reader.Skip(); + return null; + } + + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) + { + writer.WriteStringValue(value); + } +} diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index a034c63691..02da7a94e9 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -785,6 +785,7 @@ private async Task>> GetInternalAsync(AppFilter sf, T ProjectName = projectNames.GetValueOrDefault(e.ProjectId), Tags = e.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], Type = e.Type, + Environment = e.Environment, Version = e.GetVersion(), Data = summaryData.Data }; @@ -833,7 +834,7 @@ private async Task>> GetInternalAsync(AppFilter sf, T string[] stackIds = stackTerms.Buckets.Skip(skip).Take(limit + 1).Select(t => t.Key).ToArray(); var stacks = (await stackRepository.GetByIdsAsync(stackIds)).Select(s => s.ApplyOffset(ti.Offset)).ToList(); - var stackSummaries = await GetStackSummariesAsync(stacks, stackTerms.Buckets, sf, ti); + var stackSummaries = await GetStackSummariesAsync(stacks, stackTerms.Buckets, sf, ti, filter); double? totalStackCount = countResponse.Aggregations.Cardinality("cardinality_stack_id")?.Value; long? total = includeTotal && totalStackCount.HasValue ? Convert.ToInt64(totalStackCount.Value) : null; @@ -915,14 +916,14 @@ private Task> GetEventsInternalAsync(AppFilter? sys : o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit).TrackTotalHits(includeTotal)); } - private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti) + private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti, string? filter) { if (stacks.Count == 0) return new List(0); var projects = await projectRepository.GetByIdsAsync(stacks.Select(s => s.ProjectId).Distinct().ToArray(), o => o.Cache()); var projectNames = projects.ToDictionary(p => p.Id, p => p.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); + var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd, filter); return stacks.Join(stackTerms, s => s.Id, tk => tk.Key, (stack, term) => { var data = formattingPluginManager.GetStackSummaryData(stack); @@ -948,9 +949,10 @@ private async Task> GetStackSummariesAsync(List> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) + private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd, string? filter) { - using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); + filter = await EventEnvironmentFilter.GetAsync(filter); + using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}:{filter?.ToSHA1()}"); var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList(); var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); @@ -958,7 +960,7 @@ private async Task> GetUserCountByProjectIdsAsync(ICo if (totals.Count == projectIds.Count) return totals; - var systemFilter = new RepositoryQuery().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); + var systemFilter = new RepositoryQuery().AppFilter(sf).FilterExpression(filter).EnforceEventStackFilter().DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); var projects = cachedTotals .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts new file mode 100644 index 0000000000..adcc50dae4 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts @@ -0,0 +1,81 @@ +import { createReferenceId, expect, test } from '../fixtures/e2e-test'; +import { getVisibleText } from '../support/page-helpers'; + +test('environment filters include unspecified events and persist in saved views', async ({ e2eApi, e2eScenario, page }) => { + const environments = ['production', 'staging', undefined]; + const messages = environments.map((environment) => `Environment ${environment ?? 'unspecified'} ${e2eScenario.run}`); + for (const [index, environment] of environments.entries()) { + const referenceId = createReferenceId(e2eScenario.run, `-env-${index}`); + await e2eApi.submitEvent(e2eScenario.projectId, e2eScenario.projectToken, { + environment, + message: messages[index], + reference_id: referenceId, + source: 'environment-filter-test', + type: 'log' + }); + await e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId); + } + + await page.goto(`/next/event?project=${e2eScenario.projectId}&time=all`); + for (const message of messages) await expect(getVisibleText(page, message)).toBeVisible({ timeout: 30_000 }); + + await page.getByRole('button', { name: 'Manage filters' }).click(); + await page.getByPlaceholder('Search...').fill('Environment'); + await page.getByText('Environment', { exact: true }).click(); + await page.getByRole('option', { exact: true, name: 'production' }).click(); + await page.keyboard.press('Escape'); + await expect(getVisibleText(page, messages[0]!)).toBeVisible(); + await expect(getVisibleText(page, messages[1]!)).toBeHidden(); + await expect(getVisibleText(page, messages[2]!)).toBeHidden(); + + await page.getByRole('button', { name: /^Environment\s+production/ }).click(); + await page.getByRole('option', { exact: true, name: 'Unspecified' }).click(); + await page.keyboard.press('Escape'); + await expect(getVisibleText(page, messages[2]!)).toBeVisible(); + await expect(getVisibleText(page, messages[1]!)).toBeHidden(); + + const viewName = `Environments ${e2eScenario.run.slice(-24)}`; + await page.getByRole('button', { name: /^View/ }).filter({ visible: true }).first().click(); + await page.getByRole('menuitem', { name: 'Save As...' }).click(); + const dialog = page.getByRole('dialog', { name: 'Save View' }); + await dialog.getByLabel('Name', { exact: true }).fill(viewName); + await expect(dialog.getByLabel('Name', { exact: true })).toHaveValue(viewName); + const savedResponse = page.waitForResponse((response) => response.request().method() === 'POST' && response.url().includes('/saved-views')); + await dialog.getByRole('button', { name: 'Save' }).click(); + const response = await savedResponse; + expect(response.ok(), await response.text()).toBe(true); + await expect(dialog).toBeHidden(); + await expect(page.getByRole('heading', { name: viewName })).toBeVisible(); + await page.reload(); + await expect(getVisibleText(page, messages[0]!)).toBeVisible(); + await expect(getVisibleText(page, messages[2]!)).toBeVisible(); + await expect(getVisibleText(page, messages[1]!)).toBeHidden(); + + await page.getByRole('button', { name: /^Environment\s/ }).click(); + await page.getByPlaceholder('Environment', { exact: true }).fill(' Preview-West '); + await page.getByRole('option', { name: 'Use preview-west' }).click(); + await page.keyboard.press('Escape'); + await page.reload(); + await page.getByRole('button', { name: /^Environment\s/ }).click(); + await expect(page.getByRole('option', { exact: true, name: 'preview-west' })).toBeVisible(); + await page.getByRole('button', { name: 'Remove filter' }).click(); + for (const message of messages) await expect(getVisibleText(page, message)).toBeVisible(); + await page.reload(); + for (const message of messages) await expect(getVisibleText(page, message)).toBeVisible(); +}); + +test('stack, session, and stream filters retain names with no current events', async ({ e2eScenario, page }) => { + for (const route of ['stack', 'sessions', 'stream']) { + await page.goto(`/next/${route}?project=${e2eScenario.projectId}`); + await page.getByRole('button', { name: 'Manage filters' }).click(); + await page.getByPlaceholder('Search...').fill('Environment'); + await page.getByText('Environment', { exact: true }).click(); + await page.getByPlaceholder('Environment', { exact: true }).fill('qa,east'); + await page.getByRole('option', { name: 'Use qa,east' }).click(); + await page.keyboard.press('Escape'); + await expect(page.getByRole('button', { name: /^Environment\s+qa,east/ })).toBeVisible(); + await expect.poll(() => new URL(page.url()).searchParams.get(route === 'stream' ? 'filters' : 'environment')).toContain('qa,east'); + await page.reload(); + await expect(page.getByRole('button', { name: /^Environment\s+qa,east/ })).toBeVisible(); + } +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-overview.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-overview.svelte index 8bd8a218c6..eac53884de 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-overview.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/events-overview.svelte @@ -429,7 +429,7 @@ ondrop={(event) => handlePromotedTabDrop(event, tab)} ondragend={handlePromotedTabDragEnd} title={isPromotedTab(tab) ? 'Drag to reorder custom tab' : undefined} - value={tab}>{tab}{tab === 'Environment' ? 'Machine & runtime' : tab} {/each} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-builder.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-builder.svelte new file mode 100644 index 0000000000..6eeaa924c3 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-builder.svelte @@ -0,0 +1,14 @@ + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-trigger.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-trigger.svelte new file mode 100644 index 0000000000..0e42cc6183 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter-trigger.svelte @@ -0,0 +1,22 @@ + + + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte new file mode 100644 index 0000000000..dc94151cbe --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte @@ -0,0 +1,79 @@ + + + { + filter.value = values; + filterChanged(filter); + }} + {createOption} + emptyText="All environments" + hidden={filter.hidden} + loading={countQuery.isLoading} + {options} + remove={() => filterRemoved(filter)} + {title} + toggleHidden={() => { + filter.hidden = !filter.hidden; + filterChanged(filter); + }} + values={filter.value} +/> diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts new file mode 100644 index 0000000000..47ec84589c --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; + +import { filterUsesPremiumFeatures } from '../../premium-filter'; +import { deserializeFilters, serializeFilters, toFilter } from './helpers.svelte'; +import { EnvironmentFilter, ProjectFilter } from './models.svelte'; + +describe('EnvironmentFilter', () => { + it('normalizes names and supports all, multiple, and unspecified environments', () => { + expect(new EnvironmentFilter().toFilter()).toBe(''); + expect(new EnvironmentFilter([' Production ', 'production']).toFilter()).toBe('environment:production'); + expect(new EnvironmentFilter(['production', 'staging']).toFilter()).toBe('(environment:production OR environment:staging)'); + expect(new EnvironmentFilter(['']).toFilter()).toBe('_missing_:environment'); + expect(new EnvironmentFilter(['production', '']).toFilter()).toBe('(environment:production OR _missing_:environment)'); + }); + + it('quotes custom values and combines the environment choice with other filters', () => { + expect(toFilter([new ProjectFilter(['project-1']), new EnvironmentFilter(['qa west'])])).toContain('environment:"qa west"'); + expect(new EnvironmentFilter(['qa:west']).toFilter()).toBe('environment:"qa:west"'); + const name = '"qa" OR _exists_:message'; + expect(new EnvironmentFilter([name]).toFilter()).toBe(`environment:${JSON.stringify(name.toLowerCase())}`); + }); + + it('preserves selections and hidden state through saved-view and URL serialization', () => { + const filter = new EnvironmentFilter(['production', '']); + filter.hidden = true; + const restored = deserializeFilters(serializeFilters([filter])); + expect(restored[0]).toBeInstanceOf(EnvironmentFilter); + expect(restored[0]?.hidden).toBe(true); + expect(toFilter(restored)).toBe(filter.toFilter()); + const clone = filter.clone(); + clone.value.push('staging'); + expect(filter.value).toEqual(['production', '']); + }); + + it('allows environment searches on free plans and keeps custom data premium', () => { + for (const resource of ['event', 'event-stack'] as const) { + expect(filterUsesPremiumFeatures('environment:production', resource)).toBe(false); + expect(filterUsesPremiumFeatures('_missing_:environment', resource)).toBe(false); + expect(filterUsesPremiumFeatures('environment:production data.customer:123', resource)).toBe(true); + } + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts index f23e3a37c9..0443ac2a12 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts @@ -9,6 +9,7 @@ import { SvelteMap } from 'svelte/reactivity'; import { BooleanFilter, DateFilter, + EnvironmentFilter, KeywordFilter, LevelFilter, NumberFilter, @@ -296,6 +297,9 @@ function reconstructFilter(data: SerializedFilter): IFilter | null { case 'date': filter = new DateFilter(data.term, data.value as Date | string | undefined); break; + case 'environment': + filter = new EnvironmentFilter(Array.isArray(data.value) ? data.value.filter((value): value is string => typeof value === 'string') : []); + break; case 'keyword': filter = new KeywordFilter(data.value as string | undefined); break; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/index.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/index.ts index 8812a07dbe..01dbde7f0a 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/index.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/index.ts @@ -4,6 +4,8 @@ import BooleanFacetedFilter from './boolean-faceted-filter.svelte'; import DateFacetedFilterBuilder from './date-faceted-filter-builder.svelte'; import DateFacetedFilterTrigger from './date-faceted-filter-trigger.svelte'; import DateFacetedFilter from './date-faceted-filter.svelte'; +import EnvironmentFacetedFilterBuilder from './environment-faceted-filter-builder.svelte'; +import EnvironmentFacetedFilterTrigger from './environment-faceted-filter-trigger.svelte'; import KeywordFacetedFilterBuilder from './keyword-faceted-filter-builder.svelte'; import KeywordFacetedFilter from './keyword-faceted-filter.svelte'; import LevelFacetedFilterBuilder from './level-faceted-filter-builder.svelte'; @@ -50,6 +52,8 @@ export { DateFacetedFilterBuilder, DateFacetedFilterTrigger, DateFacetedFilterTrigger as DateTrigger, + EnvironmentFacetedFilterBuilder as EnvironmentBuilder, + EnvironmentFacetedFilterTrigger as EnvironmentTrigger, KeywordFacetedFilter as Keyword, KeywordFacetedFilterBuilder as KeywordBuilder, KeywordFacetedFilter, @@ -119,6 +123,7 @@ export { export { BooleanFilter, DateFilter, + EnvironmentFilter, KeywordFilter, LevelFilter, NumberFilter, diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts index ce9d78ac2b..879e465750 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts @@ -3,6 +3,8 @@ import type { PersistentEventKnownTypes } from '$features/events/models'; import type { LogLevel } from '$features/events/models/event-data'; import type { StackStatus } from '$features/stacks/models'; +import { SvelteSet } from 'svelte/reactivity'; + import { quoteIfSpecialCharacters } from './helpers.svelte'; export class BooleanFilter implements IFilter { @@ -80,6 +82,32 @@ export class DateFilter implements IFilter { } } +export class EnvironmentFilter implements IFilter { + public hidden = $state(false); + public id: string = crypto.randomUUID(); + public readonly key = 'environment'; + public readonly type = 'environment'; + public value = $state([]); + + constructor(value: string[] = []) { + this.value = [...new SvelteSet(value.map((name) => name.trim().toLowerCase()))]; + } + + public clone(): EnvironmentFilter { + const filter = new EnvironmentFilter([...this.value]); + filter.hidden = this.hidden; + filter.id = this.id; + return filter; + } + + public toFilter(): string { + const clauses = this.value.map((value) => + value === '' ? '_missing_:environment' : `environment:${/^[a-z0-9][a-z0-9_.-]*$/.test(value) ? value : JSON.stringify(value)}` + ); + return clauses.length > 1 ? `(${clauses.join(' OR ')})` : (clauses[0] ?? ''); + } +} + export class KeywordFilter implements IFilter { public hidden = $state(false); public id: string = crypto.randomUUID(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/organization-defaults-faceted-filter-builder.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/organization-defaults-faceted-filter-builder.svelte index 844b446f73..4d909536f8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/organization-defaults-faceted-filter-builder.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/organization-defaults-faceted-filter-builder.svelte @@ -230,6 +230,7 @@ {/each} + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/index.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/index.ts index 257536b972..3fb60851a2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/index.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/summary/index.ts @@ -61,6 +61,7 @@ export interface EventSummaryData { export interface EventSummaryModel extends SummaryModel { /** @format date-time */ date: string; + environment?: string; project_id: string; project_name?: string; tags: string[]; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/event-environment-cell.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/event-environment-cell.svelte new file mode 100644 index 0000000000..e257dd9c02 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/event-environment-cell.svelte @@ -0,0 +1,17 @@ + + +{#if changed} + +{:else} + {environment ?? 'Unspecified'} +{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts index 1775dde949..bd6a99bb5c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts @@ -11,12 +11,14 @@ import type { EventSummaryModel, StackSummaryModel, SummaryModel, SummaryTemplat import LogLevel from '../log-level.svelte'; import Summary from '../summary/summary.svelte'; +import EventEnvironmentCell from './event-environment-cell.svelte'; import EventTagsSummaryCell from './event-tags-summary-cell.svelte'; import EventsUserIdentitySummaryCell from './events-user-identity-summary-cell.svelte'; import StackStatusCell from './stack-status-cell.svelte'; import StackUsersSummaryCell from './stack-users-summary-cell.svelte'; export const defaultEventColumnVisibility: ColumnVisibilityState = { + environment: false, exception_type: false, level: false, message: false, @@ -35,7 +37,7 @@ export const defaultStackColumnVisibility: ColumnVisibilityState = { export function getColumns>( mode: GetEventsMode = 'summary', - options?: { onTagClick?: (tag: string) => Promise | void; showType?: boolean } + options?: { onEnvironmentClick?: (environment: string) => Promise | void; onTagClick?: (tag: string) => Promise | void; showType?: boolean } ): ColumnDef[] { const showType = options?.showType ?? true; const columns: ColumnDef[] = [ @@ -182,6 +184,19 @@ export function getColumns>('environment'), + cell: (prop) => + renderComponent(EventEnvironmentCell, { + changed: options?.onEnvironmentClick, + environment: prop.getValue() + }), + header: 'Environment', + id: 'environment', + maxSize: 640, + minSize: 96, + size: 144 + }, { accessorKey: nameof>('version'), cell: (prop) => formatTextColumn(prop.getValue()), diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte index b51cc07c65..11bdd2486c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte @@ -73,6 +73,11 @@ + + Environment + + {event.environment ?? 'Unspecified'} + {#if isSessionStart} Duration diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts index c29cce9862..febd37d6ba 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts @@ -3,6 +3,7 @@ export type SearchResource = 'event' | 'event-stack' | 'stack'; // Alias and indexed-field variants intentionally mirror the backend validators. const EVENT_FREE_QUERY_FIELDS = new Set([ 'date', + 'environment', 'organization', 'organization_id', 'project', diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/session-table-columns.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/session-table-columns.ts index 8bc9cdbfae..4d7a84edb6 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/session-table-columns.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/sessions/components/session-table-columns.ts @@ -8,7 +8,7 @@ import { type ColumnDef, type ColumnVisibilityState, renderComponent, type Stock import SessionDurationCell from './session-duration-cell.svelte'; -export const defaultSessionColumnVisibility: ColumnVisibilityState = {}; +export const defaultSessionColumnVisibility: ColumnVisibilityState = { environment: false }; export function getSessionColumns(): ColumnDef, unknown>[] { return [ @@ -74,6 +74,15 @@ export function getSessionColumns(): ColumnDef prop.getValue() ?? 'Unspecified', + header: 'Environment', + id: 'environment', + maxSize: 640, + minSize: 96, + size: 144 + }, { accessorFn: (row) => row.date, cell: (prop) => renderComponent(TimeAgo, { value: prop.getValue() }), diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-builder.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-builder.svelte index be99e2e38a..a12adb830e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-builder.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-builder.svelte @@ -12,19 +12,31 @@ import Eye from '@lucide/svelte/icons/eye'; import EyeOff from '@lucide/svelte/icons/eye-off'; import { computeCommandScore } from 'bits-ui'; + import { setContext } from 'svelte'; import type { FacetedFilter, IFilter } from './models'; import { builderContext, type FacetFilterBuilder } from './faceted-filter-builder-context.svelte'; + import { type FilterScope, filterScopeKey } from './filter-scope'; interface Props { changed: (filter: IFilter) => void; children?: Snippet; filters: IFilter[]; remove: (filter?: IFilter) => void; + time?: null | string; } - let { changed, children, filters, remove }: Props = $props(); + let { changed, children, filters, remove, time }: Props = $props(); + + setContext(filterScopeKey, { + get filters() { + return filters; + }, + get time() { + return time; + } + }); const CREATE_KEYWORD_FILTER_COMMAND_ITEM = 'CREATE_KEYWORD_FILTER_COMMAND_ITEM'; let open = $state(false); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-multi-select.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-multi-select.svelte index 83dacc93e3..7c6ae28999 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-multi-select.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/faceted-filter/faceted-filter-multi-select.svelte @@ -15,6 +15,8 @@ interface Props { changed: (values: string[]) => void; + createOption?: (search: string) => Option | undefined; + emptyText?: string; hidden?: boolean; loading?: boolean; noOptionsText?: string; @@ -28,6 +30,8 @@ let { changed, + createOption, + emptyText = 'No Value', hidden = false, loading = false, noOptionsText = 'No results found.', @@ -39,6 +43,10 @@ values }: Props = $props(); + let search = $state(''); + const customOption = $derived(createOption?.(search)); + const showCustomOption = $derived(customOption && !options.some((option) => option.value === customOption.value)); + // eslint-disable-next-line svelte/prefer-writable-derived let updatedValues = $state([]); let displayValues = $derived.by(() => { @@ -79,6 +87,8 @@ } function filter(value: string, search: string) { + search = search.trim().toLowerCase(); + value = value.toLowerCase(); if (value.includes(search)) { return 1; } @@ -107,23 +117,30 @@ {/snippet} {:else} - No Value + {emptyText} {/if} {/snippet} e.preventDefault()}> - + {noOptionsText} + {#if showCustomOption && customOption} + + customOption && onValueSelected(customOption.value)}> + Use {customOption.label} + + + {/if} {#if loading}
Loading...
{/if} {#if options.length > 0} {#each options as option (option.value)} - onValueSelected(option.value)} value={option.value}> + onValueSelected(option.value)} value={option.value || option.label}>
; + /** + * The deployment environment, such as production, staging, or development. + * Missing or invalid names remain unspecified. Machine and runtime information is stored separately in data.@environment. + */ + environment?: null | string; /** * The event type (ie. error, log message, feature usage). Check KnownTypes for standard event types. * Nullable in transit; the pipeline infers a default before save. Validated as required on repository save. diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index ec58c99678..055d423467 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -568,6 +568,11 @@ export const PersistentEventSchema = object({ is_first_occurrence: boolean(), created_utc: iso.datetime(), idx: record(string(), unknown()).nullable().optional(), + environment: string() + .min(1, "Environment is required") + .max(64, "Environment must be at most 64 characters") + .nullable() + .optional(), type: string() .min(1, "Type is required") .max(100, "Type must be at most 100 characters") diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte index f74ef8aec2..58e23de9ea 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte @@ -18,6 +18,7 @@ import { BooleanFilter, DateFilter, + EnvironmentFilter, LevelFilter, ProjectFilter, ReferenceFilter, @@ -69,6 +70,7 @@ import { ALL_TIME_QUERY_VALUE, + deserializeEnvironmentQueryParam, deserializeTimeQueryParam, getEventsNavigationOptionsForFilter, getListFilterQueryParams, @@ -102,6 +104,7 @@ after: undefined as string | undefined, before: undefined as string | undefined, bot: undefined as string | undefined, + environment: undefined as string | undefined, filter: undefined as string | undefined, first: undefined as string | undefined, level: undefined as string | undefined, @@ -161,6 +164,10 @@ filters.push(new BooleanFilter('first', first)); } + if (params.environment) { + filters.push(new EnvironmentFilter(deserializeEnvironmentQueryParam(params.environment))); + } + if (params.level) { filters.push(new LevelFilter(splitQueryParam(params.level) as never[])); } @@ -227,6 +234,7 @@ after: 'string', before: 'string', bot: 'string', + environment: 'string', filter: 'string', first: 'string', level: 'string', @@ -375,6 +383,10 @@ removedKeys.push(...savedViewFilters.filter((filter) => filter.type !== 'date' && !isQueryParamFilter(filter)).map((filter) => filter.key)); } + if (params.environment === '') { + removedKeys.push('environment'); + } + if (params.level === '') { removedKeys.push('level'); } @@ -503,6 +515,7 @@ newTimeParam !== queryParams.time || queryFilterParams.bot !== queryParams.bot || queryFilterParams.first !== queryParams.first || + queryFilterParams.environment !== queryParams.environment || queryFilterParams.level !== queryParams.level || queryFilterParams.project !== queryParams.project || queryFilterParams.reference !== queryParams.reference || @@ -533,6 +546,7 @@ after: shouldClearPaginationForFilter ? null : queryParams.after, before: shouldClearPaginationForFilter ? null : queryParams.before, bot: queryFilterParams.bot, + environment: queryFilterParams.environment, filter: newFilterParam, first: queryFilterParams.first, level: queryFilterParams.level, @@ -572,6 +586,7 @@ function getQueryFilterParams(filters: FacetedFilter.IFilter[]) { const botFilter = filters.find((f): f is BooleanFilter => f instanceof BooleanFilter && f.term === 'bot'); const firstFilter = filters.find((f): f is BooleanFilter => f instanceof BooleanFilter && f.term === 'first'); + const environmentFilter = filters.find((filter): filter is EnvironmentFilter => filter.type === 'environment'); const levelFilter = filters.find((f): f is LevelFilter => f.type === 'level'); const projectFilter = filters.find((f): f is ProjectFilter => f.type === 'project'); const referenceFilter = filters.find((f): f is ReferenceFilter => f.type === 'reference'); @@ -584,6 +599,7 @@ return { bot: botFilter?.value === undefined ? null : String(botFilter.value), + environment: environmentFilter?.value.length ? JSON.stringify(environmentFilter.value) : null, first: firstFilter?.value === undefined ? null : String(firstFilter.value), level: levelFilter?.value.length ? levelFilter.value.join(',') : null, project: projectFilter?.value.length ? projectFilter.value.join(',') : null, @@ -608,6 +624,7 @@ return { bot: getDelta(currentParams.bot, baseParams.bot), + environment: getDelta(currentParams.environment, baseParams.environment), first: getDelta(currentParams.first, baseParams.first), level: getDelta(currentParams.level, baseParams.level), project: getDelta(currentParams.project, baseParams.project), @@ -634,7 +651,7 @@ return false; } - return ['level', 'project', 'reference', 'session', 'status', 'tag', 'type', 'version'].includes(filter.type); + return ['environment', 'level', 'project', 'reference', 'session', 'status', 'tag', 'type', 'version'].includes(filter.type); } function getPageSize(): number { @@ -728,6 +745,7 @@ columnPersistenceKey: 'events-column-visibility', get columns() { return getColumns>(eventsQueryParameters.mode, { + onEnvironmentClick: (environment) => onFilterChanged(new EnvironmentFilter([environment])), onTagClick: (tag) => onFilterChanged(new TagFilter([tag])), showType: !hasSingleTypeFilter(eventsQueryParameters.filter) }); @@ -961,7 +979,7 @@

{pageTitle}

- +
diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts index 7d4148e1c9..ff6504537a 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts @@ -1,4 +1,4 @@ -import { DateFilter, ProjectFilter, StatusFilter, StringFilter } from '$features/events/components/filters/models.svelte'; +import { DateFilter, EnvironmentFilter, ProjectFilter, StatusFilter, StringFilter } from '$features/events/components/filters/models.svelte'; import { describe, expect, it, vi } from 'vitest'; vi.mock('$app/navigation', () => ({ @@ -10,6 +10,14 @@ vi.mock('$app/paths', () => ({ })); describe('redirect-to-events', () => { + it('preserves environment values including commas and unspecified through navigation', async () => { + const { buildListPageHref, deserializeEnvironmentQueryParam } = await import('./redirect-to-events.svelte'); + const names = ['production', '', 'qa,east']; + const url = new URL(buildListPageHref('events', 'organization-1', [new EnvironmentFilter(names)]), 'http://localhost'); + expect(deserializeEnvironmentQueryParam(url.searchParams.get('environment')!)).toEqual(names); + expect(url.searchParams.has('filter')).toBe(false); + expect(deserializeEnvironmentQueryParam('staging')).toEqual(['staging']); + }); it('snapshots list filter query parameters from shared reactive state', async () => { // Arrange const { getListFilterQueryParams } = await import('./redirect-to-events.svelte'); @@ -55,6 +63,7 @@ describe('redirect-to-events', () => { // Assert expect(queryParams).toEqual({ bot: null, + environment: null, filter: null, first: null, level: null, diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts index 3cf98ec3ce..bc5ac481df 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts @@ -15,6 +15,7 @@ export const ALL_TIME_QUERY_VALUE = 'all'; const LIST_FILTER_QUERY_PARAM_NAMES = [ 'bot', + 'environment', 'filter', 'first', 'level', @@ -66,6 +67,18 @@ export const LIST_FILTER_QUERY_PARAM_RESET = Object.fromEntries(LIST_FILTER_QUER null >; +export function deserializeEnvironmentQueryParam(value: string): string[] { + try { + const values: unknown = JSON.parse(value); + if (Array.isArray(values)) { + return values.filter((name): name is string => typeof name === 'string'); + } + } catch { + // Single names are also accepted in hand-written URLs. + } + return [value]; +} + export function deserializeTimeQueryParam(time: string): string { const trimmed = time.trim(); const shortcutMatch = TIME_SHORTCUT_PATTERN.exec(trimmed); @@ -101,6 +114,7 @@ export function getEventsNavigationOptionsForFilter(filter: IFilter): ListNaviga export function getListFilterQueryParams(source: ListFilterQueryParams): ListFilterQueryParamSnapshot { return { bot: source.bot ?? null, + environment: source.environment ?? null, filter: source.filter ?? null, first: source.first ?? null, level: source.level ?? null, @@ -157,6 +171,11 @@ function trySetRegisteredFilterQueryParam(queryParams: SvelteURLSearchParams, fi return true; } + if (filter.type === 'environment' && 'value' in filter && Array.isArray(filter.value) && filter.value.length > 0) { + queryParams.set('environment', JSON.stringify(filter.value)); + return true; + } + if (filter.type === 'project' && 'value' in filter && Array.isArray(filter.value) && filter.value.length > 0) { queryParams.set('project', filter.value.join(',')); return true; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte index a8b9180868..6bea03fc19 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte @@ -15,6 +15,7 @@ import { BooleanFilter, DateFilter, + EnvironmentFilter, LevelFilter, ProjectFilter, ReferenceFilter, @@ -62,6 +63,7 @@ import { ALL_TIME_QUERY_VALUE, + deserializeEnvironmentQueryParam, deserializeTimeQueryParam, getListFilterQueryParams, LIST_FILTER_QUERY_PARAM_RESET, @@ -82,6 +84,7 @@ after: undefined as string | undefined, before: undefined as string | undefined, bot: undefined as string | undefined, + environment: undefined as string | undefined, filter: undefined as string | undefined, filters: undefined as string | undefined, first: undefined as string | undefined, @@ -163,6 +166,10 @@ queryFilters.push(new BooleanFilter('first', first)); } + if (params.environment) { + queryFilters.push(new EnvironmentFilter(deserializeEnvironmentQueryParam(params.environment))); + } + if (params.level) { queryFilters.push(new LevelFilter(splitQueryParam(params.level) as never[])); } @@ -223,6 +230,7 @@ after: 'string', before: 'string', bot: 'string', + environment: 'string', filter: 'string', filters: 'string', first: 'string', @@ -383,6 +391,10 @@ removedKeys.push('boolean-first'); } + if (params.environment === '') { + removedKeys.push('environment'); + } + if (params.level === '') { removedKeys.push('level'); } @@ -518,6 +530,7 @@ newTimeParam !== queryParams.time || queryFilterParams.bot !== queryParams.bot || queryFilterParams.first !== queryParams.first || + queryFilterParams.environment !== queryParams.environment || queryFilterParams.level !== queryParams.level || queryFilterParams.project !== queryParams.project || queryFilterParams.reference !== queryParams.reference || @@ -544,6 +557,7 @@ after: shouldClearPaginationForFilter ? null : queryParams.after, before: shouldClearPaginationForFilter ? null : queryParams.before, bot: queryFilterParams.bot, + environment: queryFilterParams.environment, filter: null, filters: newFiltersParam, first: queryFilterParams.first, @@ -584,6 +598,7 @@ function getQueryFilterParams(currentFilters: FacetedFilter.IFilter[]) { const botFilter = currentFilters.find((filter): filter is BooleanFilter => filter instanceof BooleanFilter && filter.term === 'bot'); const firstFilter = currentFilters.find((filter): filter is BooleanFilter => filter instanceof BooleanFilter && filter.term === 'first'); + const environmentFilter = currentFilters.find((filter): filter is EnvironmentFilter => filter.type === 'environment'); const levelFilter = currentFilters.find((filter): filter is LevelFilter => filter.type === 'level'); const projectFilter = currentFilters.find((filter): filter is ProjectFilter => filter.type === 'project'); const referenceFilter = currentFilters.find((filter): filter is ReferenceFilter => filter.type === 'reference'); @@ -595,6 +610,7 @@ return { bot: botFilter?.value === undefined ? null : String(botFilter.value), + environment: environmentFilter?.value.length ? JSON.stringify(environmentFilter.value) : null, first: firstFilter?.value === undefined ? null : String(firstFilter.value), level: levelFilter?.value.length ? levelFilter.value.join(',') : null, project: projectFilter?.value.length ? projectFilter.value.join(',') : null, @@ -617,6 +633,7 @@ return { bot: getDelta(currentParams.bot, baseParams.bot), + environment: getDelta(currentParams.environment, baseParams.environment), first: getDelta(currentParams.first, baseParams.first), level: getDelta(currentParams.level, baseParams.level), project: getDelta(currentParams.project, baseParams.project), @@ -641,7 +658,7 @@ if (filter.type === 'version' && filter instanceof VersionFilter && filter.term !== 'version') { return false; } - return ['level', 'project', 'reference', 'session', 'status', 'tag', 'version'].includes(filter.type); + return ['environment', 'level', 'project', 'reference', 'session', 'status', 'tag', 'version'].includes(filter.type); } const viewActive = $derived( @@ -896,7 +913,7 @@

{pageTitle}

- +
diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte index b14ea3a0e2..61fe0a3244 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte @@ -20,6 +20,7 @@ import { BooleanFilter, DateFilter, + EnvironmentFilter, LevelFilter, ProjectFilter, ReferenceFilter, @@ -68,6 +69,7 @@ import { ALL_TIME_QUERY_VALUE, + deserializeEnvironmentQueryParam, deserializeTimeQueryParam, getEventsNavigationOptionsForFilter, getListFilterQueryParams, @@ -107,6 +109,7 @@ const pageSizePreference = createPageSizePreference(PAGE_SIZE_PREFERENCE_KEY); const DEFAULT_PARAMS = { bot: undefined as string | undefined, + environment: undefined as string | undefined, filter: undefined as string | undefined, first: undefined as string | undefined, level: undefined as string | undefined, @@ -165,6 +168,10 @@ filters.push(new BooleanFilter('first', first)); } + if (params.environment) { + filters.push(new EnvironmentFilter(deserializeEnvironmentQueryParam(params.environment))); + } + if (params.level) { filters.push(new LevelFilter(splitQueryParam(params.level) as never[])); } @@ -221,6 +228,7 @@ history: 'push', schema: { bot: 'string', + environment: 'string', filter: 'string', first: 'string', level: 'string', @@ -366,6 +374,10 @@ removedKeys.push(...savedViewFilters.filter((filter) => filter.type !== 'date' && !isQueryParamFilter(filter)).map((filter) => filter.key)); } + if (params.environment === '') { + removedKeys.push('environment'); + } + if (params.level === '') { removedKeys.push('level'); } @@ -499,6 +511,7 @@ newTimeParam !== queryParams.time || queryFilterParams.bot !== queryParams.bot || queryFilterParams.first !== queryParams.first || + queryFilterParams.environment !== queryParams.environment || queryFilterParams.level !== queryParams.level || queryFilterParams.project !== queryParams.project || queryFilterParams.reference !== queryParams.reference || @@ -527,6 +540,7 @@ queryParams.update( { bot: queryFilterParams.bot, + environment: queryFilterParams.environment, filter: newFilterParam, first: queryFilterParams.first, level: queryFilterParams.level, @@ -566,6 +580,7 @@ function getQueryFilterParams(filters: FacetedFilter.IFilter[]) { const botFilter = filters.find((f): f is BooleanFilter => f instanceof BooleanFilter && f.term === 'bot'); const firstFilter = filters.find((f): f is BooleanFilter => f instanceof BooleanFilter && f.term === 'first'); + const environmentFilter = filters.find((filter): filter is EnvironmentFilter => filter.type === 'environment'); const levelFilter = filters.find((f): f is LevelFilter => f.type === 'level'); const projectFilter = filters.find((f): f is ProjectFilter => f.type === 'project'); const referenceFilter = filters.find((f): f is ReferenceFilter => f.type === 'reference'); @@ -578,6 +593,7 @@ return { bot: botFilter?.value === undefined ? null : String(botFilter.value), + environment: environmentFilter?.value.length ? JSON.stringify(environmentFilter.value) : null, first: firstFilter?.value === undefined ? null : String(firstFilter.value), level: levelFilter?.value.length ? levelFilter.value.join(',') : null, project: projectFilter?.value.length ? projectFilter.value.join(',') : null, @@ -602,6 +618,7 @@ return { bot: getDelta(currentParams.bot, baseParams.bot), + environment: getDelta(currentParams.environment, baseParams.environment), first: getDelta(currentParams.first, baseParams.first), level: getDelta(currentParams.level, baseParams.level), project: getDelta(currentParams.project, baseParams.project), @@ -628,7 +645,7 @@ return false; } - return ['level', 'project', 'reference', 'session', 'status', 'tag', 'type', 'version'].includes(filter.type); + return ['environment', 'level', 'project', 'reference', 'session', 'status', 'tag', 'type', 'version'].includes(filter.type); } function getPageSize(): number { @@ -864,7 +881,7 @@

{pageTitle}

- +
diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stream/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stream/+page.svelte index bc30777ca3..8e3be234eb 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stream/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stream/+page.svelte @@ -11,9 +11,10 @@ import { showBillingDialogOnUpgradeProblem } from '$features/billing/upgrade-required.svelte'; import { PERSISTENT_EVENT_DELETE_RECONCILE_EVENT } from '$features/events/api.svelte'; import EventDetailSheet from '$features/events/components/event-detail-sheet.svelte'; - import { ProjectFilter, StatusFilter, TagFilter } from '$features/events/components/filters'; + import { EnvironmentFilter, ProjectFilter, StatusFilter, TagFilter } from '$features/events/components/filters'; import { buildFilterCacheKey, + deserializeFilters, filterChanged, filterRemoved, getFiltersFromCache, @@ -60,6 +61,7 @@ const DEFAULT_FILTERS = [new ProjectFilter([]), new StatusFilter([StackStatus.Open, StackStatus.Regressed])]; const DEFAULT_PARAMS = { filter: '(status:open OR status:regressed)', + filters: undefined as string | undefined, limit: DEFAULT_LIMIT, saved: undefined as string | undefined }; @@ -74,6 +76,7 @@ history: 'push', schema: { filter: 'string', + filters: 'string', limit: 'number', saved: 'string' } @@ -121,7 +124,10 @@ watch( () => organization.current, - () => { + (_currentOrganizationId, previousOrganizationId) => { + if (previousOrganizationId === undefined) { + return; + } updateFilterCache(filterCacheKey(DEFAULT_PARAMS.filter), DEFAULT_FILTERS); queryParams.update(DEFAULT_PARAMS); paused = false; @@ -131,11 +137,22 @@ } ); - let filters = $state(getFiltersFromCache(filterCacheKey(queryParams.filter), queryParams.filter)); + function getQueryFilters() { + const cached = getFiltersFromCache(filterCacheKey(queryParams.filter), queryParams.filter); + if (queryParams.filters && serializeFilters(cached) !== queryParams.filters) { + const restored = deserializeFilters(queryParams.filters); + if (toFilter(restored) === queryParams.filter) { + return restored; + } + } + return cached; + } + + let filters = $state(getQueryFilters()); watch( - [() => queryParams.filter], - ([filter]) => { - filters = getFiltersFromCache(filterCacheKey(filter), filter); + [() => queryParams.filter, () => queryParams.filters], + () => { + filters = getQueryFilters(); }, { lazy: true @@ -173,7 +190,8 @@ updateFilterCache(filterCacheKey(filter), updatedFilters); queryParams.update( { - filter + filter, + filters: updatedFilters.some((filter) => filter.type === 'environment') ? serializeFilters(updatedFilters) : null }, { history: options.history @@ -209,6 +227,7 @@ columnPersistenceKey: 'stream-column-visibility', get columns() { return getColumns>(eventsQueryParameters.mode, { + onEnvironmentClick: (environment) => onFilterChanged(new EnvironmentFilter([environment])), onTagClick: (tag) => onFilterChanged(new TagFilter([tag])), showType: !hasSingleTypeFilter(eventsQueryParameters.filter) }) diff --git a/src/Exceptionless.Web/Mcp/ExceptionlessMcpTools.cs b/src/Exceptionless.Web/Mcp/ExceptionlessMcpTools.cs index 48a3128499..0be1686490 100644 --- a/src/Exceptionless.Web/Mcp/ExceptionlessMcpTools.cs +++ b/src/Exceptionless.Web/Mcp/ExceptionlessMcpTools.cs @@ -40,11 +40,11 @@ public sealed class ExceptionlessMcpTools private const string LastDescription = "Optional relative time range such as 24h, 7d, or 30m. Do not combine with startUtc or endUtc."; private const string StartUtcDescription = "Optional inclusive UTC start time, for example 2026-06-25T00:00:00Z. Do not combine with last."; private const string EndUtcDescription = "Optional exclusive UTC end time, for example 2026-06-25T01:00:00Z. Do not combine with last."; - private const string EventGroupByDescription = "Optional dimension to group counts by. Supported values: version, type, source, status, tag, stack, user, level, error.type, error.code, os, os.version, browser. Multi-value fields such as tag, error.type, and error.code can place one event into multiple groups, so group totals may sum higher than the overall event total."; + private const string EventGroupByDescription = "Optional dimension to group counts by. Supported values: environment, version, type, source, status, tag, stack, user, level, error.type, error.code, os, os.version, browser. Multi-value fields such as tag, error.type, and error.code can place one event into multiple groups, so group totals may sum higher than the overall event total."; private const string SnoozeDurationDescription = "Optional relative snooze duration such as 2h, 3d, or 1w. Do not combine with snoozeUntilUtc."; private const string ProjectFilterDescription = "Optional Exceptionless filter expression applied to projects. Supported fields: id, name, organization_id, created_utc, updated_utc, last_event_date_utc."; private const string StackFilterDescription = "Optional Exceptionless filter expression. Supported fields include: stack, project, project_id, organization, organization_id, type, status, title, description, tag, tags, references, fixed, hidden, regressed, error, first, first_occurrence, last, last_occurrence, occurrences, total_occurrences."; - private const string EventFilterDescription = "Optional Exceptionless filter expression applied to events. Supported fields include: id, project, project_id, stack, stack_id, organization, organization_id, type, source, message, date, tag, tags, user, user.name, user.email, path, error, error.type, error.message, error.code, status, data.*. data.* works for custom data values that were indexed for search; arbitrary event detail data is returned by get_event but is not searchable unless indexed."; + private const string EventFilterDescription = "Optional Exceptionless filter expression applied to events. Supported fields include: id, project, project_id, stack, stack_id, organization, organization_id, type, environment, source, message, date, tag, tags, user, user.name, user.email, path, error, error.type, error.message, error.code, status, data.*. data.* works for custom data values that were indexed for search; arbitrary event detail data is returned by get_event but is not searchable unless indexed."; private const string IndexedDataFilterNote = "data.* filters work for custom data values that were indexed for search. Arbitrary event detail data is returned by get_event but is not searchable unless indexed."; @@ -1655,7 +1655,8 @@ private McpEventResult ToEventResult(PersistentEvent ev, bool includeDetails = f ev.Source, ev.Message, ev.ReferenceId, - includeDetails ? ToEventDetails(ev, maxDetailSize) : null); + includeDetails ? ToEventDetails(ev, maxDetailSize) : null, + ev.Environment); } private McpEventDetails ToEventDetails(PersistentEvent ev, int maxDetailSize) @@ -1832,6 +1833,7 @@ private static double GetNumericAggregationValue(object? value, double defaultVa private static readonly string[] EventGroupByAllowedFields = [ + "environment", "version", "type", "source", @@ -1849,6 +1851,7 @@ private static double GetNumericAggregationValue(object? value, double defaultVa private static readonly IReadOnlyDictionary EventGroupByFields = new Dictionary(StringComparer.OrdinalIgnoreCase) { + ["environment"] = new("environment", EventIndex.Alias.Environment), ["version"] = new("version", EventIndex.Alias.Version), ["type"] = new("type", EventIndex.Alias.Type), ["source"] = new("source", EventIndex.Alias.Source), @@ -1922,6 +1925,7 @@ private static double GetNumericAggregationValue(object? value, double defaultVa private static readonly HashSet EventSortFields = new(StringComparer.OrdinalIgnoreCase) { + EventIndex.Alias.Environment, EventIndex.Alias.Date, EventIndex.Alias.Type, EventIndex.Alias.Source, diff --git a/src/Exceptionless.Web/Mcp/McpModels.cs b/src/Exceptionless.Web/Mcp/McpModels.cs index 38ac10f42c..48050cbdad 100644 --- a/src/Exceptionless.Web/Mcp/McpModels.cs +++ b/src/Exceptionless.Web/Mcp/McpModels.cs @@ -159,7 +159,8 @@ public sealed record McpEventResult( string? Source = null, string? Message = null, string? ReferenceId = null, - McpEventDetails? Details = null); + McpEventDetails? Details = null, + string? Environment = null); public sealed record McpEventDetails( bool IsTruncated = false, diff --git a/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs b/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs index 97f2dee003..aae8629299 100644 --- a/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs +++ b/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs @@ -15,10 +15,10 @@ public record NewSavedView : IOwnedByOrganization, IValidatableObject public static readonly IReadOnlyDictionary> ValidColumnIds = new Dictionary> { - ["events"] = new HashSet { "summary", "user", "date", "project", "tags", "message", "type", "version", "exception_type", "source", "name", "level" }, - ["sessions"] = new HashSet { "summary", "duration", "user", "date" }, + ["events"] = new HashSet { "summary", "user", "date", "project", "tags", "message", "type", "version", "exception_type", "source", "name", "level", "environment" }, + ["sessions"] = new HashSet { "summary", "duration", "user", "date", "environment" }, ["stacks"] = new HashSet { "summary", "project", "tags", "status", "users", "events", "first", "last" }, - ["stream"] = new HashSet { "summary", "user", "date", "project", "tags", "message", "type", "version", "exception_type", "source", "name", "level" } + ["stream"] = new HashSet { "summary", "user", "date", "project", "tags", "message", "type", "version", "exception_type", "source", "name", "level", "environment" } }; /// Union of all valid column IDs across all views. diff --git a/src/Exceptionless.Web/Utility/OpenApi/EventEnvironmentSchemaTransformer.cs b/src/Exceptionless.Web/Utility/OpenApi/EventEnvironmentSchemaTransformer.cs new file mode 100644 index 0000000000..1083a4665e --- /dev/null +++ b/src/Exceptionless.Web/Utility/OpenApi/EventEnvironmentSchemaTransformer.cs @@ -0,0 +1,26 @@ +using Exceptionless.Core.Models; +using Microsoft.AspNetCore.OpenApi; +using Microsoft.OpenApi; + +namespace Exceptionless.Web.Utility.OpenApi; + +public sealed class EventEnvironmentSchemaTransformer : IOpenApiSchemaTransformer +{ + public Task TransformAsync(OpenApiSchema schema, OpenApiSchemaTransformerContext context, CancellationToken cancellationToken) + { + if (!typeof(Event).IsAssignableFrom(context.JsonTypeInfo.Type) || schema.Properties is null) + return Task.CompletedTask; + + var property = context.JsonTypeInfo.Type.GetProperty(nameof(Event.Environment))!; + string? name = JsonPropertyNameResolver.GetJsonPropertyName(context.JsonTypeInfo, property); + if (name is not null && schema.Properties.TryGetValue(name, out var propertySchema) && propertySchema is OpenApiSchema environment) + { + // The tolerant JSON converter accepts malformed input, but responses contain only a string or null. + environment.Type = JsonSchemaType.String | JsonSchemaType.Null; + environment.MaxLength = 64; + schema.Required?.Remove(name); + } + + return Task.CompletedTask; + } +} diff --git a/src/Exceptionless.Web/Utility/OpenApi/ExceptionlessOpenApiServiceCollectionExtensions.cs b/src/Exceptionless.Web/Utility/OpenApi/ExceptionlessOpenApiServiceCollectionExtensions.cs index 9fe098e181..829c3a2243 100644 --- a/src/Exceptionless.Web/Utility/OpenApi/ExceptionlessOpenApiServiceCollectionExtensions.cs +++ b/src/Exceptionless.Web/Utility/OpenApi/ExceptionlessOpenApiServiceCollectionExtensions.cs @@ -26,6 +26,7 @@ public static IServiceCollection AddExceptionlessOpenApi(this IServiceCollection options.AddSchemaTransformer(); options.AddSchemaTransformer(); options.AddSchemaTransformer(); + options.AddSchemaTransformer(); options.AddSchemaTransformer(); options.AddSchemaTransformer(); }); diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index 6e88d09679..79e4f67045 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -13038,6 +13038,15 @@ "additionalProperties": {}, "description": "Used to store primitive data type custom data values for searching the event." }, + "environment": { + "maxLength": 64, + "minLength": 0, + "type": [ + "null", + "string" + ], + "description": "The deployment environment, such as production, staging, or development.\nMissing or invalid names remain unspecified. Machine and runtime information is stored separately in data.@environment." + }, "type": { "maxLength": 100, "minLength": 1, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs new file mode 100644 index 0000000000..c418fe3c4b --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -0,0 +1,77 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Extensions; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Utility; +using Foundatio.Repositories.Models; +using Xunit; + +namespace Exceptionless.Tests.Api.Endpoints; + +public partial class EventEndpointTests +{ + [Fact] + public async Task GetStacks_EnvironmentFilter_ScopesUserCountsAndTheirCache() + { + await CreateDataAsync(data => + { + var first = data.Event().FreeProject().Type(Event.KnownTypes.Error).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity("production-0"); }); + for (int i = 1; i < 12; i++) + { + string identity = $"production-{i}"; + data.Event().FreeProject().Type(Event.KnownTypes.Error).Stack(first).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity(identity); }); + } + data.Event().FreeProject().Type(Event.KnownTypes.Error).Stack(first).Mutate(ev => { ev.Environment = "staging"; ev.SetUserIdentity("staging-user"); }); + data.Event().FreeProject().Type(Event.KnownTypes.Log).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity("unaffected-production-user"); }); + }); + + foreach (var (filter, expected, totalUsers) in new[] { ("environment:production", 12, 13), ("environment:staging", 1, 1), ("environment:production", 12, 13), ("", 13, 14) }) + { + var stacks = await SendRequestAsAsync>(request => request + .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") + .QueryString("filter", $"type:error {filter}").StatusCodeShouldBeOk()); + var stack = Assert.Single(Assert.IsType>(stacks)); + Assert.Equal(expected, stack.Total); + Assert.Equal(expected, stack.Users); + Assert.Equal(totalUsers, stack.TotalUsers); + } + } + + [Fact] + public async Task GetEvents_EnvironmentOnFreePlan_FiltersEventsSummariesAndStacks() + { + await CreateDataAsync(data => + { + var production = data.Event().FreeProject().Mutate(ev => ev.Environment = " Production "); + data.Event().FreeProject().Stack(production).Mutate(ev => ev.Environment = "staging"); + data.Event().FreeProject().Stack(production); + data.Event().TestProject().Mutate(ev => ev.Environment = "private-environment"); + }); + + var events = await SendRequestAsAsync>(request => request + .AsFreeOrganizationUser().AppendPath("events") + .QueryString("filter", "environment:PRODUCTION").StatusCodeShouldBeOk()); + Assert.Equal("production", Assert.Single(Assert.IsType>(events)).Environment); + + var summaries = await SendRequestAsAsync>(request => request + .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "summary") + .QueryString("filter", "environment:production").StatusCodeShouldBeOk()); + Assert.Equal("production", Assert.Single(Assert.IsType>(summaries)).Environment); + + var stacks = await SendRequestAsAsync>(request => request + .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") + .QueryString("filter", "environment:production").StatusCodeShouldBeOk()); + Assert.Equal(1, Assert.Single(Assert.IsType>(stacks)).Total); + + var missing = await SendRequestAsAsync>(request => request + .AsFreeOrganizationUser().AppendPath("events") + .QueryString("filter", "_missing_:environment").StatusCodeShouldBeOk()); + Assert.Null(Assert.Single(Assert.IsType>(missing)).Environment); + + var count = await SendRequestAsAsync(request => request + .AsFreeOrganizationUser().AppendPaths("events", "count") + .QueryString("aggregations", "terms:(environment~100)").StatusCodeShouldBeOk()); + Assert.NotNull(count); + Assert.Equal(3, count.Total); + Assert.Equal(new[] { "production", "staging" }, count.Aggregations.Terms("terms_environment")!.Buckets.Select(bucket => bucket.Key).Order().ToArray()); + } +} diff --git a/tests/Exceptionless.Tests/Api/Endpoints/SavedViewEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/SavedViewEndpointTests.cs index 6ea6bd8be8..b18138251a 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/SavedViewEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/SavedViewEndpointTests.cs @@ -431,7 +431,8 @@ public async Task PostAsync_StructuredColumns_PersistsAllSettings() var columnSettings = new Dictionary { ["summary"] = new() { AutoFill = true, Position = 0, Visible = true, Wrap = true }, - ["project"] = new() { Position = 1, Visible = true, Width = 240 } + ["project"] = new() { Position = 1, Visible = true, Width = 240 }, + ["environment"] = new() { Visible = false } }; // Act @@ -452,6 +453,7 @@ public async Task PostAsync_StructuredColumns_PersistsAllSettings() // Assert Assert.NotNull(result); Assert.Equal(240, result.Columns?["project"].Width); + Assert.False(result.Columns?["environment"].Visible); Assert.True(result.Columns?["project"].Visible); Assert.Equal(1, result.Columns?["project"].Position); Assert.True(result.Columns?["summary"].AutoFill); diff --git a/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs b/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs index 9d85fa6956..4db2a0e1af 100644 --- a/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs +++ b/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs @@ -96,6 +96,11 @@ public async Task GetOpenApiJson_Default_ContainsExpectedSchemasAndSecuritySchem // Assert Assert.True(schemas.TryGetProperty("Login", out _)); + var persistentEvent = schemas.GetProperty("PersistentEvent"); + var eventEnvironment = persistentEvent.GetProperty("properties").GetProperty("environment"); + Assert.Contains(eventEnvironment.GetProperty("type").EnumerateArray(), type => type.GetString() == "string"); + Assert.Equal(64, eventEnvironment.GetProperty("maxLength").GetInt32()); + Assert.DoesNotContain(persistentEvent.GetProperty("required").EnumerateArray(), name => name.GetString() == "environment"); Assert.True(schemas.TryGetProperty("Signup", out _)); Assert.True(schemas.TryGetProperty("NewProject", out _)); Assert.True(schemas.TryGetProperty("SavedViewColumnSettings", out var savedViewColumnSettings)); diff --git a/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs b/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs index 41069da579..c7e0c2b28d 100644 --- a/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs +++ b/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs @@ -465,6 +465,7 @@ public async Task GetEventAsync_EventsScope_ReturnsEventDetails() Path = "/broken" }; ev.Data!["custom"] = "custom-value"; + ev.Environment = "production"; await _eventRepository.SaveAsync(ev, o => o.ImmediateConsistency()); await RefreshDataAsync(); var tools = await CreateToolsAsync(AuthorizationRoles.McpRead, AuthorizationRoles.EventsRead); @@ -481,6 +482,7 @@ public async Task GetEventAsync_EventsScope_ReturnsEventDetails() Assert.Equal("at Test.Throw() in Test.cs:line 42", error.StackTrace); Assert.Equal("/broken", item.Details.Request?.Path); Assert.Equal("custom-value", item.Details.Data?["custom"]); + Assert.Equal("production", item.Environment); } [Fact] @@ -703,6 +705,23 @@ public async Task CountEventsAsync_GroupByVersion_ReturnsVersionCounts() Assert.Contains(data.Groups, g => g.Key == "1.0.3" && g.Events >= 1); } + [Fact] + public async Task CountEventsAsync_GroupByEnvironment_ReturnsFilteredDeploymentCounts() + { + const string referenceId = "mcp-count-environments"; + await CreateDataAsync(data => + { + data.Event().TestProject().ReferenceId(referenceId).Mutate(ev => ev.Environment = "production"); + data.Event().TestProject().ReferenceId(referenceId).Mutate(ev => ev.Environment = "staging"); + }); + var tools = await CreateToolsAsync(AuthorizationRoles.McpRead, AuthorizationRoles.EventsRead); + var result = await tools.CountEventsAsync(TestConstants.ProjectId, filter: $"reference:{referenceId} environment:production", groupBy: "environment"); + Assert.True(result.Ok); + var group = Assert.Single(Data(result).Groups!); + Assert.Equal("production", group.Key); + Assert.Equal(1, group.Events); + } + [Fact] public async Task CountEventsAsync_GroupByVersionAndInterval_ReturnsGroupedTrend() { diff --git a/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs new file mode 100644 index 0000000000..d020050c90 --- /dev/null +++ b/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs @@ -0,0 +1,47 @@ +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core.Migrations; +using Exceptionless.Core.Repositories.Configuration; +using Foundatio.Lock; +using Foundatio.Repositories.Migrations; +using Foundatio.Utility; +using Xunit; + +namespace Exceptionless.Tests.Migrations; + +public sealed class AddEventEnvironmentMigrationTests : IntegrationTestsBase +{ + public AddEventEnvironmentMigrationTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { } + + [Fact] + public async Task RunAsync_LegacyDailyIndex_AddsMappingAndCanRunAgain() + { + var configuration = GetService(); + string indexName = $"{configuration.Events.Name}-v0-2000.01.01"; + var client = configuration.Client; + try + { + var created = await client.Indices.CreateAsync(indexName, index => index + .Settings(settings => settings.Analysis(analysis => analysis.Analyzers(analyzers => analyzers + .Custom("lowerkeyword", analyzer => analyzer.Filter("lowercase").Tokenizer("keyword"))))) + .Mappings(mapping => mapping.Properties(properties => properties.Keyword("legacy_field"))), TestCancellationToken); + Assert.True(created.IsValidResponse, created.DebugInformation); + + var migration = new AddEventEnvironment(configuration, GetService()); + var context = new MigrationContext(EmptyLock.Empty, _logger, TestCancellationToken); + await migration.RunAsync(context); + await migration.RunAsync(context); + + var mapping = await client.Indices.GetMappingAsync(indexName, TestCancellationToken); + Assert.True(mapping.IsValidResponse, mapping.DebugInformation); + var properties = Assert.Single(mapping.Mappings).Value.Mappings.Properties!; + var environment = Assert.IsType(properties["environment"]); + Assert.Equal("lowerkeyword", environment.Analyzer); + Assert.IsType(environment.Fields!["keyword"]); + Assert.IsType(properties["legacy_field"]); + } + finally + { + await client.Indices.DeleteAsync(indexName, TestCancellationToken); + } + } +} diff --git a/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs b/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs index 622190eec1..c7239d330d 100644 --- a/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs @@ -6,6 +6,14 @@ namespace Exceptionless.Tests.Migrations; public sealed class MigrationRegistrationTests : TestWithServices { + [Fact] + public void MigrationRegistration_EventEnvironment_IsVersionedAndResumable() + { + var migration = Assert.Single(GetService>().OfType().DistinctBy(migration => migration.GetType())); + Assert.Equal(MigrationType.VersionedAndResumable, migration.MigrationType); + Assert.Equal(10, migration.Version); + } + public MigrationRegistrationTests(ITestOutputHelper output) : base(output) { } diff --git a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs index 18afb37434..9f44643e8f 100644 --- a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs +++ b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs @@ -85,6 +85,36 @@ public Task CreateAutoSessionAsync() return CreateAutoSessionInternalAsync(DateTimeOffset.Now); } + [Fact] + public async Task RunAsync_SameUserInDifferentEnvironments_SharesStackAndSeparatesAutomaticSessions() + { + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + var project = _projectData.GenerateSampleProject(); + var production = GenerateEvent(DateTimeOffset.Now.AddMinutes(-3), "same-user"); + production.Environment = "production"; + var staging = GenerateEvent(DateTimeOffset.Now.AddMinutes(-2), "same-user"); + staging.Data![Event.KnownDataKeys.Error] = production.Data![Event.KnownDataKeys.Error]; + staging.Environment = "staging"; + var laterProduction = GenerateEvent(DateTimeOffset.Now.AddMinutes(-1), "same-user"); + laterProduction.Data![Event.KnownDataKeys.Error] = production.Data[Event.KnownDataKeys.Error]; + laterProduction.Environment = "production"; + + foreach (var ev in new[] { production, staging, laterProduction }) + { + var context = await _pipeline.RunAsync(ev, organization, project); + Assert.False(context.HasError, context.ErrorMessage); + Assert.True(context.IsProcessed); + } + + Assert.Equal(production.StackId, staging.StackId); + Assert.Equal(production.GetSessionId(), laterProduction.GetSessionId()); + Assert.NotEqual(production.GetSessionId(), staging.GetSessionId()); + await RefreshDataAsync(); + var sessions = await _eventRepository.FindAsync(query => query.FilterExpression("type:session")); + Assert.Equal(2, sessions.Total); + Assert.Equal(new[] { "production", "staging" }, sessions.Documents.Select(ev => ev.Environment).Order().ToArray()); + } + private async Task CreateAutoSessionInternalAsync(DateTimeOffset date) { var ev = GenerateEvent(date, "blake@exceptionless.io"); @@ -1043,6 +1073,7 @@ public async Task CanDiscardStackEventsBasedOnEventVersion(StackStatus expectedS Assert.NotNull(project); var ev = _eventData.GenerateEvent(organizationId: organization.Id, projectId: project.Id, type: Event.KnownTypes.Log, source: "test", occurrenceDate: DateTimeOffset.Now); + ev.Environment = "staging"; var context = await _pipeline.RunAsync(ev, organization, project); var stack = context.Stack; @@ -1061,6 +1092,7 @@ public async Task CanDiscardStackEventsBasedOnEventVersion(StackStatus expectedS await RefreshDataAsync(); ev = _eventData.GenerateEvent(organizationId: organization.Id, projectId: project.Id, type: Event.KnownTypes.Log, source: "test", occurrenceDate: DateTimeOffset.Now, semver: eventSemanticVersion); + ev.Environment = "production"; context = await _pipeline.RunAsync(ev, organization, project); stack = context.Stack; diff --git a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs index f0872829ae..6bb1bf8cd0 100644 --- a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs +++ b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs @@ -50,6 +50,15 @@ public async Task CreateFromEventAsync(string version, bool expectData) } } + [Fact] + public async Task CreateFromEventAsync_VersionTwo_PreservesDeploymentEnvironment() + { + var context = GetWebHookDataContext(WebHook.KnownVersions.Version2); + context.Event!.Environment = "production"; + var data = Assert.IsType(await _webHookData.CreateFromEventAsync(context)); + Assert.Equal("production", data.Environment); + } + [Theory] [MemberData(nameof(WebHookData))] public async Task CanCreateFromStackAsync(string version, bool expectData) diff --git a/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs new file mode 100644 index 0000000000..4ea6a1f1a3 --- /dev/null +++ b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs @@ -0,0 +1,23 @@ +using Exceptionless.Core.Repositories.Queries; +using Xunit; + +namespace Exceptionless.Tests.Search; + +public sealed class EventEnvironmentFilterTests +{ + [Theory] + [InlineData(null, null)] + [InlineData("type:error status:open", null)] + [InlineData("type:error environment:production", "environment:production")] + [InlineData("environment:production OR type:error", null)] + [InlineData("type:error (environment:production OR environment:staging)", "(environment:production OR environment:staging)")] + [InlineData("type:error (_missing_:environment OR environment:production)", "(_missing_:environment OR environment:production)")] + [InlineData("type:error _exists_:environment", "_exists_:environment")] + [InlineData("type:error NOT environment:staging", "NOT environment:staging")] + [InlineData("NOT (environment:staging OR type:error)", "NOT (environment:staging)")] + [InlineData("NOT (environment:staging AND type:error)", null)] + public async Task GetAsync_EnvironmentScope_PreservesUsersOutsideTheEventType(string? filter, string? expected) + { + Assert.Equal(expected, await EventEnvironmentFilter.GetAsync(filter)); + } +} diff --git a/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs b/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs index 5209f02d71..50f7b229a7 100644 --- a/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs +++ b/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs @@ -44,6 +44,9 @@ public PersistentEventQueryValidatorTests(ITestOutputHelper output) : base(outpu [InlineData("data.age:[* TO 10]", "idx.age-n:[* TO 10]", true, true)] [InlineData("type:404 AND data.age:(>30 AND <=40)", "type:404 AND idx.age-n:(>30 AND <=40)", true, true)] [InlineData("type:404", "type:404", true, false)] + [InlineData("environment:production", "environment:production", true, false)] + [InlineData("environment:(production OR staging)", "environment:(production OR staging)", true, false)] + [InlineData("_missing_:environment", "_missing_:environment", true, false)] [InlineData("reference:404", "reference:404", true, false)] [InlineData("organization:404", "organization:404", true, false)] [InlineData("project:404", "project:404", true, false)] @@ -97,6 +100,8 @@ public async Task CanProcessQueryAsync(string query, string expected, bool isVal [InlineData("cardinality:stack", true, false)] [InlineData("cardinality:user", true, false)] [InlineData("cardinality:type", true, false)] + [InlineData("terms:(environment~100)", true, false)] + [InlineData("cardinality:environment", true, false)] [InlineData("cardinality:source", true, true)] [InlineData("cardinality:tags", true, true)] [InlineData("cardinality:geo", true, true)] diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs new file mode 100644 index 0000000000..e77b750e16 --- /dev/null +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -0,0 +1,68 @@ +using Exceptionless.Core; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Plugins.EventParser; +using Foundatio.Serializer; +using Xunit; + +namespace Exceptionless.Tests.Serializer.Models; + +public sealed class EventEnvironmentTests : TestWithServices +{ + public EventEnvironmentTests(ITestOutputHelper output) : base(output) { } + + [Theory] + [InlineData("environment")] + [InlineData("Environment")] + [InlineData("ENVIRONMENT")] + public void Deserialize_DeploymentEnvironment_NormalizesNameAndPreservesRuntimeMetadata(string property) + { + var serializer = GetService(); + var ev = serializer.Deserialize("""{"PROPERTY":" Production ","data":{"@environment":{"machine_name":"worker-1"},"environment":{"custom":true}}}""".Replace("PROPERTY", property)); + + Assert.NotNull(ev); + Assert.Equal("production", ev.Environment); + Assert.Equal("worker-1", ev.GetEnvironmentInfo(serializer, _logger)?.MachineName); + Assert.NotNull(ev.Data?["environment"]); + string json = serializer.SerializeToString(ev)!; + Assert.Contains("\"environment\":\"production\"", json); + Assert.Equal("production", serializer.Deserialize(json)?.Environment); + } + + [Theory] + [InlineData("null")] + [InlineData("42")] + [InlineData("true")] + [InlineData("{\"name\":\"production\"}")] + [InlineData("[\"production\"]")] + [InlineData("\" \"")] + [InlineData("\"production\\ninvalid\"")] + public void ParseEvents_InvalidEnvironment_PreservesSubmissionBatch(string environment) + { + var parser = GetService(); + var events = parser.ParseEvents($$"""[{"type":"error","message":"first","environment":{{environment}}},{"type":"log","message":"second","environment":"staging"}]""", 2, null); + + Assert.NotNull(events); + Assert.Equal(2, events.Count); + Assert.Null(events[0].Environment); + Assert.Equal("first", events[0].Message); + Assert.Equal("staging", events[1].Environment); + } + + [Fact] + public void Serialize_UnspecifiedEnvironment_OmitsProperty() + { + var serializer = GetService(); + Assert.DoesNotContain("environment", serializer.SerializeToString(new Event())!); + Assert.Null(serializer.Deserialize("{}")?.Environment); + Assert.Null(new Event { Environment = new string('x', 65) }.Environment); + Assert.Equal(new string('x', 64), new Event { Environment = new string('X', 64) }.Environment); + } + + [Fact] + public void Equals_DifferentEnvironments_DistinguishesEvents() + { + Assert.NotEqual(new Event { Environment = "production", Data = null }, new Event { Environment = "staging", Data = null }); + Assert.Equal(new Event { Environment = " Production ", Data = null }, new Event { Environment = "production", Data = null }); + } +} diff --git a/tests/http/events.http b/tests/http/events.http index dda0f774f9..140b316e7a 100644 --- a/tests/http/events.http +++ b/tests/http/events.http @@ -29,6 +29,30 @@ Authorization: Bearer {{token}} GET {{apiUrl}}/events?mode=stack_frequent&limit=10 Authorization: Bearer {{token}} +### Deployment environment (available on all plans) +GET {{apiUrl}}/events?filter=environment:production&mode=summary +Authorization: Bearer {{token}} + +### Environment names for this project +GET {{apiUrl}}/projects/{{projectId}}/events/count?aggregations=terms:(environment~100) +Authorization: Bearer {{token}} + +### Historical and unspecified environments +GET {{apiUrl}}/events?filter=_missing_:environment +Authorization: Bearer {{token}} + +### Submit a staging event (same stack across environments) +POST {{apiUrl}}/events +Authorization: Bearer {{clientToken}} +Content-Type: application/json + +{ + "type": "log", + "source": "environment-example", + "message": "Deployment complete", + "environment": "staging" +} + ### @eventId = {{allEvents.response.body.$[0].id}} @stackId = {{allEvents.response.body.$[0].stack_id}} From ac99281fca116f69bd6980f9a3a61f004c30efc6 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 22:47:07 -0500 Subject: [PATCH 02/24] Keep migration seed fixture current with registered migrations --- .../MigrateSavedViewColumnsIntegrationTests.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/Exceptionless.Tests/Migrations/MigrateSavedViewColumnsIntegrationTests.cs b/tests/Exceptionless.Tests/Migrations/MigrateSavedViewColumnsIntegrationTests.cs index d4a00fc1aa..91be39c2dd 100644 --- a/tests/Exceptionless.Tests/Migrations/MigrateSavedViewColumnsIntegrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/MigrateSavedViewColumnsIntegrationTests.cs @@ -95,12 +95,16 @@ public async Task DataSeedStartupAction_OnlyRepeatableMigrationsPending_SeedsDat { // Arrange // Mark all current versioned migrations complete so this test isolates repeatable data seeding. + var latestMigration = GetService>() + .Where(migration => migration.MigrationType is MigrationType.Versioned or MigrationType.VersionedAndResumable) + .MaxBy(migration => migration.Version)!; + int latestVersion = latestMigration.Version!.Value; var migrationStateRepository = GetService(); await migrationStateRepository.AddAsync(new MigrationState { - Id = "9", - Version = 9, - MigrationType = MigrationType.VersionedAndResumable, + Id = latestVersion.ToString(), + Version = latestVersion, + MigrationType = latestMigration.MigrationType, StartedUtc = DateTime.UtcNow, CompletedUtc = DateTime.UtcNow }); From 05efc27c6123648b7ec7b251a273fffedcdd68a1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 22:52:20 -0500 Subject: [PATCH 03/24] Scope environment choices to the active saved view time range --- .../e2e/tests/event-environments.e2e.ts | 39 +++++++++++++++++++ .../src/routes/(app)/event/+page.svelte | 2 +- .../src/routes/(app)/sessions/+page.svelte | 2 +- .../src/routes/(app)/stack/+page.svelte | 2 +- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts index adcc50dae4..b8dcac268b 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts @@ -79,3 +79,42 @@ test('stack, session, and stream filters retain names with no current events', a await expect(page.getByRole('button', { name: /^Environment\s+qa,east/ })).toBeVisible(); } }); + +test('environment choices use the saved view time range', async ({ e2eScenario, page, request }) => { + const time = '[now-7d TO now]'; + for (const [route, viewType] of [ + ['event', 'events'], + ['stack', 'stacks'], + ['sessions', 'sessions'] + ]) { + const slug = `environment-time-${route}-${e2eScenario.run.slice(-20)}` + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + const name = `Environment time ${route}`; + const response = await request.post(`/api/v2/organizations/${e2eScenario.organizationId}/saved-views`, { + data: { + filter: `project:${e2eScenario.projectId}`, + name, + organization_id: e2eScenario.organizationId, + slug, + time, + view_type: viewType + }, + headers: { Authorization: `Bearer ${e2eScenario.userToken}` } + }); + expect(response.status(), await response.text()).toBe(201); + await page.goto(`/next/${route}/${slug}`); + await expect(page.getByRole('heading', { exact: true, name })).toBeVisible(); + expect(new URL(page.url()).searchParams.get('time')).toBeNull(); + await page.getByRole('button', { name: 'Manage filters' }).click(); + await page.getByPlaceholder('Search...').fill('Environment'); + const facetRequest = page.waitForRequest((request) => { + const url = new URL(request.url()); + return url.pathname.endsWith('/events/count') && url.searchParams.get('aggregations') === 'terms:(environment~100)'; + }); + await page.getByText('Environment', { exact: true }).click(); + expect(new URL((await facetRequest).url()).searchParams.get('time')).toBe(time); + await page.keyboard.press('Escape'); + } +}); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte index 58e23de9ea..0b0f9b75ea 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte @@ -979,7 +979,7 @@

{pageTitle}

- +
diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte index 6bea03fc19..8e333099c1 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/sessions/+page.svelte @@ -913,7 +913,7 @@

{pageTitle}

- +
diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte index 61fe0a3244..90269d5ac3 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte @@ -881,7 +881,7 @@

{pageTitle}

- +
From abe2060564ca5be48cfb6e30a9d3915580e73112 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 23:23:31 -0500 Subject: [PATCH 04/24] Update browser tests for the machine and runtime tab label --- .../ClientApp/e2e/support/exceptionless-journey.ts | 4 ++-- .../ClientApp/e2e/tests/session-investigation.e2e.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/e2e/support/exceptionless-journey.ts b/src/Exceptionless.Web/ClientApp/e2e/support/exceptionless-journey.ts index 5844adad6f..1a829fd7b5 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/support/exceptionless-journey.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/support/exceptionless-journey.ts @@ -135,7 +135,7 @@ export class ExceptionlessE2EJourney { await expect(this.page.getByRole('tab', { name: 'Overview' })).toBeVisible(); await expect(this.page.getByRole('tab', { name: 'Exception' })).toBeVisible(); await expect(this.page.getByRole('tab', { name: 'Request' })).toBeVisible(); - await expect(this.page.getByRole('tab', { name: 'Environment' })).toBeVisible(); + await expect(this.page.getByRole('tab', { name: 'Machine & runtime' })).toBeVisible(); await expect(this.page.getByRole('tab', { name: 'Extended Data' })).toBeVisible(); await expect(getVisibleRow(this.page, new RegExp(`^Reference\\s+${escapeRegExp(this.referenceId)}$`))).toBeVisible(); @@ -152,7 +152,7 @@ export class ExceptionlessE2EJourney { await expect(getVisibleRow(this.page, 'URL', '/e2e/onboarding')).toBeVisible(); await expect(getVisibleRow(this.page, 'User Agent', 'Exceptionless Playwright E2E')).toBeVisible(); - await this.page.getByRole('tab', { name: 'Environment' }).click(); + await this.page.getByRole('tab', { name: 'Machine & runtime' }).click(); await expect(getVisibleRow(this.page, 'Machine Name', 'playwright-runner')).toBeVisible(); await expect(getVisibleRow(this.page, 'Process Name', 'e2e-tests')).toBeVisible(); diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/session-investigation.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/session-investigation.e2e.ts index 09f93c49b7..d25b850e51 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/session-investigation.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/session-investigation.e2e.ts @@ -115,7 +115,7 @@ test('operator can find and inspect a user session', async ({ e2eApi, e2eScenari await page.getByRole('tab', { name: 'Exception' }).click(); await expectWrappedMessageWithoutMeaningfulOverflow(); - await page.getByRole('tab', { name: 'Environment' }).click(); + await page.getByRole('tab', { name: 'Machine & runtime' }).click(); const machineNameRow = activePanel().getByRole('row').filter({ hasText: 'Machine Name' }); await expect(machineNameRow).toBeVisible(); await expect From dd1470b9fa6c6eaf856a499500774fa70e856ef1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 9 Sep 2026 23:39:13 -0500 Subject: [PATCH 05/24] Support environment on GET event submissions --- .../Api/Endpoints/EventEndpoints.cs | 1 + .../Api/Handlers/EventHandler.cs | 3 + .../Exceptionless.Tests/Api/Data/openapi.json | 64 +++++++++++++++++++ .../EventEndpointTests.Environment.cs | 32 +++++++++- .../Api/OpenApiSnapshotTests.cs | 15 +++++ .../Search/EventEnvironmentFilterTests.cs | 2 + tests/http/events.http | 2 +- 7 files changed, 116 insertions(+), 3 deletions(-) diff --git a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs index 05ad674605..86f75d2602 100644 --- a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs @@ -910,6 +910,7 @@ internal static class EventEndpointHelpers ///
public static readonly List SubmitGetAdditionalParameters = [ + new("environment", "query", Description: "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified."), new("source", "query", Description: "The event source (ie. machine name, log name, feature name)."), new("message", "query", Description: "The event message."), new("reference", "query", Description: "An optional identifier to be used for referencing this event instance at a later time."), diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 02da7a94e9..fc0b87a0d2 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -472,6 +472,9 @@ public async Task Handle(SubmitEventByGet message) case "source": ev.Source = kvp.Value.FirstOrDefault(); break; + case "environment": + ev.Environment = kvp.Value.FirstOrDefault(); + break; case "message": ev.Message = kvp.Value.FirstOrDefault(); break; diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index 79e4f67045..e2ff95abcf 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -530,6 +530,14 @@ "Event" ], "parameters": [ + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", @@ -693,6 +701,14 @@ "type": "string" } }, + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", @@ -856,6 +872,14 @@ "type": "string" } }, + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", @@ -1028,6 +1052,14 @@ "type": "string" } }, + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", @@ -10645,6 +10677,14 @@ "type": "string" } }, + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", @@ -10810,6 +10850,14 @@ "type": "string" } }, + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", @@ -10982,6 +11030,14 @@ "type": "string" } }, + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", @@ -11157,6 +11213,14 @@ "type": "string" } }, + { + "name": "environment", + "in": "query", + "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "schema": { + "type": "string" + } + }, { "name": "source", "in": "query", diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index c418fe3c4b..d81ef62eb9 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -1,7 +1,9 @@ -using Exceptionless.Core.Models; using Exceptionless.Core.Extensions; +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; using Exceptionless.Tests.Extensions; using Exceptionless.Tests.Utility; +using Foundatio.Jobs; using Foundatio.Repositories.Models; using Xunit; @@ -9,6 +11,30 @@ namespace Exceptionless.Tests.Api.Endpoints; public partial class EventEndpointTests { + [Theory] + [InlineData(" Production ", "production")] + [InlineData("preview-42", "preview-42")] + [InlineData(" ", null)] + [InlineData("bad\nenvironment", null)] + [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null)] + public async Task GetSubmitEvent_EnvironmentParameter_PersistsNormalizedEnvironment(string environment, string? expected) + { + await SendRequestAsync(request => request + .AsTestOrganizationClientUser().AppendPaths("events", "submit") + .QueryString("message", "GET environment submission") + .QueryString("reference", "get-environment-reference") + .QueryString("environment", environment).StatusCodeShouldBeOk()); + + await GetService().RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + var ev = Assert.Single((await _eventRepository.GetAllAsync()).Documents, + item => item.ReferenceId == "get-environment-reference"); + Assert.Equal(expected, ev.Environment); + Assert.NotNull(ev.Data); + Assert.False(ev.Data.ContainsKey("environment")); + } + [Fact] public async Task GetStacks_EnvironmentFilter_ScopesUserCountsAndTheirCache() { @@ -22,9 +48,11 @@ await CreateDataAsync(data => } data.Event().FreeProject().Type(Event.KnownTypes.Error).Stack(first).Mutate(ev => { ev.Environment = "staging"; ev.SetUserIdentity("staging-user"); }); data.Event().FreeProject().Type(Event.KnownTypes.Log).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity("unaffected-production-user"); }); + data.Event().FreeProject().Type(Event.KnownTypes.Error).Stack(first).Mutate(ev => { ev.Environment = "development"; ev.SetUserIdentity("development-user"); }); + data.Event().FreeProject().Type(Event.KnownTypes.Log).Mutate(ev => { ev.Environment = "development"; ev.SetUserIdentity("unaffected-development-user"); }); }); - foreach (var (filter, expected, totalUsers) in new[] { ("environment:production", 12, 13), ("environment:staging", 1, 1), ("environment:production", 12, 13), ("", 13, 14) }) + foreach (var (filter, expected, totalUsers) in new[] { ("environment:production", 12, 13), ("environment:staging", 1, 1), ("environment:production", 12, 13), ("environment:(production OR staging)", 13, 14), ("", 14, 16) }) { var stacks = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") diff --git a/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs b/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs index 4db2a0e1af..7992bd67f4 100644 --- a/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs +++ b/tests/Exceptionless.Tests/Api/OpenApiSnapshotTests.cs @@ -81,6 +81,21 @@ public async Task GetOpenApiJson_Default_ContainsExpectedRoutesOperationsAndResp .GetProperty("application/problem+json") .GetProperty("schema"); Assert.Equal("#/components/schemas/HttpValidationProblemDetails", validationProblemSchema.GetProperty("$ref").GetString()); + + foreach (string route in new[] + { + "/api/v2/events/submit", + "/api/v2/events/submit/{type}", + "/api/v2/projects/{projectId}/events/submit", + "/api/v2/projects/{projectId}/events/submit/{type}" + }) + { + var environmentParameter = Assert.Single(paths.GetProperty(route).GetProperty("get").GetProperty("parameters").EnumerateArray(), + parameter => parameter.GetProperty("name").GetString() == "environment"); + Assert.Equal("query", environmentParameter.GetProperty("in").GetString()); + Assert.Equal("string", environmentParameter.GetProperty("schema").GetProperty("type").GetString()); + Assert.False(environmentParameter.TryGetProperty("required", out var required) && required.GetBoolean()); + } } [Fact] diff --git a/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs index 4ea6a1f1a3..382e48f97b 100644 --- a/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs +++ b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs @@ -9,6 +9,8 @@ public sealed class EventEnvironmentFilterTests [InlineData(null, null)] [InlineData("type:error status:open", null)] [InlineData("type:error environment:production", "environment:production")] + [InlineData("type:error environment:(production OR staging)", "environment:(production OR staging)")] + [InlineData("type:error NOT environment:(production OR staging)", "NOT environment:(production OR staging)")] [InlineData("environment:production OR type:error", null)] [InlineData("type:error (environment:production OR environment:staging)", "(environment:production OR environment:staging)")] [InlineData("type:error (_missing_:environment OR environment:production)", "(_missing_:environment OR environment:production)")] diff --git a/tests/http/events.http b/tests/http/events.http index 140b316e7a..1d658fe6fb 100644 --- a/tests/http/events.http +++ b/tests/http/events.http @@ -152,7 +152,7 @@ Content-Type: application/json } ### GET Submit Random Parameters -GET {{apiUrl}}/events/submit?access_token={{clientToken}}&foo=bar&edit=&spam=eggs=ham&tags=blue&tags=red&message=foo +GET {{apiUrl}}/events/submit?access_token={{clientToken}}&environment=production&foo=bar&edit=&spam=eggs=ham&tags=blue&tags=red&message=foo ### Raygun Post POST {{apiUrl}}/events?access_token={{clientToken}} From cc1cfc152f5699aec89f45ddfaf20429e0eb5319 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 10 Sep 2026 00:09:54 -0500 Subject: [PATCH 06/24] Preserve supplied environment casing --- docs/docs/environments.md | 2 +- .../Migrations/010_AddEventEnvironment.cs | 4 ++-- src/Exceptionless.Core/Models/Event.cs | 2 +- .../EventProcessor/Default/70_SessionPlugin.cs | 2 +- .../Configuration/Indexes/EventIndex.cs | 3 ++- .../Api/Endpoints/EventEndpoints.cs | 2 +- tests/Exceptionless.Tests/Api/Data/openapi.json | 16 ++++++++-------- .../Endpoints/EventEndpointTests.Environment.cs | 14 ++++++++------ .../AddEventEnvironmentMigrationTests.cs | 2 +- .../Pipeline/EventPipelineTests.cs | 8 +++++--- .../Serializer/Models/EventEnvironmentTests.cs | 13 +++++++------ tests/http/events.http | 6 +++--- 12 files changed, 40 insertions(+), 34 deletions(-) diff --git a/docs/docs/environments.md b/docs/docs/environments.md index b73b79a5c5..0cc418d0d3 100644 --- a/docs/docs/environments.md +++ b/docs/docs/environments.md @@ -40,7 +40,7 @@ For direct API submissions: } ``` -Names are trimmed and lowercased, with a maximum of 64 characters. Custom names such as `qa-west` are supported. Empty, oversized, control-character, and non-string values are treated as unspecified. Historical events and older clients without this field remain unspecified; they are never assumed to be production. +Names are trimmed, with a maximum of 64 characters. Events preserve the supplied casing: `" Production "` is stored and returned as `"Production"`. Filtering is case-insensitive, and aggregation keys are normalized to lowercase, so `Production` and `production` share one environment bucket. Custom names such as `qa-west` are supported. Empty, oversized, control-character, and non-string values are treated as unspecified. Historical events and older clients without this field remain unspecified; they are never assumed to be production. ## Filter your data diff --git a/src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs b/src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs index ed0a58b8f4..ca0582251b 100644 --- a/src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs +++ b/src/Exceptionless.Core/Migrations/010_AddEventEnvironment.cs @@ -1,7 +1,6 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; -using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Migrations; using Microsoft.Extensions.Logging; @@ -26,7 +25,8 @@ public override async Task RunAsync(MigrationContext context) .AllowNoIndices(true) .IgnoreUnavailable(true) .Properties(properties => properties.Text(ev => ev.Environment, - text => text.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField())), context.CancellationToken); + text => text.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER) + .Fields(fields => fields.Keyword("keyword", keyword => keyword.Normalizer("lowercase"))))), context.CancellationToken); _logger.LogRequest(response); if (!response.IsValidResponse) { diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index defefeddd2..20e1560b4d 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -27,7 +27,7 @@ public string? Environment string? name = value?.Trim(); _environment = String.IsNullOrEmpty(name) || name.Length > 64 || name.Any(Char.IsControl) ? null - : name.ToLowerInvariant(); + : name; } } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs index b13b920bed..380f6fbaef 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs @@ -125,7 +125,7 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts) { var identityGroups = contexts .OrderBy(c => c.Event.Date) - .GroupBy(c => (c.Event.ProjectId, c.Event.Environment, Identity: c.Event.GetUserIdentity(_serializer, _logger)?.Identity)); + .GroupBy(c => (c.Event.ProjectId, Environment: c.Event.Environment?.ToLowerInvariant(), Identity: c.Event.GetUserIdentity(_serializer, _logger)?.Identity)); foreach (var identityGroup in identityGroups) { diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs index f75c9b0b07..b60ff5e842 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs @@ -70,7 +70,8 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor e.ReferenceId) .FieldAlias(Alias.ReferenceId, a => a.Path(f => f.ReferenceId)) .Text(e => e.Type, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Text(e => e.Environment, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Text(e => e.Environment, t => t.Analyzer(LOWER_KEYWORD_ANALYZER) + .Fields(fields => fields.Keyword("keyword", keyword => keyword.Normalizer("lowercase")))) .Text(e => e.Source, t => t.Analyzer(STANDARDPLUS_ANALYZER).SearchAnalyzer(WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) .Date(e => e.Date) .Text(e => e.Message) diff --git a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs index 86f75d2602..26ab7a1726 100644 --- a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs @@ -910,7 +910,7 @@ internal static class EventEndpointHelpers /// public static readonly List SubmitGetAdditionalParameters = [ - new("environment", "query", Description: "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified."), + new("environment", "query", Description: "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified."), new("source", "query", Description: "The event source (ie. machine name, log name, feature name)."), new("message", "query", Description: "The event message."), new("reference", "query", Description: "An optional identifier to be used for referencing this event instance at a later time."), diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index e2ff95abcf..7d1bbcd5d8 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -533,7 +533,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } @@ -704,7 +704,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } @@ -875,7 +875,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } @@ -1055,7 +1055,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } @@ -10680,7 +10680,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } @@ -10853,7 +10853,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } @@ -11033,7 +11033,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } @@ -11216,7 +11216,7 @@ { "name": "environment", "in": "query", - "description": "The deployment environment (for example, production or staging). Names are trimmed, lowercased, and limited to 64 characters; missing or invalid names remain unspecified.", + "description": "The deployment environment (for example, production or staging). Names are trimmed and limited to 64 characters, preserving the supplied casing; missing or invalid names remain unspecified.", "schema": { "type": "string" } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index d81ef62eb9..cd095be959 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -12,12 +12,12 @@ namespace Exceptionless.Tests.Api.Endpoints; public partial class EventEndpointTests { [Theory] - [InlineData(" Production ", "production")] + [InlineData(" Production ", "Production")] [InlineData("preview-42", "preview-42")] [InlineData(" ", null)] [InlineData("bad\nenvironment", null)] [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null)] - public async Task GetSubmitEvent_EnvironmentParameter_PersistsNormalizedEnvironment(string environment, string? expected) + public async Task GetSubmitEvent_EnvironmentParameter_TrimsAndPreservesCasing(string environment, string? expected) { await SendRequestAsync(request => request .AsTestOrganizationClientUser().AppendPaths("events", "submit") @@ -70,6 +70,7 @@ public async Task GetEvents_EnvironmentOnFreePlan_FiltersEventsSummariesAndStack await CreateDataAsync(data => { var production = data.Event().FreeProject().Mutate(ev => ev.Environment = " Production "); + data.Event().FreeProject().Stack(production).Mutate(ev => ev.Environment = "production"); data.Event().FreeProject().Stack(production).Mutate(ev => ev.Environment = "staging"); data.Event().FreeProject().Stack(production); data.Event().TestProject().Mutate(ev => ev.Environment = "private-environment"); @@ -78,17 +79,17 @@ await CreateDataAsync(data => var events = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events") .QueryString("filter", "environment:PRODUCTION").StatusCodeShouldBeOk()); - Assert.Equal("production", Assert.Single(Assert.IsType>(events)).Environment); + Assert.Equal(new[] { "Production", "production" }, Assert.IsType>(events).Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); var summaries = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "summary") .QueryString("filter", "environment:production").StatusCodeShouldBeOk()); - Assert.Equal("production", Assert.Single(Assert.IsType>(summaries)).Environment); + Assert.Equal(new[] { "Production", "production" }, Assert.IsType>(summaries).Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); var stacks = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") .QueryString("filter", "environment:production").StatusCodeShouldBeOk()); - Assert.Equal(1, Assert.Single(Assert.IsType>(stacks)).Total); + Assert.Equal(2, Assert.Single(Assert.IsType>(stacks)).Total); var missing = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events") @@ -99,7 +100,8 @@ await CreateDataAsync(data => .AsFreeOrganizationUser().AppendPaths("events", "count") .QueryString("aggregations", "terms:(environment~100)").StatusCodeShouldBeOk()); Assert.NotNull(count); - Assert.Equal(3, count.Total); + Assert.Equal(4, count.Total); Assert.Equal(new[] { "production", "staging" }, count.Aggregations.Terms("terms_environment")!.Buckets.Select(bucket => bucket.Key).Order().ToArray()); + Assert.Equal(2, count.Aggregations.Terms("terms_environment")!.Buckets.Single(bucket => bucket.Key == "production").Total); } } diff --git a/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs index d020050c90..828b89998f 100644 --- a/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs @@ -36,7 +36,7 @@ public async Task RunAsync_LegacyDailyIndex_AddsMappingAndCanRunAgain() var properties = Assert.Single(mapping.Mappings).Value.Mappings.Properties!; var environment = Assert.IsType(properties["environment"]); Assert.Equal("lowerkeyword", environment.Analyzer); - Assert.IsType(environment.Fields!["keyword"]); + Assert.Equal("lowercase", Assert.IsType(environment.Fields!["keyword"]).Normalizer); Assert.IsType(properties["legacy_field"]); } finally diff --git a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs index 9f44643e8f..1fcc10552b 100644 --- a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs +++ b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs @@ -91,13 +91,13 @@ public async Task RunAsync_SameUserInDifferentEnvironments_SharesStackAndSeparat var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); var project = _projectData.GenerateSampleProject(); var production = GenerateEvent(DateTimeOffset.Now.AddMinutes(-3), "same-user"); - production.Environment = "production"; + production.Environment = " Production "; var staging = GenerateEvent(DateTimeOffset.Now.AddMinutes(-2), "same-user"); staging.Data![Event.KnownDataKeys.Error] = production.Data![Event.KnownDataKeys.Error]; staging.Environment = "staging"; var laterProduction = GenerateEvent(DateTimeOffset.Now.AddMinutes(-1), "same-user"); laterProduction.Data![Event.KnownDataKeys.Error] = production.Data[Event.KnownDataKeys.Error]; - laterProduction.Environment = "production"; + laterProduction.Environment = "PRODUCTION"; foreach (var ev in new[] { production, staging, laterProduction }) { @@ -109,10 +109,12 @@ public async Task RunAsync_SameUserInDifferentEnvironments_SharesStackAndSeparat Assert.Equal(production.StackId, staging.StackId); Assert.Equal(production.GetSessionId(), laterProduction.GetSessionId()); Assert.NotEqual(production.GetSessionId(), staging.GetSessionId()); + Assert.Equal("Production", production.Environment); + Assert.Equal("PRODUCTION", laterProduction.Environment); await RefreshDataAsync(); var sessions = await _eventRepository.FindAsync(query => query.FilterExpression("type:session")); Assert.Equal(2, sessions.Total); - Assert.Equal(new[] { "production", "staging" }, sessions.Documents.Select(ev => ev.Environment).Order().ToArray()); + Assert.Equal(new[] { "Production", "staging" }, sessions.Documents.Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); } private async Task CreateAutoSessionInternalAsync(DateTimeOffset date) diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index e77b750e16..2446f2e73c 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -15,18 +15,18 @@ public EventEnvironmentTests(ITestOutputHelper output) : base(output) { } [InlineData("environment")] [InlineData("Environment")] [InlineData("ENVIRONMENT")] - public void Deserialize_DeploymentEnvironment_NormalizesNameAndPreservesRuntimeMetadata(string property) + public void Deserialize_DeploymentEnvironment_TrimsNameAndPreservesCasingAndRuntimeMetadata(string property) { var serializer = GetService(); var ev = serializer.Deserialize("""{"PROPERTY":" Production ","data":{"@environment":{"machine_name":"worker-1"},"environment":{"custom":true}}}""".Replace("PROPERTY", property)); Assert.NotNull(ev); - Assert.Equal("production", ev.Environment); + Assert.Equal("Production", ev.Environment); Assert.Equal("worker-1", ev.GetEnvironmentInfo(serializer, _logger)?.MachineName); Assert.NotNull(ev.Data?["environment"]); string json = serializer.SerializeToString(ev)!; - Assert.Contains("\"environment\":\"production\"", json); - Assert.Equal("production", serializer.Deserialize(json)?.Environment); + Assert.Contains("\"environment\":\"Production\"", json); + Assert.Equal("Production", serializer.Deserialize(json)?.Environment); } [Theory] @@ -56,13 +56,14 @@ public void Serialize_UnspecifiedEnvironment_OmitsProperty() Assert.DoesNotContain("environment", serializer.SerializeToString(new Event())!); Assert.Null(serializer.Deserialize("{}")?.Environment); Assert.Null(new Event { Environment = new string('x', 65) }.Environment); - Assert.Equal(new string('x', 64), new Event { Environment = new string('X', 64) }.Environment); + Assert.Equal(new string('X', 64), new Event { Environment = new string('X', 64) }.Environment); } [Fact] public void Equals_DifferentEnvironments_DistinguishesEvents() { Assert.NotEqual(new Event { Environment = "production", Data = null }, new Event { Environment = "staging", Data = null }); - Assert.Equal(new Event { Environment = " Production ", Data = null }, new Event { Environment = "production", Data = null }); + Assert.Equal(new Event { Environment = " Production ", Data = null }, new Event { Environment = "Production", Data = null }); + Assert.NotEqual(new Event { Environment = "Production", Data = null }, new Event { Environment = "production", Data = null }); } } diff --git a/tests/http/events.http b/tests/http/events.http index 1d658fe6fb..1d82bd07a7 100644 --- a/tests/http/events.http +++ b/tests/http/events.http @@ -41,7 +41,7 @@ Authorization: Bearer {{token}} GET {{apiUrl}}/events?filter=_missing_:environment Authorization: Bearer {{token}} -### Submit a staging event (same stack across environments) +### Submit a staging event (preserves casing; same stack across environments) POST {{apiUrl}}/events Authorization: Bearer {{clientToken}} Content-Type: application/json @@ -50,7 +50,7 @@ Content-Type: application/json "type": "log", "source": "environment-example", "message": "Deployment complete", - "environment": "staging" + "environment": "Staging" } ### @@ -152,7 +152,7 @@ Content-Type: application/json } ### GET Submit Random Parameters -GET {{apiUrl}}/events/submit?access_token={{clientToken}}&environment=production&foo=bar&edit=&spam=eggs=ham&tags=blue&tags=red&message=foo +GET {{apiUrl}}/events/submit?access_token={{clientToken}}&environment=Production&foo=bar&edit=&spam=eggs=ham&tags=blue&tags=red&message=foo ### Raygun Post POST {{apiUrl}}/events?access_token={{clientToken}} From 438c801d107a012b5cab48bb56a17b10ad5e1ac3 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 12 Sep 2026 17:18:41 -0500 Subject: [PATCH 07/24] Preserve legacy environment custom data --- docs/docs/environments.md | 2 +- src/Exceptionless.Core/Models/Event.cs | 1 - .../EventEnvironmentConverter.cs | 58 ++++++++++++++++--- .../JsonSerializerOptionsExtensions.cs | 1 + .../EventEndpointTests.Environment.cs | 26 +++++++++ .../Models/EventEnvironmentTests.cs | 37 ++++++++++++ 6 files changed, 116 insertions(+), 9 deletions(-) diff --git a/docs/docs/environments.md b/docs/docs/environments.md index 0cc418d0d3..dddf9a84b9 100644 --- a/docs/docs/environments.md +++ b/docs/docs/environments.md @@ -66,4 +66,4 @@ Use [fixed in version](/docs/versioning/) when environments run different releas ## Rollout -Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. +Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. Legacy non-string root `environment` values are preserved as custom event data and leave the deployment environment unspecified. diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 20e1560b4d..4eea605131 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -18,7 +18,6 @@ public class Event : IData, IJsonOnDeserialized /// Missing or invalid names remain unspecified. Machine and runtime information is stored separately in data.@environment. /// [StringLength(64)] - [JsonConverter(typeof(EventEnvironmentConverter))] public string? Environment { get => _environment; diff --git a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs index 9a6ff454d6..0b1c29545e 100644 --- a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs +++ b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs @@ -1,26 +1,70 @@ +using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using Exceptionless.Core.Models; namespace Exceptionless.Core.Serialization; /// -/// Ignores malformed optional deployment metadata without rejecting an event or its submission batch. +/// Reads deployment names while preserving non-string legacy root values as custom event data. /// -public sealed class EventEnvironmentConverter : JsonConverter +public sealed class EventEnvironmentConverter : JsonConverter { - public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + public override bool HandleNull => true; + + public static void ConfigureProperty(JsonTypeInfo typeInfo) + { + if (typeInfo.Kind != JsonTypeInfoKind.Object || !typeof(Event).IsAssignableFrom(typeInfo.Type)) + { + return; + } + + for (int i = 0; i < typeInfo.Properties.Count; i++) + { + var property = typeInfo.Properties[i]; + if (property.AttributeProvider is not PropertyInfo { Name: nameof(Event.Environment) }) + { + continue; + } + + var environment = typeInfo.CreateJsonPropertyInfo(typeof(object), property.Name); + environment.AttributeProvider = property.AttributeProvider; + environment.CustomConverter = new EventEnvironmentConverter(); + environment.Get = property.Get; + environment.Set = (instance, value) => + { + if (value is JsonElement legacyValue) + { + var ev = (Event)instance; + ev.Environment = null; + // Merge after all properties have been read so a later "data" property cannot overwrite it. + ev.ExtensionData ??= []; + ev.ExtensionData[property.Name] = legacyValue; + } + else + { + property.Set!(instance, value); + } + }; + typeInfo.Properties[i] = environment; + return; + } + } + + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.String) { return reader.GetString(); } - reader.Skip(); - return null; + using var document = JsonDocument.ParseValue(ref reader); + return document.RootElement.Clone(); } - public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) { - writer.WriteStringValue(value); + writer.WriteStringValue((string?)value); } } diff --git a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs index cd27fb785e..6bc18ffda5 100644 --- a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs +++ b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs @@ -67,6 +67,7 @@ private static JsonSerializerOptions ConfigureExceptionlessDefaults(JsonSerializ options.RespectNullableAnnotations = true; var resolver = new DefaultJsonTypeInfoResolver(); + resolver.Modifiers.Add(EventEnvironmentConverter.ConfigureProperty); if (skipEmptyCollections) resolver.Modifiers.Add(EmptyCollectionModifier.SkipEmptyCollections); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index cd095be959..c161c59e82 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; @@ -11,6 +12,31 @@ namespace Exceptionless.Tests.Api.Endpoints; public partial class EventEndpointTests { + [Fact] + public async Task PostEvent_LegacyRootEnvironment_PreservesCustomDataThroughStorageAndApi() + { + await SendRequestAsync(request => request.Post() + .AsTestOrganizationClientUser().AppendPath("events") + .Content(new + { + type = "log", + message = "Legacy environment metadata", + reference_id = "legacy-environment-reference", + environment = new { region = "west" } + }).StatusCodeShouldBeAccepted()); + + await GetService().RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + var events = await SendRequestAsAsync>(request => request + .AsTestOrganizationUser().AppendPath("events") + .QueryString("filter", "reference:legacy-environment-reference").StatusCodeShouldBeOk()); + var ev = Assert.Single(Assert.IsType>(events)); + Assert.Null(ev.Environment); + Assert.NotNull(ev.Data); + Assert.Equal("west", JsonSerializer.SerializeToElement(ev.Data["environment"]).GetProperty("region").GetString()); + } + [Theory] [InlineData(" Production ", "Production")] [InlineData("preview-42", "preview-42")] diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index 2446f2e73c..557c1ab877 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using Exceptionless.Core; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; @@ -59,6 +60,42 @@ public void Serialize_UnspecifiedEnvironment_OmitsProperty() Assert.Equal(new string('X', 64), new Event { Environment = new string('X', 64) }.Environment); } + [Theory] + [InlineData("null")] + [InlineData("42")] + [InlineData("true")] + [InlineData("{\"region\":\"west\"}")] + [InlineData("[\"production\",\"staging\"]")] + public void Deserialize_LegacyRootEnvironment_PreservesCustomDataRegardlessOfPropertyOrder(string environment) + { + var serializer = GetService(); + using var expected = JsonDocument.Parse(environment); + + foreach (var (environmentFirst, duplicateKey) in new[] { (true, false), (false, false), (true, true), (false, true) }) + { + string environmentProperty = $"\"environment\":{environment}"; + string dataProperty = duplicateKey + ? "\"data\":{\"environment\":\"nested\",\"kept\":true}" + : "\"data\":{\"kept\":true}"; + string json = environmentFirst + ? $"{{{environmentProperty},{dataProperty}}}" + : $"{{{dataProperty},{environmentProperty}}}"; + var ev = serializer.Deserialize(json); + + Assert.NotNull(ev); + Assert.Null(ev.Environment); + using var result = JsonDocument.Parse(serializer.SerializeToString(ev)!); + var data = result.RootElement.GetProperty("data"); + Assert.True(data.GetProperty("kept").GetBoolean()); + Assert.True(JsonElement.DeepEquals(expected.RootElement, data.GetProperty(duplicateKey ? "environment1" : "environment"))); + if (duplicateKey) + { + Assert.Equal("nested", data.GetProperty("environment").GetString()); + } + Assert.False(result.RootElement.TryGetProperty("environment", out _)); + } + } + [Fact] public void Equals_DifferentEnvironments_DistinguishesEvents() { From 62139384434553a590e3582a028d87eaeaca5f29 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 5 Oct 2026 13:13:27 -0500 Subject: [PATCH 08/24] Preserve legacy environment strings during event ingestion --- docs/docs/environments.md | 2 +- .../Default/JsonEventParserPlugin.cs | 11 +++++- .../EventEnvironmentConverter.cs | 20 ++++++++-- .../Api/Handlers/EventHandler.cs | 21 +++++++++- .../e2e/tests/event-environments.e2e.ts | 6 +-- .../EventEndpointTests.Environment.cs | 19 +++++++--- .../Models/EventEnvironmentTests.cs | 38 +++++++++++++++++++ 7 files changed, 100 insertions(+), 17 deletions(-) diff --git a/docs/docs/environments.md b/docs/docs/environments.md index dddf9a84b9..393498e867 100644 --- a/docs/docs/environments.md +++ b/docs/docs/environments.md @@ -66,4 +66,4 @@ Use [fixed in version](/docs/versioning/) when environments run different releas ## Rollout -Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. Legacy non-string root `environment` values are preserved as custom event data and leave the deployment environment unspecified. +Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. Root `environment` values are also preserved as legacy custom event data, including the original whitespace in strings. Non-string values leave the deployment environment unspecified. If `data.environment` already exists, the root value uses the next available key (`environment1`, `environment2`, and so on). diff --git a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs index 327c507c6b..53ee3ec476 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs @@ -1,7 +1,9 @@ using System.Text.Json; +using System.Text.Json.Serialization.Metadata; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Serialization; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventParser; @@ -16,7 +18,14 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptio // Create lenient parsing options — inbound events from older SDK clients may omit // non-nullable properties. We must not reject structurally valid events; the pipeline // handles missing/null values gracefully downstream. - _jsonOptions = new JsonSerializerOptions(jsonOptions) { RespectNullableAnnotations = false }; + _jsonOptions = new JsonSerializerOptions(jsonOptions) + { + RespectNullableAnnotations = false, + // Preserve the original root value once, at ingestion. Repeating this during + // storage or API deserialization would accumulate duplicate custom data. + TypeInfoResolver = (jsonOptions.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver()) + .WithAddedModifier(EventEnvironmentConverter.ConfigureIngestionProperty) + }; } public List? ParseEvents(string input, int apiVersion, string? userAgent) diff --git a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs index 0b1c29545e..e820be10fb 100644 --- a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs +++ b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs @@ -7,13 +7,25 @@ namespace Exceptionless.Core.Serialization; /// -/// Reads deployment names while preserving non-string legacy root values as custom event data. +/// Reads deployment names while preserving legacy root values as custom event data. /// public sealed class EventEnvironmentConverter : JsonConverter { + private bool PreserveStringValues { get; init; } + public override bool HandleNull => true; public static void ConfigureProperty(JsonTypeInfo typeInfo) + { + ConfigureProperty(typeInfo, preserveStringValues: false); + } + + public static void ConfigureIngestionProperty(JsonTypeInfo typeInfo) + { + ConfigureProperty(typeInfo, preserveStringValues: true); + } + + private static void ConfigureProperty(JsonTypeInfo typeInfo, bool preserveStringValues) { if (typeInfo.Kind != JsonTypeInfoKind.Object || !typeof(Event).IsAssignableFrom(typeInfo.Type)) { @@ -30,14 +42,14 @@ public static void ConfigureProperty(JsonTypeInfo typeInfo) var environment = typeInfo.CreateJsonPropertyInfo(typeof(object), property.Name); environment.AttributeProvider = property.AttributeProvider; - environment.CustomConverter = new EventEnvironmentConverter(); + environment.CustomConverter = new EventEnvironmentConverter { PreserveStringValues = preserveStringValues }; environment.Get = property.Get; environment.Set = (instance, value) => { if (value is JsonElement legacyValue) { var ev = (Event)instance; - ev.Environment = null; + ev.Environment = legacyValue.ValueKind == JsonValueKind.String ? legacyValue.GetString() : null; // Merge after all properties have been read so a later "data" property cannot overwrite it. ev.ExtensionData ??= []; ev.ExtensionData[property.Name] = legacyValue; @@ -54,7 +66,7 @@ public static void ConfigureProperty(JsonTypeInfo typeInfo) public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - if (reader.TokenType == JsonTokenType.String) + if (reader.TokenType == JsonTokenType.String && !PreserveStringValues) { return reader.GetString(); } diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index fc0b87a0d2..3076055f07 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Text.Json.Nodes; using Exceptionless.Core; using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; @@ -460,6 +461,7 @@ public async Task Handle(SubmitEventByGet message) string? identity = null; string? identityName = null; + string? environment = null; var exclusions = project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions).ToList(); foreach (var kvp in filteredParameters) @@ -473,7 +475,12 @@ public async Task Handle(SubmitEventByGet message) ev.Source = kvp.Value.FirstOrDefault(); break; case "environment": - ev.Environment = kvp.Value.FirstOrDefault(); + if (kvp.Key.AnyWildcardMatches(exclusions, true)) + continue; + + environment = kvp.Value.FirstOrDefault(); + if (kvp.Value.Count > 1) + ev.Data![kvp.Key] = kvp.Value; break; case "message": ev.Message = kvp.Value.FirstOrDefault(); @@ -530,7 +537,17 @@ public async Task Handle(SubmitEventByGet message) charSet = contentTypeHeader.Charset.ToString(); } - using var stream = new MemoryStream(ev.GetBytes(serializer)); + byte[] eventBytes = ev.GetBytes(serializer); + if (environment is not null) + { + // Keep the original query value until ingestion, where both the normalized + // deployment name and legacy custom data are populated together. + var submission = JsonNode.Parse(eventBytes)!; + submission["environment"] = environment; + eventBytes = Encoding.UTF8.GetBytes(submission.ToJsonString()); + } + + using var stream = new MemoryStream(eventBytes); await eventPostService.EnqueueAsync(new EventPost(appOptions.EnableArchive) { ApiVersion = message.ApiVersion, diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts index b8dcac268b..e39d19b328 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts @@ -16,7 +16,7 @@ test('environment filters include unspecified events and persist in saved views' await e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId); } - await page.goto(`/next/event?project=${e2eScenario.projectId}&time=all`); + await page.goto(`/event?project=${e2eScenario.projectId}&time=all`); for (const message of messages) await expect(getVisibleText(page, message)).toBeVisible({ timeout: 30_000 }); await page.getByRole('button', { name: 'Manage filters' }).click(); @@ -66,7 +66,7 @@ test('environment filters include unspecified events and persist in saved views' test('stack, session, and stream filters retain names with no current events', async ({ e2eScenario, page }) => { for (const route of ['stack', 'sessions', 'stream']) { - await page.goto(`/next/${route}?project=${e2eScenario.projectId}`); + await page.goto(`/${route}?project=${e2eScenario.projectId}`); await page.getByRole('button', { name: 'Manage filters' }).click(); await page.getByPlaceholder('Search...').fill('Environment'); await page.getByText('Environment', { exact: true }).click(); @@ -104,7 +104,7 @@ test('environment choices use the saved view time range', async ({ e2eScenario, headers: { Authorization: `Bearer ${e2eScenario.userToken}` } }); expect(response.status(), await response.text()).toBe(201); - await page.goto(`/next/${route}/${slug}`); + await page.goto(`/${route}/${slug}`); await expect(page.getByRole('heading', { exact: true, name })).toBeVisible(); expect(new URL(page.url()).searchParams.get('time')).toBeNull(); await page.getByRole('button', { name: 'Manage filters' }).click(); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index c161c59e82..bb67440f83 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -12,8 +12,13 @@ namespace Exceptionless.Tests.Api.Endpoints; public partial class EventEndpointTests { - [Fact] - public async Task PostEvent_LegacyRootEnvironment_PreservesCustomDataThroughStorageAndApi() + [Theory] + [InlineData("{\"region\":\"west\"}", null)] + [InlineData("\"west\"", "west")] + [InlineData("\" Production \"", "Production")] + [InlineData("\" \"", null)] + [InlineData("\"bad\\nenvironment\"", null)] + public async Task PostEvent_LegacyRootEnvironment_PreservesCustomDataThroughStorageAndApi(string environment, string? expected) { await SendRequestAsync(request => request.Post() .AsTestOrganizationClientUser().AppendPath("events") @@ -22,7 +27,7 @@ await SendRequestAsync(request => request.Post() type = "log", message = "Legacy environment metadata", reference_id = "legacy-environment-reference", - environment = new { region = "west" } + environment = JsonSerializer.Deserialize(environment) }).StatusCodeShouldBeAccepted()); await GetService().RunAsync(TestCancellationToken); @@ -32,9 +37,10 @@ await SendRequestAsync(request => request.Post() .AsTestOrganizationUser().AppendPath("events") .QueryString("filter", "reference:legacy-environment-reference").StatusCodeShouldBeOk()); var ev = Assert.Single(Assert.IsType>(events)); - Assert.Null(ev.Environment); + Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); - Assert.Equal("west", JsonSerializer.SerializeToElement(ev.Data["environment"]).GetProperty("region").GetString()); + Assert.True(JsonElement.DeepEquals(JsonSerializer.Deserialize(environment), JsonSerializer.SerializeToElement(ev.Data["environment"]))); + Assert.False(ev.Data.ContainsKey("environment1")); } [Theory] @@ -58,7 +64,8 @@ await SendRequestAsync(request => request item => item.ReferenceId == "get-environment-reference"); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); - Assert.False(ev.Data.ContainsKey("environment")); + Assert.Equal(environment, ev.Data["environment"]); + Assert.False(ev.Data.ContainsKey("environment1")); } [Fact] diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index 557c1ab877..4232eb2825 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -103,4 +103,42 @@ public void Equals_DifferentEnvironments_DistinguishesEvents() Assert.Equal(new Event { Environment = " Production ", Data = null }, new Event { Environment = "Production", Data = null }); Assert.NotEqual(new Event { Environment = "Production", Data = null }, new Event { Environment = "production", Data = null }); } + + [Theory] + [InlineData("west", "west")] + [InlineData(" Production ", "Production")] + [InlineData(" ", null)] + [InlineData("bad\nenvironment", null)] + [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null)] + public void ParseEvents_LegacyStringEnvironment_PreservesOriginalDataAcrossRoundTrips(string environment, string? expected) + { + var parser = GetService(); + var serializer = GetService(); + + foreach (var (environmentFirst, duplicateKey) in new[] { (true, false), (false, false), (true, true), (false, true) }) + { + string environmentProperty = $"\"environment\":{JsonSerializer.Serialize(environment)}"; + string dataProperty = duplicateKey + ? "\"data\":{\"environment\":\"nested\",\"kept\":true}" + : "\"data\":{\"kept\":true}"; + string json = environmentFirst + ? $"{{{environmentProperty},{dataProperty}}}" + : $"{{{dataProperty},{environmentProperty}}}"; + + var ev = Assert.Single(Assert.IsType>(parser.ParseEvents(json, 2, null))); + Assert.Equal(expected, ev.Environment); + Assert.Equal(environment, ev.Data![duplicateKey ? "environment1" : "environment"]); + if (duplicateKey) + Assert.Equal("nested", ev.Data["environment"]); + + string serialized = serializer.SerializeToString(ev)!; + for (int i = 0; i < 3; i++) + { + ev = serializer.Deserialize(serialized)!; + string roundTrip = serializer.SerializeToString(ev)!; + Assert.Equal(serialized, roundTrip); + Assert.Equal(expected, ev.Environment); + } + } + } } From bd760b766e784d3a529eee7765782ae0ead3205f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 5 Oct 2026 13:23:29 -0500 Subject: [PATCH 09/24] Retain submitted environment key casing during ingestion --- docs/docs/environments.md | 2 +- src/Exceptionless.Core/Models/Event.cs | 5 +++ .../EventEnvironmentConverter.cs | 22 ++++++---- .../Api/Handlers/EventHandler.cs | 4 +- .../EventEndpointTests.Environment.cs | 41 +++++++++++-------- .../Models/EventEnvironmentTests.cs | 37 +++++++++++++++++ 6 files changed, 83 insertions(+), 28 deletions(-) diff --git a/docs/docs/environments.md b/docs/docs/environments.md index 393498e867..a36d8e56fd 100644 --- a/docs/docs/environments.md +++ b/docs/docs/environments.md @@ -66,4 +66,4 @@ Use [fixed in version](/docs/versioning/) when environments run different releas ## Rollout -Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. Root `environment` values are also preserved as legacy custom event data, including the original whitespace in strings. Non-string values leave the deployment environment unspecified. If `data.environment` already exists, the root value uses the next available key (`environment1`, `environment2`, and so on). +Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. Root `environment` values are also preserved as legacy custom event data, including submitted key casing and the original whitespace in strings. Non-string values leave the deployment environment unspecified. If `data.environment` already exists, the root value uses the next available key (`environment1`, `environment2`, and so on). diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 4eea605131..6466816e56 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -121,6 +121,11 @@ void IJsonOnDeserialized.OnDeserialized() Data ??= []; foreach (var kvp in ExtensionData) { + // Ingestion leaves deployment environments in extension data to retain the + // original key casing and value, including case-distinct legacy properties. + if (String.Equals(kvp.Key, nameof(Environment), StringComparison.OrdinalIgnoreCase)) + Environment = kvp.Value.ValueKind == JsonValueKind.String ? kvp.Value.GetString() : null; + object? value = JsonElementConverter.Convert(kvp.Value); EventDataNormalizer.Set(Data, kvp.Key, value); } diff --git a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs index e820be10fb..2d83b04b77 100644 --- a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs +++ b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs @@ -11,21 +11,19 @@ namespace Exceptionless.Core.Serialization; /// public sealed class EventEnvironmentConverter : JsonConverter { - private bool PreserveStringValues { get; init; } - public override bool HandleNull => true; public static void ConfigureProperty(JsonTypeInfo typeInfo) { - ConfigureProperty(typeInfo, preserveStringValues: false); + ConfigureProperty(typeInfo, ingestion: false); } public static void ConfigureIngestionProperty(JsonTypeInfo typeInfo) { - ConfigureProperty(typeInfo, preserveStringValues: true); + ConfigureProperty(typeInfo, ingestion: true); } - private static void ConfigureProperty(JsonTypeInfo typeInfo, bool preserveStringValues) + private static void ConfigureProperty(JsonTypeInfo typeInfo, bool ingestion) { if (typeInfo.Kind != JsonTypeInfoKind.Object || !typeof(Event).IsAssignableFrom(typeInfo.Type)) { @@ -40,16 +38,24 @@ private static void ConfigureProperty(JsonTypeInfo typeInfo, bool preserveString continue; } + if (ingestion) + { + // Let extension data retain the exact submitted names and case-distinct keys. + // Event.OnDeserialized promotes the deployment name before merging the legacy data. + typeInfo.Properties.RemoveAt(i); + return; + } + var environment = typeInfo.CreateJsonPropertyInfo(typeof(object), property.Name); environment.AttributeProvider = property.AttributeProvider; - environment.CustomConverter = new EventEnvironmentConverter { PreserveStringValues = preserveStringValues }; + environment.CustomConverter = new EventEnvironmentConverter(); environment.Get = property.Get; environment.Set = (instance, value) => { if (value is JsonElement legacyValue) { var ev = (Event)instance; - ev.Environment = legacyValue.ValueKind == JsonValueKind.String ? legacyValue.GetString() : null; + ev.Environment = null; // Merge after all properties have been read so a later "data" property cannot overwrite it. ev.ExtensionData ??= []; ev.ExtensionData[property.Name] = legacyValue; @@ -66,7 +72,7 @@ private static void ConfigureProperty(JsonTypeInfo typeInfo, bool preserveString public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - if (reader.TokenType == JsonTokenType.String && !PreserveStringValues) + if (reader.TokenType == JsonTokenType.String) { return reader.GetString(); } diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 3076055f07..6468963dd7 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -462,6 +462,7 @@ public async Task Handle(SubmitEventByGet message) string? identity = null; string? identityName = null; string? environment = null; + string? environmentPropertyName = null; var exclusions = project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions).ToList(); foreach (var kvp in filteredParameters) @@ -479,6 +480,7 @@ public async Task Handle(SubmitEventByGet message) continue; environment = kvp.Value.FirstOrDefault(); + environmentPropertyName = kvp.Key; if (kvp.Value.Count > 1) ev.Data![kvp.Key] = kvp.Value; break; @@ -543,7 +545,7 @@ public async Task Handle(SubmitEventByGet message) // Keep the original query value until ingestion, where both the normalized // deployment name and legacy custom data are populated together. var submission = JsonNode.Parse(eventBytes)!; - submission["environment"] = environment; + submission[environmentPropertyName!] = environment; eventBytes = Encoding.UTF8.GetBytes(submission.ToJsonString()); } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index bb67440f83..3e5dd44449 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -13,21 +13,23 @@ namespace Exceptionless.Tests.Api.Endpoints; public partial class EventEndpointTests { [Theory] - [InlineData("{\"region\":\"west\"}", null)] - [InlineData("\"west\"", "west")] - [InlineData("\" Production \"", "Production")] - [InlineData("\" \"", null)] - [InlineData("\"bad\\nenvironment\"", null)] - public async Task PostEvent_LegacyRootEnvironment_PreservesCustomDataThroughStorageAndApi(string environment, string? expected) + [InlineData("{\"region\":\"west\"}", null, "environment")] + [InlineData("{\"region\":\"west\"}", null, "Environment")] + [InlineData("\"west\"", "west", "environment")] + [InlineData("\" Production \"", "Production", "environment")] + [InlineData("\" Production \"", "Production", "ENVIRONMENT")] + [InlineData("\" \"", null, "environment")] + [InlineData("\"bad\\nenvironment\"", null, "environment")] + public async Task PostEvent_LegacyRootEnvironment_PreservesCustomDataThroughStorageAndApi(string environment, string? expected, string propertyName) { await SendRequestAsync(request => request.Post() .AsTestOrganizationClientUser().AppendPath("events") - .Content(new + .Content(new Dictionary { - type = "log", - message = "Legacy environment metadata", - reference_id = "legacy-environment-reference", - environment = JsonSerializer.Deserialize(environment) + ["type"] = "log", + ["message"] = "Legacy environment metadata", + ["reference_id"] = "legacy-environment-reference", + [propertyName] = JsonSerializer.Deserialize(environment) }).StatusCodeShouldBeAccepted()); await GetService().RunAsync(TestCancellationToken); @@ -39,23 +41,25 @@ await SendRequestAsync(request => request.Post() var ev = Assert.Single(Assert.IsType>(events)); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); + Assert.Contains(propertyName, ev.Data.Keys, StringComparer.Ordinal); Assert.True(JsonElement.DeepEquals(JsonSerializer.Deserialize(environment), JsonSerializer.SerializeToElement(ev.Data["environment"]))); Assert.False(ev.Data.ContainsKey("environment1")); } [Theory] - [InlineData(" Production ", "Production")] - [InlineData("preview-42", "preview-42")] - [InlineData(" ", null)] - [InlineData("bad\nenvironment", null)] - [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null)] - public async Task GetSubmitEvent_EnvironmentParameter_TrimsAndPreservesCasing(string environment, string? expected) + [InlineData(" Production ", "Production", "environment")] + [InlineData(" Production ", "Production", "Environment")] + [InlineData("preview-42", "preview-42", "environment")] + [InlineData(" ", null, "environment")] + [InlineData("bad\nenvironment", null, "environment")] + [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null, "environment")] + public async Task GetSubmitEvent_EnvironmentParameter_TrimsAndPreservesCasing(string environment, string? expected, string propertyName) { await SendRequestAsync(request => request .AsTestOrganizationClientUser().AppendPaths("events", "submit") .QueryString("message", "GET environment submission") .QueryString("reference", "get-environment-reference") - .QueryString("environment", environment).StatusCodeShouldBeOk()); + .QueryString(propertyName, environment).StatusCodeShouldBeOk()); await GetService().RunAsync(TestCancellationToken); await RefreshDataAsync(); @@ -64,6 +68,7 @@ await SendRequestAsync(request => request item => item.ReferenceId == "get-environment-reference"); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); + Assert.Contains(propertyName, ev.Data.Keys, StringComparer.Ordinal); Assert.Equal(environment, ev.Data["environment"]); Assert.False(ev.Data.ContainsKey("environment1")); } diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index 4232eb2825..a67b66e218 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -104,6 +104,43 @@ public void Equals_DifferentEnvironments_DistinguishesEvents() Assert.NotEqual(new Event { Environment = "Production", Data = null }, new Event { Environment = "production", Data = null }); } + [Theory] + [InlineData("Environment")] + [InlineData("ENVIRONMENT")] + [InlineData("environment")] + public void ParseEvents_LegacyEnvironmentKey_PreservesSubmittedCasing(string propertyName) + { + var parser = GetService(); + var serializer = GetService(); + + var ev = Assert.Single(Assert.IsType>(parser.ParseEvents($$"""{"{{propertyName}}":"west"}""", 2, null))); + + Assert.Equal("west", ev.Environment); + using var serialized = JsonDocument.Parse(serializer.SerializeToString(ev)!); + var property = Assert.Single(serialized.RootElement.GetProperty("data").EnumerateObject()); + Assert.Equal(propertyName, property.Name); + Assert.Equal("west", property.Value.GetString()); + } + + [Fact] + public void ParseEvents_CaseDistinctEnvironmentKeys_PreservesAllLegacyValues() + { + var parser = GetService(); + var serializer = GetService(); + const string json = """{"Environment":"west","environment":"east","ENVIRONMENT":{"region":"north"},"data":{"environment":"nested"}}"""; + + var ev = Assert.Single(Assert.IsType>(parser.ParseEvents(json, 2, null))); + + Assert.Null(ev.Environment); + using var serialized = JsonDocument.Parse(serializer.SerializeToString(ev)!); + var data = serialized.RootElement.GetProperty("data"); + Assert.Equal(4, data.EnumerateObject().Count()); + Assert.Equal("nested", data.GetProperty("environment").GetString()); + Assert.Equal("west", data.GetProperty("Environment1").GetString()); + Assert.Equal("east", data.GetProperty("environment2").GetString()); + Assert.Equal("north", data.GetProperty("ENVIRONMENT3").GetProperty("region").GetString()); + } + [Theory] [InlineData("west", "west")] [InlineData(" Production ", "Production")] From 591d22c14b3092bb4b6dd36ab55836823bb40ac9 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 5 Oct 2026 13:37:37 -0500 Subject: [PATCH 10/24] Avoid duplicate environment data in GET submissions and retries --- src/Exceptionless.Core/Jobs/EventPostsJob.cs | 5 ++- .../Models/Queues/EventPostInfo.cs | 8 +++- .../Default/JsonEventParserPlugin.cs | 19 ++++++-- .../EventParser/EventParserPluginManager.cs | 33 ++++++++++---- .../Api/Handlers/EventHandler.cs | 22 ++------- .../EventEndpointTests.Environment.cs | 32 ++++++++++++- .../Jobs/EventPostJobTests.cs | 45 +++++++++++++++++++ .../Models/EventEnvironmentTests.cs | 15 +++++++ 8 files changed, 145 insertions(+), 34 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/EventPostsJob.cs b/src/Exceptionless.Core/Jobs/EventPostsJob.cs index 40db24f038..495ac37c92 100644 --- a/src/Exceptionless.Core/Jobs/EventPostsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventPostsJob.cs @@ -267,7 +267,9 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex AppDiagnostics.PostsParsingTime.Time(() => { string input = encoding.GetString(uncompressedData); - events = _eventParserPluginManager.ParseEvents(input, ep.ApiVersion, ep.UserAgent); + events = ep.IsNormalized + ? _eventParserPluginManager.ParseNormalizedEvents(input) + : _eventParserPluginManager.ParseEvents(input, ep.ApiVersion, ep.UserAgent); foreach (var ev in events) { ev.CreatedUtc = createdUtc; @@ -310,6 +312,7 @@ private async Task RetryEventsAsync(List eventsToRetry, EventPo // Put this single event back into the queue so we can retry it separately. await _eventPostService.EnqueueAsync(new EventPost(false) { + IsNormalized = true, ApiVersion = ep.ApiVersion, CharSet = ep.CharSet, ClientKeyHash = ep.ClientKeyHash, diff --git a/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs b/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs index 69223071d8..92c4341dfd 100644 --- a/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs +++ b/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs @@ -1,4 +1,6 @@ -namespace Exceptionless.Core.Queues.Models; +using System.Text.Json.Serialization; + +namespace Exceptionless.Core.Queues.Models; public record EventPostInfo { @@ -7,6 +9,10 @@ public record EventPostInfo public string? CharSet { get; init; } public string? MediaType { get; init; } public int ApiVersion { get; init; } + // Internal GET submissions and retries already contain normalized event data. + // Missing metadata on older queued posts retains the normal ingestion path. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool IsNormalized { get; init; } public string? UserAgent { get; init; } public string? ContentEncoding { get; init; } public string? IpAddress { get; init; } diff --git a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs index 53ee3ec476..c598ba757b 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs @@ -12,15 +12,16 @@ namespace Exceptionless.Core.Plugins.EventParser; public class JsonEventParserPlugin : PluginBase, IEventParserPlugin { private readonly JsonSerializerOptions _jsonOptions; + private readonly JsonSerializerOptions _normalizedJsonOptions; public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptions, ILoggerFactory loggerFactory) : base(options, loggerFactory) { // Create lenient parsing options — inbound events from older SDK clients may omit // non-nullable properties. We must not reject structurally valid events; the pipeline // handles missing/null values gracefully downstream. - _jsonOptions = new JsonSerializerOptions(jsonOptions) + _normalizedJsonOptions = new JsonSerializerOptions(jsonOptions) { RespectNullableAnnotations = false }; + _jsonOptions = new JsonSerializerOptions(_normalizedJsonOptions) { - RespectNullableAnnotations = false, // Preserve the original root value once, at ingestion. Repeating this during // storage or API deserialization would accumulate duplicate custom data. TypeInfoResolver = (jsonOptions.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver()) @@ -29,6 +30,16 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptio } public List? ParseEvents(string input, int apiVersion, string? userAgent) + { + return ParseEvents(input, apiVersion, _jsonOptions); + } + + internal List? ParseNormalizedEvents(string input) + { + return ParseEvents(input, 2, _normalizedJsonOptions); + } + + private List? ParseEvents(string input, int apiVersion, JsonSerializerOptions jsonOptions) { if (apiVersion < 2) return null; @@ -40,7 +51,7 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptio { try { - var ev = JsonSerializer.Deserialize(input, _jsonOptions); + var ev = JsonSerializer.Deserialize(input, jsonOptions); if (ev is not null) events.Add(ev); } @@ -56,7 +67,7 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptio { try { - var parsedEvents = JsonSerializer.Deserialize(input, _jsonOptions); + var parsedEvents = JsonSerializer.Deserialize(input, jsonOptions); if (parsedEvents is { Length: > 0 }) events.AddRange(parsedEvents.Where(e => e is not null)); } diff --git a/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs b/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs index 588bc5bd75..acde9a2ccf 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs @@ -1,4 +1,5 @@ using Exceptionless.Core.Models; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventParser; @@ -30,15 +31,7 @@ public List ParseEvents(string input, int apiVersion, string? u if (events is null) continue; - // Set required event properties - events.ForEach(e => - { - if (e.Date == DateTimeOffset.MinValue) - e.Date = _timeProvider.GetLocalNow(); - - if (String.IsNullOrWhiteSpace(e.Type)) - e.Type = e.HasError() || e.HasSimpleError() ? Event.KnownTypes.Error : Event.KnownTypes.Log; - }); + SetRequiredProperties(events); return events; } @@ -50,4 +43,26 @@ public List ParseEvents(string input, int apiVersion, string? u return new List(); } + + public List ParseNormalizedEvents(string input) + { + var parser = _serviceProvider.GetRequiredService(); + List? events = null; + AppDiagnostics.Time(() => events = parser.ParseNormalizedEvents(input), String.Concat("events.parse.", parser.Name.ToLower())); + events ??= []; + SetRequiredProperties(events); + return events; + } + + private void SetRequiredProperties(List events) + { + foreach (var ev in events) + { + if (ev.Date == DateTimeOffset.MinValue) + ev.Date = _timeProvider.GetLocalNow(); + + if (String.IsNullOrWhiteSpace(ev.Type)) + ev.Type = ev.HasError() || ev.HasSimpleError() ? Event.KnownTypes.Error : Event.KnownTypes.Log; + } + } } diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 6468963dd7..efa223e7f0 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -1,5 +1,4 @@ using System.Text; -using System.Text.Json.Nodes; using Exceptionless.Core; using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; @@ -461,8 +460,6 @@ public async Task Handle(SubmitEventByGet message) string? identity = null; string? identityName = null; - string? environment = null; - string? environmentPropertyName = null; var exclusions = project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions).ToList(); foreach (var kvp in filteredParameters) @@ -479,10 +476,8 @@ public async Task Handle(SubmitEventByGet message) if (kvp.Key.AnyWildcardMatches(exclusions, true)) continue; - environment = kvp.Value.FirstOrDefault(); - environmentPropertyName = kvp.Key; - if (kvp.Value.Count > 1) - ev.Data![kvp.Key] = kvp.Value; + ev.Environment = kvp.Value.FirstOrDefault(); + ev.Data![kvp.Key] = kvp.Value.Count > 1 ? kvp.Value : kvp.Value.FirstOrDefault(); break; case "message": ev.Message = kvp.Value.FirstOrDefault(); @@ -539,19 +534,10 @@ public async Task Handle(SubmitEventByGet message) charSet = contentTypeHeader.Charset.ToString(); } - byte[] eventBytes = ev.GetBytes(serializer); - if (environment is not null) - { - // Keep the original query value until ingestion, where both the normalized - // deployment name and legacy custom data are populated together. - var submission = JsonNode.Parse(eventBytes)!; - submission[environmentPropertyName!] = environment; - eventBytes = Encoding.UTF8.GetBytes(submission.ToJsonString()); - } - - using var stream = new MemoryStream(eventBytes); + using var stream = new MemoryStream(ev.GetBytes(serializer)); await eventPostService.EnqueueAsync(new EventPost(appOptions.EnableArchive) { + IsNormalized = true, ApiVersion = message.ApiVersion, CharSet = charSet, ContentEncoding = contentEncoding, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index 3e5dd44449..2f4ea1f8f9 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -88,9 +88,16 @@ await CreateDataAsync(data => data.Event().FreeProject().Type(Event.KnownTypes.Log).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity("unaffected-production-user"); }); data.Event().FreeProject().Type(Event.KnownTypes.Error).Stack(first).Mutate(ev => { ev.Environment = "development"; ev.SetUserIdentity("development-user"); }); data.Event().FreeProject().Type(Event.KnownTypes.Log).Mutate(ev => { ev.Environment = "development"; ev.SetUserIdentity("unaffected-development-user"); }); + data.Event().FreeProject().Type(Event.KnownTypes.Error).Stack(first).Mutate(ev => ev.SetUserIdentity("unspecified-user")); + data.Event().FreeProject().Type(Event.KnownTypes.Log).Mutate(ev => ev.SetUserIdentity("unaffected-unspecified-user")); }); - foreach (var (filter, expected, totalUsers) in new[] { ("environment:production", 12, 13), ("environment:staging", 1, 1), ("environment:production", 12, 13), ("environment:(production OR staging)", 13, 14), ("", 14, 16) }) + foreach (var (filter, expected, totalUsers) in new[] + { + ("environment:production", 12, 13), ("environment:staging", 1, 1), ("environment:production", 12, 13), + ("environment:(production OR staging)", 13, 14), ("_missing_:environment", 1, 2), ("_exists_:environment", 14, 16), + ("(_missing_:environment OR environment:production)", 13, 15), ("", 15, 18) + }) { var stacks = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") @@ -102,6 +109,29 @@ await CreateDataAsync(data => } } + [Theory] + [InlineData("environment")] + [InlineData("Environment")] + public async Task GetSubmitEvent_RepeatedEnvironments_PreservesCollectionWithoutDuplicateData(string propertyName) + { + await SendRequestAsync(request => request + .AsTestOrganizationClientUser().AppendPaths("events", "submit") + .QueryString("message", "Repeated GET environments") + .QueryString("reference", "repeated-environment-reference") + .QueryStrings(propertyName, new[] { " Production ", "staging" }).StatusCodeShouldBeOk()); + + await GetService().RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + var ev = Assert.Single((await _eventRepository.GetAllAsync()).Documents, + item => item.ReferenceId == "repeated-environment-reference"); + Assert.Equal("Production", ev.Environment); + Assert.NotNull(ev.Data); + Assert.Contains(propertyName, ev.Data.Keys, StringComparer.Ordinal); + Assert.Equal(new[] { " Production ", "staging" }, JsonSerializer.SerializeToElement(ev.Data[propertyName]).EnumerateArray().Select(value => value.GetString())); + Assert.False(ev.Data.ContainsKey("environment1")); + } + [Fact] public async Task GetEvents_EnvironmentOnFreePlan_FiltersEventsSummariesAndStacks() { diff --git a/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs b/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs index fb07d885e1..aa8557fa5c 100644 --- a/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs @@ -4,6 +4,8 @@ using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Billing; +using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Plugins.EventProcessor; using Exceptionless.Core.Queues.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; @@ -150,6 +152,49 @@ public async Task CanRunJobWithDiscardedEventUsage() Assert.Equal(0, usage.CurrentUsage.Blocked); } + [Fact] + public async Task RunAsync_RetryingParsedBatch_PreservesEnvironmentDataWithoutDuplicates() + { + var production = GenerateEvent(type: Event.KnownTypes.Log); + production.Environment = "Production"; + var staging = GenerateEvent(type: Event.KnownTypes.Log); + staging.Environment = "Staging"; + await EnqueueEventPostAsync([production, staging]); + + var services = GetService(); + var failingPipeline = new FailingPipeline(services, _options, Log); + var failingJob = ActivatorUtilities.CreateInstance(services, failingPipeline); + Assert.True((await failingJob.RunAsync(TestCancellationToken)).IsSuccess); + Assert.Equal(3, (await _eventQueue.GetQueueStatsAsync()).Enqueued); + + Assert.True((await _job.RunAsync(TestCancellationToken)).IsSuccess); + Assert.True((await _job.RunAsync(TestCancellationToken)).IsSuccess); + await RefreshDataAsync(); + + var events = (await _eventRepository.GetAllAsync()).Documents; + Assert.Equal(2, events.Count); + Assert.All(events, ev => + { + Assert.NotNull(ev.Data); + Assert.Equal(ev.Environment, ev.Data["environment"]); + Assert.False(ev.Data.ContainsKey("environment1")); + }); + } + + private sealed class FailingPipeline(IServiceProvider services, AppOptions options, ILoggerFactory loggerFactory) + : EventPipeline(services, options, loggerFactory) + { + protected override IList GetActionTypes() => []; + + public override Task> RunAsync(ICollection contexts) + { + foreach (var context in contexts) + context.SetError("Simulated transient batch failure", new InvalidOperationException("Retry this event")); + + return Task.FromResult(contexts); + } + } + [Fact] public async Task CanRunJobWithMassiveEventAsync() { diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index a67b66e218..a98a0ce1ce 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -3,6 +3,7 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.EventParser; +using Exceptionless.Core.Queues.Models; using Foundatio.Serializer; using Xunit; @@ -141,6 +142,20 @@ public void ParseEvents_CaseDistinctEnvironmentKeys_PreservesAllLegacyValues() Assert.Equal("north", data.GetProperty("ENVIRONMENT3").GetProperty("region").GetString()); } + [Fact] + public void EventPostMetadata_NormalizedFlag_DefaultsToLegacyIngestionAndRoundTrips() + { + var serializer = GetService(); + const string legacyJson = """{"organization_id":"organization","project_id":"project","api_version":2}"""; + var legacy = serializer.Deserialize(legacyJson); + + Assert.NotNull(legacy); + Assert.False(legacy.IsNormalized); + Assert.DoesNotContain("is_normalized", serializer.SerializeToString(legacy)!); + var normalized = legacy with { IsNormalized = true }; + Assert.True(serializer.Deserialize(serializer.SerializeToString(normalized)!)!.IsNormalized); + } + [Theory] [InlineData("west", "west")] [InlineData(" Production ", "Production")] From a50f93b8c5384f56160169c551365d3d65022bc9 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 16 Oct 2026 11:23:59 -0500 Subject: [PATCH 11/24] Store valid root environments only as the deployment environment A valid root environment string is stored only as Event.Environment instead of also being copied into data.environment. Any other root value keeps main's behavior and is stored as custom data. Parsing is now the same on every path and gives the same result when repeated, so the ingestion-only serializer, the normalized parse path, and the EventPostInfo.IsNormalized queue flag are removed. Co-Authored-By: Claude Opus 5.5 --- docs/docs/environments.md | 4 +- src/Exceptionless.Core/Jobs/EventPostsJob.cs | 5 +- src/Exceptionless.Core/Models/Event.cs | 24 ++--- .../Models/Queues/EventPostInfo.cs | 8 +- .../Default/JsonEventParserPlugin.cs | 26 +---- .../EventParser/EventParserPluginManager.cs | 33 ++---- .../EventEnvironmentConverter.cs | 29 +---- .../Api/Handlers/EventHandler.cs | 9 +- .../EventEndpointTests.Environment.cs | 28 +++-- .../Jobs/EventPostJobTests.cs | 11 +- .../Models/EventEnvironmentTests.cs | 102 ++++-------------- 11 files changed, 78 insertions(+), 201 deletions(-) diff --git a/docs/docs/environments.md b/docs/docs/environments.md index a36d8e56fd..92f00c0039 100644 --- a/docs/docs/environments.md +++ b/docs/docs/environments.md @@ -66,4 +66,6 @@ Use [fixed in version](/docs/versioning/) when environments run different releas ## Rollout -Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. Root `environment` values are also preserved as legacy custom event data, including submitted key casing and the original whitespace in strings. Non-string values leave the deployment environment unspecified. If `data.environment` already exists, the root value uses the next available key (`environment1`, `environment2`, and so on). +Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. + +Before this release, the server stored an unrecognized root `environment` property as custom data in `data.environment`. A valid root `environment` string is now stored only as the event's environment, trimmed, so read it from `environment` instead of `data.environment`. Any other root value, such as an object, number, blank string, or oversized name, is still stored as `data.environment` exactly as before. If `data.environment` already exists, that value uses the next available key (`environment1`, `environment2`, and so on). diff --git a/src/Exceptionless.Core/Jobs/EventPostsJob.cs b/src/Exceptionless.Core/Jobs/EventPostsJob.cs index 495ac37c92..40db24f038 100644 --- a/src/Exceptionless.Core/Jobs/EventPostsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventPostsJob.cs @@ -267,9 +267,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex AppDiagnostics.PostsParsingTime.Time(() => { string input = encoding.GetString(uncompressedData); - events = ep.IsNormalized - ? _eventParserPluginManager.ParseNormalizedEvents(input) - : _eventParserPluginManager.ParseEvents(input, ep.ApiVersion, ep.UserAgent); + events = _eventParserPluginManager.ParseEvents(input, ep.ApiVersion, ep.UserAgent); foreach (var ev in events) { ev.CreatedUtc = createdUtc; @@ -312,7 +310,6 @@ private async Task RetryEventsAsync(List eventsToRetry, EventPo // Put this single event back into the queue so we can retry it separately. await _eventPostService.EnqueueAsync(new EventPost(false) { - IsNormalized = true, ApiVersion = ep.ApiVersion, CharSet = ep.CharSet, ClientKeyHash = ep.ClientKeyHash, diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 6466816e56..a0bc454ee5 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -21,13 +21,18 @@ public class Event : IData, IJsonOnDeserialized public string? Environment { get => _environment; - set - { - string? name = value?.Trim(); - _environment = String.IsNullOrEmpty(name) || name.Length > 64 || name.Any(Char.IsControl) - ? null - : name; - } + set => _environment = NormalizeEnvironment(value); + } + + /// + /// Returns the trimmed deployment name, or null when the value cannot be a deployment name. + /// + public static string? NormalizeEnvironment(string? value) + { + string? name = value?.Trim(); + return String.IsNullOrEmpty(name) || name.Length > 64 || name.Any(Char.IsControl) + ? null + : name; } /// @@ -121,11 +126,6 @@ void IJsonOnDeserialized.OnDeserialized() Data ??= []; foreach (var kvp in ExtensionData) { - // Ingestion leaves deployment environments in extension data to retain the - // original key casing and value, including case-distinct legacy properties. - if (String.Equals(kvp.Key, nameof(Environment), StringComparison.OrdinalIgnoreCase)) - Environment = kvp.Value.ValueKind == JsonValueKind.String ? kvp.Value.GetString() : null; - object? value = JsonElementConverter.Convert(kvp.Value); EventDataNormalizer.Set(Data, kvp.Key, value); } diff --git a/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs b/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs index 92c4341dfd..69223071d8 100644 --- a/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs +++ b/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs @@ -1,6 +1,4 @@ -using System.Text.Json.Serialization; - -namespace Exceptionless.Core.Queues.Models; +namespace Exceptionless.Core.Queues.Models; public record EventPostInfo { @@ -9,10 +7,6 @@ public record EventPostInfo public string? CharSet { get; init; } public string? MediaType { get; init; } public int ApiVersion { get; init; } - // Internal GET submissions and retries already contain normalized event data. - // Missing metadata on older queued posts retains the normal ingestion path. - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public bool IsNormalized { get; init; } public string? UserAgent { get; init; } public string? ContentEncoding { get; init; } public string? IpAddress { get; init; } diff --git a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs index c598ba757b..327c507c6b 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs @@ -1,9 +1,7 @@ using System.Text.Json; -using System.Text.Json.Serialization.Metadata; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; -using Exceptionless.Core.Serialization; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventParser; @@ -12,34 +10,16 @@ namespace Exceptionless.Core.Plugins.EventParser; public class JsonEventParserPlugin : PluginBase, IEventParserPlugin { private readonly JsonSerializerOptions _jsonOptions; - private readonly JsonSerializerOptions _normalizedJsonOptions; public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptions, ILoggerFactory loggerFactory) : base(options, loggerFactory) { // Create lenient parsing options — inbound events from older SDK clients may omit // non-nullable properties. We must not reject structurally valid events; the pipeline // handles missing/null values gracefully downstream. - _normalizedJsonOptions = new JsonSerializerOptions(jsonOptions) { RespectNullableAnnotations = false }; - _jsonOptions = new JsonSerializerOptions(_normalizedJsonOptions) - { - // Preserve the original root value once, at ingestion. Repeating this during - // storage or API deserialization would accumulate duplicate custom data. - TypeInfoResolver = (jsonOptions.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver()) - .WithAddedModifier(EventEnvironmentConverter.ConfigureIngestionProperty) - }; + _jsonOptions = new JsonSerializerOptions(jsonOptions) { RespectNullableAnnotations = false }; } public List? ParseEvents(string input, int apiVersion, string? userAgent) - { - return ParseEvents(input, apiVersion, _jsonOptions); - } - - internal List? ParseNormalizedEvents(string input) - { - return ParseEvents(input, 2, _normalizedJsonOptions); - } - - private List? ParseEvents(string input, int apiVersion, JsonSerializerOptions jsonOptions) { if (apiVersion < 2) return null; @@ -51,7 +31,7 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptio { try { - var ev = JsonSerializer.Deserialize(input, jsonOptions); + var ev = JsonSerializer.Deserialize(input, _jsonOptions); if (ev is not null) events.Add(ev); } @@ -67,7 +47,7 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptio { try { - var parsedEvents = JsonSerializer.Deserialize(input, jsonOptions); + var parsedEvents = JsonSerializer.Deserialize(input, _jsonOptions); if (parsedEvents is { Length: > 0 }) events.AddRange(parsedEvents.Where(e => e is not null)); } diff --git a/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs b/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs index acde9a2ccf..588bc5bd75 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/EventParserPluginManager.cs @@ -1,5 +1,4 @@ using Exceptionless.Core.Models; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventParser; @@ -31,7 +30,15 @@ public List ParseEvents(string input, int apiVersion, string? u if (events is null) continue; - SetRequiredProperties(events); + // Set required event properties + events.ForEach(e => + { + if (e.Date == DateTimeOffset.MinValue) + e.Date = _timeProvider.GetLocalNow(); + + if (String.IsNullOrWhiteSpace(e.Type)) + e.Type = e.HasError() || e.HasSimpleError() ? Event.KnownTypes.Error : Event.KnownTypes.Log; + }); return events; } @@ -43,26 +50,4 @@ public List ParseEvents(string input, int apiVersion, string? u return new List(); } - - public List ParseNormalizedEvents(string input) - { - var parser = _serviceProvider.GetRequiredService(); - List? events = null; - AppDiagnostics.Time(() => events = parser.ParseNormalizedEvents(input), String.Concat("events.parse.", parser.Name.ToLower())); - events ??= []; - SetRequiredProperties(events); - return events; - } - - private void SetRequiredProperties(List events) - { - foreach (var ev in events) - { - if (ev.Date == DateTimeOffset.MinValue) - ev.Date = _timeProvider.GetLocalNow(); - - if (String.IsNullOrWhiteSpace(ev.Type)) - ev.Type = ev.HasError() || ev.HasSimpleError() ? Event.KnownTypes.Error : Event.KnownTypes.Log; - } - } } diff --git a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs index 2d83b04b77..6a2d0b930c 100644 --- a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs +++ b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs @@ -7,23 +7,12 @@ namespace Exceptionless.Core.Serialization; /// -/// Reads deployment names while preserving legacy root values as custom event data. +/// Reads valid deployment names into . Any other root value keeps +/// its legacy behavior and is stored as custom event data. /// public sealed class EventEnvironmentConverter : JsonConverter { - public override bool HandleNull => true; - public static void ConfigureProperty(JsonTypeInfo typeInfo) - { - ConfigureProperty(typeInfo, ingestion: false); - } - - public static void ConfigureIngestionProperty(JsonTypeInfo typeInfo) - { - ConfigureProperty(typeInfo, ingestion: true); - } - - private static void ConfigureProperty(JsonTypeInfo typeInfo, bool ingestion) { if (typeInfo.Kind != JsonTypeInfoKind.Object || !typeof(Event).IsAssignableFrom(typeInfo.Type)) { @@ -38,14 +27,6 @@ private static void ConfigureProperty(JsonTypeInfo typeInfo, bool ingestion) continue; } - if (ingestion) - { - // Let extension data retain the exact submitted names and case-distinct keys. - // Event.OnDeserialized promotes the deployment name before merging the legacy data. - typeInfo.Properties.RemoveAt(i); - return; - } - var environment = typeInfo.CreateJsonPropertyInfo(typeof(object), property.Name); environment.AttributeProvider = property.AttributeProvider; environment.CustomConverter = new EventEnvironmentConverter(); @@ -54,9 +35,9 @@ private static void ConfigureProperty(JsonTypeInfo typeInfo, bool ingestion) { if (value is JsonElement legacyValue) { + // Merged into data by Event.OnDeserialized after all properties have been read, + // so a later "data" property cannot overwrite it. var ev = (Event)instance; - ev.Environment = null; - // Merge after all properties have been read so a later "data" property cannot overwrite it. ev.ExtensionData ??= []; ev.ExtensionData[property.Name] = legacyValue; } @@ -72,7 +53,7 @@ private static void ConfigureProperty(JsonTypeInfo typeInfo, bool ingestion) public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - if (reader.TokenType == JsonTokenType.String) + if (reader.TokenType == JsonTokenType.String && Event.NormalizeEnvironment(reader.GetString()) is not null) { return reader.GetString(); } diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index efa223e7f0..1c0e4d53f3 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -472,12 +472,8 @@ public async Task Handle(SubmitEventByGet message) case "source": ev.Source = kvp.Value.FirstOrDefault(); break; - case "environment": - if (kvp.Key.AnyWildcardMatches(exclusions, true)) - continue; - - ev.Environment = kvp.Value.FirstOrDefault(); - ev.Data![kvp.Key] = kvp.Value.Count > 1 ? kvp.Value : kvp.Value.FirstOrDefault(); + case "environment" when kvp.Value.Count == 1 && Event.NormalizeEnvironment(kvp.Value[0]) is not null: + ev.Environment = kvp.Value[0]; break; case "message": ev.Message = kvp.Value.FirstOrDefault(); @@ -537,7 +533,6 @@ public async Task Handle(SubmitEventByGet message) using var stream = new MemoryStream(ev.GetBytes(serializer)); await eventPostService.EnqueueAsync(new EventPost(appOptions.EnableArchive) { - IsNormalized = true, ApiVersion = message.ApiVersion, CharSet = charSet, ContentEncoding = contentEncoding, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index 2f4ea1f8f9..9a42314af8 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -20,7 +20,7 @@ public partial class EventEndpointTests [InlineData("\" Production \"", "Production", "ENVIRONMENT")] [InlineData("\" \"", null, "environment")] [InlineData("\"bad\\nenvironment\"", null, "environment")] - public async Task PostEvent_LegacyRootEnvironment_PreservesCustomDataThroughStorageAndApi(string environment, string? expected, string propertyName) + public async Task PostEvent_RootEnvironment_StoresValidNamesAndPreservesOtherValuesAsCustomData(string environment, string? expected, string propertyName) { await SendRequestAsync(request => request.Post() .AsTestOrganizationClientUser().AppendPath("events") @@ -41,8 +41,14 @@ await SendRequestAsync(request => request.Post() var ev = Assert.Single(Assert.IsType>(events)); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); - Assert.Contains(propertyName, ev.Data.Keys, StringComparer.Ordinal); - Assert.True(JsonElement.DeepEquals(JsonSerializer.Deserialize(environment), JsonSerializer.SerializeToElement(ev.Data["environment"]))); + if (expected is not null) + { + Assert.False(ev.Data.ContainsKey("environment")); + } + else + { + Assert.True(JsonElement.DeepEquals(JsonSerializer.Deserialize(environment), JsonSerializer.SerializeToElement(ev.Data["environment"]))); + } Assert.False(ev.Data.ContainsKey("environment1")); } @@ -53,7 +59,7 @@ await SendRequestAsync(request => request.Post() [InlineData(" ", null, "environment")] [InlineData("bad\nenvironment", null, "environment")] [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null, "environment")] - public async Task GetSubmitEvent_EnvironmentParameter_TrimsAndPreservesCasing(string environment, string? expected, string propertyName) + public async Task GetSubmitEvent_EnvironmentParameter_StoresValidNamesAndPreservesOtherValuesAsCustomData(string environment, string? expected, string propertyName) { await SendRequestAsync(request => request .AsTestOrganizationClientUser().AppendPaths("events", "submit") @@ -68,8 +74,14 @@ await SendRequestAsync(request => request item => item.ReferenceId == "get-environment-reference"); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); - Assert.Contains(propertyName, ev.Data.Keys, StringComparer.Ordinal); - Assert.Equal(environment, ev.Data["environment"]); + if (expected is not null) + { + Assert.False(ev.Data.ContainsKey(propertyName)); + } + else + { + Assert.Equal(environment, ev.Data[propertyName]); + } Assert.False(ev.Data.ContainsKey("environment1")); } @@ -112,7 +124,7 @@ await CreateDataAsync(data => [Theory] [InlineData("environment")] [InlineData("Environment")] - public async Task GetSubmitEvent_RepeatedEnvironments_PreservesCollectionWithoutDuplicateData(string propertyName) + public async Task GetSubmitEvent_RepeatedEnvironments_PreservesCollectionAsCustomData(string propertyName) { await SendRequestAsync(request => request .AsTestOrganizationClientUser().AppendPaths("events", "submit") @@ -125,7 +137,7 @@ await SendRequestAsync(request => request var ev = Assert.Single((await _eventRepository.GetAllAsync()).Documents, item => item.ReferenceId == "repeated-environment-reference"); - Assert.Equal("Production", ev.Environment); + Assert.Null(ev.Environment); Assert.NotNull(ev.Data); Assert.Contains(propertyName, ev.Data.Keys, StringComparer.Ordinal); Assert.Equal(new[] { " Production ", "staging" }, JsonSerializer.SerializeToElement(ev.Data[propertyName]).EnumerateArray().Select(value => value.GetString())); diff --git a/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs b/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs index aa8557fa5c..d62f71d9b4 100644 --- a/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs @@ -153,7 +153,7 @@ public async Task CanRunJobWithDiscardedEventUsage() } [Fact] - public async Task RunAsync_RetryingParsedBatch_PreservesEnvironmentDataWithoutDuplicates() + public async Task RunAsync_RetryingParsedBatch_PreservesEnvironmentWithoutCustomData() { var production = GenerateEvent(type: Event.KnownTypes.Log); production.Environment = "Production"; @@ -172,13 +172,8 @@ public async Task RunAsync_RetryingParsedBatch_PreservesEnvironmentDataWithoutDu await RefreshDataAsync(); var events = (await _eventRepository.GetAllAsync()).Documents; - Assert.Equal(2, events.Count); - Assert.All(events, ev => - { - Assert.NotNull(ev.Data); - Assert.Equal(ev.Environment, ev.Data["environment"]); - Assert.False(ev.Data.ContainsKey("environment1")); - }); + Assert.Equal(new[] { "Production", "Staging" }, events.Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); + Assert.All(events, ev => Assert.False(ev.Data?.ContainsKey("environment") ?? false)); } private sealed class FailingPipeline(IServiceProvider services, AppOptions options, ILoggerFactory loggerFactory) diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index a98a0ce1ce..9e0482006a 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -3,7 +3,6 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.EventParser; -using Exceptionless.Core.Queues.Models; using Foundatio.Serializer; using Xunit; @@ -62,12 +61,14 @@ public void Serialize_UnspecifiedEnvironment_OmitsProperty() } [Theory] - [InlineData("null")] [InlineData("42")] [InlineData("true")] [InlineData("{\"region\":\"west\"}")] [InlineData("[\"production\",\"staging\"]")] - public void Deserialize_LegacyRootEnvironment_PreservesCustomDataRegardlessOfPropertyOrder(string environment) + [InlineData("\" \"")] + [InlineData("\"bad\\nenvironment\"")] + [InlineData("\"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm\"")] + public void Deserialize_InvalidRootEnvironment_PreservesCustomDataRegardlessOfPropertyOrder(string environment) { var serializer = GetService(); using var expected = JsonDocument.Parse(environment); @@ -85,7 +86,8 @@ public void Deserialize_LegacyRootEnvironment_PreservesCustomDataRegardlessOfPro Assert.NotNull(ev); Assert.Null(ev.Environment); - using var result = JsonDocument.Parse(serializer.SerializeToString(ev)!); + string serialized = serializer.SerializeToString(ev)!; + using var result = JsonDocument.Parse(serialized); var data = result.RootElement.GetProperty("data"); Assert.True(data.GetProperty("kept").GetBoolean()); Assert.True(JsonElement.DeepEquals(expected.RootElement, data.GetProperty(duplicateKey ? "environment1" : "environment"))); @@ -94,6 +96,7 @@ public void Deserialize_LegacyRootEnvironment_PreservesCustomDataRegardlessOfPro Assert.Equal("nested", data.GetProperty("environment").GetString()); } Assert.False(result.RootElement.TryGetProperty("environment", out _)); + Assert.Equal(serialized, serializer.SerializeToString(serializer.Deserialize(serialized))); } } @@ -106,91 +109,24 @@ public void Equals_DifferentEnvironments_DistinguishesEvents() } [Theory] - [InlineData("Environment")] - [InlineData("ENVIRONMENT")] - [InlineData("environment")] - public void ParseEvents_LegacyEnvironmentKey_PreservesSubmittedCasing(string propertyName) - { - var parser = GetService(); - var serializer = GetService(); - - var ev = Assert.Single(Assert.IsType>(parser.ParseEvents($$"""{"{{propertyName}}":"west"}""", 2, null))); - - Assert.Equal("west", ev.Environment); - using var serialized = JsonDocument.Parse(serializer.SerializeToString(ev)!); - var property = Assert.Single(serialized.RootElement.GetProperty("data").EnumerateObject()); - Assert.Equal(propertyName, property.Name); - Assert.Equal("west", property.Value.GetString()); - } - - [Fact] - public void ParseEvents_CaseDistinctEnvironmentKeys_PreservesAllLegacyValues() + [InlineData("environment", "west", "west")] + [InlineData("Environment", "west", "west")] + [InlineData("ENVIRONMENT", " Production ", "Production")] + public void ParseEvents_ValidEnvironment_StoresDeploymentNameOnlyAndRoundTrips(string propertyName, string environment, string expected) { var parser = GetService(); var serializer = GetService(); - const string json = """{"Environment":"west","environment":"east","ENVIRONMENT":{"region":"north"},"data":{"environment":"nested"}}"""; - - var ev = Assert.Single(Assert.IsType>(parser.ParseEvents(json, 2, null))); - - Assert.Null(ev.Environment); - using var serialized = JsonDocument.Parse(serializer.SerializeToString(ev)!); - var data = serialized.RootElement.GetProperty("data"); - Assert.Equal(4, data.EnumerateObject().Count()); - Assert.Equal("nested", data.GetProperty("environment").GetString()); - Assert.Equal("west", data.GetProperty("Environment1").GetString()); - Assert.Equal("east", data.GetProperty("environment2").GetString()); - Assert.Equal("north", data.GetProperty("ENVIRONMENT3").GetProperty("region").GetString()); - } - - [Fact] - public void EventPostMetadata_NormalizedFlag_DefaultsToLegacyIngestionAndRoundTrips() - { - var serializer = GetService(); - const string legacyJson = """{"organization_id":"organization","project_id":"project","api_version":2}"""; - var legacy = serializer.Deserialize(legacyJson); - - Assert.NotNull(legacy); - Assert.False(legacy.IsNormalized); - Assert.DoesNotContain("is_normalized", serializer.SerializeToString(legacy)!); - var normalized = legacy with { IsNormalized = true }; - Assert.True(serializer.Deserialize(serializer.SerializeToString(normalized)!)!.IsNormalized); - } - [Theory] - [InlineData("west", "west")] - [InlineData(" Production ", "Production")] - [InlineData(" ", null)] - [InlineData("bad\nenvironment", null)] - [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null)] - public void ParseEvents_LegacyStringEnvironment_PreservesOriginalDataAcrossRoundTrips(string environment, string? expected) - { - var parser = GetService(); - var serializer = GetService(); + var ev = Assert.Single(Assert.IsType>(parser.ParseEvents($$$"""{"{{{propertyName}}}":{{{JsonSerializer.Serialize(environment)}}},"data":{"kept":true}}""", 2, null))); - foreach (var (environmentFirst, duplicateKey) in new[] { (true, false), (false, false), (true, true), (false, true) }) + Assert.Equal(expected, ev.Environment); + Assert.NotNull(ev.Data); + Assert.False(ev.Data.ContainsKey("environment")); + string serialized = serializer.SerializeToString(ev)!; + for (int i = 0; i < 3; i++) { - string environmentProperty = $"\"environment\":{JsonSerializer.Serialize(environment)}"; - string dataProperty = duplicateKey - ? "\"data\":{\"environment\":\"nested\",\"kept\":true}" - : "\"data\":{\"kept\":true}"; - string json = environmentFirst - ? $"{{{environmentProperty},{dataProperty}}}" - : $"{{{dataProperty},{environmentProperty}}}"; - - var ev = Assert.Single(Assert.IsType>(parser.ParseEvents(json, 2, null))); - Assert.Equal(expected, ev.Environment); - Assert.Equal(environment, ev.Data![duplicateKey ? "environment1" : "environment"]); - if (duplicateKey) - Assert.Equal("nested", ev.Data["environment"]); - - string serialized = serializer.SerializeToString(ev)!; - for (int i = 0; i < 3; i++) - { - ev = serializer.Deserialize(serialized)!; - string roundTrip = serializer.SerializeToString(ev)!; - Assert.Equal(serialized, roundTrip); - Assert.Equal(expected, ev.Environment); - } + var roundTrip = Assert.Single(Assert.IsType>(parser.ParseEvents(serialized, 2, null))); + Assert.Equal(serialized, serializer.SerializeToString(roundTrip)); } } } From fc32e9039f0ff74584a3712a590826fd39c34d3f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 16 Oct 2026 11:45:49 -0500 Subject: [PATCH 12/24] Retain explicit null root environments as custom data Co-Authored-By: Claude Opus 5.5 --- docs/docs/environments.md | 2 +- .../Serialization/EventEnvironmentConverter.cs | 3 +++ .../Api/Endpoints/EventEndpointTests.Environment.cs | 1 + .../Serializer/Models/EventEnvironmentTests.cs | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/docs/environments.md b/docs/docs/environments.md index 92f00c0039..f08e16256b 100644 --- a/docs/docs/environments.md +++ b/docs/docs/environments.md @@ -68,4 +68,4 @@ Use [fixed in version](/docs/versioning/) when environments run different releas Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. -Before this release, the server stored an unrecognized root `environment` property as custom data in `data.environment`. A valid root `environment` string is now stored only as the event's environment, trimmed, so read it from `environment` instead of `data.environment`. Any other root value, such as an object, number, blank string, or oversized name, is still stored as `data.environment` exactly as before. If `data.environment` already exists, that value uses the next available key (`environment1`, `environment2`, and so on). +Before this release, the server stored an unrecognized root `environment` property as custom data in `data.environment`. A valid root `environment` string is now stored only as the event's environment, trimmed, so read it from `environment` instead of `data.environment`. Any other root value, such as null, an object, a number, a blank string, or an oversized name, is still stored as `data.environment`. If `data.environment` already exists, that value uses the next available key (`environment1`, `environment2`, and so on). diff --git a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs index 6a2d0b930c..a8107b509d 100644 --- a/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs +++ b/src/Exceptionless.Core/Serialization/EventEnvironmentConverter.cs @@ -12,6 +12,9 @@ namespace Exceptionless.Core.Serialization; /// public sealed class EventEnvironmentConverter : JsonConverter { + // Read null values too, so an explicit null is retained as custom data like other invalid values. + public override bool HandleNull => true; + public static void ConfigureProperty(JsonTypeInfo typeInfo) { if (typeInfo.Kind != JsonTypeInfoKind.Object || !typeof(Event).IsAssignableFrom(typeInfo.Type)) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index 9a42314af8..579b776029 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -13,6 +13,7 @@ namespace Exceptionless.Tests.Api.Endpoints; public partial class EventEndpointTests { [Theory] + [InlineData("null", null, "environment")] [InlineData("{\"region\":\"west\"}", null, "environment")] [InlineData("{\"region\":\"west\"}", null, "Environment")] [InlineData("\"west\"", "west", "environment")] diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index 9e0482006a..bdd817e2bf 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -61,6 +61,7 @@ public void Serialize_UnspecifiedEnvironment_OmitsProperty() } [Theory] + [InlineData("null")] [InlineData("42")] [InlineData("true")] [InlineData("{\"region\":\"west\"}")] From 61a0b9ed285bb3d21ee23fed8cb5a98cd1dd82ec Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 16 Oct 2026 15:36:13 -0500 Subject: [PATCH 13/24] Fix environment-scoped user counts, cross-environment session closing, and ! negation - Combine the environment filter with the project filter for stack user counts; a filter expression on the system filter replaced the project filter, so organizations with more than ten busy projects could report zero total users. - Treat one user's open sessions in different environments as distinct when closing duplicate heartbeat sessions. - Recognize ! as a negated group when scoping user counts by environment. - Hide the event overview Environment row when the event has none. - Document that notifications and the New view are not environment-aware, the shared heartbeat behavior, and running the mapping migration before processing events. Co-Authored-By: Claude Opus 5.5 --- docs/docs/environments.md | 8 +++-- .../Jobs/CloseInactiveSessionsJob.cs | 5 ++- .../Queries/EventEnvironmentFilter.cs | 3 +- .../Api/Handlers/EventHandler.cs | 7 ++-- .../events/components/views/overview.svelte | 12 ++++--- .../EventEndpointTests.Environment.cs | 29 ++++++++++++++++ .../Jobs/CloseInactiveSessionsJobTests.cs | 33 +++++++++++++++++++ .../Search/EventEnvironmentFilterTests.cs | 2 ++ 8 files changed, 85 insertions(+), 14 deletions(-) diff --git a/docs/docs/environments.md b/docs/docs/environments.md index f08e16256b..fc1e066a9a 100644 --- a/docs/docs/environments.md +++ b/docs/docs/environments.md @@ -46,7 +46,7 @@ Names are trimmed, with a maximum of 64 characters. Events preserve the supplied Choose **Manage filters → Environment** on Events, Stacks, Sessions, or Stream. Select one or more names, or **Unspecified** for events without an environment. Clear the selection to include all environments. The picker discovers names for the selected projects and time range; you can also enter a name that has no current events. -Environment selections persist in URLs and saved views. Events, sessions, and stream tables have an optional Environment column. Event details show Environment in the overview; **Machine & runtime** contains the existing `data.@environment` diagnostics. +Environment selections persist in URLs and saved views. Events, sessions, and stream tables have an optional Environment column. Event details show Environment in the overview when the event has one; **Machine & runtime** contains the existing `data.@environment` diagnostics. Environment searches and aggregations are available on every plan: @@ -58,7 +58,9 @@ Environment searches and aggregations are available on every plan: | `_exists_:environment` | Events with an environment | | `environment:"qa west"` | A custom name containing spaces | -Stack dashboard counts and charts use events matching the filter. Stack status remains shared, so changing status while viewing production also changes the same stack seen in staging. Automatic session detection separates the same user's activity by environment. +Stack dashboard counts and charts use events matching the filter. Stack status remains shared, so changing status while viewing production also changes the same stack seen in staging. Automatic session detection separates the same user's activity by environment. Session heartbeats do not include an environment, so a heartbeat keeps the user's open sessions active in every environment. + +Notifications and the **New** view are not environment-aware yet. A stack is new once, when its first event arrives from any environment. An error first seen in development triggers the new error notification then, and is not reported as new again when it reaches production; the **New** view also dates it from that first occurrence. Critical error and regression notifications are sent for events from every environment. ## Fixing stacks across deployments @@ -66,6 +68,6 @@ Use [fixed in version](/docs/versioning/) when environments run different releas ## Rollout -Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. +Deploy the server before adopting SDK versions that expose the new setting. The server adds mappings to existing event indices without rewriting historical events. Self-hosted installs should let the migration job finish before the new API and job instances process events: events written to an existing index before the mapping is added keep their environment but cannot be filtered or aggregated by it. Clients that do not send an environment continue to work. `data.@environment` and custom `data.environment` values retain their existing meanings. Before this release, the server stored an unrecognized root `environment` property as custom data in `data.environment`. A valid root `environment` string is now stored only as the event's environment, trimmed, so read it from `environment` instead of `data.environment`. Any other root value, such as null, an object, a number, a blank string, or an oversized name, is still stored as `data.environment`. If `data.environment` already exists, that value uses the next available key (`environment1`, `environment2`, and so on). diff --git a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs index 16262c3739..722cf19f79 100644 --- a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs +++ b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs @@ -61,9 +61,8 @@ protected override async Task RunInternalAsync(JobContext context) { var lastActivityUtc = sessionStart.Date.UtcDateTime.AddSeconds((double)sessionStart.Value.GetValueOrDefault()); - bool closeDuplicate = heartbeatResult?.CacheKey is not null && existingSessionHeartbeatIds.Contains(heartbeatResult.CacheKey); - if (heartbeatResult?.CacheKey is not null && !closeDuplicate) - existingSessionHeartbeatIds.Add(heartbeatResult.CacheKey); + // Heartbeats are not scoped to an environment, so one user's sessions in different environments are not duplicates. + bool closeDuplicate = heartbeatResult?.CacheKey is not null && !existingSessionHeartbeatIds.Add($"{heartbeatResult.CacheKey}:{sessionStart.Environment}"); if (heartbeatResult is not null && (closeDuplicate || heartbeatResult.Close || heartbeatResult.ActivityUtc > lastActivityUtc)) sessionStart.UpdateSessionStart(heartbeatResult.ActivityUtc, isSessionEnd: closeDuplicate || heartbeatResult.Close || heartbeatResult.ActivityUtc <= inactivePeriodUtc); diff --git a/src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs b/src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs index e4c0d73708..211cf0ba88 100644 --- a/src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs +++ b/src/Exceptionless.Core/Repositories/Queries/EventEnvironmentFilter.cs @@ -1,4 +1,5 @@ using Foundatio.Parsers.LuceneQueries; +using Foundatio.Parsers.LuceneQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; namespace Exceptionless.Core.Repositories.Queries; @@ -35,7 +36,7 @@ public static class EventEnvironmentFilter return null; } - negate ^= group.IsNegated == true || group.Prefix == "-"; + negate ^= group.IsExcluded(); if (group.Left is null) { return GetConstraint(group.Right, negate); diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 1c0e4d53f3..1b9a8f4029 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -963,12 +963,15 @@ private async Task> GetUserCountByProjectIdsAsync(ICo if (totals.Count == projectIds.Count) return totals; - var systemFilter = new RepositoryQuery().AppFilter(sf).FilterExpression(filter).EnforceEventStackFilter().DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); + var systemFilter = new RepositoryQuery().AppFilter(sf).EnforceEventStackFilter().DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); var projects = cachedTotals .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) .ToList(); - var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).EnforceEventStackFilter().AggregationsExpression("terms:(project_id cardinality:user)")); + // A filter expression on the system filter replaces this one, so both constraints are combined here. + string projectFilter = projects.BuildFilter(); + string countFilter = filter is null ? projectFilter : $"({projectFilter}) AND ({filter})"; + var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(countFilter).EnforceEventStackFilter().AggregationsExpression("terms:(project_id cardinality:user)")); var projectTerms = countResult.Aggregations.Terms("terms_project_id")?.Buckets ?? []; var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte index 50330dd2a1..7000be7e34 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte @@ -74,11 +74,13 @@ - - Environment - - {event.environment ?? 'Unspecified'} - + {#if event.environment} + + Environment + + {event.environment} + + {/if} {#if isSessionStart} Duration diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index 579b776029..753d5aec3b 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; +using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; using Exceptionless.Tests.Utility; using Foundatio.Jobs; @@ -122,6 +123,34 @@ await CreateDataAsync(data => } } + [Fact] + public async Task GetStacks_EnvironmentFilter_CountsUsersOnlyForTheStackProjects() + { + await CreateDataAsync(data => + { + data.Event().FreeProject().Type(Event.KnownTypes.Error).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity("stack-user"); }); + // More busy projects than the project terms aggregation returns by default. + for (int project = 0; project < 11; project++) + { + string projectId = $"{project:D24}"; + for (int user = 0; user < 3; user++) + { + string identity = $"other-{project}-{user}"; + data.Event().Organization(SampleDataService.FREE_ORG_ID).Project(projectId).Type(Event.KnownTypes.Log) + .Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity(identity); }); + } + } + }); + + var stacks = await SendRequestAsAsync>(request => request + .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") + .QueryString("filter", "type:error environment:production").StatusCodeShouldBeOk()); + + var stack = Assert.Single(Assert.IsType>(stacks)); + Assert.Equal(1, stack.Users); + Assert.Equal(1, stack.TotalUsers); + } + [Theory] [InlineData("environment")] [InlineData("Environment")] diff --git a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs index ab17d7f345..42d48b3d76 100644 --- a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs @@ -88,6 +88,39 @@ public async Task CloseDuplicateIdentitySessions() Assert.Equal(1, sessionStarts.Count(e => e.HasSessionEndTime())); } + [Fact] + public async Task WillNotCloseIdentitySessionsInDifferentEnvironmentsAsDuplicates() + { + const string userId = "blake@exceptionless.io"; + var event1 = GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId); + event1.Environment = "Production"; + var event2 = GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId); + event2.Environment = "staging"; + var event3 = GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId); + event3.Environment = "production"; + + var contexts = await _pipeline.RunAsync(new[] { event1, event2, event3 }, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()); + Assert.True(contexts.All(c => !c.HasError)); + Assert.True(contexts.All(c => !c.IsCancelled)); + Assert.True(contexts.All(c => c.IsProcessed)); + + await RefreshDataAsync(); + var events = await _eventRepository.GetAllAsync(); + var sessionStarts = events.Documents.Where(e => e.IsSessionStart()).ToList(); + Assert.Equal(2, sessionStarts.Count); + Assert.DoesNotContain(sessionStarts, e => e.HasSessionEndTime()); + + await _cache.SetAsync($"Project:{sessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}", DateTime.UtcNow.SubtractMinutes(1)); + + _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); + Assert.Equal(JobResult.Success, await _job.RunAsync(TestCancellationToken)); + await RefreshDataAsync(); + + sessionStarts = (await _eventRepository.GetAllAsync()).Documents.Where(e => e.IsSessionStart()).ToList(); + Assert.Equal(2, sessionStarts.Count); + Assert.DoesNotContain(sessionStarts, e => e.HasSessionEndTime()); + } + [Fact] public async Task WillNotCloseDuplicateIdentitySessionsWithSessionIdHeartbeat() { diff --git a/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs index 382e48f97b..a733a22d00 100644 --- a/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs +++ b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs @@ -17,6 +17,8 @@ public sealed class EventEnvironmentFilterTests [InlineData("type:error _exists_:environment", "_exists_:environment")] [InlineData("type:error NOT environment:staging", "NOT environment:staging")] [InlineData("NOT (environment:staging OR type:error)", "NOT (environment:staging)")] + [InlineData("-(environment:staging OR type:error)", "NOT (environment:staging)")] + [InlineData("!(environment:staging OR type:error)", "NOT (environment:staging)")] [InlineData("NOT (environment:staging AND type:error)", null)] public async Task GetAsync_EnvironmentScope_PreservesUsersOutsideTheEventType(string? filter, string? expected) { From 90595465bf622c89b3946b469db5890edec4e776 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 19 Oct 2026 23:24:24 -0500 Subject: [PATCH 14/24] Keep heartbeats across session pages and scope environment discovery to raw project terms - Consume session heartbeats after every page is processed, and detect duplicate sessions across pages, so a user's heartbeat keeps their sessions open in every environment even when those sessions are returned on different pages. - Include positive project terms from a raw search, such as legacy saved views or hand-written URLs, when discovering environment names. Co-Authored-By: Claude Opus 5.5 --- .../Jobs/CloseInactiveSessionsJob.cs | 13 +++++--- .../filters/environment-faceted-filter.svelte | 4 +-- .../filters/environment-filter.svelte.test.ts | 18 ++++++++-- .../components/filters/helpers.svelte.ts | 21 ++++++++++++ .../Jobs/CloseInactiveSessionsJobTests.cs | 33 +++++++++++++++++++ 5 files changed, 80 insertions(+), 9 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs index 722cf19f79..af362d4053 100644 --- a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs +++ b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs @@ -50,12 +50,14 @@ protected override async Task RunInternalAsync(JobContext context) if (results.Documents.Count == 0) return JobResult.Success; + // A user's heartbeat applies to their sessions in every environment, and those sessions can be on + // different pages, so heartbeats are consumed only after every page has been processed. + var cacheKeysToRemove = new HashSet(StringComparer.OrdinalIgnoreCase); + var existingSessionHeartbeatIds = new HashSet(StringComparer.OrdinalIgnoreCase); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { var inactivePeriodUtc = _timeProvider.GetUtcNow().UtcDateTime.Subtract(DefaultInactivePeriod); var sessionsToUpdate = new List(results.Documents.Count); - var cacheKeysToRemove = new List(results.Documents.Count * 2); - var existingSessionHeartbeatIds = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var (sessionStart, heartbeatResult) in await GetHeartbeatsBatchAsync(results.Documents)) { @@ -88,9 +90,6 @@ protected override async Task RunInternalAsync(JobContext context) if (sessionsToUpdate.Count > 0) await _eventRepository.SaveAsync(sessionsToUpdate); - if (cacheKeysToRemove.Count > 0) - await _cache.RemoveAllAsync(cacheKeysToRemove); - _logger.LogInformation("Closing {SessionClosedCount} of {SessionCount} sessions", sessionsToUpdate.Count, results.Documents.Count); // Sleep so we are not hammering the backend. @@ -105,6 +104,10 @@ protected override async Task RunInternalAsync(JobContext context) _lastActivity = _timeProvider.GetUtcNow().UtcDateTime; } } + + if (cacheKeysToRemove.Count > 0) + await _cache.RemoveAllAsync(cacheKeysToRemove); + _logger.LogInformation("Done checking active sessions. Closed {SessionClosedCount} of {SessionCount} sessions", sessionsClosed, totalSessions); return JobResult.Success; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte index 0d04858bf7..1f4802ffd0 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-faceted-filter.svelte @@ -9,7 +9,7 @@ import { organization } from '#features/organizations/context.svelte.ts'; import { terms } from '#features/shared/api/aggregations.ts'; - import { toFilter } from './helpers.svelte'; + import { getProjectScopeFilter } from './helpers.svelte'; import { EnvironmentFilter } from './models.svelte'; let { filter, filterChanged, filterRemoved, open = $bindable(false), title = 'Environment' }: FacetedFilterProps = $props(); @@ -19,7 +19,7 @@ params: { aggregations: 'terms:(environment~100)', get filter() { - return toFilter(scope?.filters.filter((filter) => filter.type === 'project') ?? []); + return getProjectScopeFilter(scope?.filters ?? []); }, get time() { return scope?.time ?? undefined; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts index 47ec84589c..ebab2f9217 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from 'vitest'; import { filterUsesPremiumFeatures } from '../../premium-filter'; -import { deserializeFilters, serializeFilters, toFilter } from './helpers.svelte'; -import { EnvironmentFilter, ProjectFilter } from './models.svelte'; +import { deserializeFilters, getProjectScopeFilter, serializeFilters, toFilter } from './helpers.svelte'; +import { EnvironmentFilter, KeywordFilter, ProjectFilter, TypeFilter } from './models.svelte'; describe('EnvironmentFilter', () => { it('normalizes names and supports all, multiple, and unspecified environments', () => { @@ -32,6 +32,20 @@ describe('EnvironmentFilter', () => { expect(filter.value).toEqual(['production', '']); }); + it('scopes environment discovery to picked projects and project terms in a raw search', () => { + const first = '537650f3b77efe23a47914f3'; + const second = '537650f3b77efe23a47914f4'; + const third = '537650f3b77efe23a47914f5'; + + expect(getProjectScopeFilter([new TypeFilter(['error'])])).toBe(''); + expect(getProjectScopeFilter([new ProjectFilter([first]), new EnvironmentFilter(['production'])])).toBe(`project:${first}`); + expect(getProjectScopeFilter([new KeywordFilter(`type:error project:${first}`)])).toBe(`project:${first}`); + expect(getProjectScopeFilter([new KeywordFilter(`(project:${first} OR project_id:${second})`)])).toBe(`(project:${first} OR project:${second})`); + expect(getProjectScopeFilter([new KeywordFilter(`project:(${first} OR ${second})`)])).toBe(`(project:${first} OR project:${second})`); + expect(getProjectScopeFilter([new KeywordFilter(`-project:${first} !project:${second} NOT project:${third}`)])).toBe(''); + expect(getProjectScopeFilter([new ProjectFilter([first]), new KeywordFilter(`project:${second}`)])).toBe(`(project:${first} OR project:${second})`); + }); + it('allows environment searches on free plans and keeps custom data premium', () => { for (const resource of ['event', 'event-stack'] as const) { expect(filterUsesPremiumFeatures('environment:production', resource)).toBe(false); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts index 2e394f3d2e..47fca7ab22 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts @@ -111,6 +111,27 @@ export function getProjectFilter(filters: IFilter[]): ProjectFilter { return filters.find((f) => f.type === 'project') as ProjectFilter; } +/** + * Projects that scope facet discovery: the project picker plus positive project terms in a raw + * search, such as legacy saved views or hand-written URLs. + */ +export function getProjectScopeFilter(filters: IFilter[]): string { + const projectIds = [...(getProjectFilter(filters)?.value ?? [])]; + for (const filter of filters) { + if (filter.type !== 'keyword') { + continue; + } + + for (const [term] of ((filter as KeywordFilter).value ?? '').matchAll(/(? projectIds.indexOf(projectId) === index)).toFilter(); +} + export function getStackFilter(filters: IFilter[]): StringFilter | undefined { return filters.find((f) => f.type === 'string') as StringFilter; } diff --git a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs index 42d48b3d76..ac83666d9a 100644 --- a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs @@ -121,6 +121,39 @@ public async Task WillNotCloseIdentitySessionsInDifferentEnvironmentsAsDuplicate Assert.DoesNotContain(sessionStarts, e => e.HasSessionEndTime()); } + [Fact] + public async Task WillApplyIdentityHeartbeatToEnvironmentSessionsOnDifferentPages() + { + const string userId = "blake@exceptionless.io"; + var production = GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId); + production.Environment = "production"; + var staging = GenerateEvent(DateTimeOffset.Now.SubtractMinutes(10), userId); + staging.Environment = "staging"; + + // Open sessions are paged newest first, 100 at a time, so these place the staging session on the second page. + var events = new List { production, staging }; + for (int i = 0; i < 100; i++) + events.Add(GenerateEvent(DateTimeOffset.Now.SubtractMinutes(6).SubtractSeconds(i), $"user-{i}@exceptionless.io")); + + var contexts = await _pipeline.RunAsync(events, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()); + Assert.True(contexts.All(c => !c.HasError)); + + await RefreshDataAsync(); + var sessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); + Assert.Equal(102, sessionStarts.Count); + + await _cache.SetAsync($"Project:{sessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}", DateTime.UtcNow.SubtractMinutes(1)); + + _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); + Assert.Equal(JobResult.Success, await _job.RunAsync(TestCancellationToken)); + await RefreshDataAsync(); + + sessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); + var openSessions = sessionStarts.Where(e => !e.HasSessionEndTime()).ToList(); + Assert.Equal(2, openSessions.Count); + Assert.Equal(["production", "staging"], openSessions.Select(e => e.Environment).Order()); + } + [Fact] public async Task WillNotCloseDuplicateIdentitySessionsWithSessionIdHeartbeat() { From 17e987a31088ff681d4151ba3f7ea51779793f30 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 19 Oct 2026 23:35:38 -0500 Subject: [PATCH 15/24] Ignore negated project groups and keep heartbeats when session paging is cancelled - Skip project terms beneath negated groups, such as NOT (project:id) or -(project:id), when scoping environment discovery. - Leave consumed heartbeats in the cache when a session run is cancelled before every page is processed, so sessions on the remaining pages still see them on the next run. Co-Authored-By: Claude Opus 5.5 --- .../Jobs/CloseInactiveSessionsJob.cs | 3 ++- .../filters/environment-filter.svelte.test.ts | 4 +++ .../components/filters/helpers.svelte.ts | 26 ++++++++++++++++++- .../Jobs/CloseInactiveSessionsJobTests.cs | 20 ++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs index af362d4053..9459294d9f 100644 --- a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs +++ b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs @@ -105,7 +105,8 @@ protected override async Task RunInternalAsync(JobContext context) } } - if (cacheKeysToRemove.Count > 0) + // Sessions on unprocessed pages still need the heartbeats, so a cancelled run leaves them for the next run. + if (cacheKeysToRemove.Count > 0 && !context.CancellationToken.IsCancellationRequested) await _cache.RemoveAllAsync(cacheKeysToRemove); _logger.LogInformation("Done checking active sessions. Closed {SessionClosedCount} of {SessionCount} sessions", sessionsClosed, totalSessions); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts index ebab2f9217..00daf1ac83 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts @@ -44,6 +44,10 @@ describe('EnvironmentFilter', () => { expect(getProjectScopeFilter([new KeywordFilter(`project:(${first} OR ${second})`)])).toBe(`(project:${first} OR project:${second})`); expect(getProjectScopeFilter([new KeywordFilter(`-project:${first} !project:${second} NOT project:${third}`)])).toBe(''); expect(getProjectScopeFilter([new ProjectFilter([first]), new KeywordFilter(`project:${second}`)])).toBe(`(project:${first} OR project:${second})`); + expect(getProjectScopeFilter([new KeywordFilter(`NOT (project:${first}) -(type:error OR project:${second}) !(project:${third})`)])).toBe(''); + expect(getProjectScopeFilter([new KeywordFilter(`NOT (status:fixed) (project:${first} OR -(project:${second})) "a (" project:${third}`)])).toBe( + `(project:${first} OR project:${third})` + ); }); it('allows environment searches on free plans and keeps custom data premium', () => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts index 47fca7ab22..13a6a9e6b2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts @@ -122,7 +122,12 @@ export function getProjectScopeFilter(filters: IFilter[]): string { continue; } - for (const [term] of ((filter as KeywordFilter).value ?? '').matchAll(/(? /(?:^|[\s(])(?:[-!]|NOT\s+)$/.test(search.slice(0, position)); + const groups: boolean[] = []; + let quoted = false; + for (let position = 0; position < index; position++) { + const char = search[position]; + if (char === '"' && search[position - 1] !== '\\') { + quoted = !quoted; + } else if (!quoted && char === '(') { + groups.push(groups.at(-1) === true || isNegatedHere(position)); + } else if (!quoted && char === ')') { + groups.pop(); + } + } + + return groups.at(-1) === true || isNegatedHere(index); +} + const TYPE_FILTER_REGEX = /\btype:(\w+)\b/g; export function hasSingleTypeFilter(filter: null | string | undefined): boolean { diff --git a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs index ac83666d9a..bc1efc3920 100644 --- a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs @@ -154,6 +154,26 @@ public async Task WillApplyIdentityHeartbeatToEnvironmentSessionsOnDifferentPage Assert.Equal(["production", "staging"], openSessions.Select(e => e.Environment).Order()); } + [Fact] + public async Task WillKeepHeartbeatsWhenRunIsCancelledBeforeEveryPageIsProcessed() + { + const string userId = "blake@exceptionless.io"; + var contexts = await _pipeline.RunAsync(new[] { GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId) }, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()); + Assert.True(contexts.All(c => !c.HasError)); + await RefreshDataAsync(); + + string heartbeatKey = $"Project:{TestConstants.ProjectId}:heartbeat:{userId.ToSHA1()}"; + await _cache.SetAsync(heartbeatKey, DateTime.UtcNow.SubtractMinutes(1)); + + // The job pauses after each page, so this cancels it before it checks for another page. + using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + cancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(1)); + _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); + await _job.RunAsync(cancellationTokenSource.Token); + + Assert.True(await _cache.ExistsAsync(heartbeatKey)); + } + [Fact] public async Task WillNotCloseDuplicateIdentitySessionsWithSessionIdHeartbeat() { From 81c2365d496396d75133c4108d9f49565b83859d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 19 Oct 2026 23:48:33 -0500 Subject: [PATCH 16/24] Only narrow environment discovery to projects that certainly limit a raw search Collecting every positive project term ignored OR semantics, so a search such as (project:id OR type:error) restricted the environment picker to one project even though the other branch includes every project. A raw search now narrows discovery only when its project terms, or project picker groups such as (project:a OR project:b), are combined with the rest of the search by AND. Searches with any other OR branch or negation keep organization-wide discovery, which can only show extra names. Co-Authored-By: Claude Opus 5.5 --- .../filters/environment-filter.svelte.test.ts | 28 +++++++++---- .../components/filters/helpers.svelte.ts | 39 +++++++------------ 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts index 00daf1ac83..c491cc0f26 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts @@ -32,22 +32,34 @@ describe('EnvironmentFilter', () => { expect(filter.value).toEqual(['production', '']); }); - it('scopes environment discovery to picked projects and project terms in a raw search', () => { + it('scopes environment discovery to picked projects and project terms that limit a raw search', () => { const first = '537650f3b77efe23a47914f3'; const second = '537650f3b77efe23a47914f4'; - const third = '537650f3b77efe23a47914f5'; expect(getProjectScopeFilter([new TypeFilter(['error'])])).toBe(''); expect(getProjectScopeFilter([new ProjectFilter([first]), new EnvironmentFilter(['production'])])).toBe(`project:${first}`); expect(getProjectScopeFilter([new KeywordFilter(`type:error project:${first}`)])).toBe(`project:${first}`); - expect(getProjectScopeFilter([new KeywordFilter(`(project:${first} OR project_id:${second})`)])).toBe(`(project:${first} OR project:${second})`); + expect(getProjectScopeFilter([new KeywordFilter(`type:error (project:${first} OR project_id:${second}) status:open`)])).toBe( + `(project:${first} OR project:${second})` + ); expect(getProjectScopeFilter([new KeywordFilter(`project:(${first} OR ${second})`)])).toBe(`(project:${first} OR project:${second})`); - expect(getProjectScopeFilter([new KeywordFilter(`-project:${first} !project:${second} NOT project:${third}`)])).toBe(''); + expect(getProjectScopeFilter([new KeywordFilter(`"a OR b" project:${first}`)])).toBe(`project:${first}`); expect(getProjectScopeFilter([new ProjectFilter([first]), new KeywordFilter(`project:${second}`)])).toBe(`(project:${first} OR project:${second})`); - expect(getProjectScopeFilter([new KeywordFilter(`NOT (project:${first}) -(type:error OR project:${second}) !(project:${third})`)])).toBe(''); - expect(getProjectScopeFilter([new KeywordFilter(`NOT (status:fixed) (project:${first} OR -(project:${second})) "a (" project:${third}`)])).toBe( - `(project:${first} OR project:${third})` - ); + + // Other OR branches and negations may include other projects. + for (const search of [ + `(project:${first} OR type:error)`, + `project:${first} OR type:error`, + `project:${first} || type:error`, + `-project:${first}`, + `!project:${first}`, + `NOT project:${first}`, + `NOT (project:${first})`, + `-(type:error OR project:${first})` + ]) { + expect(getProjectScopeFilter([new KeywordFilter(search)])).toBe(''); + expect(getProjectScopeFilter([new ProjectFilter([second]), new KeywordFilter(search)])).toBe(`project:${second}`); + } }); it('allows environment searches on free plans and keeps custom data premium', () => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts index 13a6a9e6b2..835e9f21df 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts @@ -26,6 +26,10 @@ import { const filterCache = new SvelteMap(); +const PROJECT_TERM = String.raw`project(?:_id)?:(?:\([^)]*\)|[^\s()]+)`; +// A project term, or a group of project terms joined by OR such as the project picker writes. +const PROJECT_SCOPE_REGEX = new RegExp(String.raw`\(\s*${PROJECT_TERM}(?:\s+OR\s+${PROJECT_TERM})*\s*\)|(? term); + const remainder = terms.reduce((value, term) => value.replace(term, ' '), search).replace(/"(?:[^"\\]|\\.)*"/g, '""'); + if (terms.length === 0 || /\b(?:OR|NOT)\b|\|\||(?:^|[\s(])[-!]/i.test(remainder)) { + continue; + } + for (const term of terms) { for (const [projectId] of term.matchAll(/\b[a-f0-9]{24}\b/gi)) { projectIds.push(projectId.toLowerCase()); } @@ -247,25 +255,6 @@ export function shouldRefreshPersistentEventRemoval( return removedFromTable || shouldRefreshPersistentEventChanged(filters, filter, organization_id, project_id, stack_id, id); } -/** Whether the term at index is negated directly or by an enclosing group, such as `NOT (type:error OR project:id)`. */ -function isNegated(search: string, index: number): boolean { - const isNegatedHere = (position: number) => /(?:^|[\s(])(?:[-!]|NOT\s+)$/.test(search.slice(0, position)); - const groups: boolean[] = []; - let quoted = false; - for (let position = 0; position < index; position++) { - const char = search[position]; - if (char === '"' && search[position - 1] !== '\\') { - quoted = !quoted; - } else if (!quoted && char === '(') { - groups.push(groups.at(-1) === true || isNegatedHere(position)); - } else if (!quoted && char === ')') { - groups.pop(); - } - } - - return groups.at(-1) === true || isNegatedHere(index); -} - const TYPE_FILTER_REGEX = /\btype:(\w+)\b/g; export function hasSingleTypeFilter(filter: null | string | undefined): boolean { From 50eb8486a4da83cdf96e98c288d7c1112f634180 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 19 Oct 2026 23:54:12 -0500 Subject: [PATCH 17/24] Parse raw searches to find the projects that scope environment discovery Checking the text outside project terms for any OR or negation discarded the project scope for searches such as project:id (status:open OR status:regressed), where the unrelated OR group is still constrained to the project. A small Boolean parser now follows grouping: an unrelated AND clause keeps the scope, an OR branch without one removes it, and negated clauses never add one, matching how the server scopes environment user counts. Co-Authored-By: Claude Opus 5.5 --- .../filters/environment-filter.svelte.test.ts | 4 + .../components/filters/helpers.svelte.ts | 113 +++++++++++++++--- 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts index c491cc0f26..3fbb8612b3 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts @@ -44,6 +44,10 @@ describe('EnvironmentFilter', () => { ); expect(getProjectScopeFilter([new KeywordFilter(`project:(${first} OR ${second})`)])).toBe(`(project:${first} OR project:${second})`); expect(getProjectScopeFilter([new KeywordFilter(`"a OR b" project:${first}`)])).toBe(`project:${first}`); + expect(getProjectScopeFilter([new KeywordFilter(`project:${first} (status:open OR status:regressed) NOT type:log`)])).toBe(`project:${first}`); + expect(getProjectScopeFilter([new KeywordFilter(`(project:${first} type:error) OR (project:${second} -status:fixed)`)])).toBe( + `(project:${first} OR project:${second})` + ); expect(getProjectScopeFilter([new ProjectFilter([first]), new KeywordFilter(`project:${second}`)])).toBe(`(project:${first} OR project:${second})`); // Other OR branches and negations may include other projects. diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts index 835e9f21df..60b62cc99a 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts @@ -26,10 +26,6 @@ import { const filterCache = new SvelteMap(); -const PROJECT_TERM = String.raw`project(?:_id)?:(?:\([^)]*\)|[^\s()]+)`; -// A project term, or a group of project terms joined by OR such as the project picker writes. -const PROJECT_SCOPE_REGEX = new RegExp(String.raw`\(\s*${PROJECT_TERM}(?:\s+OR\s+${PROJECT_TERM})*\s*\)|(? term); - const remainder = terms.reduce((value, term) => value.replace(term, ' '), search).replace(/"(?:[^"\\]|\\.)*"/g, '""'); - if (terms.length === 0 || /\b(?:OR|NOT)\b|\|\||(?:^|[\s(])[-!]/i.test(remainder)) { - continue; - } - - for (const term of terms) { - for (const [projectId] of term.matchAll(/\b[a-f0-9]{24}\b/gi)) { - projectIds.push(projectId.toLowerCase()); - } - } + projectIds.push(...(getSearchProjectScope((filter as KeywordFilter).value ?? '') ?? [])); } return new ProjectFilter(projectIds.filter((projectId, index) => projectIds.indexOf(projectId) === index)).toFilter(); @@ -255,6 +238,100 @@ export function shouldRefreshPersistentEventRemoval( return removedFromTable || shouldRefreshPersistentEventChanged(filters, filter, organization_id, project_id, stack_id, id); } +/** + * Projects that a raw search certainly limits results to, or null when it may include any project. An + * unrelated AND clause keeps the scope, an OR branch without one removes it, and negations never add one. + */ +function getSearchProjectScope(search: string): null | string[] { + let position = 0; + + function readTerm(): string { + const start = position; + let depth = 0; + let quoted = false; + for (; position < search.length; position++) { + const char = search[position]!; + if (quoted) { + if (char === '\\') { + position++; + } else if (char === '"') { + quoted = false; + } + } else if (char === '"') { + quoted = true; + } else if (char === '(') { + depth++; + } else if (char === ')' && depth-- === 0) { + break; + } else if (/\s/.test(char) && depth === 0) { + break; + } + } + + return search.slice(start, position); + } + + function readGroup(): null | string[] { + const clauses: (null | string[])[] = []; + let isOr = false; + let negated = false; + while (position < search.length) { + const char = search[position]!; + if (char === ')') { + position++; + break; + } + + if (/\s/.test(char) || char === '+') { + position++; + continue; + } + + if (char === '-' || char === '!') { + negated = true; + position++; + continue; + } + + let scope: null | string[]; + if (char === '(') { + position++; + scope = readGroup(); + } else { + const term = readTerm(); + if (term === 'OR' || term === '||') { + isOr = true; + continue; + } + + if (term === 'NOT') { + negated = true; + continue; + } + + if (term === 'AND' || term === '&&') { + continue; + } + + scope = /^project(?:_id)?:/i.test(term) ? [...term.matchAll(/\b[a-f0-9]{24}\b/gi)].map(([projectId]) => projectId.toLowerCase()) : null; + } + + clauses.push(negated ? null : scope); + negated = false; + } + + // Mixed AND and OR groups are treated as OR, which can only widen the scope. + if (isOr) { + return clauses.length > 0 && clauses.every((scope) => scope !== null) ? clauses.flat() : null; + } + + const scoped = clauses.filter((scope) => scope !== null); + return scoped.length > 0 ? scoped.flat() : null; + } + + return readGroup(); +} + const TYPE_FILTER_REGEX = /\btype:(\w+)\b/g; export function hasSingleTypeFilter(filter: null | string | undefined): boolean { From c4c200b1e2a2418485190fc2e11e92cd339100fc Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 20 Oct 2026 00:00:03 -0500 Subject: [PATCH 18/24] Intersect project scopes combined by AND when scoping environment discovery The project picker and a raw search's project terms are combined by AND, as are AND clauses within a search, so their project scopes now intersect instead of being unioned. Co-Authored-By: Claude Opus 5.5 --- .../filters/environment-filter.svelte.test.ts | 3 +- .../components/filters/helpers.svelte.ts | 37 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts index 3fbb8612b3..e0130a7869 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts @@ -48,7 +48,8 @@ describe('EnvironmentFilter', () => { expect(getProjectScopeFilter([new KeywordFilter(`(project:${first} type:error) OR (project:${second} -status:fixed)`)])).toBe( `(project:${first} OR project:${second})` ); - expect(getProjectScopeFilter([new ProjectFilter([first]), new KeywordFilter(`project:${second}`)])).toBe(`(project:${first} OR project:${second})`); + expect(getProjectScopeFilter([new ProjectFilter([first, second]), new KeywordFilter(`type:error project:${second}`)])).toBe(`project:${second}`); + expect(getProjectScopeFilter([new KeywordFilter(`(project:${first} OR project:${second}) project:${first}`)])).toBe(`project:${first}`); // Other OR branches and negations may include other projects. for (const search of [ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts index 60b62cc99a..9e06d39d70 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts @@ -112,20 +112,18 @@ export function getProjectFilter(filters: IFilter[]): ProjectFilter { } /** - * Projects that scope facet discovery: the project picker plus project terms in a raw search, such - * as legacy saved views or hand-written URLs. + * Projects that scope facet discovery: the project picker and any project terms in a raw search, such + * as legacy saved views or hand-written URLs. Filters are combined by AND, so their scopes intersect. */ export function getProjectScopeFilter(filters: IFilter[]): string { - const projectIds = [...(getProjectFilter(filters)?.value ?? [])]; - for (const filter of filters) { - if (filter.type !== 'keyword') { - continue; - } - - projectIds.push(...(getSearchProjectScope((filter as KeywordFilter).value ?? '') ?? [])); - } - - return new ProjectFilter(projectIds.filter((projectId, index) => projectIds.indexOf(projectId) === index)).toFilter(); + const projectFilter = getProjectFilter(filters); + const scopes = [ + projectFilter?.value.length ? projectFilter.value : null, + ...filters.filter((filter) => filter.type === 'keyword').map((filter) => getSearchProjectScope((filter as KeywordFilter).value ?? '')) + ]; + + // Disjoint scopes match no events, so discovery falls back to the whole organization. + return new ProjectFilter(intersectProjectScopes(scopes) ?? []).toFilter(); } export function getStackFilter(filters: IFilter[]): StringFilter | undefined { @@ -238,6 +236,10 @@ export function shouldRefreshPersistentEventRemoval( return removedFromTable || shouldRefreshPersistentEventChanged(filters, filter, organization_id, project_id, stack_id, id); } +function distinct(values: string[]): string[] { + return values.filter((value, index) => values.indexOf(value) === index); +} + /** * Projects that a raw search certainly limits results to, or null when it may include any project. An * unrelated AND clause keeps the scope, an OR branch without one removes it, and negations never add one. @@ -322,16 +324,21 @@ function getSearchProjectScope(search: string): null | string[] { // Mixed AND and OR groups are treated as OR, which can only widen the scope. if (isOr) { - return clauses.length > 0 && clauses.every((scope) => scope !== null) ? clauses.flat() : null; + return clauses.length > 0 && clauses.every((scope) => scope !== null) ? distinct(clauses.flat()) : null; } - const scoped = clauses.filter((scope) => scope !== null); - return scoped.length > 0 ? scoped.flat() : null; + return intersectProjectScopes(clauses); } return readGroup(); } +/** Projects allowed by every scoped clause of an AND, or null when no clause limits the projects. */ +function intersectProjectScopes(scopes: (null | string[])[]): null | string[] { + const scoped = scopes.filter((scope) => scope !== null); + return scoped.length > 0 ? distinct(scoped.reduce((projectIds, scope) => projectIds.filter((projectId) => scope.includes(projectId)))) : null; +} + const TYPE_FILTER_REGEX = /\btype:(\w+)\b/g; export function hasSingleTypeFilter(filter: null | string | undefined): boolean { From 39a80937a9a79bdd0ed469dd36b414ae924eb70c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 20 Oct 2026 00:06:15 -0500 Subject: [PATCH 19/24] Treat project terms other than exact IDs as unscoped in environment discovery A wildcard, range, or partial project ID can match other projects, but it was treated as an empty bounded scope, so (project:id OR project:537*) narrowed environment discovery to the exact project. Only exact project IDs, alone or in an OR group, now limit the scope. Co-Authored-By: Claude Opus 5.5 --- .../filters/environment-filter.svelte.test.ts | 5 +++++ .../events/components/filters/helpers.svelte.ts | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts index e0130a7869..c74bd310fd 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/environment-filter.svelte.test.ts @@ -43,6 +43,7 @@ describe('EnvironmentFilter', () => { `(project:${first} OR project:${second})` ); expect(getProjectScopeFilter([new KeywordFilter(`project:(${first} OR ${second})`)])).toBe(`(project:${first} OR project:${second})`); + expect(getProjectScopeFilter([new KeywordFilter(`project:"${first}" project_id:(${first} || ${second})`)])).toBe(`project:${first}`); expect(getProjectScopeFilter([new KeywordFilter(`"a OR b" project:${first}`)])).toBe(`project:${first}`); expect(getProjectScopeFilter([new KeywordFilter(`project:${first} (status:open OR status:regressed) NOT type:log`)])).toBe(`project:${first}`); expect(getProjectScopeFilter([new KeywordFilter(`(project:${first} type:error) OR (project:${second} -status:fixed)`)])).toBe( @@ -60,6 +61,10 @@ describe('EnvironmentFilter', () => { `!project:${first}`, `NOT project:${first}`, `NOT (project:${first})`, + `(project:${first} OR project:537*)`, + `project:${first.slice(0, 20)}`, + `project:[${first} TO ${second}]`, + `project:(${first} OR 537*)`, `-(type:error OR project:${first})` ]) { expect(getProjectScopeFilter([new KeywordFilter(search)])).toBe(''); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts index 9e06d39d70..e03c4e88e4 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.ts @@ -315,7 +315,7 @@ function getSearchProjectScope(search: string): null | string[] { continue; } - scope = /^project(?:_id)?:/i.test(term) ? [...term.matchAll(/\b[a-f0-9]{24}\b/gi)].map(([projectId]) => projectId.toLowerCase()) : null; + scope = getProjectTermScope(term); } clauses.push(negated ? null : scope); @@ -333,6 +333,18 @@ function getSearchProjectScope(search: string): null | string[] { return readGroup(); } +const PROJECT_ID = String.raw`"?[a-f0-9]{24}"?`; +// Exact project IDs, alone or in an OR group. Wildcards, ranges, and partial IDs can match other projects. +const EXACT_PROJECT_TERM_REGEX = new RegExp( + String.raw`^project(?:_id)?:(?:${PROJECT_ID}|\(\s*${PROJECT_ID}(?:\s+(?:OR\s+|\|\|\s+)?${PROJECT_ID})*\s*\))$`, + 'i' +); + +/** Projects that a project term limits results to, or null for any other term. */ +function getProjectTermScope(term: string): null | string[] { + return EXACT_PROJECT_TERM_REGEX.test(term) ? [...term.matchAll(/[a-f0-9]{24}/gi)].map(([projectId]) => projectId.toLowerCase()) : null; +} + /** Projects allowed by every scoped clause of an AND, or null when no clause limits the projects. */ function intersectProjectScopes(scopes: (null | string[])[]): null | string[] { const scoped = scopes.filter((scope) => scope !== null); From 96ade495c2180d3c325fea50b57b8315f3bb07af Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 20 Oct 2026 00:14:26 -0500 Subject: [PATCH 20/24] Keep session heartbeats written while the inactive session job runs Consumed heartbeats are now removed only if they still hold the value the job applied, so a heartbeat written while later pages are processed is left for the next run instead of being deleted unapplied. The cancellation test now cancels after the job saves its first page instead of on a wall-clock timer, and a new test writes a heartbeat at that point and checks it survives. Co-Authored-By: Claude Opus 5.5 --- .../Jobs/CloseInactiveSessionsJob.cs | 16 +++++-- .../Jobs/CloseInactiveSessionsJobTests.cs | 47 +++++++++++++++---- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs index 9459294d9f..d507bb4b7b 100644 --- a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs +++ b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs @@ -52,7 +52,8 @@ protected override async Task RunInternalAsync(JobContext context) // A user's heartbeat applies to their sessions in every environment, and those sessions can be on // different pages, so heartbeats are consumed only after every page has been processed. - var cacheKeysToRemove = new HashSet(StringComparer.OrdinalIgnoreCase); + var consumedHeartbeats = new Dictionary(StringComparer.OrdinalIgnoreCase); + var closeKeysToRemove = new HashSet(StringComparer.OrdinalIgnoreCase); var existingSessionHeartbeatIds = new HashSet(StringComparer.OrdinalIgnoreCase); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { @@ -76,9 +77,9 @@ protected override async Task RunInternalAsync(JobContext context) sessionsToUpdate.Add(sessionStart); if (heartbeatResult?.CacheKey is not null) { - cacheKeysToRemove.Add(heartbeatResult.CacheKey); + consumedHeartbeats[heartbeatResult.CacheKey] = heartbeatResult.ActivityUtc; if (heartbeatResult.Close) - cacheKeysToRemove.Add($"{heartbeatResult.CacheKey}-close"); + closeKeysToRemove.Add($"{heartbeatResult.CacheKey}-close"); } Debug.Assert(sessionStart.Value is not null && sessionStart.Value >= 0, "Session start value cannot be a negative number."); @@ -106,8 +107,13 @@ protected override async Task RunInternalAsync(JobContext context) } // Sessions on unprocessed pages still need the heartbeats, so a cancelled run leaves them for the next run. - if (cacheKeysToRemove.Count > 0 && !context.CancellationToken.IsCancellationRequested) - await _cache.RemoveAllAsync(cacheKeysToRemove); + if (!context.CancellationToken.IsCancellationRequested) + { + // A heartbeat written while later pages were processed has not been applied, so only unchanged values are removed. + await Task.WhenAll(consumedHeartbeats.Select(heartbeat => _cache.RemoveIfEqualAsync(heartbeat.Key, heartbeat.Value))); + if (closeKeysToRemove.Count > 0) + await _cache.RemoveAllAsync(closeKeysToRemove); + } _logger.LogInformation("Done checking active sessions. Closed {SessionClosedCount} of {SessionCount} sessions", sessionsClosed, totalSessions); diff --git a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs index bc1efc3920..075bd3e0f9 100644 --- a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs @@ -156,6 +156,44 @@ public async Task WillApplyIdentityHeartbeatToEnvironmentSessionsOnDifferentPage [Fact] public async Task WillKeepHeartbeatsWhenRunIsCancelledBeforeEveryPageIsProcessed() + { + string heartbeatKey = await CreateSessionWithHeartbeatAsync(); + + // Cancels after the job saves its first page, before it checks for another page. + using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + int savedSessions = 0; + using var handler = _eventRepository.DocumentsSaved.AddHandler((_, args) => + { + savedSessions += args.Documents.Count; + cancellationTokenSource.Cancel(); + return Task.CompletedTask; + }); + + _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); + await _job.RunAsync(cancellationTokenSource.Token); + + Assert.Equal(1, savedSessions); + Assert.True(await _cache.ExistsAsync(heartbeatKey)); + } + + [Fact] + public async Task WillKeepHeartbeatsWrittenWhileSessionPagesAreProcessed() + { + string heartbeatKey = await CreateSessionWithHeartbeatAsync(); + + // A heartbeat arrives after the job has applied the one it read. + var latestHeartbeatUtc = DateTime.UtcNow; + using var handler = _eventRepository.DocumentsSaved.AddHandler(async (_, _) => await _cache.SetAsync(heartbeatKey, latestHeartbeatUtc)); + + _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); + Assert.Equal(JobResult.Success, await _job.RunAsync(TestCancellationToken)); + + var heartbeat = await _cache.GetAsync(heartbeatKey); + Assert.True(heartbeat.HasValue); + Assert.Equal(latestHeartbeatUtc, heartbeat.Value); + } + + private async Task CreateSessionWithHeartbeatAsync() { const string userId = "blake@exceptionless.io"; var contexts = await _pipeline.RunAsync(new[] { GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId) }, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()); @@ -164,14 +202,7 @@ public async Task WillKeepHeartbeatsWhenRunIsCancelledBeforeEveryPageIsProcessed string heartbeatKey = $"Project:{TestConstants.ProjectId}:heartbeat:{userId.ToSHA1()}"; await _cache.SetAsync(heartbeatKey, DateTime.UtcNow.SubtractMinutes(1)); - - // The job pauses after each page, so this cancels it before it checks for another page. - using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); - cancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(1)); - _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); - await _job.RunAsync(cancellationTokenSource.Token); - - Assert.True(await _cache.ExistsAsync(heartbeatKey)); + return heartbeatKey; } [Fact] From 0f720377832b4eff30aefa49a88c27633301611d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 20 Oct 2026 00:36:57 -0500 Subject: [PATCH 21/24] Use Arrange, Act, and Assert sections in the environment tests Restructure the C# tests added by this branch into explicit Arrange, Act, and Assert sections with assertions kept out of the action, as the repository's test conventions require. Scenario tests that send several requests now capture the results in Act and assert on them afterward. The environment round-trip test parses its serialized output once instead of three identical times. Co-Authored-By: Claude Opus 5.5 --- .../EventEndpointTests.Environment.cs | 64 ++++++++++---- .../Jobs/CloseInactiveSessionsJobTests.cs | 48 ++++++---- .../Jobs/EventPostJobTests.cs | 18 ++-- .../Mcp/ExceptionlessMcpToolsTests.cs | 5 ++ .../AddEventEnvironmentMigrationTests.cs | 6 +- .../Migrations/MigrationRegistrationTests.cs | 9 +- .../Pipeline/EventPipelineTests.cs | 13 ++- .../Plugins/WebHookDataTests.cs | 8 +- .../Search/EventEnvironmentFilterTests.cs | 8 +- .../Models/EventEnvironmentTests.cs | 87 ++++++++++++++----- 10 files changed, 203 insertions(+), 63 deletions(-) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs index 753d5aec3b..86a2e4db0a 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.Environment.cs @@ -24,15 +24,19 @@ public partial class EventEndpointTests [InlineData("\"bad\\nenvironment\"", null, "environment")] public async Task PostEvent_RootEnvironment_StoresValidNamesAndPreservesOtherValuesAsCustomData(string environment, string? expected, string propertyName) { + // Arrange + var content = new Dictionary + { + ["type"] = "log", + ["message"] = "Legacy environment metadata", + ["reference_id"] = "legacy-environment-reference", + [propertyName] = JsonSerializer.Deserialize(environment) + }; + + // Act await SendRequestAsync(request => request.Post() .AsTestOrganizationClientUser().AppendPath("events") - .Content(new Dictionary - { - ["type"] = "log", - ["message"] = "Legacy environment metadata", - ["reference_id"] = "legacy-environment-reference", - [propertyName] = JsonSerializer.Deserialize(environment) - }).StatusCodeShouldBeAccepted()); + .Content(content).StatusCodeShouldBeAccepted()); await GetService().RunAsync(TestCancellationToken); await RefreshDataAsync(); @@ -40,6 +44,8 @@ await SendRequestAsync(request => request.Post() var events = await SendRequestAsAsync>(request => request .AsTestOrganizationUser().AppendPath("events") .QueryString("filter", "reference:legacy-environment-reference").StatusCodeShouldBeOk()); + + // Assert var ev = Assert.Single(Assert.IsType>(events)); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); @@ -63,6 +69,9 @@ await SendRequestAsync(request => request.Post() [InlineData("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm", null, "environment")] public async Task GetSubmitEvent_EnvironmentParameter_StoresValidNamesAndPreservesOtherValuesAsCustomData(string environment, string? expected, string propertyName) { + // Arrange is provided by the theory data. + + // Act await SendRequestAsync(request => request .AsTestOrganizationClientUser().AppendPaths("events", "submit") .QueryString("message", "GET environment submission") @@ -71,8 +80,10 @@ await SendRequestAsync(request => request await GetService().RunAsync(TestCancellationToken); await RefreshDataAsync(); + var storedEvents = (await _eventRepository.GetAllAsync()).Documents; - var ev = Assert.Single((await _eventRepository.GetAllAsync()).Documents, + // Assert + var ev = Assert.Single(storedEvents, item => item.ReferenceId == "get-environment-reference"); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); @@ -90,6 +101,7 @@ await SendRequestAsync(request => request [Fact] public async Task GetStacks_EnvironmentFilter_ScopesUserCountsAndTheirCache() { + // Arrange await CreateDataAsync(data => { var first = data.Event().FreeProject().Type(Event.KnownTypes.Error).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity("production-0"); }); @@ -106,16 +118,26 @@ await CreateDataAsync(data => data.Event().FreeProject().Type(Event.KnownTypes.Log).Mutate(ev => ev.SetUserIdentity("unaffected-unspecified-user")); }); - foreach (var (filter, expected, totalUsers) in new[] + var cases = new[] { ("environment:production", 12, 13), ("environment:staging", 1, 1), ("environment:production", 12, 13), ("environment:(production OR staging)", 13, 14), ("_missing_:environment", 1, 2), ("_exists_:environment", 14, 16), ("(_missing_:environment OR environment:production)", 13, 15), ("", 15, 18) - }) + }; + + // Act + var results = new List<(int Expected, int TotalUsers, List? Stacks)>(); + foreach (var (filter, expected, totalUsers) in cases) { var stacks = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") .QueryString("filter", $"type:error {filter}").StatusCodeShouldBeOk()); + results.Add((expected, totalUsers, stacks)); + } + + // Assert + foreach (var (expected, totalUsers, stacks) in results) + { var stack = Assert.Single(Assert.IsType>(stacks)); Assert.Equal(expected, stack.Total); Assert.Equal(expected, stack.Users); @@ -126,6 +148,7 @@ await CreateDataAsync(data => [Fact] public async Task GetStacks_EnvironmentFilter_CountsUsersOnlyForTheStackProjects() { + // Arrange await CreateDataAsync(data => { data.Event().FreeProject().Type(Event.KnownTypes.Error).Mutate(ev => { ev.Environment = "production"; ev.SetUserIdentity("stack-user"); }); @@ -142,10 +165,12 @@ await CreateDataAsync(data => } }); + // Act var stacks = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") .QueryString("filter", "type:error environment:production").StatusCodeShouldBeOk()); + // Assert var stack = Assert.Single(Assert.IsType>(stacks)); Assert.Equal(1, stack.Users); Assert.Equal(1, stack.TotalUsers); @@ -156,6 +181,9 @@ await CreateDataAsync(data => [InlineData("Environment")] public async Task GetSubmitEvent_RepeatedEnvironments_PreservesCollectionAsCustomData(string propertyName) { + // Arrange is provided by the theory data. + + // Act await SendRequestAsync(request => request .AsTestOrganizationClientUser().AppendPaths("events", "submit") .QueryString("message", "Repeated GET environments") @@ -164,8 +192,10 @@ await SendRequestAsync(request => request await GetService().RunAsync(TestCancellationToken); await RefreshDataAsync(); + var storedEvents = (await _eventRepository.GetAllAsync()).Documents; - var ev = Assert.Single((await _eventRepository.GetAllAsync()).Documents, + // Assert + var ev = Assert.Single(storedEvents, item => item.ReferenceId == "repeated-environment-reference"); Assert.Null(ev.Environment); Assert.NotNull(ev.Data); @@ -177,6 +207,7 @@ await SendRequestAsync(request => request [Fact] public async Task GetEvents_EnvironmentOnFreePlan_FiltersEventsSummariesAndStacks() { + // Arrange await CreateDataAsync(data => { var production = data.Event().FreeProject().Mutate(ev => ev.Environment = " Production "); @@ -186,29 +217,32 @@ await CreateDataAsync(data => data.Event().TestProject().Mutate(ev => ev.Environment = "private-environment"); }); + // Act var events = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events") .QueryString("filter", "environment:PRODUCTION").StatusCodeShouldBeOk()); - Assert.Equal(new[] { "Production", "production" }, Assert.IsType>(events).Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); var summaries = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "summary") .QueryString("filter", "environment:production").StatusCodeShouldBeOk()); - Assert.Equal(new[] { "Production", "production" }, Assert.IsType>(summaries).Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); var stacks = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events").QueryString("mode", "stack_frequent") .QueryString("filter", "environment:production").StatusCodeShouldBeOk()); - Assert.Equal(2, Assert.Single(Assert.IsType>(stacks)).Total); var missing = await SendRequestAsAsync>(request => request .AsFreeOrganizationUser().AppendPath("events") .QueryString("filter", "_missing_:environment").StatusCodeShouldBeOk()); - Assert.Null(Assert.Single(Assert.IsType>(missing)).Environment); var count = await SendRequestAsAsync(request => request .AsFreeOrganizationUser().AppendPaths("events", "count") .QueryString("aggregations", "terms:(environment~100)").StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(new[] { "Production", "production" }, Assert.IsType>(events).Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); + Assert.Equal(new[] { "Production", "production" }, Assert.IsType>(summaries).Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); + Assert.Equal(2, Assert.Single(Assert.IsType>(stacks)).Total); + Assert.Null(Assert.Single(Assert.IsType>(missing)).Environment); Assert.NotNull(count); Assert.Equal(4, count.Total); Assert.Equal(new[] { "production", "staging" }, count.Aggregations.Terms("terms_environment")!.Buckets.Select(bucket => bucket.Key).Order().ToArray()); diff --git a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs index 075bd3e0f9..aa8cf921bd 100644 --- a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs @@ -91,6 +91,7 @@ public async Task CloseDuplicateIdentitySessions() [Fact] public async Task WillNotCloseIdentitySessionsInDifferentEnvironmentsAsDuplicates() { + // Arrange const string userId = "blake@exceptionless.io"; var event1 = GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId); event1.Environment = "Production"; @@ -106,17 +107,20 @@ public async Task WillNotCloseIdentitySessionsInDifferentEnvironmentsAsDuplicate await RefreshDataAsync(); var events = await _eventRepository.GetAllAsync(); - var sessionStarts = events.Documents.Where(e => e.IsSessionStart()).ToList(); - Assert.Equal(2, sessionStarts.Count); - Assert.DoesNotContain(sessionStarts, e => e.HasSessionEndTime()); - - await _cache.SetAsync($"Project:{sessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}", DateTime.UtcNow.SubtractMinutes(1)); + var initialSessionStarts = events.Documents.Where(e => e.IsSessionStart()).ToList(); + Assert.Equal(2, initialSessionStarts.Count); + Assert.DoesNotContain(initialSessionStarts, e => e.HasSessionEndTime()); + await _cache.SetAsync($"Project:{initialSessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}", DateTime.UtcNow.SubtractMinutes(1)); _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); - Assert.Equal(JobResult.Success, await _job.RunAsync(TestCancellationToken)); + + // Act + var result = await _job.RunAsync(TestCancellationToken); await RefreshDataAsync(); + var sessionStarts = (await _eventRepository.GetAllAsync()).Documents.Where(e => e.IsSessionStart()).ToList(); - sessionStarts = (await _eventRepository.GetAllAsync()).Documents.Where(e => e.IsSessionStart()).ToList(); + // Assert + Assert.Equal(JobResult.Success, result); Assert.Equal(2, sessionStarts.Count); Assert.DoesNotContain(sessionStarts, e => e.HasSessionEndTime()); } @@ -124,6 +128,7 @@ public async Task WillNotCloseIdentitySessionsInDifferentEnvironmentsAsDuplicate [Fact] public async Task WillApplyIdentityHeartbeatToEnvironmentSessionsOnDifferentPages() { + // Arrange const string userId = "blake@exceptionless.io"; var production = GenerateEvent(DateTimeOffset.Now.SubtractMinutes(5), userId); production.Environment = "production"; @@ -139,16 +144,19 @@ public async Task WillApplyIdentityHeartbeatToEnvironmentSessionsOnDifferentPage Assert.True(contexts.All(c => !c.HasError)); await RefreshDataAsync(); - var sessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); - Assert.Equal(102, sessionStarts.Count); - - await _cache.SetAsync($"Project:{sessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}", DateTime.UtcNow.SubtractMinutes(1)); + var initialSessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); + Assert.Equal(102, initialSessionStarts.Count); + await _cache.SetAsync($"Project:{initialSessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}", DateTime.UtcNow.SubtractMinutes(1)); _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); - Assert.Equal(JobResult.Success, await _job.RunAsync(TestCancellationToken)); + + // Act + var result = await _job.RunAsync(TestCancellationToken); await RefreshDataAsync(); + var sessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); - sessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); + // Assert + Assert.Equal(JobResult.Success, result); var openSessions = sessionStarts.Where(e => !e.HasSessionEndTime()).ToList(); Assert.Equal(2, openSessions.Count); Assert.Equal(["production", "staging"], openSessions.Select(e => e.Environment).Order()); @@ -157,6 +165,7 @@ public async Task WillApplyIdentityHeartbeatToEnvironmentSessionsOnDifferentPage [Fact] public async Task WillKeepHeartbeatsWhenRunIsCancelledBeforeEveryPageIsProcessed() { + // Arrange string heartbeatKey = await CreateSessionWithHeartbeatAsync(); // Cancels after the job saves its first page, before it checks for another page. @@ -170,15 +179,20 @@ public async Task WillKeepHeartbeatsWhenRunIsCancelledBeforeEveryPageIsProcessed }); _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); + + // Act await _job.RunAsync(cancellationTokenSource.Token); + bool heartbeatExists = await _cache.ExistsAsync(heartbeatKey); + // Assert Assert.Equal(1, savedSessions); - Assert.True(await _cache.ExistsAsync(heartbeatKey)); + Assert.True(heartbeatExists); } [Fact] public async Task WillKeepHeartbeatsWrittenWhileSessionPagesAreProcessed() { + // Arrange string heartbeatKey = await CreateSessionWithHeartbeatAsync(); // A heartbeat arrives after the job has applied the one it read. @@ -186,9 +200,13 @@ public async Task WillKeepHeartbeatsWrittenWhileSessionPagesAreProcessed() using var handler = _eventRepository.DocumentsSaved.AddHandler(async (_, _) => await _cache.SetAsync(heartbeatKey, latestHeartbeatUtc)); _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); - Assert.Equal(JobResult.Success, await _job.RunAsync(TestCancellationToken)); + // Act + var result = await _job.RunAsync(TestCancellationToken); var heartbeat = await _cache.GetAsync(heartbeatKey); + + // Assert + Assert.Equal(JobResult.Success, result); Assert.True(heartbeat.HasValue); Assert.Equal(latestHeartbeatUtc, heartbeat.Value); } diff --git a/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs b/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs index d62f71d9b4..9bbddb2f9e 100644 --- a/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/EventPostJobTests.cs @@ -155,6 +155,7 @@ public async Task CanRunJobWithDiscardedEventUsage() [Fact] public async Task RunAsync_RetryingParsedBatch_PreservesEnvironmentWithoutCustomData() { + // Arrange var production = GenerateEvent(type: Event.KnownTypes.Log); production.Environment = "Production"; var staging = GenerateEvent(type: Event.KnownTypes.Log); @@ -164,14 +165,21 @@ public async Task RunAsync_RetryingParsedBatch_PreservesEnvironmentWithoutCustom var services = GetService(); var failingPipeline = new FailingPipeline(services, _options, Log); var failingJob = ActivatorUtilities.CreateInstance(services, failingPipeline); - Assert.True((await failingJob.RunAsync(TestCancellationToken)).IsSuccess); - Assert.Equal(3, (await _eventQueue.GetQueueStatsAsync()).Enqueued); - Assert.True((await _job.RunAsync(TestCancellationToken)).IsSuccess); - Assert.True((await _job.RunAsync(TestCancellationToken)).IsSuccess); - await RefreshDataAsync(); + // Act + var failingResult = await failingJob.RunAsync(TestCancellationToken); + var queueStats = await _eventQueue.GetQueueStatsAsync(); + var firstRetryResult = await _job.RunAsync(TestCancellationToken); + var secondRetryResult = await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); var events = (await _eventRepository.GetAllAsync()).Documents; + + // Assert + Assert.True(failingResult.IsSuccess); + Assert.Equal(3, queueStats.Enqueued); + Assert.True(firstRetryResult.IsSuccess); + Assert.True(secondRetryResult.IsSuccess); Assert.Equal(new[] { "Production", "Staging" }, events.Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); Assert.All(events, ev => Assert.False(ev.Data?.ContainsKey("environment") ?? false)); } diff --git a/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs b/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs index c7e0c2b28d..4607efbf3b 100644 --- a/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs +++ b/tests/Exceptionless.Tests/Mcp/ExceptionlessMcpToolsTests.cs @@ -708,6 +708,7 @@ public async Task CountEventsAsync_GroupByVersion_ReturnsVersionCounts() [Fact] public async Task CountEventsAsync_GroupByEnvironment_ReturnsFilteredDeploymentCounts() { + // Arrange const string referenceId = "mcp-count-environments"; await CreateDataAsync(data => { @@ -715,7 +716,11 @@ await CreateDataAsync(data => data.Event().TestProject().ReferenceId(referenceId).Mutate(ev => ev.Environment = "staging"); }); var tools = await CreateToolsAsync(AuthorizationRoles.McpRead, AuthorizationRoles.EventsRead); + + // Act var result = await tools.CountEventsAsync(TestConstants.ProjectId, filter: $"reference:{referenceId} environment:production", groupBy: "environment"); + + // Assert Assert.True(result.Ok); var group = Assert.Single(Data(result).Groups!); Assert.Equal("production", group.Key); diff --git a/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs index 828b89998f..95ad091b41 100644 --- a/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/AddEventEnvironmentMigrationTests.cs @@ -15,6 +15,7 @@ public AddEventEnvironmentMigrationTests(ITestOutputHelper output, AppWebHostFac [Fact] public async Task RunAsync_LegacyDailyIndex_AddsMappingAndCanRunAgain() { + // Arrange var configuration = GetService(); string indexName = $"{configuration.Events.Name}-v0-2000.01.01"; var client = configuration.Client; @@ -28,10 +29,13 @@ public async Task RunAsync_LegacyDailyIndex_AddsMappingAndCanRunAgain() var migration = new AddEventEnvironment(configuration, GetService()); var context = new MigrationContext(EmptyLock.Empty, _logger, TestCancellationToken); + + // Act await migration.RunAsync(context); await migration.RunAsync(context); - var mapping = await client.Indices.GetMappingAsync(indexName, TestCancellationToken); + + // Assert Assert.True(mapping.IsValidResponse, mapping.DebugInformation); var properties = Assert.Single(mapping.Mappings).Value.Mappings.Properties!; var environment = Assert.IsType(properties["environment"]); diff --git a/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs b/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs index c7239d330d..7c51fdb5ed 100644 --- a/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/MigrationRegistrationTests.cs @@ -9,7 +9,14 @@ public sealed class MigrationRegistrationTests : TestWithServices [Fact] public void MigrationRegistration_EventEnvironment_IsVersionedAndResumable() { - var migration = Assert.Single(GetService>().OfType().DistinctBy(migration => migration.GetType())); + // Arrange + var migrations = GetService>(); + + // Act + var registeredMigrations = migrations.OfType().DistinctBy(migration => migration.GetType()).ToList(); + + // Assert + var migration = Assert.Single(registeredMigrations); Assert.Equal(MigrationType.VersionedAndResumable, migration.MigrationType); Assert.Equal(10, migration.Version); } diff --git a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs index 1fcc10552b..9b42e36895 100644 --- a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs +++ b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs @@ -88,6 +88,7 @@ public Task CreateAutoSessionAsync() [Fact] public async Task RunAsync_SameUserInDifferentEnvironments_SharesStackAndSeparatesAutomaticSessions() { + // Arrange var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); var project = _projectData.GenerateSampleProject(); var production = GenerateEvent(DateTimeOffset.Now.AddMinutes(-3), "same-user"); @@ -99,9 +100,17 @@ public async Task RunAsync_SameUserInDifferentEnvironments_SharesStackAndSeparat laterProduction.Data![Event.KnownDataKeys.Error] = production.Data[Event.KnownDataKeys.Error]; laterProduction.Environment = "PRODUCTION"; + // Act + var contexts = new List(); foreach (var ev in new[] { production, staging, laterProduction }) + contexts.Add(await _pipeline.RunAsync(ev, organization, project)); + + await RefreshDataAsync(); + var sessions = await _eventRepository.FindAsync(query => query.FilterExpression("type:session")); + + // Assert + foreach (var context in contexts) { - var context = await _pipeline.RunAsync(ev, organization, project); Assert.False(context.HasError, context.ErrorMessage); Assert.True(context.IsProcessed); } @@ -111,8 +120,6 @@ public async Task RunAsync_SameUserInDifferentEnvironments_SharesStackAndSeparat Assert.NotEqual(production.GetSessionId(), staging.GetSessionId()); Assert.Equal("Production", production.Environment); Assert.Equal("PRODUCTION", laterProduction.Environment); - await RefreshDataAsync(); - var sessions = await _eventRepository.FindAsync(query => query.FilterExpression("type:session")); Assert.Equal(2, sessions.Total); Assert.Equal(new[] { "Production", "staging" }, sessions.Documents.Select(ev => ev.Environment).Order(StringComparer.Ordinal).ToArray()); } diff --git a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs index feae829c89..26e2271238 100644 --- a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs +++ b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs @@ -58,9 +58,15 @@ public async Task CreateFromEventAsync_VersionAndBaseUrl_PreservesPayloadContrac [Fact] public async Task CreateFromEventAsync_VersionTwo_PreservesDeploymentEnvironment() { + // Arrange var context = GetWebHookDataContext(WebHook.KnownVersions.Version2); context.Event!.Environment = "production"; - var data = Assert.IsType(await _webHookData.CreateFromEventAsync(context)); + + // Act + object? result = await _webHookData.CreateFromEventAsync(context); + + // Assert + var data = Assert.IsType(result); Assert.Equal("production", data.Environment); } diff --git a/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs index a733a22d00..63d5a7aac7 100644 --- a/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs +++ b/tests/Exceptionless.Tests/Search/EventEnvironmentFilterTests.cs @@ -22,6 +22,12 @@ public sealed class EventEnvironmentFilterTests [InlineData("NOT (environment:staging AND type:error)", null)] public async Task GetAsync_EnvironmentScope_PreservesUsersOutsideTheEventType(string? filter, string? expected) { - Assert.Equal(expected, await EventEnvironmentFilter.GetAsync(filter)); + // Arrange is provided by the theory data. + + // Act + string? result = await EventEnvironmentFilter.GetAsync(filter); + + // Assert + Assert.Equal(expected, result); } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs index bdd817e2bf..5a532cf7a1 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EventEnvironmentTests.cs @@ -18,16 +18,22 @@ public EventEnvironmentTests(ITestOutputHelper output) : base(output) { } [InlineData("ENVIRONMENT")] public void Deserialize_DeploymentEnvironment_TrimsNameAndPreservesCasingAndRuntimeMetadata(string property) { + // Arrange var serializer = GetService(); - var ev = serializer.Deserialize("""{"PROPERTY":" Production ","data":{"@environment":{"machine_name":"worker-1"},"environment":{"custom":true}}}""".Replace("PROPERTY", property)); + string input = """{"PROPERTY":" Production ","data":{"@environment":{"machine_name":"worker-1"},"environment":{"custom":true}}}""".Replace("PROPERTY", property); + // Act + var ev = serializer.Deserialize(input); + string json = serializer.SerializeToString(ev)!; + var roundTrip = serializer.Deserialize(json); + + // Assert Assert.NotNull(ev); Assert.Equal("Production", ev.Environment); Assert.Equal("worker-1", ev.GetEnvironmentInfo(serializer, _logger)?.MachineName); Assert.NotNull(ev.Data?["environment"]); - string json = serializer.SerializeToString(ev)!; Assert.Contains("\"environment\":\"Production\"", json); - Assert.Equal("Production", serializer.Deserialize(json)?.Environment); + Assert.Equal("Production", roundTrip?.Environment); } [Theory] @@ -40,9 +46,14 @@ public void Deserialize_DeploymentEnvironment_TrimsNameAndPreservesCasingAndRunt [InlineData("\"production\\ninvalid\"")] public void ParseEvents_InvalidEnvironment_PreservesSubmissionBatch(string environment) { + // Arrange var parser = GetService(); - var events = parser.ParseEvents($$"""[{"type":"error","message":"first","environment":{{environment}}},{"type":"log","message":"second","environment":"staging"}]""", 2, null); + string input = $$"""[{"type":"error","message":"first","environment":{{environment}}},{"type":"log","message":"second","environment":"staging"}]"""; + + // Act + var events = parser.ParseEvents(input, 2, null); + // Assert Assert.NotNull(events); Assert.Equal(2, events.Count); Assert.Null(events[0].Environment); @@ -53,11 +64,20 @@ public void ParseEvents_InvalidEnvironment_PreservesSubmissionBatch(string envir [Fact] public void Serialize_UnspecifiedEnvironment_OmitsProperty() { + // Arrange var serializer = GetService(); - Assert.DoesNotContain("environment", serializer.SerializeToString(new Event())!); - Assert.Null(serializer.Deserialize("{}")?.Environment); - Assert.Null(new Event { Environment = new string('x', 65) }.Environment); - Assert.Equal(new string('X', 64), new Event { Environment = new string('X', 64) }.Environment); + + // Act + string serialized = serializer.SerializeToString(new Event())!; + var deserialized = serializer.Deserialize("{}"); + var tooLong = new Event { Environment = new string('x', 65) }; + var maxLength = new Event { Environment = new string('X', 64) }; + + // Assert + Assert.DoesNotContain("environment", serialized); + Assert.Null(deserialized?.Environment); + Assert.Null(tooLong.Environment); + Assert.Equal(new string('X', 64), maxLength.Environment); } [Theory] @@ -71,10 +91,14 @@ public void Serialize_UnspecifiedEnvironment_OmitsProperty() [InlineData("\"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm\"")] public void Deserialize_InvalidRootEnvironment_PreservesCustomDataRegardlessOfPropertyOrder(string environment) { + // Arrange var serializer = GetService(); using var expected = JsonDocument.Parse(environment); + var variants = new[] { (true, false), (false, false), (true, true), (false, true) }; - foreach (var (environmentFirst, duplicateKey) in new[] { (true, false), (false, false), (true, true), (false, true) }) + // Act + var results = new List<(bool DuplicateKey, Event? Event, string Serialized, string? Reserialized)>(); + foreach (var (environmentFirst, duplicateKey) in variants) { string environmentProperty = $"\"environment\":{environment}"; string dataProperty = duplicateKey @@ -84,10 +108,16 @@ public void Deserialize_InvalidRootEnvironment_PreservesCustomDataRegardlessOfPr ? $"{{{environmentProperty},{dataProperty}}}" : $"{{{dataProperty},{environmentProperty}}}"; var ev = serializer.Deserialize(json); + string serialized = serializer.SerializeToString(ev)!; + string? reserialized = serializer.SerializeToString(serializer.Deserialize(serialized)); + results.Add((duplicateKey, ev, serialized, reserialized)); + } + // Assert + foreach (var (duplicateKey, ev, serialized, reserialized) in results) + { Assert.NotNull(ev); Assert.Null(ev.Environment); - string serialized = serializer.SerializeToString(ev)!; using var result = JsonDocument.Parse(serialized); var data = result.RootElement.GetProperty("data"); Assert.True(data.GetProperty("kept").GetBoolean()); @@ -97,16 +127,28 @@ public void Deserialize_InvalidRootEnvironment_PreservesCustomDataRegardlessOfPr Assert.Equal("nested", data.GetProperty("environment").GetString()); } Assert.False(result.RootElement.TryGetProperty("environment", out _)); - Assert.Equal(serialized, serializer.SerializeToString(serializer.Deserialize(serialized))); + Assert.Equal(serialized, reserialized); } } [Fact] public void Equals_DifferentEnvironments_DistinguishesEvents() { - Assert.NotEqual(new Event { Environment = "production", Data = null }, new Event { Environment = "staging", Data = null }); - Assert.Equal(new Event { Environment = " Production ", Data = null }, new Event { Environment = "Production", Data = null }); - Assert.NotEqual(new Event { Environment = "Production", Data = null }, new Event { Environment = "production", Data = null }); + // Arrange + var production = new Event { Environment = "production", Data = null }; + var staging = new Event { Environment = "staging", Data = null }; + var paddedProduction = new Event { Environment = " Production ", Data = null }; + var capitalizedProduction = new Event { Environment = "Production", Data = null }; + + // Act + bool productionEqualsStaging = production.Equals(staging); + bool paddedEqualsCapitalized = paddedProduction.Equals(capitalizedProduction); + bool capitalizedEqualsProduction = capitalizedProduction.Equals(production); + + // Assert + Assert.False(productionEqualsStaging); + Assert.True(paddedEqualsCapitalized); + Assert.False(capitalizedEqualsProduction); } [Theory] @@ -115,19 +157,22 @@ public void Equals_DifferentEnvironments_DistinguishesEvents() [InlineData("ENVIRONMENT", " Production ", "Production")] public void ParseEvents_ValidEnvironment_StoresDeploymentNameOnlyAndRoundTrips(string propertyName, string environment, string expected) { + // Arrange var parser = GetService(); var serializer = GetService(); + string input = $$$"""{"{{{propertyName}}}":{{{JsonSerializer.Serialize(environment)}}},"data":{"kept":true}}"""; - var ev = Assert.Single(Assert.IsType>(parser.ParseEvents($$$"""{"{{{propertyName}}}":{{{JsonSerializer.Serialize(environment)}}},"data":{"kept":true}}""", 2, null))); + // Act + var parsed = parser.ParseEvents(input, 2, null); + string? serialized = serializer.SerializeToString(parsed?.SingleOrDefault()); + var reparsed = parser.ParseEvents(serialized!, 2, null); + // Assert + var ev = Assert.Single(Assert.IsType>(parsed)); Assert.Equal(expected, ev.Environment); Assert.NotNull(ev.Data); Assert.False(ev.Data.ContainsKey("environment")); - string serialized = serializer.SerializeToString(ev)!; - for (int i = 0; i < 3; i++) - { - var roundTrip = Assert.Single(Assert.IsType>(parser.ParseEvents(serialized, 2, null))); - Assert.Equal(serialized, serializer.SerializeToString(roundTrip)); - } + var roundTrip = Assert.Single(Assert.IsType>(reparsed)); + Assert.Equal(serialized, serializer.SerializeToString(roundTrip)); } } From 15dea4a2f307c2dff2e725f841e3ce4df733ecb3 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 20 Oct 2026 00:43:26 -0500 Subject: [PATCH 22/24] Keep the first heartbeat value read across session pages When a newer heartbeat arrived between pages, a later page recorded the new value and the final compare-and-remove deleted it, even though sessions on earlier pages had only applied the old value. The job now keeps the first value read for each heartbeat key, so the newer heartbeat is left for the next run. Co-Authored-By: Claude Opus 5.5 --- .../Jobs/CloseInactiveSessionsJob.cs | 3 ++- .../Jobs/CloseInactiveSessionsJobTests.cs | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs index d507bb4b7b..b0cfce0e67 100644 --- a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs +++ b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs @@ -77,7 +77,8 @@ protected override async Task RunInternalAsync(JobContext context) sessionsToUpdate.Add(sessionStart); if (heartbeatResult?.CacheKey is not null) { - consumedHeartbeats[heartbeatResult.CacheKey] = heartbeatResult.ActivityUtc; + // Keep the first value read: sessions on earlier pages only applied that one. + consumedHeartbeats.TryAdd(heartbeatResult.CacheKey, heartbeatResult.ActivityUtc); if (heartbeatResult.Close) closeKeysToRemove.Add($"{heartbeatResult.CacheKey}-close"); } diff --git a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs index aa8cf921bd..be9f123ed4 100644 --- a/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CloseInactiveSessionsJobTests.cs @@ -147,19 +147,37 @@ public async Task WillApplyIdentityHeartbeatToEnvironmentSessionsOnDifferentPage var initialSessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); Assert.Equal(102, initialSessionStarts.Count); - await _cache.SetAsync($"Project:{initialSessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}", DateTime.UtcNow.SubtractMinutes(1)); + string heartbeatKey = $"Project:{initialSessionStarts.First().ProjectId}:heartbeat:{userId.ToSHA1()}"; + await _cache.SetAsync(heartbeatKey, DateTime.UtcNow.SubtractMinutes(1)); _job.DefaultInactivePeriod = TimeSpan.FromMinutes(3); + // A newer heartbeat arrives after the first page applied the original one, so the second page reads it. + var latestHeartbeatUtc = DateTime.UtcNow; + bool isFirstPage = true; + using var handler = _eventRepository.DocumentsSaved.AddHandler(async (_, _) => + { + if (isFirstPage) + { + isFirstPage = false; + await _cache.SetAsync(heartbeatKey, latestHeartbeatUtc); + } + }); + // Act var result = await _job.RunAsync(TestCancellationToken); await RefreshDataAsync(); var sessionStarts = (await _eventRepository.GetAllAsync(o => o.PageLimit(500))).Documents.Where(e => e.IsSessionStart()).ToList(); + var heartbeat = await _cache.GetAsync(heartbeatKey); // Assert Assert.Equal(JobResult.Success, result); + Assert.False(isFirstPage); var openSessions = sessionStarts.Where(e => !e.HasSessionEndTime()).ToList(); Assert.Equal(2, openSessions.Count); Assert.Equal(["production", "staging"], openSessions.Select(e => e.Environment).Order()); + // The production session has not applied the newer heartbeat yet, so it is left for the next run. + Assert.True(heartbeat.HasValue); + Assert.Equal(latestHeartbeatUtc, heartbeat.Value); } [Fact] From f15ee8d83b9e4811535554c31c7d3799401fc6c1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 20 Oct 2026 00:55:37 -0500 Subject: [PATCH 23/24] Stop forwarding API requests before the API-origin e2e test ends The API-origin scenario forwards requests through page.route. A background saved-views query could still be forwarding when the assertions finished, and Playwright reported the in-flight route.fetch as an error outside any test, failing the shard. Remove the routes and ignore in-flight handlers before the test ends. Co-Authored-By: Claude Opus 5.5 --- src/Exceptionless.Web/ClientApp/e2e/tests/api-origin.e2e.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/api-origin.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/api-origin.e2e.ts index 901481f43b..407fffeae6 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/api-origin.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/api-origin.e2e.ts @@ -123,4 +123,7 @@ test('the application sends API requests and push connections to its configured await expect.poll(() => pushConnections.length).toBeGreaterThan(0); await expect.poll(() => telemetryRequests).toContain('/api/v2/projects/config'); expect(sameOriginApiRequests).toEqual([]); + + // Background queries can still be forwarding through the API route when the assertions finish. + await page.unrouteAll({ behavior: 'ignoreErrors' }); }); From 3ce65012caedab34c8ba4c01ab4bb797176c1417 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 20 Oct 2026 22:59:46 -0500 Subject: [PATCH 24/24] Let an environment in a saved view link override the saved filter The saved-view hook did not treat `environment` as a URL override, so a shared `/event/?environment=[...]` link showed the view's saved environment, and back/forward between environments left the modified state stale. A quoted single name in a hand-written URL also kept its quotes. Co-Authored-By: Claude Opus 5.5 --- .../e2e/tests/event-environments.e2e.ts | 43 +++++++++++++++++++ .../saved-views/use-saved-views.svelte.ts | 3 ++ .../saved-views/use-saved-views.test.ts | 16 ++++--- .../(app)/redirect-to-events.svelte.test.ts | 1 + .../routes/(app)/redirect-to-events.svelte.ts | 4 ++ 5 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts index e39d19b328..58c901f779 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/event-environments.e2e.ts @@ -64,6 +64,49 @@ test('environment filters include unspecified events and persist in saved views' for (const message of messages) await expect(getVisibleText(page, message)).toBeVisible(); }); +test('an environment in a saved view link overrides the saved environment', async ({ e2eApi, e2eScenario, page, request }) => { + const environments = ['production', 'staging']; + const messages = environments.map((environment) => `Saved view override ${environment} ${e2eScenario.run}`); + for (const [index, environment] of environments.entries()) { + const referenceId = createReferenceId(e2eScenario.run, `-env-override-${index}`); + await e2eApi.submitEvent(e2eScenario.projectId, e2eScenario.projectToken, { + environment, + message: messages[index], + reference_id: referenceId, + source: 'environment-override-test', + type: 'log' + }); + await e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId); + } + + const name = `Environment override ${e2eScenario.run.slice(-20)}`; + const slug = name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + const response = await request.post(`/api/v2/organizations/${e2eScenario.organizationId}/saved-views`, { + data: { + filter: `project:${e2eScenario.projectId} environment:production`, + filter_definitions: JSON.stringify([ + { type: 'project', value: [e2eScenario.projectId] }, + { type: 'environment', value: ['production'] } + ]), + name, + organization_id: e2eScenario.organizationId, + slug, + view_type: 'events' + }, + headers: { Authorization: `Bearer ${e2eScenario.userToken}` } + }); + expect(response.status(), await response.text()).toBe(201); + + await page.goto(`/event/${slug}?environment=${encodeURIComponent(JSON.stringify(['staging']))}`); + await expect(page.getByRole('heading', { exact: true, name })).toBeVisible(); + await expect(getVisibleText(page, messages[1]!)).toBeVisible({ timeout: 30_000 }); + await expect(getVisibleText(page, messages[0]!)).toBeHidden(); + await expect(page.getByRole('button', { name: /^Environment\s+staging/ })).toBeVisible(); +}); + test('stack, session, and stream filters retain names with no current events', async ({ e2eScenario, page }) => { for (const route of ['stack', 'sessions', 'stream']) { await page.goto(`/${route}?project=${e2eScenario.projectId}`); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.svelte.ts index f38e289f60..61e3527a00 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.svelte.ts @@ -152,6 +152,7 @@ type SavedViewDraftFilterPageState = App.PageState & { const SAVED_VIEW_QUERY_PARAMETER_FILTER_KEYS: readonly string[] = [ 'boolean-bot', 'boolean-first', + 'environment', 'level', 'project', 'reference', @@ -315,6 +316,7 @@ export function isSavedViewDraftFilterHistoryEntry(entry: unknown, viewId: strin const SAVED_VIEW_OVERRIDE_QUERY_PARAMETERS = [ 'bot', + 'environment', 'filter', 'filters', 'first', @@ -575,6 +577,7 @@ export function useSavedViews(options: UseSavedViewsOptions): UseSavedViewsRetur const keys: string[] = []; const queryParameterKeys = [ ['bot', 'boolean-bot'], + ['environment', 'environment'], ['first', 'boolean-first'], ['level', 'level'], ['project', 'project'], diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.test.ts index a16d9c65c0..463ef65831 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/saved-views/use-saved-views.test.ts @@ -2,7 +2,7 @@ import { QueryClient } from '@tanstack/svelte-query'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { serializeFilters } from '#features/events/components/filters/helpers.svelte.ts'; -import { DateFilter, ProjectFilter, StringFilter } from '#features/events/components/filters/index.ts'; +import { DateFilter, EnvironmentFilter, ProjectFilter, StringFilter } from '#features/events/components/filters/index.ts'; import { ChangeType } from '#features/websockets/models.ts'; import type { SavedView } from './models'; @@ -160,6 +160,9 @@ describe('useSavedViews', () => { expect(getSavedViewOverrideSignature(new URL('https://example.test/event/errors?time=30d&status=open'))).not.toBe(baseline); expect(getSavedViewOverrideSignature(new URL('https://example.test/event/errors?time=90d'))).not.toBe(baseline); expect(getSavedViewOverrideSignature(new URL('https://example.test/event/errors?time=90d&status=open&sort=type'))).not.toBe(baseline); + expect(getSavedViewOverrideSignature(new URL('https://example.test/event/errors?time=90d&status=open&environment=%5B%22staging%22%5D'))).not.toBe( + baseline + ); }); }); @@ -173,10 +176,13 @@ describe('useSavedViews', () => { version: 1 as const }; - expect(getExpressionFilterOverrideKeys([new StringFilter('error.type', 'OldException')], [new ProjectFilter(['project-1'])], draft)).toEqual([ - 'string-error.type', - 'keyword' - ]); + expect( + getExpressionFilterOverrideKeys( + [new StringFilter('error.type', 'OldException')], + [new ProjectFilter(['project-1']), new EnvironmentFilter(['staging'])], + draft + ) + ).toEqual(['string-error.type', 'keyword']); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts index a77634b2e7..a4516aa383 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.test.ts @@ -18,6 +18,7 @@ describe('redirect-to-events', () => { expect(deserializeEnvironmentQueryParam(url.searchParams.get('environment')!)).toEqual(names); expect(url.searchParams.has('filter')).toBe(false); expect(deserializeEnvironmentQueryParam('staging')).toEqual(['staging']); + expect(deserializeEnvironmentQueryParam('"qa west"')).toEqual(['qa west']); }); it('snapshots list filter query parameters from shared reactive state', async () => { // Arrange diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts index 0bd92ca290..62b7ca77a1 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/redirect-to-events.svelte.ts @@ -74,6 +74,10 @@ export function deserializeEnvironmentQueryParam(value: string): string[] { if (Array.isArray(values)) { return values.filter((name): name is string => typeof name === 'string'); } + + if (typeof values === 'string') { + return [values]; + } } catch { // Single names are also accepted in hand-written URLs. }