From 5064c00e6b50c65ea3d1ca2ea7bb08f7007e4fb3 Mon Sep 17 00:00:00 2001
From: Mae Beale <maebeale@gmail.com>
Date: Thu, 13 Aug 2026 18:38:20 -0400
Subject: [PATCH] Drop benign SolidQueue shutdown connection errors from
 Honeybadger

A terminating worker pod raises ActiveRecord::ConnectionNotEstablished
(TRILOGY_CLOSED_CONNECTION) in at_exit while deregistering its SolidQueue
Process record after the DB pool has closed. It pages on every deploy/restart,
affects no users, and loses no work. Halt only closed-connection errors raised
outside any request so genuine mid-request DB outages still report.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 config/initializers/honeybadger.rb            | 23 +++++++++
 .../honeybadger_shutdown_filter_spec.rb       | 48 +++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 config/initializers/honeybadger.rb
 create mode 100644 spec/initializers/honeybadger_shutdown_filter_spec.rb

diff --git a/config/initializers/honeybadger.rb b/config/initializers/honeybadger.rb
new file mode 100644
index 000000000..d6401e7a9
--- /dev/null
+++ b/config/initializers/honeybadger.rb
@@ -0,0 +1,23 @@
+# Suppress benign shutdown noise. When a pod is terminated (deploy, restart,
+# scale-down) the DB connection pool closes while SolidQueue's supervisor is
+# still trying to deregister its Process record, so that final query fails with
+# a closed connection. There's no request, no user, and nothing is lost — it's
+# just restart noise, so we drop it rather than page on it.
+#
+# Kept narrow on purpose: only closed-connection errors with no component (i.e.
+# outside any web request or job) are dropped, so a genuine mid-request DB
+# outage still reports.
+Honeybadger.configure do |config|
+  shutdown_error_classes = %w[
+    ActiveRecord::ConnectionNotEstablished
+    Trilogy::EOFError
+  ]
+
+  config.before_notify do |notice|
+    connection_error = shutdown_error_classes.include?(notice.error_class)
+    closed_connection = notice.error_message.to_s.include?("TRILOGY_CLOSED_CONNECTION")
+    outside_request = notice.component.blank?
+
+    notice.halt! if connection_error && closed_connection && outside_request
+  end
+end
diff --git a/spec/initializers/honeybadger_shutdown_filter_spec.rb b/spec/initializers/honeybadger_shutdown_filter_spec.rb
new file mode 100644
index 000000000..f136d5f98
--- /dev/null
+++ b/spec/initializers/honeybadger_shutdown_filter_spec.rb
@@ -0,0 +1,48 @@
+require "rails_helper"
+
+# Exercises the before_notify hook registered in config/initializers/honeybadger.rb,
+# which drops the benign closed-connection error a SolidQueue pod raises while
+# deregistering itself during shutdown, without hiding real DB outages.
+RSpec.describe "Honeybadger shutdown filter" do
+  def run_hooks(error_class:, error_message:, component:)
+    notice = instance_double(
+      Honeybadger::Notice,
+      error_class: error_class,
+      error_message: error_message,
+      component: component
+    )
+    allow(notice).to receive(:halt!)
+    Honeybadger.config.before_notify_hooks.each { |hook| hook.call(notice) }
+    notice
+  end
+
+  it "drops a closed-connection error raised outside any request (pod shutdown noise)" do
+    notice = run_hooks(
+      error_class: "ActiveRecord::ConnectionNotEstablished",
+      error_message: "trilogy_connect - unable to connect to db:25060: TRILOGY_CLOSED_CONNECTION",
+      component: nil
+    )
+
+    expect(notice).to have_received(:halt!)
+  end
+
+  it "keeps a closed-connection error raised during a request" do
+    notice = run_hooks(
+      error_class: "ActiveRecord::ConnectionNotEstablished",
+      error_message: "trilogy_connect - ...: TRILOGY_CLOSED_CONNECTION",
+      component: "GrantsController"
+    )
+
+    expect(notice).not_to have_received(:halt!)
+  end
+
+  it "keeps a genuine can't-reach-the-database error" do
+    notice = run_hooks(
+      error_class: "ActiveRecord::ConnectionNotEstablished",
+      error_message: "trilogy_connect - unable to connect: Connection refused",
+      component: nil
+    )
+
+    expect(notice).not_to have_received(:halt!)
+  end
+end