From 6be1032f41f1eb8cf321320fb8bcd26bcd86e873 Mon Sep 17 00:00:00 2001 From: Slawosz Slawinski Date: Tue, 1 Sep 2026 10:28:21 +0100 Subject: [PATCH 1/5] Remove airbyte config json from gem config, it is now generated dynamically during airbyte rake tasks --- config/locales/en.yml | 4 -- lib/dfe/analytics.rb | 6 -- lib/dfe/analytics/airbyte_stream_config.rb | 4 -- lib/dfe/analytics/config.rb | 3 +- lib/dfe/analytics/fields.rb | 14 ---- lib/dfe/analytics/tasks/fields.rake | 8 --- .../dfe/analytics/install_generator.rb | 4 -- .../analytics/airbyte_stream_config_spec.rb | 9 --- spec/dfe/analytics/config_spec.rb | 4 +- .../generators/install_generator_spec.rb | 8 --- spec/dfe/analytics_spec.rb | 66 ------------------- 11 files changed, 3 insertions(+), 127 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 287acc4b..eb94de5c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -116,10 +116,6 @@ en: description: | Perform airbyte checks on startup and allow airbyte config generation default: false - airbyte_stream_config_path: - description: | - Path of airbyte stream config file relative to the App root (Rails.root) - default: /terraform/aks/workspace-variables/airbyte_stream_config.json airbyte_internal_dataset: description: | Airbyte internal dataset for service diff --git a/lib/dfe/analytics.rb b/lib/dfe/analytics.rb index 32272588..414d5d10 100644 --- a/lib/dfe/analytics.rb +++ b/lib/dfe/analytics.rb @@ -120,12 +120,6 @@ def self.event_debug_filters {} end - def self.airbyte_stream_config - JSON.parse(File.read(DfE::Analytics.config.airbyte_stream_config_path)).deep_symbolize_keys - rescue RuntimeError - {} - end - def self.environment config.environment end diff --git a/lib/dfe/analytics/airbyte_stream_config.rb b/lib/dfe/analytics/airbyte_stream_config.rb index 49031215..1954ce36 100644 --- a/lib/dfe/analytics/airbyte_stream_config.rb +++ b/lib/dfe/analytics/airbyte_stream_config.rb @@ -13,10 +13,6 @@ class AirbyteStreamConfig AIRBYTE_HEARTBEAT_ATTRIBUTES = %w[id last_heartbeat].freeze AIRBYTE_HEARTBEAT_ENTITY_ATTRIBUTES = { AIRBYTE_HEARTBEAT_ENTITY.to_sym => AIRBYTE_HEARTBEAT_ATTRIBUTES }.freeze - def self.generate_pretty_json_for(table_attributes) - JSON.pretty_generate(generate_for(table_attributes)) - end - def self.generate_for(table_attributes) { configurations: { streams: streams_for(table_attributes) } } end diff --git a/lib/dfe/analytics/config.rb b/lib/dfe/analytics/config.rb index 834ed61c..bc321e87 100644 --- a/lib/dfe/analytics/config.rb +++ b/lib/dfe/analytics/config.rb @@ -76,11 +76,10 @@ def self.configure(config) config.airbyte_server_url ||= ENV.fetch('AIRBYTE_SERVER_URL', nil) config.airbyte_configuration ||= JSON.parse(ENV.fetch('AIRBYTE_CONFIGURATION', '{}')).transform_keys(&:underscore).symbolize_keys + config.airbyte_stream_config_path ||= nil config.async = true if config.async.nil? - config.airbyte_stream_config_path = File.join(Rails.root, config.airbyte_stream_config_path) if config.airbyte_stream_config_path.present? - return unless config.azure_federated_auth config.azure_client_id ||= ENV.fetch('AZURE_CLIENT_ID', nil) diff --git a/lib/dfe/analytics/fields.rb b/lib/dfe/analytics/fields.rb index 3e885063..52443bf4 100644 --- a/lib/dfe/analytics/fields.rb +++ b/lib/dfe/analytics/fields.rb @@ -45,20 +45,6 @@ def self.check! HEREDOC end - if DfE::Analytics.airbyte_enabled? && airbyte_conflicting_fields.any? - errors << <<~HEREDOC - Differences detected between analytics.yml and #{File.basename(DfE::Analytics.config.airbyte_stream_config_path)}! - - The following field differences exist. To upgrade the airbyte stream config, run: - - SUPPRESS_DFE_ANALYTICS_INIT=1 bundle exec rails dfe:analytics:regenerate_airbyte_stream_config - - Field Differences: - - #{airbyte_conflicting_fields.to_yaml} - HEREDOC - end - configuration_errors = errors.join("\n\n----------------\n\n") raise(ConfigurationError, configuration_errors) if errors.any? diff --git a/lib/dfe/analytics/tasks/fields.rake b/lib/dfe/analytics/tasks/fields.rake index 5df1bd3b..997c6990 100644 --- a/lib/dfe/analytics/tasks/fields.rake +++ b/lib/dfe/analytics/tasks/fields.rake @@ -12,13 +12,5 @@ namespace :dfe do { shared: DfE::Analytics::Fields.generate_blocklist }.to_yaml ) end - - desc 'Generate a new airbyte stream config listing all fields to be send to Bigquery from analytics.yml' - task regenerate_airbyte_stream_config: :environment do - File.write( - DfE::Analytics.config.airbyte_stream_config_path, - DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(DfE::Analytics.allowlist) - ) - end end end diff --git a/lib/generators/dfe/analytics/install_generator.rb b/lib/generators/dfe/analytics/install_generator.rb index a0e1644b..5b9c3697 100644 --- a/lib/generators/dfe/analytics/install_generator.rb +++ b/lib/generators/dfe/analytics/install_generator.rb @@ -13,10 +13,6 @@ def install create_file 'config/analytics.yml', { 'shared' => {} }.to_yaml create_file 'config/analytics_hidden_pii.yml', { 'shared' => {} }.to_yaml create_file 'config/analytics_blocklist.yml', { 'shared' => {} }.to_yaml - create_file( - DfE::Analytics.config.airbyte_stream_config_path, - DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(table1: %w[id field1 field2]) - ) end private diff --git a/spec/dfe/analytics/airbyte_stream_config_spec.rb b/spec/dfe/analytics/airbyte_stream_config_spec.rb index 3aacf642..b4c0e47b 100644 --- a/spec/dfe/analytics/airbyte_stream_config_spec.rb +++ b/spec/dfe/analytics/airbyte_stream_config_spec.rb @@ -56,15 +56,6 @@ end end - describe '.generate_pretty_json_for' do - let(:entity_attributes) { { users: %w[id name] } } - - it 'returns a pretty JSON version of generate_for' do - expect(described_class.generate_pretty_json_for(entity_attributes)) - .to eq(JSON.pretty_generate(described_class.generate_for(entity_attributes))) - end - end - describe '.entity_attributes' do context 'when config is empty' do before { allow(DfE::Analytics).to receive(:airbyte_stream_config).and_return({}) } diff --git a/spec/dfe/analytics/config_spec.rb b/spec/dfe/analytics/config_spec.rb index e48638c2..9adb83aa 100644 --- a/spec/dfe/analytics/config_spec.rb +++ b/spec/dfe/analytics/config_spec.rb @@ -73,8 +73,8 @@ described_class.configure(config) end - it 'resolves the full path from Rails.root' do - expect(config.airbyte_stream_config_path).to eq(Rails.root.join('config/airbyte.json').to_s) + it 'still stores the value' do + expect(config.airbyte_stream_config_path).to eq('config/airbyte.json') end end end diff --git a/spec/dfe/analytics/generators/install_generator_spec.rb b/spec/dfe/analytics/generators/install_generator_spec.rb index 786b5c6d..676295c1 100644 --- a/spec/dfe/analytics/generators/install_generator_spec.rb +++ b/spec/dfe/analytics/generators/install_generator_spec.rb @@ -64,14 +64,6 @@ { 'shared' => {} }.to_yaml ) end - it 'creates the airbyte stream config file' do - generator.install - - expect(generator).to have_received(:create_file).with( - airbyte_stream_config_path, - generated_stream_config - ) - end it 'uses the generated config options in the initializer' do generator.install diff --git a/spec/dfe/analytics_spec.rb b/spec/dfe/analytics_spec.rb index db375db6..9ce77b5f 100644 --- a/spec/dfe/analytics_spec.rb +++ b/spec/dfe/analytics_spec.rb @@ -427,70 +427,4 @@ end end end - - describe '.airbyte_stream_config' do - let(:mock_path) { '/fake/path/airbyte_stream_config.json' } - - let(:config_double) do - instance_double( - 'DfE::Analytics.config', - airbyte_stream_config_path: mock_path - ) - end - - before do - allow(DfE::Analytics).to receive(:config).and_return(config_double) - end - - context 'when the JSON file is valid' do - let(:json_data) do - { - configurations: { - streams: [ - { - name: 'teachers', - syncMode: 'incremental_append', - selectedFields: [{ fieldPath: ['id'] }] - } - ] - } - }.to_json - end - - before do - allow(File).to receive(:read).with(mock_path).and_return(json_data) - end - - it 'returns the parsed JSON with symbolized keys' do - result = described_class.airbyte_stream_config - - expect(result).to eq( - configurations: { - streams: [ - { - name: 'teachers', - syncMode: 'incremental_append', - selectedFields: [{ fieldPath: ['id'] }] - } - ] - } - ) - - # verify keys are deep symbolized - expect(result.keys).to all(be_a(Symbol)) - expect(result[:configurations][:streams].first[:name]).to eq('teachers') - expect(result[:configurations][:streams].first[:selectedFields].first[:fieldPath]).to eq(['id']) - end - end - - context 'when File.read raises a RuntimeError' do - before do - allow(File).to receive(:read).with(mock_path).and_raise(RuntimeError) - end - - it 'returns an empty hash' do - expect(described_class.airbyte_stream_config).to eq({}) - end - end - end end From fda166e4a277ff11a4f15e3d4e389cae4ddd55b6 Mon Sep 17 00:00:00 2001 From: Slawosz Slawinski Date: Wed, 2 Sep 2026 13:24:17 +0100 Subject: [PATCH 2/5] Add depriciation warning for airbyte_stream_config_path config option --- lib/dfe/analytics/config.rb | 2 ++ spec/dfe/analytics/config_spec.rb | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/lib/dfe/analytics/config.rb b/lib/dfe/analytics/config.rb index bc321e87..d518990b 100644 --- a/lib/dfe/analytics/config.rb +++ b/lib/dfe/analytics/config.rb @@ -78,6 +78,8 @@ def self.configure(config) JSON.parse(ENV.fetch('AIRBYTE_CONFIGURATION', '{}')).transform_keys(&:underscore).symbolize_keys config.airbyte_stream_config_path ||= nil + Rails.logger.warn("DEPRECATION WARNING: DfE::Analytics airbyte_stream_config_path config option is no longer used; the airbyte stream config is now generated dynamically. You're running v#{DfE::Analytics::VERSION}. To silence this warning, remove airbyte_stream_config_path from your DfE::Analytics initializer.") if config.airbyte_stream_config_path.present? + config.async = true if config.async.nil? return unless config.azure_federated_auth diff --git a/spec/dfe/analytics/config_spec.rb b/spec/dfe/analytics/config_spec.rb index 9adb83aa..fac68020 100644 --- a/spec/dfe/analytics/config_spec.rb +++ b/spec/dfe/analytics/config_spec.rb @@ -69,6 +69,7 @@ context 'when airbyte_stream_config_path is set' do before do + allow(Rails.logger).to receive(:warn) config.airbyte_stream_config_path = 'config/airbyte.json' described_class.configure(config) end @@ -76,6 +77,21 @@ it 'still stores the value' do expect(config.airbyte_stream_config_path).to eq('config/airbyte.json') end + + it 'logs a deprecation warning' do + expect(Rails.logger).to have_received(:warn).with(/DEPRECATION WARNING/) + end + end + + context 'when airbyte_stream_config_path is not set' do + before do + allow(Rails.logger).to receive(:warn) + described_class.configure(config) + end + + it 'does not log a deprecation warning' do + expect(Rails.logger).not_to have_received(:warn).with(/DEPRECATION WARNING/) + end end end From 4ce84f207a37c9060c7e126fb5183c4e69442549 Mon Sep 17 00:00:00 2001 From: Slawosz Slawinski Date: Wed, 2 Sep 2026 13:52:35 +0100 Subject: [PATCH 3/5] Address PR comments --- lib/dfe/analytics/fields.rb | 7 ------- lib/dfe/analytics/tasks/fields.rake | 8 ++++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/dfe/analytics/fields.rb b/lib/dfe/analytics/fields.rb index 52443bf4..4e35719a 100644 --- a/lib/dfe/analytics/fields.rb +++ b/lib/dfe/analytics/fields.rb @@ -88,13 +88,6 @@ def self.conflicting_fields diff_between(allowlist, diff_between(allowlist, blocklist)) end - def self.airbyte_conflicting_fields - diff_between( - allowlist.merge(DfE::Analytics::AirbyteStreamConfig::AIRBYTE_HEARTBEAT_ENTITY_ATTRIBUTES).transform_values(&:uniq), - DfE::Analytics::AirbyteStreamConfig.entity_attributes - ) - end - # extract and concatenate the fields associated with an entity in 1 or # more entity->field lists def self.extract_entity_attributes_from_lists(entity, *lists) diff --git a/lib/dfe/analytics/tasks/fields.rake b/lib/dfe/analytics/tasks/fields.rake index 997c6990..953fd6d3 100644 --- a/lib/dfe/analytics/tasks/fields.rake +++ b/lib/dfe/analytics/tasks/fields.rake @@ -12,5 +12,13 @@ namespace :dfe do { shared: DfE::Analytics::Fields.generate_blocklist }.to_yaml ) end + + desc 'Generate a debug airbyte stream config listing all fields to be send to Bigquery from analytics.yml. Generates debug_airbyte_config.json.' + task airbyte_stream_config_for_debug: :environment do + File.write( + "debug_airbyte_config.json", + DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(DfE::Analytics.allowlist) + ) + end end end From de71a694a795ac8c4a65b90272f1f1702b72e075 Mon Sep 17 00:00:00 2001 From: Slawosz Slawinski Date: Wed, 2 Sep 2026 13:56:14 +0100 Subject: [PATCH 4/5] Fix rake task --- lib/dfe/analytics/airbyte_stream_config.rb | 4 ++++ lib/dfe/analytics/tasks/fields.rake | 2 +- lib/dfe/analytics/testing/helpers.rb | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/dfe/analytics/airbyte_stream_config.rb b/lib/dfe/analytics/airbyte_stream_config.rb index 1954ce36..49031215 100644 --- a/lib/dfe/analytics/airbyte_stream_config.rb +++ b/lib/dfe/analytics/airbyte_stream_config.rb @@ -13,6 +13,10 @@ class AirbyteStreamConfig AIRBYTE_HEARTBEAT_ATTRIBUTES = %w[id last_heartbeat].freeze AIRBYTE_HEARTBEAT_ENTITY_ATTRIBUTES = { AIRBYTE_HEARTBEAT_ENTITY.to_sym => AIRBYTE_HEARTBEAT_ATTRIBUTES }.freeze + def self.generate_pretty_json_for(table_attributes) + JSON.pretty_generate(generate_for(table_attributes)) + end + def self.generate_for(table_attributes) { configurations: { streams: streams_for(table_attributes) } } end diff --git a/lib/dfe/analytics/tasks/fields.rake b/lib/dfe/analytics/tasks/fields.rake index 953fd6d3..7b876842 100644 --- a/lib/dfe/analytics/tasks/fields.rake +++ b/lib/dfe/analytics/tasks/fields.rake @@ -16,7 +16,7 @@ namespace :dfe do desc 'Generate a debug airbyte stream config listing all fields to be send to Bigquery from analytics.yml. Generates debug_airbyte_config.json.' task airbyte_stream_config_for_debug: :environment do File.write( - "debug_airbyte_config.json", + 'debug_airbyte_config.json', DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(DfE::Analytics.allowlist) ) end diff --git a/lib/dfe/analytics/testing/helpers.rb b/lib/dfe/analytics/testing/helpers.rb index 5cf47b51..b133dd32 100644 --- a/lib/dfe/analytics/testing/helpers.rb +++ b/lib/dfe/analytics/testing/helpers.rb @@ -311,7 +311,7 @@ def with_analytics_config(options) end def test_dummy_config - config = DfE::Analytics.config.members.each_with_object({}) { |key, mem| mem[key] = 'dummy_value' } + config = DfE::Analytics.config.members.to_h { |key| [key, 'dummy_value'] } config[:google_cloud_credentials] = '{ "dummy_value": 1 }' config[:bigquery_api_json_key] = '{ "dummy_value": 1 }' config From dd3449d3633ef9853769126e39cfe37eae15fe6c Mon Sep 17 00:00:00 2001 From: Slawosz Slawinski Date: Wed, 2 Sep 2026 14:05:35 +0100 Subject: [PATCH 5/5] Fix specs --- spec/dfe/analytics/fields_spec.rb | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/spec/dfe/analytics/fields_spec.rb b/spec/dfe/analytics/fields_spec.rb index 9815ed8c..7aab2808 100644 --- a/spec/dfe/analytics/fields_spec.rb +++ b/spec/dfe/analytics/fields_spec.rb @@ -77,34 +77,6 @@ end end - describe '.airbyte_conflicting_fields' do - context 'when fields conflict' do - let(:existing_allowlist) { { Candidate.table_name.to_sym => %w[email_address id first_name dob] } } - let(:existing_airbytelist) { { Candidate.table_name.to_sym => %w[email_address first_name] } } - - it 'returns the conflicting fields' do - conflicts = described_class.airbyte_conflicting_fields - expect(conflicts[Candidate.table_name.to_sym]).to eq(%w[id dob]) - end - - describe '.check!' do - it 'raises an error' do - expect { DfE::Analytics::Fields.check! }.to raise_error(DfE::Analytics::ConfigurationError, /Conflict detected/) - end - end - end - - context 'when there are no conflicts' do - let(:existing_allowlist) { { Candidate.table_name.to_sym => %w[id email_address] } } - let(:existing_airbytelist) { { Candidate.table_name.to_sym => %w[email_address id] } } - - it 'returns nothing' do - conflicts = described_class.airbyte_conflicting_fields - expect(conflicts).to be_empty - end - end - end - describe '.generate_blocklist' do it 'returns all the fields in the model that aren’t in the allowlist' do fields = described_class.generate_blocklist[Candidate.table_name.to_sym]