Remove airbyte config json from gem config, it is now generated dynam…

config/locales/en.yml

@@ -116,10 +116,6 @@ en:
           description: |
             Perform airbyte checks on startup and allow airbyte config generation
           default: false
-        airbyte_stream_config_path:
-          description: |
-            Path of airbyte stream config file relative to the App root (Rails.root)
-          default: /terraform/aks/workspace-variables/airbyte_stream_config.json
         airbyte_internal_dataset:
           description: |
             Airbyte internal dataset for service

lib/dfe/analytics.rb

@@ -120,12 +120,6 @@ def self.event_debug_filters
       {}
     end
 
-    def self.airbyte_stream_config
-      JSON.parse(File.read(DfE::Analytics.config.airbyte_stream_config_path)).deep_symbolize_keys
-    rescue RuntimeError
-      {}
-    end
-
     def self.environment
       config.environment
     end

lib/dfe/analytics/config.rb

@@ -76,10 +76,11 @@ def self.configure(config)
         config.airbyte_server_url               ||= ENV.fetch('AIRBYTE_SERVER_URL', nil)
         config.airbyte_configuration            ||=
           JSON.parse(ENV.fetch('AIRBYTE_CONFIGURATION', '{}')).transform_keys(&:underscore).symbolize_keys
+        config.airbyte_stream_config_path       ||= nil
 
-        config.async = true if config.async.nil?
+        Rails.logger.warn("DEPRECATION WARNING: DfE::Analytics airbyte_stream_config_path config option is no longer used; the airbyte stream config is now generated dynamically. You're running v#{DfE::Analytics::VERSION}. To silence this warning, remove airbyte_stream_config_path from your DfE::Analytics initializer.") if config.airbyte_stream_config_path.present?
 
-        config.airbyte_stream_config_path = File.join(Rails.root, config.airbyte_stream_config_path) if config.airbyte_stream_config_path.present?
+        config.async = true if config.async.nil?
 
         return unless config.azure_federated_auth
 

lib/dfe/analytics/fields.rb

@@ -45,20 +45,6 @@ def self.check!
           HEREDOC
         end
 
-        if DfE::Analytics.airbyte_enabled? && airbyte_conflicting_fields.any?
-          errors << <<~HEREDOC
-            Differences detected between analytics.yml and #{File.basename(DfE::Analytics.config.airbyte_stream_config_path)}!
-
-            The following field differences exist. To upgrade the airbyte stream config, run:
-
-            SUPPRESS_DFE_ANALYTICS_INIT=1 bundle exec rails dfe:analytics:regenerate_airbyte_stream_config
-
-            Field Differences:
-
-            #{airbyte_conflicting_fields.to_yaml}
-          HEREDOC
-        end
-
         configuration_errors = errors.join("\n\n----------------\n\n")
 
         raise(ConfigurationError, configuration_errors) if errors.any?
@@ -102,13 +88,6 @@ def self.conflicting_fields
         diff_between(allowlist, diff_between(allowlist, blocklist))
       end
 
-      def self.airbyte_conflicting_fields
-        diff_between(
-          allowlist.merge(DfE::Analytics::AirbyteStreamConfig::AIRBYTE_HEARTBEAT_ENTITY_ATTRIBUTES).transform_values(&:uniq),
-          DfE::Analytics::AirbyteStreamConfig.entity_attributes
-        )
-      end
-
       # extract and concatenate the fields associated with an entity in 1 or
       # more entity->field lists
       def self.extract_entity_attributes_from_lists(entity, *lists)

lib/dfe/analytics/tasks/fields.rake

@@ -13,10 +13,10 @@ namespace :dfe do
       )
     end
 
-    desc 'Generate a new airbyte stream config listing all fields to be send to Bigquery from analytics.yml'
-    task regenerate_airbyte_stream_config: :environment do
+    desc 'Generate a debug airbyte stream config listing all fields to be send to Bigquery from analytics.yml. Generates debug_airbyte_config.json.'
+    task airbyte_stream_config_for_debug: :environment do
       File.write(
-        DfE::Analytics.config.airbyte_stream_config_path,
+        'debug_airbyte_config.json',
         DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(DfE::Analytics.allowlist)
       )
     end

lib/dfe/analytics/testing/helpers.rb

@@ -311,7 +311,7 @@ def with_analytics_config(options)
         end
 
         def test_dummy_config
-          config = DfE::Analytics.config.members.each_with_object({}) { |key, mem| mem[key] = 'dummy_value' }
+          config = DfE::Analytics.config.members.to_h { |key| [key, 'dummy_value'] }
           config[:google_cloud_credentials] = '{ "dummy_value":  1 }'
           config[:bigquery_api_json_key] = '{ "dummy_value":  1 }'
           config

lib/generators/dfe/analytics/install_generator.rb

@@ -13,10 +13,6 @@ def install
         create_file 'config/analytics.yml', { 'shared' => {} }.to_yaml
         create_file 'config/analytics_hidden_pii.yml', { 'shared' => {} }.to_yaml
         create_file 'config/analytics_blocklist.yml', { 'shared' => {} }.to_yaml
-        create_file(
-          DfE::Analytics.config.airbyte_stream_config_path,
-          DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(table1: %w[id field1 field2])
-        )
       end
 
       private

spec/dfe/analytics/airbyte_stream_config_spec.rb

@@ -56,15 +56,6 @@
     end
   end
 
-  describe '.generate_pretty_json_for' do
-    let(:entity_attributes) { { users: %w[id name] } }
-
-    it 'returns a pretty JSON version of generate_for' do
-      expect(described_class.generate_pretty_json_for(entity_attributes))
-        .to eq(JSON.pretty_generate(described_class.generate_for(entity_attributes)))
-    end
-  end
-
   describe '.entity_attributes' do
     context 'when config is empty' do
       before { allow(DfE::Analytics).to receive(:airbyte_stream_config).and_return({}) }

spec/dfe/analytics/config_spec.rb

@@ -69,12 +69,28 @@
 
     context 'when airbyte_stream_config_path is set' do
       before do
+        allow(Rails.logger).to receive(:warn)
         config.airbyte_stream_config_path = 'config/airbyte.json'
         described_class.configure(config)
       end
 
-      it 'resolves the full path from Rails.root' do
-        expect(config.airbyte_stream_config_path).to eq(Rails.root.join('config/airbyte.json').to_s)
+      it 'still stores the value' do
+        expect(config.airbyte_stream_config_path).to eq('config/airbyte.json')
+      end
+
+      it 'logs a deprecation warning' do
+        expect(Rails.logger).to have_received(:warn).with(/DEPRECATION WARNING/)
+      end
+    end
+
+    context 'when airbyte_stream_config_path is not set' do
+      before do
+        allow(Rails.logger).to receive(:warn)
+        described_class.configure(config)
+      end
+
+      it 'does not log a deprecation warning' do
+        expect(Rails.logger).not_to have_received(:warn).with(/DEPRECATION WARNING/)
       end
     end
   end

spec/dfe/analytics/fields_spec.rb

@@ -77,34 +77,6 @@
       end
     end
 
-    describe '.airbyte_conflicting_fields' do
-      context 'when fields conflict' do
-        let(:existing_allowlist) { { Candidate.table_name.to_sym => %w[email_address id first_name dob] } }
-        let(:existing_airbytelist) { { Candidate.table_name.to_sym => %w[email_address first_name] } }
-
-        it 'returns the conflicting fields' do
-          conflicts = described_class.airbyte_conflicting_fields
-          expect(conflicts[Candidate.table_name.to_sym]).to eq(%w[id dob])
-        end
-
-        describe '.check!' do
-          it 'raises an error' do
-            expect { DfE::Analytics::Fields.check! }.to raise_error(DfE::Analytics::ConfigurationError, /Conflict detected/)
-          end
-        end
-      end
-
-      context 'when there are no conflicts' do
-        let(:existing_allowlist) { { Candidate.table_name.to_sym => %w[id email_address] } }
-        let(:existing_airbytelist) { { Candidate.table_name.to_sym => %w[email_address id] } }
-
-        it 'returns nothing' do
-          conflicts = described_class.airbyte_conflicting_fields
-          expect(conflicts).to be_empty
-        end
-      end
-    end
-
     describe '.generate_blocklist' do
       it 'returns all the fields in the model that aren’t in the allowlist' do
         fields = described_class.generate_blocklist[Candidate.table_name.to_sym]

spec/dfe/analytics/generators/install_generator_spec.rb

@@ -64,14 +64,6 @@
         { 'shared' => {} }.to_yaml
       )
     end
-    it 'creates the airbyte stream config file' do
-      generator.install
-
-      expect(generator).to have_received(:create_file).with(
-        airbyte_stream_config_path,
-        generated_stream_config
-      )
-    end
 
     it 'uses the generated config options in the initializer' do
       generator.install

spec/dfe/analytics_spec.rb

@@ -427,70 +427,4 @@
       end
     end
   end
-
-  describe '.airbyte_stream_config' do
-    let(:mock_path) { '/fake/path/airbyte_stream_config.json' }
-
-    let(:config_double) do
-      instance_double(
-        'DfE::Analytics.config',
-        airbyte_stream_config_path: mock_path
-      )
-    end
-
-    before do
-      allow(DfE::Analytics).to receive(:config).and_return(config_double)
-    end
-
-    context 'when the JSON file is valid' do
-      let(:json_data) do
-        {
-          configurations: {
-            streams: [
-              {
-                name: 'teachers',
-                syncMode: 'incremental_append',
-                selectedFields: [{ fieldPath: ['id'] }]
-              }
-            ]
-          }
-        }.to_json
-      end
-
-      before do
-        allow(File).to receive(:read).with(mock_path).and_return(json_data)
-      end
-
-      it 'returns the parsed JSON with symbolized keys' do
-        result = described_class.airbyte_stream_config
-
-        expect(result).to eq(
-          configurations: {
-            streams: [
-              {
-                name: 'teachers',
-                syncMode: 'incremental_append',
-                selectedFields: [{ fieldPath: ['id'] }]
-              }
-            ]
-          }
-        )
-
-        # verify keys are deep symbolized
-        expect(result.keys).to all(be_a(Symbol))
-        expect(result[:configurations][:streams].first[:name]).to eq('teachers')
-        expect(result[:configurations][:streams].first[:selectedFields].first[:fieldPath]).to eq(['id'])
-      end
-    end
-
-    context 'when File.read raises a RuntimeError' do
-      before do
-        allow(File).to receive(:read).with(mock_path).and_raise(RuntimeError)
-      end
-
-      it 'returns an empty hash' do
-        expect(described_class.airbyte_stream_config).to eq({})
-      end
-    end
-  end
 end