From 65a13898e2ca8b40fab4c5e290d5e522f072df70 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Tue, 11 Aug 2026 14:01:14 -0400 Subject: [PATCH] HIT L3: removed 10-minute chunking transformation, as it has been moved to HIT L2 --- imap_l3_processing/hit/l3/hit_processor.py | 5 +- .../science/sectored_products_algorithms.py | 47 ------------------- imap_l3_processing/hit/l3/utils.py | 11 +---- .../test_sectored_products_algorithms.py | 46 +----------------- tests/hit/l3/test_hit_processor.py | 9 +--- tests/hit/l3/test_utils.py | 36 +------------- 6 files changed, 8 insertions(+), 146 deletions(-) diff --git a/imap_l3_processing/hit/l3/hit_processor.py b/imap_l3_processing/hit/l3/hit_processor.py index 1537021b6..f2a9b2593 100644 --- a/imap_l3_processing/hit/l3/hit_processor.py +++ b/imap_l3_processing/hit/l3/hit_processor.py @@ -11,7 +11,7 @@ from imap_l3_processing.hit.l3.pha.science.calculate_pha import process_pha_event from imap_l3_processing.hit.l3.sectored_products.models import HitPitchAngleDataProduct from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import get_sector_unit_vectors, \ - get_hit_bin_polar_coordinates, transform_to_10_minute_chunks + get_hit_bin_polar_coordinates from imap_l3_processing.hit.quality_flags import HitL3Flags from imap_l3_processing.models import InputMetadata from imap_l3_processing.pitch_angles import calculate_unit_vector, calculate_pitch_angle, calculate_gyrophase, \ @@ -166,8 +166,7 @@ def process_pitch_angle_product(self, dependencies: HITL3SectoredDependencies) - number_of_gyrophase_bins = 15 mag_data = dependencies.mag_data - - hit_data = transform_to_10_minute_chunks(dependencies.data) + hit_data = dependencies.data input_intensity_data_by_species = { "hydrogen": (hit_data.h, hit_data.delta_plus_h, hit_data.delta_minus_h), diff --git a/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py b/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py index 1fc052e0c..35cc374f7 100644 --- a/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py +++ b/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py @@ -1,11 +1,5 @@ -import dataclasses -from datetime import datetime, timedelta - import numpy as np -from imap_l3_processing.constants import FIVE_MINUTES_IN_NANOSECONDS -from imap_l3_processing.hit.l3.models import HitL2Data - def get_hit_bin_polar_coordinates(declination_bins=8, inclination_bins=15) -> tuple[ np.ndarray, np.ndarray, np.ndarray, np.ndarray]: @@ -32,44 +26,3 @@ def get_sector_unit_vectors(declinations_degrees: np.ndarray, inclinations_degre stacked = np.stack(np.broadcast_arrays(x, y, z), axis=-1) return stacked - -def transform_to_10_minute_chunks(hit_data: HitL2Data) -> HitL2Data: - species_energy = [ - ("h", 3), - ("he4", 2), - ("cno", 2), - ("nemgsi", 2), - ("fe", 1), - ] - - transformed_hit_dict = {} - - input_hit_dict = dataclasses.asdict(hit_data) - - species_i = 0 - for species, num_energy_levels in species_energy: - input_species_data = input_hit_dict[species] - new_species_shape = (input_species_data.shape[0] // 10, *input_species_data.shape[1:]) - - transformed_hit_dict[species] = np.full(new_species_shape, np.nan) - transformed_hit_dict[f"delta_plus_{species}"] = np.full(new_species_shape, np.nan) - transformed_hit_dict[f"delta_minus_{species}"] = np.full(new_species_shape, np.nan) - - for energy_i in range(num_energy_levels): - transformed_hit_dict[species][:, energy_i] = input_species_data[species_i::10, energy_i] - transformed_hit_dict[f"delta_plus_{species}"][:, energy_i] = input_hit_dict[f"delta_plus_{species}"][species_i::10, energy_i] - transformed_hit_dict[f"delta_minus_{species}"][:, energy_i] = input_hit_dict[f"delta_minus_{species}"][species_i::10, energy_i] - species_i += 1 - - minute_cadence_epochs = input_hit_dict['epoch'] - ten_minute_cadence_epochs = minute_cadence_epochs.reshape(-1, 10) - new_epochs = [] - for chunk in ten_minute_cadence_epochs: - start_time = chunk[0] - end_time = chunk[-1] - new_epochs.append(start_time + (end_time - start_time) / 2 - timedelta(minutes=10)) - transformed_hit_dict['epoch'] = np.array(new_epochs) - transformed_hit_dict['epoch_delta'] = np.full(np.array(new_epochs).shape, timedelta(minutes=5)) - - return dataclasses.replace(hit_data, **transformed_hit_dict) - diff --git a/imap_l3_processing/hit/l3/utils.py b/imap_l3_processing/hit/l3/utils.py index 82ca2f972..051b3cf9d 100644 --- a/imap_l3_processing/hit/l3/utils.py +++ b/imap_l3_processing/hit/l3/utils.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Union -import numpy as np from spacepy.pycdf import CDF from imap_l3_processing.cdf.cdf_utils import read_numeric_variable @@ -11,17 +10,9 @@ def read_l2_hit_data(cdf_file_path: Union[str, Path]) -> HitL2Data: with CDF(str(cdf_file_path)) as cdf: - epoch = cdf["epoch"][...] - - half_epoch_diff = np.diff(epoch) / 2 - variance = np.max(half_epoch_diff) - np.min(half_epoch_diff) - assert variance < timedelta(milliseconds=1), \ - "L2 epochs are not evenly spaced, failed to synthesize epoch deltas" - fabricated_epoch_deltas = np.repeat(half_epoch_diff[0], len(epoch)) - return HitL2Data( epoch=cdf["epoch"][...], - epoch_delta=fabricated_epoch_deltas, + epoch_delta=cdf["epoch_delta"][...] / 1e9 * timedelta(seconds=1), azimuth=read_numeric_variable(cdf["azimuth"]), zenith=read_numeric_variable(cdf["zenith"]), h=read_numeric_variable(cdf["h_macropixel_intensity"]), diff --git a/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py b/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py index cb7569ef3..85dde9c03 100644 --- a/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py +++ b/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py @@ -1,12 +1,9 @@ -from datetime import datetime, timedelta from unittest import TestCase import numpy as np from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import get_sector_unit_vectors, \ - get_hit_bin_polar_coordinates, transform_to_10_minute_chunks -from imap_l3_processing.hit.l3.utils import read_l2_hit_data -from tests.test_helpers import get_test_data_path + get_hit_bin_polar_coordinates class TestSectoredProductsAlgorithms(TestCase): @@ -41,44 +38,3 @@ def test_get_hit_bin_polar_coordinates(self): np.testing.assert_array_equal(azimuth_delta, [12.0] * 15, strict=True) np.testing.assert_array_almost_equal(declinations, 11.25 + np.arange(0, 8) * 22.5) np.testing.assert_array_almost_equal(azimuths, 12 + np.arange(15) * 24) - - def test_transform_to_10_minute_chunks(self): - scuffed_l2_input = get_test_data_path("hit/imap_hit_l2_macropixel-intensity_20100106_v001.cdf") - hit_l2_data = read_l2_hit_data(scuffed_l2_input) - - transformed_hit_data = transform_to_10_minute_chunks(hit_l2_data) - - self.assertEqual((144,), transformed_hit_data.epoch.shape) - self.assertEqual((144,), transformed_hit_data.epoch_delta.shape) - self.assertTrue(np.all(transformed_hit_data.epoch_delta == timedelta(minutes=5))) - self.assertEqual(datetime(2010, 1, 5, 23, 52, 48, 500000), transformed_hit_data.epoch[0]) - - self.assertEqual((144, 3, 15, 8), transformed_hit_data.h.shape) - self.assertEqual((144, 3, 15, 8), transformed_hit_data.delta_plus_h.shape) - self.assertEqual((144, 3, 15, 8), transformed_hit_data.delta_minus_h.shape) - self.assertAlmostEqual(4.087e-4, transformed_hit_data.h[0, 0, 0, 0]) - self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.h)))) - - self.assertEqual((144, 2, 15, 8), transformed_hit_data.he4.shape) - self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_plus_he4.shape) - self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_minus_he4.shape) - self.assertAlmostEqual(6.322e-4, transformed_hit_data.he4[0, 0, 0, 0]) - self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.he4)))) - - self.assertEqual((144, 2, 15, 8), transformed_hit_data.cno.shape) - self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_plus_cno.shape) - self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_minus_cno.shape) - self.assertAlmostEqual(7.813e-4, transformed_hit_data.cno[0, 0, 0, 0]) - self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.cno)))) - - self.assertEqual((144, 2, 15, 8), transformed_hit_data.nemgsi.shape) - self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_plus_nemgsi.shape) - self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_minus_nemgsi.shape) - self.assertAlmostEqual(8.527e-4, transformed_hit_data.nemgsi[0, 0, 0, 0]) - self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.nemgsi)))) - - self.assertEqual((144, 1, 15, 8), transformed_hit_data.fe.shape) - self.assertEqual((144, 1, 15, 8), transformed_hit_data.delta_plus_fe.shape) - self.assertEqual((144, 1, 15, 8), transformed_hit_data.delta_minus_fe.shape) - self.assertAlmostEqual(2.695e-4, transformed_hit_data.fe[0, 0, 0, 0]) - self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.fe)))) diff --git a/tests/hit/l3/test_hit_processor.py b/tests/hit/l3/test_hit_processor.py index 4dbb6ffee..5d307f5e0 100644 --- a/tests/hit/l3/test_hit_processor.py +++ b/tests/hit/l3/test_hit_processor.py @@ -30,7 +30,6 @@ def test_is_a_processor(self): Processor ) - @patch('imap_l3_processing.hit.l3.hit_processor.transform_to_10_minute_chunks') @patch("imap_l3_processing.utils.spiceypy") @patch('imap_l3_processing.hit.l3.hit_processor.save_data') @patch('imap_l3_processing.hit.l3.hit_processor.HITL3SectoredDependencies.fetch_dependencies') @@ -47,7 +46,7 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit mock_get_sector_unit_vectors, mock_get_hit_bin_polar_coordinates, mock_calculate_unit_vector, mock_fetch_dependencies, mock_save_data, - mock_spiceypy, mock_transform_epochs): + mock_spiceypy): mock_spiceypy.ktotal.return_value = 0 input_metadata = InputMetadata( @@ -123,11 +122,9 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit mock_hit_data.delta_plus_nemgsi = np.array([delta_plus_NeMgSi_time1, delta_plus_NeMgSi_time2]) mock_hit_data.delta_minus_nemgsi = np.array([delta_minus_NeMgSi_time1, delta_minus_NeMgSi_time2]) - mock_dependencies.data = sentinel.pre_transform_hit_data + mock_dependencies.data = mock_hit_data mock_fetch_dependencies.return_value = mock_dependencies - mock_transform_epochs.return_value = mock_hit_data - mock_calculate_unit_vector.side_effect = [sentinel.mag_unit_vector1, sentinel.mag_unit_vector2] mock_get_hit_bin_polar_coordinates.return_value = ( sentinel.pitch_angles, sentinel.gyrophases, sentinel.pitch_angle_deltas, sentinel.gyrophase_deltas) @@ -238,8 +235,6 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit product = processor.process() mock_fetch_dependencies.assert_called_once_with(mock_processing_input_collection) - mock_transform_epochs.assert_called_once_with(sentinel.pre_transform_hit_data) - mock_mag_data.rebin_to.assert_called_once_with(mock_hit_data.epoch, mock_hit_data.epoch_delta) mock_calculate_unit_vector.assert_has_calls([ diff --git a/tests/hit/l3/test_utils.py b/tests/hit/l3/test_utils.py index d6494dcdc..07c31e2a8 100644 --- a/tests/hit/l3/test_utils.py +++ b/tests/hit/l3/test_utils.py @@ -57,7 +57,7 @@ def test_read_l2_hit_data(self): np.testing.assert_array_equal(iron_data, result.fe) np.testing.assert_array_equal(epoch_data, result.epoch) - np.testing.assert_array_equal([timedelta(minutes=2.5), timedelta(minutes=2.5), timedelta(minutes=2.5)], + np.testing.assert_array_equal([timedelta(minutes=5), timedelta(minutes=5), timedelta(minutes=5)], result.epoch_delta) np.testing.assert_array_equal([0, 1, 2], result.h_energy) @@ -128,39 +128,6 @@ def test_read_l2_hit_data_handles_fill_values(self): np.testing.assert_array_equal(result.delta_plus_fe, np.full_like(iron_data, np.nan)) np.testing.assert_array_equal(result.delta_minus_fe, np.full_like(iron_data, np.nan)) - def test_read_l2_hit_throws_error_when_it_cannot_fabricate_deltas(self): - rng = np.random.default_rng() - - start_time = datetime(2010, 1, 1, 0, 5) - - epoch_data = np.array( - [start_time, start_time + timedelta(minutes=5), start_time + timedelta(minutes=10, microseconds=2001)]) - - hydrogen_data = rng.random((3, 3, 15, 8)) - helium_data = rng.random((3, 2, 15, 8)) - cno_data = rng.random((3, 2, 15, 8)) - nemgsi_data = rng.random((3, 2, 15, 8)) - iron_data = rng.random((3, 1, 15, 8)) - - hydrogen_delta = hydrogen_data * 0.1 - helium_delta = helium_data * 0.1 - cno_delta = cno_data * 0.1 - nemgsi_delta = nemgsi_data * 0.1 - iron_delta = iron_data * 0.1 - - pathname = self.write_test_data_file(epoch_data, hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data, - hydrogen_delta, - helium_delta, - cno_delta, - nemgsi_delta, - iron_delta, - ) - with self.assertRaises(AssertionError) as exc_ctx: - read_l2_hit_data(pathname) - - expected_error_message = "L2 epochs are not evenly spaced, failed to synthesize epoch deltas" - self.assertEqual(expected_error_message, str(exc_ctx.exception)) - def write_test_data_file(self, epoch_data, hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data, hydrogen_delta, helium_delta, @@ -182,6 +149,7 @@ def write_test_data_file(self, epoch_data, hydrogen_data, helium_data, cno_data, cdf["fe_macropixel_intensity"] = iron_data cdf["epoch"] = epoch_data + cdf["epoch_delta"] = np.full(len(epoch_data), 300_000_000_000) cdf.new("h_energy_mean", np.arange(3), recVary=False) cdf.new("he4_energy_mean", np.arange(2), recVary=False)