HIT L3: moved 10-minute chunking transformation to HIT L2

imap_l3_processing/hit/l3/hit_processor.py

@@ -11,7 +11,7 @@
 from imap_l3_processing.hit.l3.pha.science.calculate_pha import process_pha_event
 from imap_l3_processing.hit.l3.sectored_products.models import HitPitchAngleDataProduct
 from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import get_sector_unit_vectors, \
-    get_hit_bin_polar_coordinates, transform_to_10_minute_chunks
+    get_hit_bin_polar_coordinates
 from imap_l3_processing.hit.quality_flags import HitL3Flags
 from imap_l3_processing.models import InputMetadata
 from imap_l3_processing.pitch_angles import calculate_unit_vector, calculate_pitch_angle, calculate_gyrophase, \
@@ -166,8 +166,7 @@ def process_pitch_angle_product(self, dependencies: HITL3SectoredDependencies) -
         number_of_gyrophase_bins = 15
 
         mag_data = dependencies.mag_data
-
-        hit_data = transform_to_10_minute_chunks(dependencies.data)
+        hit_data = dependencies.data
 
         input_intensity_data_by_species = {
             "hydrogen": (hit_data.h, hit_data.delta_plus_h, hit_data.delta_minus_h),

imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py

@@ -1,11 +1,5 @@
-import dataclasses
-from datetime import datetime, timedelta
-
 import numpy as np
 
-from imap_l3_processing.constants import FIVE_MINUTES_IN_NANOSECONDS
-from imap_l3_processing.hit.l3.models import HitL2Data
-
 
 def get_hit_bin_polar_coordinates(declination_bins=8, inclination_bins=15) -> tuple[
     np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
@@ -32,44 +26,3 @@ def get_sector_unit_vectors(declinations_degrees: np.ndarray, inclinations_degre
 
     stacked = np.stack(np.broadcast_arrays(x, y, z), axis=-1)
     return stacked
-
-def transform_to_10_minute_chunks(hit_data: HitL2Data) -> HitL2Data:
-    species_energy = [
-        ("h", 3),
-        ("he4", 2),
-        ("cno", 2),
-        ("nemgsi", 2),
-        ("fe", 1),
-    ]
-
-    transformed_hit_dict = {}
-
-    input_hit_dict = dataclasses.asdict(hit_data)
-
-    species_i = 0
-    for species, num_energy_levels in species_energy:
-        input_species_data = input_hit_dict[species]
-        new_species_shape = (input_species_data.shape[0] // 10, *input_species_data.shape[1:])
-
-        transformed_hit_dict[species] = np.full(new_species_shape, np.nan)
-        transformed_hit_dict[f"delta_plus_{species}"] = np.full(new_species_shape, np.nan)
-        transformed_hit_dict[f"delta_minus_{species}"] = np.full(new_species_shape, np.nan)
-
-        for energy_i in range(num_energy_levels):
-            transformed_hit_dict[species][:, energy_i] = input_species_data[species_i::10, energy_i]
-            transformed_hit_dict[f"delta_plus_{species}"][:, energy_i] = input_hit_dict[f"delta_plus_{species}"][species_i::10, energy_i]
-            transformed_hit_dict[f"delta_minus_{species}"][:, energy_i] = input_hit_dict[f"delta_minus_{species}"][species_i::10, energy_i]
-            species_i += 1
-
-    minute_cadence_epochs = input_hit_dict['epoch']
-    ten_minute_cadence_epochs = minute_cadence_epochs.reshape(-1, 10)
-    new_epochs = []
-    for chunk in ten_minute_cadence_epochs:
-        start_time = chunk[0]
-        end_time = chunk[-1]
-        new_epochs.append(start_time + (end_time - start_time) / 2 - timedelta(minutes=10))
-    transformed_hit_dict['epoch'] = np.array(new_epochs)
-    transformed_hit_dict['epoch_delta'] = np.full(np.array(new_epochs).shape, timedelta(minutes=5))
-
-    return dataclasses.replace(hit_data, **transformed_hit_dict)
-

imap_l3_processing/hit/l3/utils.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 from typing import Union
 
-import numpy as np
 from spacepy.pycdf import CDF
 
 from imap_l3_processing.cdf.cdf_utils import read_numeric_variable
@@ -11,17 +10,9 @@
 
 def read_l2_hit_data(cdf_file_path: Union[str, Path]) -> HitL2Data:
     with CDF(str(cdf_file_path)) as cdf:
-        epoch = cdf["epoch"][...]
-
-        half_epoch_diff = np.diff(epoch) / 2
-        variance = np.max(half_epoch_diff) - np.min(half_epoch_diff)
-        assert variance < timedelta(milliseconds=1), \
-            "L2 epochs are not evenly spaced, failed to synthesize epoch deltas"
-        fabricated_epoch_deltas = np.repeat(half_epoch_diff[0], len(epoch))
-
         return HitL2Data(
             epoch=cdf["epoch"][...],
-            epoch_delta=fabricated_epoch_deltas,
+            epoch_delta=cdf["epoch_delta"][...] / 1e9 * timedelta(seconds=1),
             azimuth=read_numeric_variable(cdf["azimuth"]),
             zenith=read_numeric_variable(cdf["zenith"]),
             h=read_numeric_variable(cdf["h_macropixel_intensity"]),

tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py

@@ -1,12 +1,9 @@
-from datetime import datetime, timedelta
 from unittest import TestCase
 
 import numpy as np
 
 from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import get_sector_unit_vectors, \
-    get_hit_bin_polar_coordinates, transform_to_10_minute_chunks
-from imap_l3_processing.hit.l3.utils import read_l2_hit_data
-from tests.test_helpers import get_test_data_path
+    get_hit_bin_polar_coordinates
 
 
 class TestSectoredProductsAlgorithms(TestCase):
@@ -41,44 +38,3 @@ def test_get_hit_bin_polar_coordinates(self):
         np.testing.assert_array_equal(azimuth_delta, [12.0] * 15, strict=True)
         np.testing.assert_array_almost_equal(declinations, 11.25 + np.arange(0, 8) * 22.5)
         np.testing.assert_array_almost_equal(azimuths, 12 + np.arange(15) * 24)
-
-    def test_transform_to_10_minute_chunks(self):
-        scuffed_l2_input = get_test_data_path("hit/imap_hit_l2_macropixel-intensity_20100106_v001.cdf")
-        hit_l2_data = read_l2_hit_data(scuffed_l2_input)
-
-        transformed_hit_data = transform_to_10_minute_chunks(hit_l2_data)
-
-        self.assertEqual((144,), transformed_hit_data.epoch.shape)
-        self.assertEqual((144,), transformed_hit_data.epoch_delta.shape)
-        self.assertTrue(np.all(transformed_hit_data.epoch_delta == timedelta(minutes=5)))
-        self.assertEqual(datetime(2010, 1, 5, 23, 52, 48, 500000), transformed_hit_data.epoch[0])
-
-        self.assertEqual((144, 3, 15, 8), transformed_hit_data.h.shape)
-        self.assertEqual((144, 3, 15, 8), transformed_hit_data.delta_plus_h.shape)
-        self.assertEqual((144, 3, 15, 8), transformed_hit_data.delta_minus_h.shape)
-        self.assertAlmostEqual(4.087e-4, transformed_hit_data.h[0, 0, 0, 0])
-        self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.h))))
-
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.he4.shape)
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_plus_he4.shape)
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_minus_he4.shape)
-        self.assertAlmostEqual(6.322e-4, transformed_hit_data.he4[0, 0, 0, 0])
-        self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.he4))))
-
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.cno.shape)
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_plus_cno.shape)
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_minus_cno.shape)
-        self.assertAlmostEqual(7.813e-4, transformed_hit_data.cno[0, 0, 0, 0])
-        self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.cno))))
-
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.nemgsi.shape)
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_plus_nemgsi.shape)
-        self.assertEqual((144, 2, 15, 8), transformed_hit_data.delta_minus_nemgsi.shape)
-        self.assertAlmostEqual(8.527e-4, transformed_hit_data.nemgsi[0, 0, 0, 0])
-        self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.nemgsi))))
-
-        self.assertEqual((144, 1, 15, 8), transformed_hit_data.fe.shape)
-        self.assertEqual((144, 1, 15, 8), transformed_hit_data.delta_plus_fe.shape)
-        self.assertEqual((144, 1, 15, 8), transformed_hit_data.delta_minus_fe.shape)
-        self.assertAlmostEqual(2.695e-4, transformed_hit_data.fe[0, 0, 0, 0])
-        self.assertTrue(np.all(np.logical_not(np.isnan(transformed_hit_data.fe))))

tests/hit/l3/test_hit_processor.py

@@ -30,7 +30,6 @@ def test_is_a_processor(self):
             Processor
         )
 
-    @patch('imap_l3_processing.hit.l3.hit_processor.transform_to_10_minute_chunks')
     @patch("imap_l3_processing.utils.spiceypy")
     @patch('imap_l3_processing.hit.l3.hit_processor.save_data')
     @patch('imap_l3_processing.hit.l3.hit_processor.HITL3SectoredDependencies.fetch_dependencies')
@@ -47,7 +46,7 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit
                                          mock_get_sector_unit_vectors, mock_get_hit_bin_polar_coordinates,
                                          mock_calculate_unit_vector,
                                          mock_fetch_dependencies, mock_save_data,
-                                         mock_spiceypy, mock_transform_epochs):
+                                         mock_spiceypy):
         mock_spiceypy.ktotal.return_value = 0
 
         input_metadata = InputMetadata(
@@ -123,11 +122,9 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit
         mock_hit_data.delta_plus_nemgsi = np.array([delta_plus_NeMgSi_time1, delta_plus_NeMgSi_time2])
         mock_hit_data.delta_minus_nemgsi = np.array([delta_minus_NeMgSi_time1, delta_minus_NeMgSi_time2])
 
-        mock_dependencies.data = sentinel.pre_transform_hit_data
+        mock_dependencies.data = mock_hit_data
         mock_fetch_dependencies.return_value = mock_dependencies
 
-        mock_transform_epochs.return_value = mock_hit_data
-
         mock_calculate_unit_vector.side_effect = [sentinel.mag_unit_vector1, sentinel.mag_unit_vector2]
         mock_get_hit_bin_polar_coordinates.return_value = (
             sentinel.pitch_angles, sentinel.gyrophases, sentinel.pitch_angle_deltas, sentinel.gyrophase_deltas)
@@ -238,8 +235,6 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit
         product = processor.process()
         mock_fetch_dependencies.assert_called_once_with(mock_processing_input_collection)
 
-        mock_transform_epochs.assert_called_once_with(sentinel.pre_transform_hit_data)
-
         mock_mag_data.rebin_to.assert_called_once_with(mock_hit_data.epoch, mock_hit_data.epoch_delta)
 
         mock_calculate_unit_vector.assert_has_calls([

tests/hit/l3/test_utils.py

@@ -57,7 +57,7 @@ def test_read_l2_hit_data(self):
                 np.testing.assert_array_equal(iron_data, result.fe)
 
                 np.testing.assert_array_equal(epoch_data, result.epoch)
-                np.testing.assert_array_equal([timedelta(minutes=2.5), timedelta(minutes=2.5), timedelta(minutes=2.5)],
+                np.testing.assert_array_equal([timedelta(minutes=5), timedelta(minutes=5), timedelta(minutes=5)],
                                               result.epoch_delta)
 
                 np.testing.assert_array_equal([0, 1, 2], result.h_energy)
@@ -128,39 +128,6 @@ def test_read_l2_hit_data_handles_fill_values(self):
                 np.testing.assert_array_equal(result.delta_plus_fe, np.full_like(iron_data, np.nan))
                 np.testing.assert_array_equal(result.delta_minus_fe, np.full_like(iron_data, np.nan))
 
-    def test_read_l2_hit_throws_error_when_it_cannot_fabricate_deltas(self):
-        rng = np.random.default_rng()
-
-        start_time = datetime(2010, 1, 1, 0, 5)
-
-        epoch_data = np.array(
-            [start_time, start_time + timedelta(minutes=5), start_time + timedelta(minutes=10, microseconds=2001)])
-
-        hydrogen_data = rng.random((3, 3, 15, 8))
-        helium_data = rng.random((3, 2, 15, 8))
-        cno_data = rng.random((3, 2, 15, 8))
-        nemgsi_data = rng.random((3, 2, 15, 8))
-        iron_data = rng.random((3, 1, 15, 8))
-
-        hydrogen_delta = hydrogen_data * 0.1
-        helium_delta = helium_data * 0.1
-        cno_delta = cno_data * 0.1
-        nemgsi_delta = nemgsi_data * 0.1
-        iron_delta = iron_data * 0.1
-
-        pathname = self.write_test_data_file(epoch_data, hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data,
-                                             hydrogen_delta,
-                                             helium_delta,
-                                             cno_delta,
-                                             nemgsi_delta,
-                                             iron_delta,
-                                             )
-        with self.assertRaises(AssertionError) as exc_ctx:
-            read_l2_hit_data(pathname)
-
-        expected_error_message = "L2 epochs are not evenly spaced, failed to synthesize epoch deltas"
-        self.assertEqual(expected_error_message, str(exc_ctx.exception))
-
     def write_test_data_file(self, epoch_data, hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data,
                              hydrogen_delta,
                              helium_delta,
@@ -182,6 +149,7 @@ def write_test_data_file(self, epoch_data, hydrogen_data, helium_data, cno_data,
             cdf["fe_macropixel_intensity"] = iron_data
 
             cdf["epoch"] = epoch_data
+            cdf["epoch_delta"] = np.full(len(epoch_data), 300_000_000_000)
 
             cdf.new("h_energy_mean", np.arange(3), recVary=False)
             cdf.new("he4_energy_mean", np.arange(2), recVary=False)