From 485f56ed80ca2f39ea69047ba045a820fb84a6d6 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Tue, 30 Jun 2026 13:20:13 -0600 Subject: [PATCH 1/3] Adding MAG L1C day boundary tests --- imap_processing/tests/mag/test_mag_l1c.py | 123 ++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/imap_processing/tests/mag/test_mag_l1c.py b/imap_processing/tests/mag/test_mag_l1c.py index 0f0020e35..a863d0b33 100644 --- a/imap_processing/tests/mag/test_mag_l1c.py +++ b/imap_processing/tests/mag/test_mag_l1c.py @@ -1147,3 +1147,126 @@ def test_cic_filter_delay_compensation(): f"{len(input_filtered_case2)} elements, vectors_filtered has " f"{len(vectors_filtered_case2)} elements" ) + + +def _build_cross_day_l1b( + day1: np.datetime64, day2: np.datetime64 +) -> tuple[xr.Dataset, xr.Dataset, xr.Dataset, dict]: + """Build a previous-day (day1) and current-day (day2) L1B scenario. + + Day 1 ends at 4 vec/s on a grid carrying a sub-cadence phase offset, so its + final NM grid does not line up with day 2's own 2 vec/s grid (requirement 1). + Day 2's NM data does not begin until 10 minutes into the day window, leaving a + gap at the start of the day (requirement 2). Day 2 burst covers that leading + gap so the simulated NM timestamps can be filled. + + Returns norm_day1, norm_day2, burst_day2, and a dict of the key constants the + continuity assertions are written against. + """ + cadence_day1 = 250_000_000 # 4 vec/s + cadence_day2 = 500_000_000 # 2 vec/s + phase_ns = 73_000_000 # sub-cadence offset, < cadence_day1 + + day2_start_ns, _ = _ttj2000_day_bounds(day2) + + # Day 1 NM ends just before day 2's window, on a 4 vec/s grid offset by phase. + day1_last = day2_start_ns - cadence_day1 + phase_ns + day1_epochs = day1_last - np.arange(9, -1, -1) * cadence_day1 + norm_day1 = _build_mag_l1b( + day1_epochs.astype(np.int64), "imap_mag_l1b_norm-mago", "0:4;" + ) + + # Day 2 NM starts 10 minutes into the window at 2 vec/s (phase 0 vs boundary). + day2_nm_start = day2_start_ns + 600 * 1_000_000_000 + day2_nm_epochs = np.arange( + day2_nm_start, + day2_nm_start + 120 * 1_000_000_000 + 1, + cadence_day2, + dtype=np.int64, + ) + norm_day2 = _build_mag_l1b(day2_nm_epochs, "imap_mag_l1b_norm-mago", "0:2") + + # Day 2 burst covers the leading gap (and a little past NM start) at 8 vec/s. + burst_day2_epochs = np.arange( + day2_start_ns, + day2_nm_start + 30 * 1_000_000_000, + 125_000_000, + dtype=np.int64, + ) + burst_day2 = _build_mag_l1b(burst_day2_epochs, "imap_mag_l1b_burst-mago", "0:8") + + meta = { + "day2_start_ns": day2_start_ns, + "day1_last": int(day1_last), + "cadence_day1": cadence_day1, + "cadence_day2": cadence_day2, + "phase_ns": phase_ns, + "day2_nm_start": int(day2_nm_start), + "day2_nm_epochs": day2_nm_epochs, + } + return norm_day1, norm_day2, burst_day2, meta + + +@pytest.mark.xfail(reason="Not implemented") +def test_process_mag_l1c_continues_previous_day_grid(): + """Leading-gap timestamps must continue the previous day's NM grid. + + When day 2 opens with a gap, the simulated + NM timeline that fills it should adopt the *previous day's* ending rate and + phase (algorithm doc 7.3.4 step 3, "regular and consistent, across boundaries + between days"), not day 2's own boundary-anchored grid. + """ + day1 = np.datetime64("2025-01-01") + day2 = np.datetime64("2025-01-02") + norm_day1, norm_day2, burst_day2, meta = _build_cross_day_l1b(day1, day2) + + result = process_mag_l1c( + norm_day2, + burst_day2, + InterpolationFunction.linear, + day2, + previous_day_dataset=norm_day1, + ) + epochs_out = result[:, 0] + + # epochs in the first 10 minute gap are filled + leading = epochs_out[epochs_out < meta["day2_nm_start"]] + assert leading.size > 0 + + # Starting point lines up with day 1's ending point + one day-1 cadence. + assert leading[0] == meta["day1_last"] + meta["cadence_day1"] + # Rate lines up with day 1's ending rate (4 vec/s), not day 2's 2 vec/s. + assert np.all(np.diff(leading) == meta["cadence_day1"]) + # Zero jitter: every leading sample sits exactly on day 1's grid. + assert np.all((leading - meta["day1_last"]) % meta["cadence_day1"] == 0) + # Day 2's real NM samples are still used where they exist + assert np.all(np.isin(meta["day2_nm_epochs"], epochs_out)) + + +@pytest.mark.xfail(reason="Not implemented") +def test_mag_l1c_continues_previous_day_grid(): + """mag_l1c() must thread the previous day's NM grid into the output. + + End-to-end companion to test_process_mag_l1c_continues_previous_day_grid: the + public entry point should accept previous_day_dataset and produce an L1C epoch + axis whose start-of-day gap fill continues day 1's rate and phase, while still + using day 2's own NM samples where they exist. + """ + day1 = np.datetime64("2025-01-01") + day2 = np.datetime64("2025-01-02") + norm_day1, norm_day2, burst_day2, meta = _build_cross_day_l1b(day1, day2) + + output = mag_l1c(norm_day2, day2, burst_day2, previous_day_dataset=norm_day1) + epochs_out = output["epoch"].data + + leading = epochs_out[epochs_out < meta["day2_nm_start"]] + assert leading.size > 0 + + # Starting point and rate continue day 1's ending grid with zero jitter. + assert leading[0] == meta["day1_last"] + meta["cadence_day1"] + assert np.all(np.diff(leading) == meta["cadence_day1"]) + assert np.all( + (leading - meta["day2_start_ns"]) % meta["cadence_day1"] == meta["phase_ns"] + ) + # Day 2's real NM samples are still used where they exist + assert np.all(np.isin(meta["day2_nm_epochs"], epochs_out)) From ac9fdb535782a08101a329b0bb50305239aca35c Mon Sep 17 00:00:00 2001 From: sapols Date: Mon, 6 Jul 2026 11:57:48 -0600 Subject: [PATCH 2/3] MAG L1C: continue previous day's timeline across the day boundary (#2925) When the processing day opens with a gap, generate the missing normal-mode timestamps on the previous day's grid - inheriting its ending cadence and phase from an optional previous-day norm L1B dataset - so the L1C timeline stays regular across the day boundary. With no (usable) previous-day file, behavior is unchanged and single-day processing still succeeds. - mag_l1c()/process_mag_l1c() gain an optional previous_day_dataset. The inherited grid anchors on the last sample within the previous 24-hour day and snaps its cadence against the known MAG rates via _is_expected_rate. With no normal-mode data at all, the whole window continues that grid. - Mag CLI splits L1C dependencies by start date with get_valid_inputs_for_start_date; the previous day's norm L1B (delivered via a date_range entry in imap_mag_dependencies.yaml) routes to previous_day_dataset. Mode/sensor validation lives in mag_l1c. - generate_missing_timestamps no longer routes integer nanosecond gap bounds through float64 (np.rint), which shifted generated timestamps by up to 64 ns at TTJ2000 scales. - Un-xfail the day-boundary tests from #3318; their phase offset moves to a multiple of 128 ns because the float64 epoch columns cannot represent other offsets exactly at 2025-era epochs. Shared test helper _ttj2000_day_bounds is replaced by the production _expected_day_ns. --- imap_processing/cli.py | 40 +++- imap_processing/mag/l1c/mag_l1c.py | 232 ++++++++++++++++++++-- imap_processing/tests/mag/test_mag_l1c.py | 149 ++++++++++++-- imap_processing/tests/test_cli.py | 78 ++++++++ 4 files changed, 459 insertions(+), 40 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 8e1bafc49..1f2c937d8 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -18,6 +18,7 @@ import re import sys from abc import ABC, abstractmethod +from datetime import datetime, timedelta from pathlib import Path from typing import final @@ -1345,16 +1346,47 @@ def do_processing( # noqa: PLR0912 ] if self.data_level == "l1c": - science_files = dependencies.get_file_paths(source="mag", data_type="l1b") + start_datetime = datetime.strptime(self.start_date, "%Y%m%d") + science_files = dependencies.get_valid_inputs_for_start_date( + start_datetime + ).get_file_paths(source="mag", data_type="l1b") input_data = [load_cdf(dep) for dep in science_files] + + # The previous day's norm L1B may arrive as an extra dependency (via the + # date_range entry in imap_mag_dependencies.yaml) so the L1C timeline can + # continue the previous day's cadence and phase across the day boundary. + previous_day_files = [ + path + for path in dependencies.get_valid_inputs_for_start_date( + start_datetime - timedelta(days=1) + ).get_file_paths(source="mag", data_type="l1b") + if "norm" in path.name + ] + previous_day_dataset = ( + load_cdf(previous_day_files[0]) if previous_day_files else None + ) + # Input datasets can be in any order, and are validated within mag_l1c if len(input_data) == 1: - datasets = [mag_l1c(input_data[0], current_day)] + datasets = [ + mag_l1c( + input_data[0], + current_day, + previous_day_dataset=previous_day_dataset, + ) + ] elif len(input_data) == 2: - datasets = [mag_l1c(input_data[0], current_day, input_data[1])] + datasets = [ + mag_l1c( + input_data[0], + current_day, + input_data[1], + previous_day_dataset=previous_day_dataset, + ) + ] else: raise ValueError( - f"Invalid dependencies found for MAG L1C:" + f"Invalid current-day dependencies found for MAG L1C:" f"{dependencies}. Expected one or two dependencies." ) if self.data_level == "l1d": diff --git a/imap_processing/mag/l1c/mag_l1c.py b/imap_processing/mag/l1c/mag_l1c.py index 861a4e047..bb7ed641b 100644 --- a/imap_processing/mag/l1c/mag_l1c.py +++ b/imap_processing/mag/l1c/mag_l1c.py @@ -22,6 +22,7 @@ def mag_l1c( first_input_dataset: xr.Dataset, day_to_process: np.datetime64, second_input_dataset: xr.Dataset = None, + previous_day_dataset: xr.Dataset = None, ) -> xr.Dataset: """ Will process MAG L1C data from L1A data. @@ -40,6 +41,12 @@ def mag_l1c( The second input dataset to process. This should be burst if first_input_dataset was norm, or norm if first_input_dataset was burst. It should match the instrument - both inputs should be mago or magi. + previous_day_dataset : xr.Dataset, optional + The previous day's normal mode L1B dataset for the same sensor. When the + current day opens with a gap, timestamps generated for that gap continue the + previous day's cadence and phase so the L1C timeline stays regular across the + day boundary. If not provided (or not usable), gaps at the start of the day + are filled on the current day's own grid, exactly as before. Returns ------- @@ -49,8 +56,6 @@ def mag_l1c( # TODO: # find missing sequences and output them # Missing burst file - just pass through norm file - # Missing norm file - go back to previous L1C file to find timestamps, then - # interpolate the entire day from burst input_logical_source_1 = first_input_dataset.attrs["Logical_source"] if isinstance(first_input_dataset.attrs["Logical_source"], list): @@ -63,10 +68,17 @@ def mag_l1c( first_input_dataset, second_input_dataset ) + if previous_day_dataset is not None: + previous_day_dataset = _validated_previous_day(previous_day_dataset, sensor) + interp_function = InterpolationFunction[configuration.L1C_INTERPOLATION_METHOD] if burst_mode_dataset is not None: full_interpolated_timeline: np.ndarray = process_mag_l1c( - normal_mode_dataset, burst_mode_dataset, interp_function, day_to_process + normal_mode_dataset, + burst_mode_dataset, + interp_function, + day_to_process, + previous_day_dataset=previous_day_dataset, ) elif normal_mode_dataset is not None: full_interpolated_timeline = fill_normal_data(normal_mode_dataset) @@ -272,11 +284,150 @@ def select_datasets( return normal_mode_dataset, burst_mode_dataset +def _validated_previous_day( + previous_day_dataset: xr.Dataset, sensor: str +) -> xr.Dataset | None: + """ + Validate the previous day's dataset, returning None if it is not usable. + + The previous day's dataset must be a normal mode L1B dataset for the same sensor + as the current day's inputs, with at least one epoch. An unusable dataset is + ignored with a warning rather than raised, so processing still succeeds with only + one day of data. + + Parameters + ---------- + previous_day_dataset : xr.Dataset + The candidate previous day dataset. + sensor : str + The sensor of the current day's inputs, "o" (mago) or "i" (magi). + + Returns + ------- + xr.Dataset or None + The validated dataset, or None if it should be ignored. + """ + logical_source = previous_day_dataset.attrs.get("Logical_source", "") + if isinstance(logical_source, list): + logical_source = logical_source[0] + + if ( + "l1b" not in logical_source + or "norm" not in logical_source + or logical_source[-1:] != sensor + ): + logger.warning( + f"Ignoring previous day dataset with logical source {logical_source}; " + f"expected normal mode L1B data for sensor mag{sensor}." + ) + return None + if previous_day_dataset["epoch"].data.size == 0: + logger.warning("Ignoring previous day dataset with no epochs.") + return None + return previous_day_dataset + + +def _expected_day_ns(day_to_process: np.datetime64) -> tuple[int, int]: + """ + Return the expected L1C processing window in TTJ2000 nanoseconds. + + The window is the 24-hour day extended by 30 minutes on each side. + + Parameters + ---------- + day_to_process : np.datetime64 + The day to process, in np.datetime64[D] format. + + Returns + ------- + tuple[int, int] + The (start, end) of the processing window in TTJ2000 nanoseconds. + """ + day_start = day_to_process.astype("datetime64[s]") - np.timedelta64(30, "m") + day_end = ( + day_to_process.astype("datetime64[s]") + + np.timedelta64(1, "D") + + np.timedelta64(30, "m") + ) + return ( + int(et_to_ttj2000ns(str_to_et(str(day_start)))), + int(et_to_ttj2000ns(str_to_et(str(day_end)))), + ) + + +def _previous_day_grid( + previous_day_dataset: xr.Dataset, midnight_ns: int, day_start_ns: int +) -> tuple[int, int] | None: + """ + Derive the timeline grid that continues the previous day's normal mode data. + + The grid is anchored at the previous day's last normal mode timestamp within its + own 24-hour day - samples in the file's trailing buffer, at or past the current + day's midnight, are not used as the anchor. The cadence is the sample spacing at + the anchor, matched against the known MAG rates. Grid points are + ``anchor + k * period``, continuing the previous day's rate and phase. + + Parameters + ---------- + previous_day_dataset : xr.Dataset + The previous day's normal mode L1B dataset. + midnight_ns : int + Start of the current 24-hour day in TTJ2000 nanoseconds. The anchor is the + last previous-day timestamp before this time. + day_start_ns : int + Start of the current processing window (midnight minus the 30 minute buffer) + in TTJ2000 nanoseconds. + + Returns + ------- + tuple[int, int] or None + ``(gap_start_ns, rate)``, where ``gap_start_ns`` is the grid point one cadence + period before the first grid point at or after ``day_start_ns`` (so a gap + starting there generates the whole in-window grid), and ``rate`` is the + inherited vectors-per-second rate. None when the previous day has fewer than + two samples before midnight, or its final spacing matches no known MAG rate. + """ + previous_epochs = previous_day_dataset["epoch"].data + anchor_index = int(np.searchsorted(previous_epochs, midnight_ns, side="left")) - 1 + if anchor_index < 1: + logger.info( + "Previous day dataset has fewer than two samples before the current day; " + "not inheriting its timeline." + ) + return None + + anchor_ns = int(previous_epochs[anchor_index]) + anchor_spacing = float( + previous_epochs[anchor_index] - previous_epochs[anchor_index - 1] + ) + + rate = None + for vecsec in VecSec: + if _is_expected_rate(anchor_spacing, vecsec.value): + rate = vecsec.value + break + if rate is None: + logger.info( + f"Previous day dataset ends with sample spacing {anchor_spacing} ns, " + f"which matches no known MAG rate; not inheriting its timeline." + ) + return None + + period_ns = int(1e9 // rate) + # Smallest number of whole periods stepping the anchor to (or past) the window + # start; at least one so the anchor itself is never a grid point in the output. + steps_to_window = max(1, -((anchor_ns - day_start_ns) // period_ns)) + grid_start_ns = anchor_ns + steps_to_window * period_ns + + return grid_start_ns - period_ns, rate + + def process_mag_l1c( normal_mode_dataset: xr.Dataset | None, burst_mode_dataset: xr.Dataset, interpolation_function: InterpolationFunction, day_to_process: np.datetime64 | None = None, + previous_day_dataset: xr.Dataset | None = None, ) -> np.ndarray: """ Create MAG L1C data from L1B datasets. @@ -307,6 +458,11 @@ def process_mag_l1c( The day to process, in np.datetime64[D] format. This is used to fill gaps at the beginning or end of the day if needed. If not included, these gaps will not be filled. + previous_day_dataset : xr.Dataset, optional + The previous day's normal mode L1B dataset. When the current day opens with a + gap, the timestamps generated for that gap continue the previous day's cadence + and phase instead of the current day's own grid, so the timeline stays regular + across the day boundary. Requires day_to_process; ignored without it. Returns ------- @@ -315,20 +471,22 @@ def process_mag_l1c( """ day_start_ns = None day_end_ns = None + inherited_grid = None if day_to_process is not None: - day_start = day_to_process.astype("datetime64[s]") - np.timedelta64(30, "m") + day_start_ns, day_end_ns = _expected_day_ns(day_to_process) - # get the end of the day plus 30 minutes - day_end = ( - day_to_process.astype("datetime64[s]") - + np.timedelta64(1, "D") - + np.timedelta64(30, "m") - ) - - day_start_ns = et_to_ttj2000ns(str_to_et(str(day_start))) - day_end_ns = et_to_ttj2000ns(str_to_et(str(day_end))) + if previous_day_dataset is not None: + # The previous day's 24-hour day ends at the current day's midnight, + # which is the window start without its 30 minute buffer. + midnight_ns = int( + et_to_ttj2000ns(str_to_et(str(day_to_process.astype("datetime64[s]")))) + ) + inherited_grid = _previous_day_grid( + previous_day_dataset, midnight_ns, day_start_ns + ) + inherited_gap_start_ns = None if normal_mode_dataset: norm_epoch = normal_mode_dataset["epoch"].data if "vectors_per_second" in normal_mode_dataset.attrs: @@ -339,6 +497,37 @@ def process_mag_l1c( normal_vecsec_dict = None gaps = find_all_gaps(norm_epoch, normal_vecsec_dict, day_start_ns, day_end_ns) + if ( + inherited_grid is not None + and gaps.shape[0] > 0 + and gaps[0][0] == day_start_ns + ): + # A gap at the beginning of the day: continue the previous day's grid up + # to the first real sample instead of the current day's own grid. + inherited_gap_start_ns, inherited_rate = inherited_grid + logger.info( + f"MAG L1C filling the gap at the start of the day on the previous " + f"day's timeline (rate {inherited_rate} vectors per second)." + ) + gaps[0] = [inherited_gap_start_ns, gaps[0][1], inherited_rate] + elif inherited_grid is not None: + # No normal mode data at all: the whole window is a gap at the beginning of + # the day, generated on the previous day's grid. + inherited_gap_start_ns, inherited_rate = inherited_grid + logger.info( + f"MAG L1C has no normal mode data; generating the full timeline on the " + f"previous day's timeline (rate {inherited_rate} vectors per second)." + ) + norm_epoch = [inherited_gap_start_ns, day_end_ns] + gaps = np.array( + [ + [ + inherited_gap_start_ns, + day_end_ns, + inherited_rate, + ] + ] + ) else: norm_epoch = [day_start_ns, day_end_ns] gaps = np.array( @@ -353,6 +542,12 @@ def process_mag_l1c( new_timeline = generate_timeline(norm_epoch, gaps) + if inherited_gap_start_ns is not None: + # The inherited gap starts one period before the first in-window grid point, + # because interpolate_gaps only fills points strictly inside a gap. That extra + # leading point must not appear in the output timeline. + new_timeline = new_timeline[new_timeline != inherited_gap_start_ns] + if normal_mode_dataset: norm_filled: np.ndarray = fill_normal_data(normal_mode_dataset, new_timeline) else: @@ -763,9 +958,16 @@ def generate_missing_timestamps(gap: np.ndarray) -> np.ndarray: # and newer (start, end, rate) gaps, which use the declared cadence. if len(gap) > 2: difference_ns = int(1e9 / int(gap[2])) + gap_start = gap[0] + gap_end = gap[1] + if not np.issubdtype(np.asarray(gap).dtype, np.integer): + # Only round non-integer bounds: np.rint returns float64, which cannot + # represent integer TTJ2000 nanoseconds exactly. + gap_start = np.rint(gap_start) + gap_end = np.rint(gap_end) output: np.ndarray = np.arange( - int(np.rint(gap[0])), - int(np.rint(gap[1])), + int(gap_start), + int(gap_end), difference_ns, dtype=np.int64, ) diff --git a/imap_processing/tests/mag/test_mag_l1c.py b/imap_processing/tests/mag/test_mag_l1c.py index a863d0b33..1b5592d10 100644 --- a/imap_processing/tests/mag/test_mag_l1c.py +++ b/imap_processing/tests/mag/test_mag_l1c.py @@ -10,6 +10,7 @@ estimate_rate, ) from imap_processing.mag.l1c.mag_l1c import ( + _expected_day_ns, fill_normal_data, find_all_gaps, find_gaps, @@ -20,7 +21,6 @@ process_mag_l1c, vectors_per_second_from_string, ) -from imap_processing.spice.time import et_to_ttj2000ns, str_to_et from imap_processing.tests.mag.conftest import ( generate_test_epoch, mag_l1a_dataset_generator, @@ -271,17 +271,6 @@ def test_mag_attributes(): assert attr in output.attrs -def _ttj2000_day_bounds(day: np.datetime64) -> tuple[int, int]: - day_start = day.astype("datetime64[s]") - np.timedelta64(30, "m") - day_end = ( - day.astype("datetime64[s]") + np.timedelta64(1, "D") + np.timedelta64(30, "m") - ) - return ( - int(et_to_ttj2000ns(str_to_et(str(day_start)))), - int(et_to_ttj2000ns(str_to_et(str(day_end)))), - ) - - def _build_mag_l1b( epochs: np.ndarray, logical_source: str, vps_attr: str ) -> xr.Dataset: @@ -299,7 +288,7 @@ def _build_mag_l1b( def _build_day_aligned_mixed_l1b(day: np.datetime64) -> tuple[xr.Dataset, xr.Dataset]: - day_start_ns, _ = _ttj2000_day_bounds(day) + day_start_ns, _ = _expected_day_ns(day) nm_start = day_start_ns + 300 * 1_000_000_000 nm_end = nm_start + 120 * 1_000_000_000 @@ -326,7 +315,7 @@ def test_process_mag_l1c_leading_burst_only_coverage(): silently dropped. """ day = np.datetime64("2025-01-01") - day_start_ns, _ = _ttj2000_day_bounds(day) + day_start_ns, _ = _expected_day_ns(day) # NM starts 5 minutes into the day window and lasts 2 minutes at 2 vec/s. nm_start = day_start_ns + 300 * 1_000_000_000 @@ -371,7 +360,7 @@ def test_process_mag_l1c_leading_burst_only_coverage(): def test_process_mag_l1c_trailing_burst_only_coverage(): """Burst coverage after the NM window must be interpolated as BURST.""" day = np.datetime64("2025-01-01") - day_start_ns, day_end_ns = _ttj2000_day_bounds(day) + day_start_ns, day_end_ns = _expected_day_ns(day) # NM sits mid-day, spanning 2 minutes at 2 vec/s. nm_center = (day_start_ns + day_end_ns) // 2 @@ -410,7 +399,7 @@ def test_mag_l1c_mixed_input_uses_day_to_process(): mag_l1c() against re-introduction of the old short-circuit to None. """ day = np.datetime64("2025-01-01") - day_start_ns, _ = _ttj2000_day_bounds(day) + day_start_ns, _ = _expected_day_ns(day) nm_start = day_start_ns + 600 * 1_000_000_000 nm_end = nm_start + 60 * 1_000_000_000 @@ -437,7 +426,7 @@ def test_mag_l1c_mixed_input_uses_day_to_process(): def test_mag_l1c_burst_only_generates_l1c_output(): """Burst-only L1C generation still fills the day window from BM data.""" day = np.datetime64("2025-01-01") - day_start_ns, _ = _ttj2000_day_bounds(day) + day_start_ns, _ = _expected_day_ns(day) # Burst covers the first 2 minutes of the day window at 8 vec/s. burst_epochs = np.arange( @@ -1165,9 +1154,14 @@ def _build_cross_day_l1b( """ cadence_day1 = 250_000_000 # 4 vec/s cadence_day2 = 500_000_000 # 2 vec/s - phase_ns = 73_000_000 # sub-cadence offset, < cadence_day1 + # Sub-cadence offset, < cadence_day1. Must be a multiple of 128 ns: the pipeline + # carries epochs in float64 columns, whose spacing at 2025-era TTJ2000 + # nanoseconds is 128 ns. A non-multiple offset makes every grid point a rounding + # tie, so the stored timestamps jitter +/- 64 ns and exact-equality assertions + # cannot hold for any implementation. + phase_ns = 73_000_064 - day2_start_ns, _ = _ttj2000_day_bounds(day2) + day2_start_ns, _ = _expected_day_ns(day2) # Day 1 NM ends just before day 2's window, on a 4 vec/s grid offset by phase. day1_last = day2_start_ns - cadence_day1 + phase_ns @@ -1207,7 +1201,6 @@ def _build_cross_day_l1b( return norm_day1, norm_day2, burst_day2, meta -@pytest.mark.xfail(reason="Not implemented") def test_process_mag_l1c_continues_previous_day_grid(): """Leading-gap timestamps must continue the previous day's NM grid. @@ -1243,7 +1236,6 @@ def test_process_mag_l1c_continues_previous_day_grid(): assert np.all(np.isin(meta["day2_nm_epochs"], epochs_out)) -@pytest.mark.xfail(reason="Not implemented") def test_mag_l1c_continues_previous_day_grid(): """mag_l1c() must thread the previous day's NM grid into the output. @@ -1270,3 +1262,118 @@ def test_mag_l1c_continues_previous_day_grid(): ) # Day 2's real NM samples are still used where they exist assert np.all(np.isin(meta["day2_nm_epochs"], epochs_out)) + + +@pytest.mark.parametrize( + "logical_source", + [ + "imap_mag_l1b_burst-mago", # not normal mode + "imap_mag_l1b_norm-magi", # wrong sensor + "imap_mag_l1c_norm-mago", # not L1B + ], +) +def test_mag_l1c_ignores_unusable_previous_day(logical_source): + """An unusable previous day dataset is ignored, not raised.""" + day1 = np.datetime64("2025-01-01") + day2 = np.datetime64("2025-01-02") + norm_day1, norm_day2, burst_day2, _ = _build_cross_day_l1b(day1, day2) + norm_day1.attrs["Logical_source"] = logical_source + + baseline = mag_l1c(norm_day2, day2, burst_day2) + output = mag_l1c(norm_day2, day2, burst_day2, previous_day_dataset=norm_day1) + + assert np.array_equal(output["epoch"].data, baseline["epoch"].data) + assert np.array_equal(output["vectors"].data, baseline["vectors"].data) + + +def test_mag_l1c_ignores_previous_day_with_unknown_cadence(): + """A previous day whose final spacing matches no MAG rate is ignored.""" + day1 = np.datetime64("2025-01-01") + day2 = np.datetime64("2025-01-02") + norm_day1, norm_day2, burst_day2, meta = _build_cross_day_l1b(day1, day2) + # Rebuild day 1 with an irregular 0.7 s spacing, matching no known rate. + day1_epochs = meta["day1_last"] - np.arange(9, -1, -1) * 700_000_000 + norm_day1 = _build_mag_l1b( + day1_epochs.astype(np.int64), "imap_mag_l1b_norm-mago", "0:2" + ) + + baseline = mag_l1c(norm_day2, day2, burst_day2) + output = mag_l1c(norm_day2, day2, burst_day2, previous_day_dataset=norm_day1) + + assert np.array_equal(output["epoch"].data, baseline["epoch"].data) + + +def test_mag_l1c_no_norm_inherits_previous_day_grid(): + """With no NM data at all, the whole timeline continues the previous day's grid.""" + day1 = np.datetime64("2025-01-01") + day2 = np.datetime64("2025-01-02") + norm_day1, _, burst_day2, meta = _build_cross_day_l1b(day1, day2) + + output = mag_l1c(burst_day2, day2, previous_day_dataset=norm_day1) + epochs_out = output["epoch"].data + + assert epochs_out.size > 0 + # Every output timestamp rides day 1's ending grid, starting one cadence after + # its last sample - not the day-aligned burst-only fallback grid. + assert epochs_out[0] == meta["day1_last"] + meta["cadence_day1"] + assert np.all((epochs_out - meta["day1_last"]) % meta["cadence_day1"] == 0) + + +def test_process_mag_l1c_previous_day_anchor_ignores_buffer_samples(): + """The inherited grid anchors on the last sample within the previous 24-hour day. + + Samples in the previous day's trailing buffer (at or past the current day's + midnight) are on a shifted grid here; if the anchor used them, the leading fill + would carry their phase instead of the pre-midnight phase. + """ + day2 = np.datetime64("2025-01-02") + window_start_ns, _ = _expected_day_ns(day2) + midnight_ns = window_start_ns + 1800 * 1_000_000_000 + cadence = 500_000_000 + + # Pre-midnight samples on a grid offset by 128 ns; buffer samples past midnight + # shifted by an extra quarter second. + pre_midnight = np.arange( + midnight_ns - 20 * 1_000_000_000 + 128, midnight_ns, cadence, dtype=np.int64 + ) + buffer_samples = np.arange( + midnight_ns + 128 + 250_000_000, + midnight_ns + 10 * 1_000_000_000, + cadence, + dtype=np.int64, + ) + previous_day = _build_mag_l1b( + np.concatenate([pre_midnight, buffer_samples]), + "imap_mag_l1b_norm-mago", + "0:2", + ) + anchor = int(pre_midnight[-1]) + + nm_epochs = np.arange( + midnight_ns + 60 * 1_000_000_000, + midnight_ns + 120 * 1_000_000_000, + cadence, + dtype=np.int64, + ) + norm_day2 = _build_mag_l1b(nm_epochs, "imap_mag_l1b_norm-mago", "0:2") + burst_day2 = _build_mag_l1b( + np.arange( + midnight_ns, midnight_ns + 30 * 1_000_000_000, 125_000_000, dtype=np.int64 + ), + "imap_mag_l1b_burst-mago", + "0:8", + ) + + result = process_mag_l1c( + norm_day2, + burst_day2, + InterpolationFunction.linear, + day2, + previous_day_dataset=previous_day, + ) + epochs_out = result[:, 0] + leading = epochs_out[epochs_out < nm_epochs[0]] + + assert leading.size > 0 + assert leading[0] == anchor + cadence + assert np.all((leading - anchor) % cadence == 0) diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index f2027522b..2105f92b0 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -28,6 +28,7 @@ Hit, Idex, Lo, + Mag, Spacecraft, Swe, Ultra, @@ -902,3 +903,80 @@ def test_post_processing( "naif0012.tls", "imap_sclk_0001.tsc", ] + + +@mock.patch("imap_processing.cli.check_epochs_within_day_offsets") +@mock.patch("imap_processing.cli.mag_l1c") +def test_mag_l1c_previous_day_routing( + mock_mag_l1c, mock_check_epochs, mock_instrument_dependencies +): + """A previous-day norm L1B dependency routes to mag_l1c's previous_day_dataset.""" + mocks = mock_instrument_dependencies + + norm_file = "imap_mag_l1b_norm-mago_20251215_v001.cdf" + burst_file = "imap_mag_l1b_burst-mago_20251215_v001.cdf" + previous_file = "imap_mag_l1b_norm-mago_20251214_v001.cdf" + input_collection = ProcessingInputCollection( + ScienceInput(norm_file), ScienceInput(burst_file), ScienceInput(previous_file) + ) + mocks["mock_pre_processing"].return_value = input_collection + + datasets_by_name = { + norm_file: xr.Dataset({}, attrs={"cdf_filename": norm_file}), + burst_file: xr.Dataset({}, attrs={"cdf_filename": burst_file}), + previous_file: xr.Dataset({}, attrs={"cdf_filename": previous_file}), + } + mocks["mock_load_cdf"].side_effect = lambda path: datasets_by_name[Path(path).name] + mock_mag_l1c.return_value = xr.Dataset( + {"epoch": ("epoch", np.array([0, 1], dtype=np.int64))}, + attrs={"cdf_filename": "l1c_out", "Logical_source": "imap_mag_l1c_norm-mago"}, + ) + mocks["mock_write_cdf"].side_effect = ["/path/to/l1c_out"] + + dependency_str = json.dumps( + [{"type": "science", "files": [norm_file, burst_file, previous_file]}] + ) + instrument = Mag( + "l1c", "norm-mago", dependency_str, "20251215", None, "v001", False + ) + instrument.process() + + assert mock_mag_l1c.call_count == 1 + call_args, call_kwargs = mock_mag_l1c.call_args + # Current-day files are the positional inputs; the previous day's norm file is + # passed separately and never treated as a current-day input. + assert call_args[0] is datasets_by_name[norm_file] + assert call_args[2] is datasets_by_name[burst_file] + assert call_kwargs["previous_day_dataset"] is datasets_by_name[previous_file] + + +@mock.patch("imap_processing.cli.check_epochs_within_day_offsets") +@mock.patch("imap_processing.cli.mag_l1c") +def test_mag_l1c_without_previous_day( + mock_mag_l1c, mock_check_epochs, mock_instrument_dependencies +): + """With only current-day dependencies, previous_day_dataset is None.""" + mocks = mock_instrument_dependencies + + norm_file = "imap_mag_l1b_norm-mago_20251215_v001.cdf" + input_collection = ProcessingInputCollection(ScienceInput(norm_file)) + mocks["mock_pre_processing"].return_value = input_collection + + norm_dataset = xr.Dataset({}, attrs={"cdf_filename": norm_file}) + mocks["mock_load_cdf"].side_effect = lambda path: norm_dataset + mock_mag_l1c.return_value = xr.Dataset( + {"epoch": ("epoch", np.array([0, 1], dtype=np.int64))}, + attrs={"cdf_filename": "l1c_out", "Logical_source": "imap_mag_l1c_norm-mago"}, + ) + mocks["mock_write_cdf"].side_effect = ["/path/to/l1c_out"] + + dependency_str = json.dumps([{"type": "science", "files": [norm_file]}]) + instrument = Mag( + "l1c", "norm-mago", dependency_str, "20251215", None, "v001", False + ) + instrument.process() + + assert mock_mag_l1c.call_count == 1 + call_args, call_kwargs = mock_mag_l1c.call_args + assert call_args[0] is norm_dataset + assert call_kwargs["previous_day_dataset"] is None From 9e5c39b28cda52f9116e19a8027737589eea8d4d Mon Sep 17 00:00:00 2001 From: sapols Date: Tue, 7 Jul 2026 13:15:28 -0600 Subject: [PATCH 3/3] MAG L1C: harden previous-day handling per independent review - Ignore (with a warning) a previous day dataset that has no epoch variable instead of raising KeyError. - Log skipped inheritance (too few pre-midnight samples, unknown cadence) at warning level to match the other unusable-neighbor paths. - Select the previous-day file by the job descriptor (norm + sensor) rather than "norm" alone, so a wrong-sensor file can never shadow the right one. - Drop the "exactly as before" claim from the mag_l1c docstring: the generate_missing_timestamps precision fix means gap fills at 8+ vec/s can differ from previously generated products by up to 128 ns even without a previous-day file. --- imap_processing/cli.py | 2 +- imap_processing/mag/l1c/mag_l1c.py | 11 +++++++---- imap_processing/tests/mag/test_mag_l1c.py | 13 +++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 1f2c937d8..e56d5c882 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -1360,7 +1360,7 @@ def do_processing( # noqa: PLR0912 for path in dependencies.get_valid_inputs_for_start_date( start_datetime - timedelta(days=1) ).get_file_paths(source="mag", data_type="l1b") - if "norm" in path.name + if self.descriptor in path.name ] previous_day_dataset = ( load_cdf(previous_day_files[0]) if previous_day_files else None diff --git a/imap_processing/mag/l1c/mag_l1c.py b/imap_processing/mag/l1c/mag_l1c.py index bb7ed641b..9f7d1d56d 100644 --- a/imap_processing/mag/l1c/mag_l1c.py +++ b/imap_processing/mag/l1c/mag_l1c.py @@ -46,7 +46,7 @@ def mag_l1c( current day opens with a gap, timestamps generated for that gap continue the previous day's cadence and phase so the L1C timeline stays regular across the day boundary. If not provided (or not usable), gaps at the start of the day - are filled on the current day's own grid, exactly as before. + are filled on the current day's own grid. Returns ------- @@ -321,7 +321,10 @@ def _validated_previous_day( f"expected normal mode L1B data for sensor mag{sensor}." ) return None - if previous_day_dataset["epoch"].data.size == 0: + if ( + "epoch" not in previous_day_dataset + or previous_day_dataset["epoch"].data.size == 0 + ): logger.warning("Ignoring previous day dataset with no epochs.") return None return previous_day_dataset @@ -390,7 +393,7 @@ def _previous_day_grid( previous_epochs = previous_day_dataset["epoch"].data anchor_index = int(np.searchsorted(previous_epochs, midnight_ns, side="left")) - 1 if anchor_index < 1: - logger.info( + logger.warning( "Previous day dataset has fewer than two samples before the current day; " "not inheriting its timeline." ) @@ -407,7 +410,7 @@ def _previous_day_grid( rate = vecsec.value break if rate is None: - logger.info( + logger.warning( f"Previous day dataset ends with sample spacing {anchor_spacing} ns, " f"which matches no known MAG rate; not inheriting its timeline." ) diff --git a/imap_processing/tests/mag/test_mag_l1c.py b/imap_processing/tests/mag/test_mag_l1c.py index 1b5592d10..680933b52 100644 --- a/imap_processing/tests/mag/test_mag_l1c.py +++ b/imap_processing/tests/mag/test_mag_l1c.py @@ -1377,3 +1377,16 @@ def test_process_mag_l1c_previous_day_anchor_ignores_buffer_samples(): assert leading.size > 0 assert leading[0] == anchor + cadence assert np.all((leading - anchor) % cadence == 0) + + +def test_mag_l1c_ignores_previous_day_without_epochs(): + """A previous day dataset with no epoch variable is ignored, not raised.""" + day1 = np.datetime64("2025-01-01") + day2 = np.datetime64("2025-01-02") + _, norm_day2, burst_day2, _ = _build_cross_day_l1b(day1, day2) + no_epochs = xr.Dataset(attrs={"Logical_source": "imap_mag_l1b_norm-mago"}) + + baseline = mag_l1c(norm_day2, day2, burst_day2) + output = mag_l1c(norm_day2, day2, burst_day2, previous_day_dataset=no_epochs) + + assert np.array_equal(output["epoch"].data, baseline["epoch"].data)