Move MAG L1A to CDF attribute manager (IMAP-Science-Operations-Center…

…#693)

Refactoring MAG CDF attribute management to combine shared code for burst/norm/mago/magi. Moved MAG L1 to use SAMMI yaml CDF attribute manager. Updating tests.

imap_processing/cdf/config/imap_mag_global_cdf_attrs.yaml

@@ -13,14 +13,14 @@ default: &default
 
 imap_mag_l1a_norm-raw:
     <<: *default
-    Data_type: L1A_raw-norm>Level-1A raw normal rate
+    Data_type: L1A_norm-raw>Level-1A raw normal rate
     Logical_source: imap_mag_l1a_norm-raw
     Logical_source_description: IMAP Mission MAG Normal Rate Instrument Level-1A Data.
     Data_level: L1A
 
 imap_mag_l1a_burst-raw:
     <<: *default
-    Data_type: L1A_raw-burst>Level-1A raw burst rate
+    Data_type: L1A_burst-raw>Level-1A raw burst rate
     Logical_source: imap_mag_l1a_burst-raw
     Logical_source_description: IMAP Mission MAG Burst Rate Instrument Level-1A Data.
     Data_level: L1A

imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml

@@ -1,3 +1,19 @@
+epoch:
+  CATDESC: Time, number of nanoseconds since J2000 with leap seconds included
+  FIELDNAM: epoch
+  LABLAXIS: epoch
+  FILLVAL: -9223372036854775808
+  FORMAT: " " # Supposedly not required, fails in xarray_to_cdf
+  VALIDMIN: -9223372036854775808
+  VALIDMAX: 9223372036854775807
+  UNITS: ns
+  VAR_TYPE: support_data
+  SCALETYP: linear
+  MONOTON: INCREASE
+  TIME_BASE: J2000
+  TIME_SCALE: Terrestrial Time
+  REFERENCE_POSITION: Rotating Earth Geoid
+
 default_attrs: &default
   # Assumed values for all variable attrs unless overwritten
   DEPEND_0: epoch
@@ -7,25 +23,34 @@ default_attrs: &default
   VALIDMIN: -9223372036854775808
   VALIDMAX: 9223372036854775807
   VAR_TYPE: data
+  UNITS: ' '
+
+default_coords: &default_coords
+  # Assumed values for all coordinate attrs unless overwritten
+  FORMAT: F2.4  # Float with 4 digits
+  VAR_TYPE: data
+  DISPLAY_TYPE: time_series
+  FILLVAL: -9223372036854775808
+  VALIDMIN: -9223372036854775808
+  VALIDMAX: 9223372036854775807
+  UNITS: counts
 
-mag_raw_vector_attrs:
-  <<: *default
+raw_vector_attrs:
+  <<: *default_coords
   CATDESC: Raw unprocessed magnetic field vector data in bytes
   DEPEND_1: direction
   FIELDNAM: Magnetic Field Vector
   LABLAXIS: Raw binary magnetic field vector data
   FORMAT: I3
-  UNITS: counts
 
 vector_attrs:
-    <<: *default
+    <<: *default_coords
     CATDESC: Magnetic field vector with x y z and sensor range varying by time
     DEPEND_1: direction
     FIELDNAM: Magnetic Field Vector
     LABLAXIS: Magnetic field vector data
     FILLVAL: 9223372036854775807
     FORMAT: I3
-    UNITS: counts
 
 mag_support_attrs: &support_default
     <<: *default
@@ -44,13 +69,13 @@ mag_flag_attrs:
     FORMAT: I1
 
 raw_direction_attrs:
-    <<: *default
+    <<: *default_coords
     CATDESC: Raw magnetic field vector binary length
     FIELDNAM: Raw magnetic field vector binary length
     LABLAXIS: Magnetic field vector directions
 
 direction_attrs:
-    <<: *default
+    <<: *default_coords
     CATDESC: Magnetic field vector
     FIELDNAM: \[xyz\] magnetic field vector
     FORMAT: I3
@@ -154,4 +179,39 @@ VECTORS:
     CATDESC: Raw binary value of MAG Science vectors before processing
     FIELDNAM: Raw vector binary
 
-
+# CCSDS attributes for MAG L1A RAW
+
+VERSION:
+  <<: *metadata_default
+  CATDESC: CCSDS Packet version number
+  FIELDNAM: CCSDS Packet version number
+
+TYPE:
+  <<: *metadata_default
+  CATDESC: CCSDS Packet type
+  FIELDNAM: CCSDS Packet type
+
+SEC_HDR_FLG:
+  <<: *metadata_default
+  CATDESC: CCSDS Packet Secondary header flag
+  FIELDNAM: CCSDS Packet Secondary header flag
+
+PKT_APID:
+  <<: *metadata_default
+  CATDESC: CCSDS Packet Application Process ID
+  FIELDNAM: CCSDS Packet Application Process ID
+
+SEQ_FLGS:
+  <<: *metadata_default
+  CATDESC: CCSDS Packet Grouping flags
+  FIELDNAM: CCSDS Packet Grouping flags
+
+SRC_SEQ_CTR:
+  <<: *metadata_default
+  CATDESC: CCSDS Packet Source Sequence Counter
+  FIELDNAM: CCSDS Packet Source Sequence Counter
+
+PKT_LEN:
+  <<: *metadata_default
+  CATDESC: CCSDS Packet Length
+  FIELDNAM: CCSDS Packet Length

imap_processing/cli.py

@@ -645,14 +645,13 @@ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
 class Mag(ProcessInstrument):
     """Process MAG."""
 
-    # TODO: fixed in later PR
-    def do_processing(self, dependencies: list) -> list[xr.Dataset]:
+    def do_processing(self, dependencies: list[Path]) -> list[xr.Dataset]:
         """
         Perform MAG specific processing.
 
         Parameters
         ----------
-        dependencies : list
+        dependencies : list[Path]
             List of dependencies to process.
 
         Returns

imap_processing/mag/__init__.py

@@ -1 +1,2 @@
 __version__ = "01"
+cdf_format_version = "v0.1"

imap_processing/mag/constants.py

@@ -0,0 +1,57 @@
+"""Collection of constant types or values for MAG."""
+
+from enum import Enum
+
+
+class DataMode(Enum):
+    """
+    Enum for MAG data modes: burst and normal (BURST + NORM).
+
+    Attributes
+    ----------
+    BURST: str
+        Burst data mode - higher frequency data
+    NORM: str
+        Normal data mode - lower frequency data (downsampled from burst)
+    """
+
+    BURST = "BURST"
+    NORM = "NORM"
+
+
+class Sensor(Enum):
+    """
+    Enum for MAG sensors: raw, MAGo, and MAGi (RAW, MAGO, MAGI).
+
+    Attributes
+    ----------
+    MAGO : str
+        MAGo sensor - for the outboard sensor. This is nominally expected to be the
+        primary sensor.
+    MAGI : str
+        MAGi sensor - for the inboard sensor.
+    RAW : str
+        RAW data - contains both sensors. Here, the vectors are unprocessed.
+    """
+
+    MAGO = "MAGO"
+    MAGI = "MAGI"
+    RAW = "RAW"
+
+
+class PrimarySensor(Enum):
+    """
+    Enum for primary sensor: MAGo and MAGi (MAGO, MAGI).
+
+    This corresponds to the PRI_SENS field in the MAG Level 0 data.
+
+    Attributes
+    ----------
+    MAGO : int
+        Primary sensor is MAGo.
+    MAGI : int
+        Primary sensor is MAGi.
+    """
+
+    MAGO = 0
+    MAGI = 1

imap_processing/mag/l0/decom_mag.py

@@ -13,9 +13,9 @@
 
 from imap_processing import imap_module_directory
 from imap_processing.ccsds.ccsds_data import CcsdsData
-from imap_processing.cdf.global_attrs import ConstantCoordinates
+from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import met_to_j2000ns
-from imap_processing.mag import mag_cdf_attrs
+from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l0.mag_l0_data import MagL0, Mode
 
 logger = logging.getLogger(__name__)
@@ -67,7 +67,9 @@ def decom_packets(packet_file_path: str | Path) -> dict[str, list[MagL0]]:
     return {"norm": norm_data, "burst": burst_data}
 
 
-def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset:
+def generate_dataset(
+    l0_data: list[MagL0], mode: DataMode, attribute_manager: ImapCdfAttributes
+) -> xr.Dataset:
     """
     Generate a CDF dataset from the sorted raw L0 MAG data.
 
@@ -76,8 +78,11 @@ def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset:
     l0_data : list[MagL0]
         List of sorted L0 MAG data.
 
-    dataset_attrs : dict
-        Global attributes for the dataset.
+    mode : DataMode
+        The mode of the CDF file - burst or norm.
+
+    attribute_manager : ImapCdfAttributes
+        Attribute manager for the dataset, including all MAG L1A attributes.
 
     Returns
     -------
@@ -115,33 +120,39 @@ def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset:
         for key, value in dataclasses.asdict(datapoint).items():
             if key not in ("ccsds_header", "VECTORS", "SHCOARSE"):
                 support_data[key].append(value)
+            if key == "ccsds_header":
+                for ccsds_key, ccsds_value in value.items():
+                    support_data[ccsds_key].append(ccsds_value)
 
     # Used in L1A vectors
     direction = xr.DataArray(
         np.arange(vector_data.shape[1]),
         name="direction",
         dims=["direction"],
-        attrs=mag_cdf_attrs.raw_direction_attrs.output(),
+        attrs=attribute_manager.get_variable_attributes("raw_direction_attrs"),
     )
+
     # TODO: Epoch here refers to the start of the sample. Confirm that this is
     # what mag is expecting, and if it is, CATDESC needs to be updated.
     epoch_time = xr.DataArray(
         shcoarse_data,
         name="epoch",
         dims=["epoch"],
-        attrs=ConstantCoordinates.EPOCH,
+        attrs=attribute_manager.get_variable_attributes("epoch"),
     )
     # TODO: raw vectors units
     raw_vectors = xr.DataArray(
         vector_data,
         name="raw_vectors",
         dims=["epoch", "direction"],
-        attrs=mag_cdf_attrs.mag_raw_vector_attrs.output(),
+        attrs=attribute_manager.get_variable_attributes("raw_vector_attrs"),
     )
 
+    logical_id = f"imap_mag_l1a_{mode.value.lower()}-raw"
+
     output = xr.Dataset(
         coords={"epoch": epoch_time, "direction": direction},
-        attrs=dataset_attrs,
+        attrs=attribute_manager.get_global_attributes(logical_id),
     )
 
     output["raw_vectors"] = raw_vectors
@@ -153,14 +164,7 @@ def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset:
                 value,
                 name=key.lower(),
                 dims=["epoch"],
-                attrs=dataclasses.replace(
-                    mag_cdf_attrs.mag_support_attrs,
-                    catdesc=mag_cdf_attrs.catdesc_fieldname_l0[key][0],
-                    fieldname=mag_cdf_attrs.catdesc_fieldname_l0[key][1],
-                    # TODO: label_axis should be as close to 6 letters as possible
-                    label_axis=key,
-                    display_type="no_plot",
-                ).output(),
+                attrs=attribute_manager.get_variable_attributes(key),
             )
 
     return output