Monthly Flarelists

stixcore/ephemeris/manager.py

@@ -388,6 +388,20 @@ def get_sun_disc_size(self, *, date):
 
         return rsun_arc
 
+    def get_earth_solo_time_shift(self, *, date):
+        """gets Time(Sun to Earth) - Time(Sun to S/C)
+
+        Returns
+        -------
+        `astropy.units.Quantity`
+            Time difference between Sun to Earth and Sun to S/C in seconds
+        """
+        et = spiceypy.scs2e(SOLAR_ORBITER_ID, str(date))
+        solo_sun_hg, sun_solo_lt = spiceypy.spkezr("SOLO", et, "SUN_EARTH_CEQU", "None", "Sun")
+        sun_earth_hee, sun_earth_lt = spiceypy.spkezr("Earth", et, "SOLO_HEE", "None", "Sun")
+
+        return (sun_earth_lt - sun_solo_lt) * u.s
+
     def get_position(self, *, date, frame):
         """
         Get the position of SolarOrbiter at the given date in the given coordinate frame.

stixcore/io/FlareListManager.py

@@ -12,6 +12,7 @@
 from astropy.time import Time
 
 from stixcore.config.config import CONFIG
+from stixcore.io.product_processors.fits.processors import CreateUtcColumn
 from stixcore.products.level3.flarelist import FlarelistSC, FlarelistSDC
 from stixcore.products.product import Product
 from stixcore.util.logging import get_logger
@@ -188,12 +189,17 @@ def get_data(self, *, start, end, fido_client):
         data["flare_id"] = Column(
             mt["flare_id"].astype(int), description=f"unique flare id for flarelist {self.flarelistname}"
         )
-        data["start_UTC"] = Column(0, description="start time of flare")
-        data["start_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]]
+        CreateUtcColumn(
+            data,
+            [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]],
+            "start_UTC",
+            description="start time of flare",
+        )
+
         data["duration"] = Column(mt["duration"].astype(float) * u.s, description="duration of flare")
-        data["end_UTC"] = Column(0, description="end time of flare")
+        data["end_UTC"] = CreateUtcColumn(description="end time of flare")
         data["end_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]]
-        data["peak_UTC"] = Column(0, description="flare peak time")
+        data["peak_UTC"] = CreateUtcColumn(description="flare peak time")
         data["peak_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]]
         data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare")
         data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV")
@@ -277,7 +283,7 @@ def get_data(self, *, start, end, fido_client):
 
         data.add_index("flare_id")
 
-        # add energy axis for the lightcurve peek time data for each flare
+        # add energy axis for the lightcurve peak time data for each flare
         # the energy bins are taken from the daily ql-lightcurve products
         # as the definition of the lc energy chanel's are will change only very seldom
         # the ql-lightcurve products assume a constant definition for an entire day.
@@ -439,13 +445,28 @@ def get_data(self, *, start, end, fido_client):
         data["flare_id"] = Column(
             mt["flare_id"].astype(int), description=f"unique flare id for flarelist {self.flarelistname}"
         )
-        data["start_UTC"] = Column(0, description="start time of flare")
-        data["start_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]]
+
+        CreateUtcColumn(
+            data,
+            [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]],
+            "start_UTC",
+            description="start time of flare",
+        )
         data["duration"] = Column(mt["duration"].astype(float) * u.s, description="duration of flare")
-        data["end_UTC"] = Column(0, description="end time of flare")
-        data["end_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]]
-        data["peak_UTC"] = Column(0, description="flare peak time")
-        data["peak_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]]
+
+        CreateUtcColumn(
+            data,
+            [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]],
+            "end_UTC",
+            description="end time of flare",
+        )
+        CreateUtcColumn(
+            data,
+            [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]],
+            "peak_UTC",
+            description="flare peak time",
+        )
+
         data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare")
         data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV")
         data["GOES_class"] = Column(
@@ -528,7 +549,7 @@ def get_data(self, *, start, end, fido_client):
 
         data.add_index("flare_id")
 
-        # add energy axis for the lightcurve peek time data for each flare
+        # add energy axis for the lightcurve peak time data for each flare
         # the energy bins are taken from the daily ql-lightcurve products
         # as the definition of the lc energy chanel's are will change only very seldom
         # the ql-lightcurve products assume a constant definition for an entire day.

stixcore/io/product_processors/fits/processors.py

@@ -58,6 +58,24 @@ def set_bscale_unsigned(table_hdu):
     return table_hdu
 
 
+def CreateUtcColumn(table, data, colname, description="UTC Time"):
+    """
+    Create UTC time column for FITS tables.
+
+    Parameters
+    ----------
+    description : `str`
+        Description for the column
+
+    Returns
+    -------
+    `astropy.table.Column`
+        Column representing UTC time
+    """
+    table[colname] = data
+    table[colname].info.description = description
+
+
 def add_default_tuint(table_hdu):
     """
     Add a default empty string tunit if not already defined
@@ -805,10 +823,11 @@ def generate_primary_header(self, filename, product, *, version=0):
             if default[0] not in soop_key_names:
                 soop_headers += tuple([default])
 
+        scet_range = product.scet_timerange
         time_headers = (
             # Name, Value, Comment
-            ("OBT_BEG", product.scet_timerange.start.as_float().value, "Start of acquisition time in OBT"),
-            ("OBT_END", product.scet_timerange.end.as_float().value, "End of acquisition time in OBT"),
+            ("OBT_BEG", scet_range.start.as_float().value, "Start of acquisition time in OBT"),
+            ("OBT_END", scet_range.end.as_float().value, "End of acquisition time in OBT"),
             ("TIMESYS", "UTC", "System used for time keywords"),
             ("LEVEL", "L1", "Processing level of the data"),
             ("DATE-OBS", product.utc_timerange.start.fits, "Start of acquisition time in UTC"),
@@ -1049,7 +1068,8 @@ def write_fits(self, prod, *, version=0):
         elif fitspath_complete.exists():
             logger.warning("Complete Fits file %s exists will be overridden", fitspath.name)
 
-        data = prod.data
+        data = prod.data.copy()
+        prod.on_serialize(data)
 
         primary_header, header_override = self.generate_primary_header(filename, prod, version=version)
         primary_hdu = fits.PrimaryHDU()
@@ -1126,7 +1146,8 @@ def write_fits(self, prod, *, version=0):
         elif fitspath_complete.exists():
             logger.warning("Complete Fits file %s exists will be overridden", fitspath.name)
 
-        data = prod.data
+        data = prod.data.copy()
+        prod.on_serialize(data)
 
         primary_header, header_override = self.generate_primary_header(filename, prod, version=version)
 
@@ -1139,7 +1160,7 @@ def write_fits(self, prod, *, version=0):
         # Add comment and history
         [primary_hdu.header.add_comment(com) for com in prod.comment]
         [primary_hdu.header.add_history(com) for com in prod.history]
-        primary_hdu.header.update({"HISTORY": "Processed by STIXCore ANC"})
+        primary_hdu.header.update({"HISTORY": "Processed by STIXCore L3"})
 
         if hasattr(prod, "maps") and len(prod.maps) > 0:
             # fig = plt.figure(figsize=(12, 6))

stixcore/processing/FLtoFL.py

@@ -15,7 +15,7 @@
 )
 from stixcore.products.level3.flarelist import (
     FlareList,
-    FlarePeekPreviewMixin,
+    FlarePeakPreviewMixin,
     FlarePositionMixin,
     FlareSOOPMixin,
 )
@@ -107,7 +107,7 @@ def test_for_processing(
         """
         try:
             c_header = fits.getheader(candidate)
-            f_data_end = datetime.fromisoformat(c_header["DATE-END"])
+            # f_data_end = datetime.fromisoformat(c_header["DATE-END"])
             f_create_date = datetime.fromisoformat(c_header["DATE"])
 
             cfn = get_complete_file_name_and_path(candidate)
@@ -128,8 +128,9 @@ def test_for_processing(
             # safety margin of 1day until we process higher products with position and pointing
             # only use flown spice kernels not predicted once as pointing information
             # can be "very off"
-            if f_data_end > (Spice.instance.get_mk_date(meta_kernel_type="flown") - timedelta(hours=24)):
-                return TestForProcessingResult.NotSuitable
+            # TODO redo
+            # if f_data_end > (Spice.instance.get_mk_date(meta_kernel_type="flown") - timedelta(hours=24)):
+            #    return TestForProcessingResult.NotSuitable
 
             # safety margin of x until we start with processing the list files
             if f_create_date >= (datetime.now() - self.cadence):
@@ -187,9 +188,9 @@ def process_fits_files(
                 if issubclass(out_product, FlareSOOPMixin) and not issubclass(in_product, FlareSOOPMixin):
                     out_product.add_soop(data)
 
-                # add peek preview images if not already present
-                if issubclass(out_product, FlarePeekPreviewMixin) and not issubclass(in_product, FlarePeekPreviewMixin):
-                    out_product.add_peek_preview(data, energy, file_path.name, fido_client, img_processor, month=month)
+                # add peak preview images if not already present
+                if issubclass(out_product, FlarePeakPreviewMixin) and not issubclass(in_product, FlarePeakPreviewMixin):
+                    out_product.add_peak_preview(data, energy, file_path.name, fido_client, img_processor, month=month)
 
                 out_prod = out_product(control=control, data=data, month=month, energy=energy)
                 out_prod.parent = file_path.name

stixcore/processing/FlareListL3.py

@@ -24,7 +24,7 @@
 class FlareListL3(SingleProductProcessingStepMixin):
     """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file."""
 
-    STARTDATE = date(2024, 1, 1)
+    STARTDATE = date(2022, 1, 1)
 
     def __init__(self, flm: FlareListManager, output_dir: Path):
         """Crates a new Processor.