diff --git a/django/processing/etl/services/task.py b/django/processing/etl/services/task.py index f08544f2..bbc47efa 100644 --- a/django/processing/etl/services/task.py +++ b/django/processing/etl/services/task.py @@ -392,6 +392,7 @@ def run( header_row=data_connection.payload.header_row, data_start_row=data_connection.payload.data_start_row, delimiter=data_connection.payload.delimiter, # noqa + identifier_type="index" if data_connection.payload.header_row is None else "name", ) elif data_connection.payload.payload_type == "JSON": diff --git a/packages/hydroserverpy/src/hydroserverpy/etl/models/timestamp.py b/packages/hydroserverpy/src/hydroserverpy/etl/models/timestamp.py index 34d34b1c..fa123f72 100644 --- a/packages/hydroserverpy/src/hydroserverpy/etl/models/timestamp.py +++ b/packages/hydroserverpy/src/hydroserverpy/etl/models/timestamp.py @@ -130,47 +130,81 @@ def parse_series_to_utc( series: pd.Series ) -> pd.Series: """ - Parse a pandas Series of timestamps and normalize them to a UTC baseline. - - Parsing Logic: - 1. If timezone_type is 'utc' or None, strings are parsed as UTC-aware. - 2. If timezone_type is 'iana' or 'offset', strings are parsed as naive. - - Conflict Resolution: - - Overwrite: If 'iana'/'offset' is configured but the data contains embedded - timezones, the embedded offsets are stripped and replaced by the config. - - Fallback: If no timezone is configured but the data is naive, UTC is assumed. - - Type Safety: Mixed offsets are flattened into a consistent datetime64[ns, UTC] - dtype to ensure the pandas .dt accessor remains available downstream. + Parse a pandas Series of timestamps and normalize them to UTC. + + Accepts uniform datetime64 Series, object-dtype Series of ISO strings, + object-dtype Series of strings with a custom timestamp_format, or + object-dtype Series of pd.Timestamp/datetime objects. Raises ValueError + on null values, mixed element types, or unsupported element types. Invalid + strings raise during parsing. The returned Series is always + datetime64[ns, UTC] with the same length as the input. + + For tz-naive inputs, the configured timezone is applied before converting + to UTC. If no timezone is configured, UTC is assumed. Tz-aware inputs are + converted to UTC from their embedded timezone without being overwritten. + + For custom timestamp formats: include %z in timestamp_format if strings + carry embedded timezone info; omit %z if strings are tz-naive. A mismatch + between the format and the actual strings raises during parsing. """ - # Ensure input is string-based for pandas parsing if not already datetime objects - if not pd.api.types.is_datetime64_any_dtype(series): - series: pd.Series = series.astype("string", copy=False).str.strip() + if len(series) == 0: + return pd.Series(dtype="datetime64[ns, UTC]") + + if series.isna().any(): + raise ValueError("Series contains null or missing values") + + tz_label = ( + self._to_pandas_offset(self.timezone) if self.timezone_type == "offset" + else (self.timezone or "UTC") + ) + + # Uniform datetime64 (all naive or all same-tz-aware) + if pd.api.types.is_datetime64_any_dtype(series): + if series.dt.tz is None: + return series.dt.tz_localize( + tz_label, ambiguous=False, nonexistent="shift_forward" + ).dt.tz_convert("UTC") + return series.dt.tz_convert("UTC") + + # Object dtype: must be uniformly strings or Timestamp/datetime objects + first_timestamp = series.iloc[0] + + if isinstance(first_timestamp, str): + series = series.str.strip() + if series.isna().any(): + raise ValueError("Series contains mixed or non-string values") + + if self.timestamp_format: + if "%z" in self.timestamp_format: + return pd.Series( + pd.to_datetime(series, utc=True, format=self.timestamp_format, errors="raise") + ) + return pd.Series( + pd.to_datetime(series, utc=False, format=self.timestamp_format, errors="raise") + .dt.tz_localize(tz_label, ambiguous=False, nonexistent="shift_forward") + .dt.tz_convert("UTC") + ) - # Determine if UTC parsing should be used directly - parse_as_utc = self.timezone_type in ["utc", None] + # ISO strings: regex detects embedded tz per element to support mixed-offset series + # (pandas cannot parse mixed tz/naive strings without coercing to NaT) + has_tz = series.str.contains(r"[Zz]$|[+-]\d{2}(?::?\d{2})?$", regex=True, na=False) + tz_aware = pd.to_datetime(series[has_tz], utc=True, errors="raise") + tz_naive_raw_series = pd.to_datetime(series[~has_tz], utc=False, errors="raise") + + elif isinstance(first_timestamp, pd.Timestamp): + has_tz = series.apply(lambda x: x.tzinfo is not None) + tz_aware = pd.to_datetime(series[has_tz], utc=True) + tz_naive_raw_series = pd.to_datetime(series[~has_tz]) - # Perform initial parsing of the values - if self.timestamp_type == "iso": - parsed_series = pd.to_datetime(series, utc=parse_as_utc, errors="coerce") - else: - parsed_series = pd.to_datetime(series, utc=parse_as_utc, format=self.timestamp_format, errors="coerce") - - # Apply fixed IANA or UTC offsets to the series if provided - if self.timezone_type in ["offset", "iana"]: - if parsed_series.dt.tz is not None or parsed_series.dtype == "object": - parsed_series = parsed_series.dt.tz_localize(None) - tz_label = self._to_pandas_offset(self.timezone) if self.timezone_type == "offset" else self.timezone - utc_series = parsed_series.dt.tz_localize( - tz_label, ambiguous=False, nonexistent="shift_forward" - ).dt.tz_convert(timezone.utc) - - # Normalize to UTC if the series timezones are embedded or naive and configured as UTC else: - utc_series = pd.to_datetime(parsed_series, utc=True) + raise ValueError(f"Unsupported series element type: {type(first_timestamp).__name__}") + + tz_naive_series = tz_naive_raw_series.dt.tz_localize( + tz_label, ambiguous=False, nonexistent="shift_forward" + ).dt.tz_convert("UTC") - return utc_series + return pd.Series(pd.concat([tz_aware, tz_naive_series])).sort_index() def to_string(self, dt: datetime) -> str: """ diff --git a/packages/hydroserverpy/tests/etl/test_timestamp_model.py b/packages/hydroserverpy/tests/etl/test_timestamp_model.py index 6cae9b30..ef0d1cac 100644 --- a/packages/hydroserverpy/tests/etl/test_timestamp_model.py +++ b/packages/hydroserverpy/tests/etl/test_timestamp_model.py @@ -189,17 +189,15 @@ def test_basic_utc_iso_strings(self, utc_iso_timestamp): assert result.iloc[0] == pd.Timestamp("2024-01-01 00:00:00", tz="UTC") assert result.iloc[1] == pd.Timestamp("2024-06-15 12:30:00", tz="UTC") - def test_invalid_strings_become_nat(self, utc_iso_timestamp): + def test_invalid_strings_raise_error(self, utc_iso_timestamp): series = pd.Series(["not-a-date", "also-bad"]) - result = utc_iso_timestamp.parse_series_to_utc(series) - assert result.isna().all() + with pytest.raises(Exception): + utc_iso_timestamp.parse_series_to_utc(series) - def test_mixed_valid_invalid(self, utc_iso_timestamp): + def test_mixed_valid_invalid_raises_error(self, utc_iso_timestamp): series = pd.Series(["2024-01-01T00:00:00", "bad-date", "2024-06-01T00:00:00"]) - result = utc_iso_timestamp.parse_series_to_utc(series) - assert pd.notna(result.iloc[0]) - assert pd.isna(result.iloc[1]) - assert pd.notna(result.iloc[2]) + with pytest.raises(Exception): + utc_iso_timestamp.parse_series_to_utc(series) def test_whitespace_is_stripped(self, utc_iso_timestamp): series = pd.Series([" 2024-01-01T12:00:00 ", "\t2024-06-01T00:00:00\n"]) @@ -249,6 +247,41 @@ def test_mixed_offset_series_is_flattened_to_utc(self): assert result.iloc[1] == pd.Timestamp("2024-01-01 05:00:00", tz="UTC") +# --------------------------------------------------------------------------- +# parse_series_to_utc – hour-only UTC offsets embedded in ISO strings (±HH) +# --------------------------------------------------------------------------- + +class TestParseSeriesHourOnlyOffset: + + def test_negative_hour_only_offset(self): + ts = Timestamp(timestamp_type="iso", timezone_type=None) + series = pd.Series(["2024-01-01T12:00:00-07"]) + result = ts.parse_series_to_utc(series) + assert result.dt.tz == timezone.utc + assert result.iloc[0] == pd.Timestamp("2024-01-01 19:00:00", tz="UTC") + + def test_positive_hour_only_offset(self): + ts = Timestamp(timestamp_type="iso", timezone_type=None) + series = pd.Series(["2024-01-01T12:00:00+05"]) + result = ts.parse_series_to_utc(series) + assert result.dt.tz == timezone.utc + assert result.iloc[0] == pd.Timestamp("2024-01-01 07:00:00", tz="UTC") + + def test_hour_only_offset_matches_full_offset(self): + ts = Timestamp(timestamp_type="iso", timezone_type=None) + hour_only = ts.parse_series_to_utc(pd.Series(["2024-06-15T08:00:00-07"])) + full_hhmm = ts.parse_series_to_utc(pd.Series(["2024-06-15T08:00:00-07:00"])) + assert hour_only.iloc[0] == full_hhmm.iloc[0] + + def test_mixed_hour_only_and_full_offsets(self): + ts = Timestamp(timestamp_type="iso", timezone_type=None) + series = pd.Series(["2024-01-01T12:00:00-07", "2024-01-01T12:00:00+05:30"]) + result = ts.parse_series_to_utc(series) + assert result.dt.tz == timezone.utc + assert result.iloc[0] == pd.Timestamp("2024-01-01 19:00:00", tz="UTC") + assert result.iloc[1] == pd.Timestamp("2024-01-01 06:30:00", tz="UTC") + + # --------------------------------------------------------------------------- # parse_series_to_utc – IANA timezone type # --------------------------------------------------------------------------- @@ -262,11 +295,18 @@ def test_naive_strings_localized_to_iana_zone(self, iana_timestamp): assert result.dt.tz == timezone.utc assert result.iloc[0] == pd.Timestamp("2024-01-01 05:00:00", tz="UTC") - def test_embedded_offsets_are_overwritten_by_iana_zone(self, iana_timestamp): - # Embedded +05:30 is stripped; America/New_York (UTC-5 in Jan) applied instead + def test_embedded_offsets_are_preserved_with_iana_zone(self, iana_timestamp): + # Embedded +05:30 is respected; America/New_York config does not overwrite it series = pd.Series(["2024-01-01T00:00:00+05:30"]) result = iana_timestamp.parse_series_to_utc(series) + assert result.iloc[0] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC") + + def test_naive_and_aware_mixed_series_with_iana_zone(self, iana_timestamp): + # Naive timestamps get America/New_York (UTC-5 in Jan); aware ones are preserved + series = pd.Series(["2024-01-01T00:00:00", "2024-01-01T00:00:00+05:30"]) + result = iana_timestamp.parse_series_to_utc(series) assert result.iloc[0] == pd.Timestamp("2024-01-01 05:00:00", tz="UTC") + assert result.iloc[1] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC") def test_dst_spring_forward(self): ts = Timestamp(timestamp_type="iso", timezone_type="iana", timezone="America/New_York") @@ -290,11 +330,18 @@ def test_naive_strings_localized_to_offset(self, offset_timestamp): assert result.dt.tz == timezone.utc assert result.iloc[0] == pd.Timestamp("2024-01-01 07:00:00", tz="UTC") - def test_embedded_offsets_are_overwritten_by_configured_offset(self, offset_timestamp): - # Embedded +05:30 is stripped; -0700 applied instead + def test_embedded_offsets_are_preserved_with_configured_offset(self, offset_timestamp): + # Embedded +05:30 is respected; -0700 config does not overwrite it series = pd.Series(["2024-01-01T00:00:00+05:30"]) result = offset_timestamp.parse_series_to_utc(series) + assert result.iloc[0] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC") + + def test_naive_and_aware_mixed_series_with_configured_offset(self, offset_timestamp): + # Naive timestamps get -0700 applied; aware ones are preserved + series = pd.Series(["2024-01-01T00:00:00", "2024-01-01T00:00:00+05:30"]) + result = offset_timestamp.parse_series_to_utc(series) assert result.iloc[0] == pd.Timestamp("2024-01-01 07:00:00", tz="UTC") + assert result.iloc[1] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC") def test_positive_offset_bare_format(self): ts = Timestamp(timestamp_type="iso", timezone_type="offset", timezone="+0530") @@ -339,10 +386,10 @@ def test_custom_format_parses_correctly(self, custom_timestamp): assert result.dt.tz == timezone.utc assert result.iloc[0] == pd.Timestamp("2024-03-15 08:00:00", tz="UTC") - def test_custom_format_wrong_data_becomes_nat(self, custom_timestamp): + def test_custom_format_wrong_data_raises_error(self, custom_timestamp): series = pd.Series(["2024-01-01T00:00:00Z"]) - result = custom_timestamp.parse_series_to_utc(series) - assert result.isna().all() + with pytest.raises(Exception): + custom_timestamp.parse_series_to_utc(series) def test_custom_format_with_iana_timezone(self): ts = Timestamp( @@ -403,3 +450,102 @@ def test_large_series_completes(self, utc_iso_timestamp): result = utc_iso_timestamp.parse_series_to_utc(pd.Series(dates)) assert len(result) == 100_000 assert result.isna().sum() == 0 + + +# --------------------------------------------------------------------------- +# parse_series_to_utc – input validation errors +# --------------------------------------------------------------------------- + +class TestParseSeriesValidation: + + def test_null_values_raise_error(self, utc_iso_timestamp): + series = pd.Series(["2024-01-01T00:00:00", None, "2024-06-01T00:00:00"]) + with pytest.raises(ValueError, match="null"): + utc_iso_timestamp.parse_series_to_utc(series) + + def test_mixed_strings_and_timestamps_raise_error(self, utc_iso_timestamp): + series = pd.Series(["2024-01-01T00:00:00Z", pd.Timestamp("2024-06-20", tz="UTC")]) + with pytest.raises(ValueError): + utc_iso_timestamp.parse_series_to_utc(series) + + def test_unsupported_element_type_raises_error(self, utc_iso_timestamp): + series = pd.Series([20240101, 20240615]) + with pytest.raises((ValueError, TypeError)): + utc_iso_timestamp.parse_series_to_utc(series) + + def test_invalid_string_raises_error(self, iana_timestamp): + series = pd.Series(["2024-01-15T08:00:00Z", "not-a-date"]) + with pytest.raises(Exception): + iana_timestamp.parse_series_to_utc(series) + + +# --------------------------------------------------------------------------- +# parse_series_to_utc – pd.Timestamp object inputs +# --------------------------------------------------------------------------- + +class TestParseSeriesTimestampObjects: + + def test_different_tz_aware_timestamps_convert_to_utc(self): + ts = Timestamp(timestamp_type="iso", timezone_type="iana", timezone="America/New_York") + series = pd.Series([ + pd.Timestamp("2024-01-15 08:00:00", tz="UTC"), + pd.Timestamp("2024-06-20 14:30:00", tz="America/New_York"), # EDT = UTC-4 + pd.Timestamp("2024-11-01 23:59:00", tz="Europe/London"), # GMT = UTC+0 + ]) + result = ts.parse_series_to_utc(series) + assert result.dt.tz == timezone.utc + assert result.iloc[0] == pd.Timestamp("2024-01-15 08:00:00", tz="UTC") + assert result.iloc[1] == pd.Timestamp("2024-06-20 18:30:00", tz="UTC") + assert result.iloc[2] == pd.Timestamp("2024-11-01 23:59:00", tz="UTC") + + def test_mixed_naive_and_aware_timestamps_apply_config_tz_to_naive(self): + ts = Timestamp(timestamp_type="iso", timezone_type="iana", timezone="America/New_York") + series = pd.Series([ + pd.Timestamp("2024-01-15 08:00:00"), # naive → America/New_York (EST = UTC-5) + pd.Timestamp("2024-01-15 08:00:00", tz="UTC"), # aware → preserved + ]) + result = ts.parse_series_to_utc(series) + assert result.iloc[0] == pd.Timestamp("2024-01-15 13:00:00", tz="UTC") + assert result.iloc[1] == pd.Timestamp("2024-01-15 08:00:00", tz="UTC") + + +# --------------------------------------------------------------------------- +# parse_series_to_utc – custom format with embedded timezone (%z) +# --------------------------------------------------------------------------- + +class TestParseSeriesCustomFormatWithTz: + + def test_custom_format_with_tz_parses_embedded_offsets(self): + ts = Timestamp( + timestamp_type="custom", + timestamp_format="%m/%d/%Y %H:%M:%S %z", + timezone_type="iana", + timezone="America/New_York", + ) + series = pd.Series(["01/15/2024 08:00:00 +0000", "11/01/2024 23:59:00 -0700"]) + result = ts.parse_series_to_utc(series) + assert result.dt.tz == timezone.utc + assert result.iloc[0] == pd.Timestamp("2024-01-15 08:00:00", tz="UTC") + assert result.iloc[1] == pd.Timestamp("2024-11-02 06:59:00", tz="UTC") + + def test_custom_format_without_tz_raises_on_tz_aware_string(self): + ts = Timestamp( + timestamp_type="custom", + timestamp_format="%m/%d/%Y %H:%M:%S", + timezone_type="iana", + timezone="America/New_York", + ) + series = pd.Series(["01/15/2024 08:00:00 +0000"]) + with pytest.raises(Exception): + ts.parse_series_to_utc(series) + + def test_custom_format_with_tz_raises_on_naive_string(self): + ts = Timestamp( + timestamp_type="custom", + timestamp_format="%m/%d/%Y %H:%M:%S %z", + timezone_type="iana", + timezone="America/New_York", + ) + series = pd.Series(["01/15/2024 08:00:00"]) + with pytest.raises(Exception): + ts.parse_series_to_utc(series)