460 etl timezone

django/processing/etl/services/task.py

@@ -392,6 +392,7 @@ def run(
                 header_row=data_connection.payload.header_row,
                 data_start_row=data_connection.payload.data_start_row,
                 delimiter=data_connection.payload.delimiter,  # noqa
+                identifier_type="index" if data_connection.payload.header_row is None else "name",
             )
 
         elif data_connection.payload.payload_type == "JSON":

packages/hydroserverpy/src/hydroserverpy/etl/models/timestamp.py

@@ -130,47 +130,81 @@ def parse_series_to_utc(
         series: pd.Series
     ) -> pd.Series:
         """
-        Parse a pandas Series of timestamps and normalize them to a UTC baseline.
-
-        Parsing Logic:
-        1. If timezone_type is 'utc' or None, strings are parsed as UTC-aware.
-        2. If timezone_type is 'iana' or 'offset', strings are parsed as naive.
-
-        Conflict Resolution:
-        - Overwrite: If 'iana'/'offset' is configured but the data contains embedded
-          timezones, the embedded offsets are stripped and replaced by the config.
-        - Fallback: If no timezone is configured but the data is naive, UTC is assumed.
-        - Type Safety: Mixed offsets are flattened into a consistent datetime64[ns, UTC]
-          dtype to ensure the pandas .dt accessor remains available downstream.
+        Parse a pandas Series of timestamps and normalize them to UTC.
+
+        Accepts uniform datetime64 Series, object-dtype Series of ISO strings,
+        object-dtype Series of strings with a custom timestamp_format, or
+        object-dtype Series of pd.Timestamp/datetime objects. Raises ValueError
+        on null values, mixed element types, or unsupported element types. Invalid
+        strings raise during parsing. The returned Series is always
+        datetime64[ns, UTC] with the same length as the input.
+
+        For tz-naive inputs, the configured timezone is applied before converting
+        to UTC. If no timezone is configured, UTC is assumed. Tz-aware inputs are
+        converted to UTC from their embedded timezone without being overwritten.
+
+        For custom timestamp formats: include %z in timestamp_format if strings
+        carry embedded timezone info; omit %z if strings are tz-naive. A mismatch
+        between the format and the actual strings raises during parsing.
         """
 
-        # Ensure input is string-based for pandas parsing if not already datetime objects
-        if not pd.api.types.is_datetime64_any_dtype(series):
-            series: pd.Series = series.astype("string", copy=False).str.strip()
+        if len(series) == 0:
+            return pd.Series(dtype="datetime64[ns, UTC]")
+
+        if series.isna().any():
+            raise ValueError("Series contains null or missing values")
+
+        tz_label = (
+            self._to_pandas_offset(self.timezone) if self.timezone_type == "offset"
+            else (self.timezone or "UTC")
+        )
+
+        # Uniform datetime64 (all naive or all same-tz-aware)
+        if pd.api.types.is_datetime64_any_dtype(series):
+            if series.dt.tz is None:
+                return series.dt.tz_localize(
+                    tz_label, ambiguous=False, nonexistent="shift_forward"
+                ).dt.tz_convert("UTC")
+            return series.dt.tz_convert("UTC")
+
+        # Object dtype: must be uniformly strings or Timestamp/datetime objects
+        first_timestamp = series.iloc[0]
+
+        if isinstance(first_timestamp, str):
+            series = series.str.strip()
+            if series.isna().any():
+                raise ValueError("Series contains mixed or non-string values")
+
+            if self.timestamp_format:
+                if "%z" in self.timestamp_format:
+                    return pd.Series(
+                        pd.to_datetime(series, utc=True, format=self.timestamp_format, errors="raise")
+                    )
+                return pd.Series(
+                    pd.to_datetime(series, utc=False, format=self.timestamp_format, errors="raise")
+                    .dt.tz_localize(tz_label, ambiguous=False, nonexistent="shift_forward")
+                    .dt.tz_convert("UTC")
+                )
 
-        # Determine if UTC parsing should be used directly
-        parse_as_utc = self.timezone_type in ["utc", None]
+            # ISO strings: regex detects embedded tz per element to support mixed-offset series
+            # (pandas cannot parse mixed tz/naive strings without coercing to NaT)
+            has_tz = series.str.contains(r"[Zz]$|[+-]\d{2}(?::?\d{2})?$", regex=True, na=False)
+            tz_aware = pd.to_datetime(series[has_tz], utc=True, errors="raise")
+            tz_naive_raw_series = pd.to_datetime(series[~has_tz], utc=False, errors="raise")
+
+        elif isinstance(first_timestamp, pd.Timestamp):
+            has_tz = series.apply(lambda x: x.tzinfo is not None)
+            tz_aware = pd.to_datetime(series[has_tz], utc=True)
+            tz_naive_raw_series = pd.to_datetime(series[~has_tz])
 
-        # Perform initial parsing of the values
-        if self.timestamp_type == "iso":
-            parsed_series = pd.to_datetime(series, utc=parse_as_utc, errors="coerce")
-        else:
-            parsed_series = pd.to_datetime(series, utc=parse_as_utc, format=self.timestamp_format, errors="coerce")
-
-        # Apply fixed IANA or UTC offsets to the series if provided
-        if self.timezone_type in ["offset", "iana"]:
-            if parsed_series.dt.tz is not None or parsed_series.dtype == "object":
-                parsed_series = parsed_series.dt.tz_localize(None)
-            tz_label = self._to_pandas_offset(self.timezone) if self.timezone_type == "offset" else self.timezone
-            utc_series = parsed_series.dt.tz_localize(
-                tz_label, ambiguous=False, nonexistent="shift_forward"
-            ).dt.tz_convert(timezone.utc)
-
-        # Normalize to UTC if the series timezones are embedded or naive and configured as UTC
         else:
-            utc_series = pd.to_datetime(parsed_series, utc=True)
+            raise ValueError(f"Unsupported series element type: {type(first_timestamp).__name__}")
+
+        tz_naive_series = tz_naive_raw_series.dt.tz_localize(
+            tz_label, ambiguous=False, nonexistent="shift_forward"
+        ).dt.tz_convert("UTC")
 
-        return utc_series
+        return pd.Series(pd.concat([tz_aware, tz_naive_series])).sort_index()
 
     def to_string(self, dt: datetime) -> str:
         """

packages/hydroserverpy/tests/etl/test_timestamp_model.py

@@ -189,17 +189,15 @@ def test_basic_utc_iso_strings(self, utc_iso_timestamp):
         assert result.iloc[0] == pd.Timestamp("2024-01-01 00:00:00", tz="UTC")
         assert result.iloc[1] == pd.Timestamp("2024-06-15 12:30:00", tz="UTC")
 
-    def test_invalid_strings_become_nat(self, utc_iso_timestamp):
+    def test_invalid_strings_raise_error(self, utc_iso_timestamp):
         series = pd.Series(["not-a-date", "also-bad"])
-        result = utc_iso_timestamp.parse_series_to_utc(series)
-        assert result.isna().all()
+        with pytest.raises(Exception):
+            utc_iso_timestamp.parse_series_to_utc(series)
 
-    def test_mixed_valid_invalid(self, utc_iso_timestamp):
+    def test_mixed_valid_invalid_raises_error(self, utc_iso_timestamp):
         series = pd.Series(["2024-01-01T00:00:00", "bad-date", "2024-06-01T00:00:00"])
-        result = utc_iso_timestamp.parse_series_to_utc(series)
-        assert pd.notna(result.iloc[0])
-        assert pd.isna(result.iloc[1])
-        assert pd.notna(result.iloc[2])
+        with pytest.raises(Exception):
+            utc_iso_timestamp.parse_series_to_utc(series)
 
     def test_whitespace_is_stripped(self, utc_iso_timestamp):
         series = pd.Series(["  2024-01-01T12:00:00  ", "\t2024-06-01T00:00:00\n"])
@@ -249,6 +247,41 @@ def test_mixed_offset_series_is_flattened_to_utc(self):
         assert result.iloc[1] == pd.Timestamp("2024-01-01 05:00:00", tz="UTC")
 
 
+# ---------------------------------------------------------------------------
+# parse_series_to_utc – hour-only UTC offsets embedded in ISO strings (±HH)
+# ---------------------------------------------------------------------------
+
+class TestParseSeriesHourOnlyOffset:
+
+    def test_negative_hour_only_offset(self):
+        ts = Timestamp(timestamp_type="iso", timezone_type=None)
+        series = pd.Series(["2024-01-01T12:00:00-07"])
+        result = ts.parse_series_to_utc(series)
+        assert result.dt.tz == timezone.utc
+        assert result.iloc[0] == pd.Timestamp("2024-01-01 19:00:00", tz="UTC")
+
+    def test_positive_hour_only_offset(self):
+        ts = Timestamp(timestamp_type="iso", timezone_type=None)
+        series = pd.Series(["2024-01-01T12:00:00+05"])
+        result = ts.parse_series_to_utc(series)
+        assert result.dt.tz == timezone.utc
+        assert result.iloc[0] == pd.Timestamp("2024-01-01 07:00:00", tz="UTC")
+
+    def test_hour_only_offset_matches_full_offset(self):
+        ts = Timestamp(timestamp_type="iso", timezone_type=None)
+        hour_only = ts.parse_series_to_utc(pd.Series(["2024-06-15T08:00:00-07"]))
+        full_hhmm = ts.parse_series_to_utc(pd.Series(["2024-06-15T08:00:00-07:00"]))
+        assert hour_only.iloc[0] == full_hhmm.iloc[0]
+
+    def test_mixed_hour_only_and_full_offsets(self):
+        ts = Timestamp(timestamp_type="iso", timezone_type=None)
+        series = pd.Series(["2024-01-01T12:00:00-07", "2024-01-01T12:00:00+05:30"])
+        result = ts.parse_series_to_utc(series)
+        assert result.dt.tz == timezone.utc
+        assert result.iloc[0] == pd.Timestamp("2024-01-01 19:00:00", tz="UTC")
+        assert result.iloc[1] == pd.Timestamp("2024-01-01 06:30:00", tz="UTC")
+
+
 # ---------------------------------------------------------------------------
 # parse_series_to_utc – IANA timezone type
 # ---------------------------------------------------------------------------
@@ -262,11 +295,18 @@ def test_naive_strings_localized_to_iana_zone(self, iana_timestamp):
         assert result.dt.tz == timezone.utc
         assert result.iloc[0] == pd.Timestamp("2024-01-01 05:00:00", tz="UTC")
 
-    def test_embedded_offsets_are_overwritten_by_iana_zone(self, iana_timestamp):
-        # Embedded +05:30 is stripped; America/New_York (UTC-5 in Jan) applied instead
+    def test_embedded_offsets_are_preserved_with_iana_zone(self, iana_timestamp):
+        # Embedded +05:30 is respected; America/New_York config does not overwrite it
         series = pd.Series(["2024-01-01T00:00:00+05:30"])
         result = iana_timestamp.parse_series_to_utc(series)
+        assert result.iloc[0] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC")
+
+    def test_naive_and_aware_mixed_series_with_iana_zone(self, iana_timestamp):
+        # Naive timestamps get America/New_York (UTC-5 in Jan); aware ones are preserved
+        series = pd.Series(["2024-01-01T00:00:00", "2024-01-01T00:00:00+05:30"])
+        result = iana_timestamp.parse_series_to_utc(series)
         assert result.iloc[0] == pd.Timestamp("2024-01-01 05:00:00", tz="UTC")
+        assert result.iloc[1] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC")
 
     def test_dst_spring_forward(self):
         ts = Timestamp(timestamp_type="iso", timezone_type="iana", timezone="America/New_York")
@@ -290,11 +330,18 @@ def test_naive_strings_localized_to_offset(self, offset_timestamp):
         assert result.dt.tz == timezone.utc
         assert result.iloc[0] == pd.Timestamp("2024-01-01 07:00:00", tz="UTC")
 
-    def test_embedded_offsets_are_overwritten_by_configured_offset(self, offset_timestamp):
-        # Embedded +05:30 is stripped; -0700 applied instead
+    def test_embedded_offsets_are_preserved_with_configured_offset(self, offset_timestamp):
+        # Embedded +05:30 is respected; -0700 config does not overwrite it
         series = pd.Series(["2024-01-01T00:00:00+05:30"])
         result = offset_timestamp.parse_series_to_utc(series)
+        assert result.iloc[0] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC")
+
+    def test_naive_and_aware_mixed_series_with_configured_offset(self, offset_timestamp):
+        # Naive timestamps get -0700 applied; aware ones are preserved
+        series = pd.Series(["2024-01-01T00:00:00", "2024-01-01T00:00:00+05:30"])
+        result = offset_timestamp.parse_series_to_utc(series)
         assert result.iloc[0] == pd.Timestamp("2024-01-01 07:00:00", tz="UTC")
+        assert result.iloc[1] == pd.Timestamp("2023-12-31 18:30:00", tz="UTC")
 
     def test_positive_offset_bare_format(self):
         ts = Timestamp(timestamp_type="iso", timezone_type="offset", timezone="+0530")
@@ -339,10 +386,10 @@ def test_custom_format_parses_correctly(self, custom_timestamp):
         assert result.dt.tz == timezone.utc
         assert result.iloc[0] == pd.Timestamp("2024-03-15 08:00:00", tz="UTC")
 
-    def test_custom_format_wrong_data_becomes_nat(self, custom_timestamp):
+    def test_custom_format_wrong_data_raises_error(self, custom_timestamp):
         series = pd.Series(["2024-01-01T00:00:00Z"])
-        result = custom_timestamp.parse_series_to_utc(series)
-        assert result.isna().all()
+        with pytest.raises(Exception):
+            custom_timestamp.parse_series_to_utc(series)
 
     def test_custom_format_with_iana_timezone(self):
         ts = Timestamp(
@@ -403,3 +450,102 @@ def test_large_series_completes(self, utc_iso_timestamp):
         result = utc_iso_timestamp.parse_series_to_utc(pd.Series(dates))
         assert len(result) == 100_000
         assert result.isna().sum() == 0
+
+
+# ---------------------------------------------------------------------------
+# parse_series_to_utc – input validation errors
+# ---------------------------------------------------------------------------
+
+class TestParseSeriesValidation:
+
+    def test_null_values_raise_error(self, utc_iso_timestamp):
+        series = pd.Series(["2024-01-01T00:00:00", None, "2024-06-01T00:00:00"])
+        with pytest.raises(ValueError, match="null"):
+            utc_iso_timestamp.parse_series_to_utc(series)
+
+    def test_mixed_strings_and_timestamps_raise_error(self, utc_iso_timestamp):
+        series = pd.Series(["2024-01-01T00:00:00Z", pd.Timestamp("2024-06-20", tz="UTC")])
+        with pytest.raises(ValueError):
+            utc_iso_timestamp.parse_series_to_utc(series)
+
+    def test_unsupported_element_type_raises_error(self, utc_iso_timestamp):
+        series = pd.Series([20240101, 20240615])
+        with pytest.raises((ValueError, TypeError)):
+            utc_iso_timestamp.parse_series_to_utc(series)
+
+    def test_invalid_string_raises_error(self, iana_timestamp):
+        series = pd.Series(["2024-01-15T08:00:00Z", "not-a-date"])
+        with pytest.raises(Exception):
+            iana_timestamp.parse_series_to_utc(series)
+
+
+# ---------------------------------------------------------------------------
+# parse_series_to_utc – pd.Timestamp object inputs
+# ---------------------------------------------------------------------------
+
+class TestParseSeriesTimestampObjects:
+
+    def test_different_tz_aware_timestamps_convert_to_utc(self):
+        ts = Timestamp(timestamp_type="iso", timezone_type="iana", timezone="America/New_York")
+        series = pd.Series([
+            pd.Timestamp("2024-01-15 08:00:00", tz="UTC"),
+            pd.Timestamp("2024-06-20 14:30:00", tz="America/New_York"),  # EDT = UTC-4
+            pd.Timestamp("2024-11-01 23:59:00", tz="Europe/London"),     # GMT = UTC+0
+        ])
+        result = ts.parse_series_to_utc(series)
+        assert result.dt.tz == timezone.utc
+        assert result.iloc[0] == pd.Timestamp("2024-01-15 08:00:00", tz="UTC")
+        assert result.iloc[1] == pd.Timestamp("2024-06-20 18:30:00", tz="UTC")
+        assert result.iloc[2] == pd.Timestamp("2024-11-01 23:59:00", tz="UTC")
+
+    def test_mixed_naive_and_aware_timestamps_apply_config_tz_to_naive(self):
+        ts = Timestamp(timestamp_type="iso", timezone_type="iana", timezone="America/New_York")
+        series = pd.Series([
+            pd.Timestamp("2024-01-15 08:00:00"),           # naive → America/New_York (EST = UTC-5)
+            pd.Timestamp("2024-01-15 08:00:00", tz="UTC"), # aware → preserved
+        ])
+        result = ts.parse_series_to_utc(series)
+        assert result.iloc[0] == pd.Timestamp("2024-01-15 13:00:00", tz="UTC")
+        assert result.iloc[1] == pd.Timestamp("2024-01-15 08:00:00", tz="UTC")
+
+
+# ---------------------------------------------------------------------------
+# parse_series_to_utc – custom format with embedded timezone (%z)
+# ---------------------------------------------------------------------------
+
+class TestParseSeriesCustomFormatWithTz:
+
+    def test_custom_format_with_tz_parses_embedded_offsets(self):
+        ts = Timestamp(
+            timestamp_type="custom",
+            timestamp_format="%m/%d/%Y %H:%M:%S %z",
+            timezone_type="iana",
+            timezone="America/New_York",
+        )
+        series = pd.Series(["01/15/2024 08:00:00 +0000", "11/01/2024 23:59:00 -0700"])
+        result = ts.parse_series_to_utc(series)
+        assert result.dt.tz == timezone.utc
+        assert result.iloc[0] == pd.Timestamp("2024-01-15 08:00:00", tz="UTC")
+        assert result.iloc[1] == pd.Timestamp("2024-11-02 06:59:00", tz="UTC")
+
+    def test_custom_format_without_tz_raises_on_tz_aware_string(self):
+        ts = Timestamp(
+            timestamp_type="custom",
+            timestamp_format="%m/%d/%Y %H:%M:%S",
+            timezone_type="iana",
+            timezone="America/New_York",
+        )
+        series = pd.Series(["01/15/2024 08:00:00 +0000"])
+        with pytest.raises(Exception):
+            ts.parse_series_to_utc(series)
+
+    def test_custom_format_with_tz_raises_on_naive_string(self):
+        ts = Timestamp(
+            timestamp_type="custom",
+            timestamp_format="%m/%d/%Y %H:%M:%S %z",
+            timezone_type="iana",
+            timezone="America/New_York",
+        )
+        series = pd.Series(["01/15/2024 08:00:00"])
+        with pytest.raises(Exception):
+            ts.parse_series_to_utc(series)