From 2e426cac0600648be760dc6cb6415898a684f981 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Mon, 13 Jul 2026 22:35:50 +0000 Subject: [PATCH 1/6] fixed curve_count and added basic to_csv() for paired_data.py --- src/hecdss/dss_csv.py | 54 +++- src/hecdss/dsspath.py | 14 +- src/hecdss/paired_data.py | 25 +- tests/test_paired_data.py | 1 + tests/test_pd_csv.py | 56 ++++ tests/test_ts_csv.py | 547 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 686 insertions(+), 11 deletions(-) create mode 100644 tests/test_pd_csv.py create mode 100644 tests/test_ts_csv.py diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index c6d8e79..7788f6d 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -4,9 +4,13 @@ from .dsspath import DssPath from .irregular_timeseries import IrregularTimeSeries +from .paired_data import PairedData from .regular_timeseries import RegularTimeSeries +METADATA_ROWS: list[str] = ["A", "B", "C", "D", "E", "F"] # Path of DSS object + + def timeseries_to_csv( series: RegularTimeSeries | IrregularTimeSeries, path: str, with_metadata: bool ) -> None: @@ -21,7 +25,6 @@ def timeseries_to_csv( if not isinstance(series, (RegularTimeSeries, IrregularTimeSeries)): raise TypeError("series must be a RegularTimeSeries or IrregularTimeSeries") - metadata_rows: list[str] = ["A", "B", "C", "D", "E", "F"] with open(path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) if with_metadata: @@ -29,8 +32,8 @@ def timeseries_to_csv( id_components: list[str] = DssPath(series.id).path_to_list() else: id_components: list[str] = [""] * 6 - for i in range(len(metadata_rows)): - row: str = metadata_rows[i] + for i in range(len(METADATA_ROWS)): + row: str = METADATA_ROWS[i] metadata_value: str = id_components[i] if row == "D": # Skip D by convention # ts pattern if metadata_value == "ts-pattern": @@ -146,6 +149,51 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] ) +def paired_data_to_csv(paired_data: PairedData, path: str, with_metadata: bool) -> None: + """ + Exports a PairedData object to a .csv file. + + Parameters: + paired_data (PairedData): Paired Data object to convert to csv. + path: (str): file path to export csv to. + with_metadata (bool): whether or not to include metadata in the csv file. + + Returns: + None + """ + if not isinstance(paired_data, PairedData): + raise TypeError("paired_data must be a PairedData!") + + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + if with_metadata: + if paired_data.id: + id_components: dict[str, str] = DssPath(paired_data.id).path_to_dict() + else: + id_components: dict[str, str] = { + 'A': '', 'B': '', 'C': '', 'D': '', 'E': '', 'F': '' + } + for letter, value in id_components.items(): + if letter == 'D': # Skipping D by convention + continue + writer.writerow([letter, "", value]) # Writing metadata rows + labels: list[str] = ["Labels", ""] + for label in paired_data.labels: + labels.append(label) + writer.writerow(labels) + writer.writerow(["Units", paired_data.units_independent, paired_data.units_dependent]) + writer.writerow(["Type", paired_data.type_independent, paired_data.type_dependent]) + + counter: int = 1 # 1st column index + x: float # x is the same as "ordinate" + y_row: list[float] # Each y_row is one row of y_values, as paired_data.values is Row-Major + for x, y_row in zip(paired_data.ordinates, paired_data.values): + full_row: list[float] = [x] + [y for y in y_row] + writer.writerow([counter] + full_row) + counter += 1 + return + + def _needs_second_precision(series: RegularTimeSeries | IrregularTimeSeries) -> bool: """ Returns True if any datetime in the series has a non-zero seconds component diff --git a/src/hecdss/dsspath.py b/src/hecdss/dsspath.py index f58d954..b086f4b 100644 --- a/src/hecdss/dsspath.py +++ b/src/hecdss/dsspath.py @@ -2,6 +2,7 @@ from .dss_type import DssType from .record_type import RecordType + class DssPath: """ Manage parts of DSS path /A/B/C/D/E/F/ @@ -11,7 +12,7 @@ class DssPath: _timeSeriesFamily = [RecordType.IrregularTimeSeries, RecordType.RegularTimeSeries, RecordType.RegularTimeSeriesProfile] - def __init__(self, path: str, recType = 0): + def __init__(self, path: str, recType=0): """ Initialize a DssPath object. @@ -95,4 +96,13 @@ def path_to_list(self) -> list[str]: Returns: list: A list containing the components of the DSS path. """ - return [self.A, self.B, self.C, self.D, self.E, self.F] \ No newline at end of file + return [self.A, self.B, self.C, self.D, self.E, self.F] + + def path_to_dict(self) -> dict[str, str]: + """ + Convert the DSS path to a dictionary of its components. + + Returns: + dict[str, str]: A dictionary containing the components of the DSS path. + """ + return {'A': self.A, 'B': self.B, 'C': self.C, 'D': self.D, 'E': self.E, 'F': self.F} diff --git a/src/hecdss/paired_data.py b/src/hecdss/paired_data.py index 49da97a..705406d 100644 --- a/src/hecdss/paired_data.py +++ b/src/hecdss/paired_data.py @@ -7,10 +7,11 @@ def __init__(self): """ Initialize a PairedData object with default values. """ - self.id = None - self.ordinates = np.empty(0) - self.values = np.empty(0) - self.labels = [] + self.id: str = None + self.ordinates: np.ndarray = np.empty(0) + # ROW-MAJOR, each sublist represents the values for each ordinate + self.values: np.ndarray = np.empty(0) + self.labels: list[str] = [] self.type_independent = "" self.type_dependent = "" self.units_independent = "" @@ -25,7 +26,19 @@ def curve_count(self): Returns: int: The number of curves. """ - return len(self.values) + return len(self.values[0]) # values is ROW-MAJOR, each sublist represents a row of curve values for each ordinate + + def to_csv(self, file_path: str, with_metadata: bool = True) -> None: + """ + Exports the PairedData instance to a .csv file. + + Parameters: + file_path (str): The path to the .csv file where the data will be exported. + with_metadata (bool): Whether to include metadata in the exported file. + """ + from .dss_csv import paired_data_to_csv + paired_data_to_csv(self, file_path, with_metadata) + print(f"Wrote PairedData to .csv file at {file_path}.") # def to_data_frame(self, include_index=False): # """ @@ -75,4 +88,4 @@ def create(x_values, y_values, labels=[], x_units="", x_type="", y_units="", y_t pd.type_independent = x_type pd.type_dependent = y_type pd.time_zone_name = time_zone_name - return pd \ No newline at end of file + return pd diff --git a/tests/test_paired_data.py b/tests/test_paired_data.py index b3c9121..79d0c29 100644 --- a/tests/test_paired_data.py +++ b/tests/test_paired_data.py @@ -63,6 +63,7 @@ def test_paired_data_create(self): assert (pd.values[4][2] == 46), f"pd.values[4][2] should be 46. is {pd.values[4][2]}" assert (len(pd.ordinates) == 5), f"len(pd.ordinates) should be 5. is {len(pd.ordinates)}" assert (pd.labels[1] == "x plus 1"), f"pd.labels[1] should be 'x plus 1'. is {pd.labels[1]}" + assert (pd.curve_count() == 3) def test_paired_data_create_store(self): """ diff --git a/tests/test_pd_csv.py b/tests/test_pd_csv.py new file mode 100644 index 0000000..b6ee39f --- /dev/null +++ b/tests/test_pd_csv.py @@ -0,0 +1,56 @@ +import unittest +from datetime import datetime +from unittest.mock import mock_open, patch + +from file_manager import FileManager + +from hecdss import HecDss +from hecdss.paired_data import PairedData + + +class TestCSV(unittest.TestCase): + + def setUp(self) -> None: + self.test_files = FileManager() + + def tearDown(self) -> None: + self.test_files.cleanup() + + def test_multiple_curves_to_csv(self): + """ + This test ensures that the data values for a PairedData object with multiple curves are correct. + This has no checks related to units or labels. + """ + num_curves: int = 3 + x_values: list[float] = [1.0, 1.5, 4.0] + y_values: list[list[float]] = [[x * i for i in range(num_curves)] for x in x_values] + labels: list[str] = [f"x times {i}" for i in range(num_curves)] + path: str = "/A/B/C///Source: I made it up/" + pd: PairedData = PairedData.create( + x_values=x_values, + y_values=y_values, + labels=labels, + path=path + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + pd.to_csv("fake_path.csv", with_metadata=True) + + mock_file.assert_called_once_with( + "fake_path.csv", "w", newline="", encoding="utf-8" + ) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + self.assertIn("A,,A", written_data) + self.assertIn("B,,B", written_data) + self.assertIn("C,,C", written_data) + self.assertIn("1,1.0,0.0,1.0,2.0", written_data) + self.assertIn("2,1.5,0.0,1.5,3.0", written_data) + self.assertIn("3,4.0,0.0,4.0,8.0", written_data) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ts_csv.py b/tests/test_ts_csv.py new file mode 100644 index 0000000..4f783a9 --- /dev/null +++ b/tests/test_ts_csv.py @@ -0,0 +1,547 @@ +import unittest +from datetime import datetime +from unittest.mock import mock_open, patch + +from file_manager import FileManager + +from hecdss import HecDss +from hecdss.irregular_timeseries import IrregularTimeSeries +from hecdss.regular_timeseries import RegularTimeSeries + + +class TestCSV(unittest.TestCase): + + def setUp(self) -> None: + self.test_files = FileManager() + + def tearDown(self) -> None: + self.test_files.cleanup() + + def test_to_csv_writes_correct_structure(self): + # Create a dummy RegularTimeSeries instance + rts = RegularTimeSeries.create( + values=[10.5, 20.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C/01Sep2021/6Hour/F/", + ) + + # Mock 'open' and capture written content + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake_path.csv", with_metadata=True) + + # Assert that open was called with correct parameters + mock_file.assert_called_once_with( + "fake_path.csv", "w", newline="", encoding="utf-8" + ) + + # Extract all written data + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + # Assertions on the CSV content structure + self.assertIn("A,,,A", written_data) + self.assertIn("B,,,B", written_data) + self.assertIn("C,,,C", written_data) + self.assertIn("Units,,,CFS", written_data) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + self.assertIn("1,01Sep2021 0600,10.5", written_data) + self.assertIn("2,01Sep2021 1200,20.0", written_data) + + def test_to_csv_without_metadata(self): + """No metadata rows should be written; only data rows.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Units", written) + self.assertNotIn("Type,Date/Time", written) + self.assertIn("1,01Sep2021 0600,1.0", written) + + def test_to_csv_empty_times(self): + rts = RegularTimeSeries.create( + values=[], + times=[], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake_path.csv", with_metadata=True) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + self.assertIn("Units,,,CFS", written_data) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + self.assertNotIn("1,", written_data) # No data rows should be present + + def test_to_csv_second_precision(self): + rts = RegularTimeSeries.create( + values=[i for i in range(10)], + times=[datetime(2021, 9, 1, 6, 0, i) for i in range(10)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//1Second/F/", + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake_path.csv", with_metadata=True) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + for i in range(10): + self.assertIn(f"{i + 1},01Sep2021 06000{i},", written_data) + + def test_to_csv_with_quality(self): + """When quality is present, header gets 'Quality' col and rows get flags.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + quality=[0, 5], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=True) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL,Quality", written) + self.assertIn("1,01Sep2021 0600,1.0,0", written) + self.assertIn("2,01Sep2021 1200,2.0,5", written) + + def read_rts_from_string(self, content): + """Helper to run read_csv against an in-memory CSV string.""" + m = mock_open(read_data=content) + with patch("builtins.open", m): + return RegularTimeSeries.read_csv("fake.csv") + + def test_read_csv_basic(self): + content = ( + "A,,,A\n" + "B,,,B\n" + "C,,,FLOW\n" + "E,,,6Hour\n" + "F,,,F\n" + "Units,,,CFS\n" + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600,10.5\n" + "2,01Sep2021 1200,20.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.units, "CFS") + self.assertEqual(rts.data_type, "INST-VAL") + self.assertEqual(rts.values.tolist(), [10.5, 20.0]) + self.assertEqual(rts.times[0], datetime(2021, 9, 1, 6, 0)) + + def test_read_csv_midnight_2400_rolls_to_next_day(self): + content = ( + "E,,,1Day\n" + "Type,Date/Time,INST-VAL\n" + "1,31Aug2021 2400,10.5\n" + "2,01Sep2021 2400,20.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual( + rts.times, [datetime(2021, 9, 1, 0, 0), datetime(2021, 9, 2, 0, 0)] + ) + self.assertEqual(rts.values.tolist(), [10.5, 20.0]) + + def test_read_csv_with_quality(self): + content = ( + "Type,Date/Time,INST-VAL,Quality\n" + "1,01Sep2021 0600,10.5,0\n" + "2,01Sep2021 1200,20.0,5\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, 20.0]) + self.assertEqual(rts.quality, [0, 5]) + + def test_read_csv_with_partial_quality(self): + content: tuple[str] = ( + "Type,Date/Time,INST-VAL,Quality\n" + "1,05Nov2004 0200,8,0\n" + "2,05Nov2004 0300,9\n" # missing quality! + "3,05Nov2004 0400,10,1\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist()[0], 8) + self.assertEqual(rts.values.tolist()[2], 10) + self.assertEqual(rts.quality[0], 0) + self.assertEqual(rts.quality[2], 1) + + def test_read_csv_skips_malformed_rows(self): + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600,10.5\n" + "2,not-a-date,20.0\n" + "3,01Sep2021 1200,not-a-number\n" + "4,01Sep2021 1800,30.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, 30.0]) + self.assertEqual( + rts.times, [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 18, 0)] + ) + + def test_read_csv_seconds_precision_basic(self): + """Reading HHMMSS (seconds-precision) timestamps with no 2400 rollover involved. + Uses a 15-second gap (a standard DSS interval) so RegularTimeSeries can infer + the interval from the deltas without a metadata E row.""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 060000,10.5\n" + "2,01Sep2021 060015,20.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, 20.0]) + self.assertEqual( + rts.times, + [datetime(2021, 9, 1, 6, 0, 0), datetime(2021, 9, 1, 6, 0, 15)], + ) + + def test_read_csv_skips_wrong_length_time(self): + """A time field that isn't 4 (minutes) or 6 (seconds) digits doesn't match + either DSS format, so the row should be skipped rather than raise.""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600,10.5\n" + "2,01Sep2021 12345,20.0\n" # 5-digit clock -- not a valid DSS format + "3,01Sep2021 1800,30.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, 30.0]) + self.assertEqual( + rts.times, [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 18, 0)] + ) + + def test_read_csv_single_digit_day_skipped(self): + """The format regex requires a zero-padded 2-digit day (matching this + library's own writer output), so a single-digit day doesn't match + and the row is skipped.""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,1Sep2021 0600,10.5\n" # single-digit day + "2,01Sep2021 1200,20.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [20.0]) + self.assertEqual(rts.times, [datetime(2021, 9, 1, 12, 0)]) + + def test_read_csv_month_case_mismatch_skipped(self): + """The format regex requires a title-case month abbreviation (matching + this library's own writer output); other casings don't match and are + skipped, even though datetime.strptime itself would accept them.""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01SEP2021 0600,10.5\n" # all-caps month + "2,01sep2021 1200,20.0\n" # all-lowercase month + "3,01Sep2021 1800,30.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [30.0]) + self.assertEqual(rts.times, [datetime(2021, 9, 1, 18, 0)]) + + def test_read_csv_2400_with_nonzero_seconds_not_treated_as_rollover(self): + """24:00:15 is not a valid DSS midnight-rollover (only 24:00:00 is) and + should be rejected as malformed rather than silently rolled to the + next day with the seconds preserved. Covers the same edge case as + test_year_is_2400, but for the RegularTimeSeries read path.""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,15Sep2021 240015,10.5\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), []) + self.assertEqual(rts.times, []) + + def test_read_csv_empty_file(self): + rts = self.read_rts_from_string("") + self.assertEqual(rts.values.tolist(), []) + self.assertEqual(rts.times, []) + + def test_read_csv_single_row_uses_path_interval(self): + content = "E,,,6Hour\n" "Type,Date/Time,INST-VAL\n" "1,01Sep2021 0600,10.5\n" + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5]) + self.assertEqual(rts.times, [datetime(2021, 9, 1, 6, 0)]) + + def test_read_csv_irregular_interval_raises(self): + """RegularTimeSeries.read_csv expects a genuinely regular interval; + data implying an irregular gap (with no usable E row) is not valid + input for this class (that's what IrregularTimeSeries is for).""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600,10.5\n" + "2,01Sep2021 0637,20.0\n" + ) + with self.assertRaises(ValueError): + self.read_rts_from_string(content) + + def test_round_trip_basic(self): + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[10.5, 20.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C/01Sep2021/6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.units, "CFS") + self.assertEqual(result.data_type, "INST-VAL") + self.assertEqual(result.interval, "6Hour") + self.assertEqual(result.id, "/A/B/C//6Hour/F/") + self.assertEqual(result.values.tolist(), [10.5, 20.0]) + self.assertEqual( + result.times, + [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + ) + + def test_round_trip_with_quality(self): + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0, 3.0], + times=[ + datetime(2021, 9, 1, 6, 0), + datetime(2021, 9, 1, 12, 0), + datetime(2021, 9, 1, 18, 0), + ], + quality=[0, 5, 10], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.values.tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(result.quality, [0, 5, 10]) + + def test_round_trip_without_metadata_infers_interval_from_times(self): + """With no metadata rows, units/data_type come back empty but values, + times, and the interval/id (inferred from the time deltas) are still + recovered correctly.""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=False) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.units, "") + self.assertEqual(result.data_type, "") + self.assertEqual(result.interval, 21600) + self.assertEqual(result.id, "/////6Hour//") + self.assertEqual(result.values.tolist(), [1.0, 2.0]) + self.assertEqual( + result.times, + [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + ) + + # IRREGULAR TIME SERIES TESTS: + + def test_basic_to_csv_irregular(self): + """ + Basic structure test for irregular time series to_csv + """ + # Create a dummy IrregularTimeSeries instance + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[ + datetime(2021, 9, 1, 0, 0), + datetime(2021, 9, 2, 0, 0), + datetime(2021, 9, 4, 0, 0), + ], # inconsistent time interval + units="CFS", + data_type="INST-VAL", + path="/A/B/C/01Sep2021/E/F/", + ) + + # Mock 'open' and capture written content + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake_path.csv", with_metadata=True) + + # Assert that open was called with correct parameters + mock_file.assert_called_once_with( + "fake_path.csv", "w", newline="", encoding="utf-8" + ) + + # Extract all written data + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + # Assertions on the CSV content structure + self.assertIn("A,,,A", written_data) + self.assertIn("B,,,B", written_data) + self.assertIn("C,,,C", written_data) + self.assertIn("Units,,,CFS", written_data) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + self.assertIn("1,01Sep2021 0000,10.5", written_data) + self.assertIn("2,02Sep2021 0000,20.0", written_data) + self.assertIn("3,04Sep2021 0000,42.0", written_data) + + def test_irregular_to_csv_without_metadata(self): + """No metadata rows should be written; only data rows.""" + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[ + datetime(2021, 9, 1, 0, 0), + datetime(2021, 9, 2, 0, 0), + datetime(2021, 9, 4, 0, 0), + ], # inconsistent time interval + units="CFS", + data_type="INST-VAL", + path="/A/B/C/01Sep2021/E/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Units", written) + self.assertNotIn("Type,Date/Time", written) + self.assertIn("1,01Sep2021 0000,10.5", written) + self.assertIn("2,02Sep2021 0000,20.0", written) + self.assertIn("3,04Sep2021 0000,42.0", written) + + def test_irregular_to_csv_empty_times(self): + its = IrregularTimeSeries.create( + values=[], + times=[], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//E/F/", + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake_path.csv", with_metadata=True) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + self.assertIn("Units,,,CFS", written_data) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + self.assertNotIn("1,", written_data) # No data rows should be present + + def test_irregular_to_csv_second_precision(self): + its = IrregularTimeSeries.create( + values=[i for i in range(3)], + times=[ + datetime(2021, 9, 1, 6, 0, 0), + datetime(2021, 9, 1, 6, 0, 1), + datetime(2021, 9, 1, 6, 0, 4), + ], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//E/F/", + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake_path.csv", with_metadata=True) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + self.assertIn("1,01Sep2021 060000,0", written_data) + self.assertIn("2,01Sep2021 060001,1", written_data) + self.assertIn("3,01Sep2021 060004,2", written_data) + + def read_its_from_string(self, content): + """Helper to run read_csv against an in-memory CSV string.""" + m = mock_open(read_data=content) + with patch("builtins.open", m): + return IrregularTimeSeries.read_csv("fake.csv") + + def test_irregular_read_csv_basic(self): + content = ( + "A,,,A\n" + "B,,,B\n" + "C,,,FLOW\n" + "E,,,IR-Year\n" + "F,,,F\n" + "Units,,,CFS\n" + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0000,10.5\n" + "2,02Sep2021 0000,20.0\n" + "3,04Sep2021 0000,20.0\n" + ) + its = self.read_its_from_string(content) + self.assertEqual(its.units, "CFS") + self.assertEqual(its.data_type, "INST-VAL") + self.assertEqual(its.values.tolist(), [10.5, 20.0, 20.0]) + self.assertEqual( + its.times, + [datetime(2021, 9, 1), datetime(2021, 9, 2), datetime(2021, 9, 4)], + ) + + # EDGE CASE TESTS: + + def test_roll_day_edge_case(self): + content = ( + "A,,,A\n" + "B,,,B\n" + "C,,,FLOW\n" + "E,,,IR-Day\n" + "F,,,F\n" + "Units,,,CFS\n" + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 002400,1\n" # 12:24 AM + "2,02Sep2021 024000,1\n" # 2:40 AM + "3,03Sep2021 240000,1\n" # 12:00 AM Next day + ) + its = self.read_its_from_string(content) + self.assertEqual(its.units, "CFS") + self.assertEqual(its.data_type, "INST-VAL") + self.assertEqual(its.values.tolist(), [1, 1, 1]) + self.assertEqual(its.times, [datetime(2021, 9, 1, 0, 24, 0), datetime( + 2021, 9, 2, 2, 40, 0), datetime(2021, 9, 4, 0, 0, 0)]) + + def test_year_is_2400(self): + content = ( + "A,,,A\n" + "B,,,B\n" + "C,,,FLOW\n" + "E,,,IR-Day\n" + "F,,,F\n" + "Units,,,CFS\n" + "Type,Date/Time,INST-VAL\n" + "1,01Sep2400 000000,1\n" + "2,01Sep2400 240000,1\n" + "3,01Sep2400 240015,1\n" # Should not be accepted + "4,01Sep2400 24000,1\n" # Should not be accepted + ) + its = self.read_its_from_string(content) + self.assertEqual(its.units, "CFS") + self.assertEqual(its.data_type, "INST-VAL") + self.assertEqual(its.values.tolist(), [1, 1]) + self.assertEqual(its.times, [datetime(2400, 9, 1, 0, 0, 0), datetime(2400, 9, 2, 0, 0, 0)]) + + +if __name__ == "__main__": + unittest.main() From d41766660f61db13fe297c6849c6b3bd9ceca8a5 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Tue, 14 Jul 2026 18:00:26 +0000 Subject: [PATCH 2/6] added read_csv for PairedData --- src/hecdss/dss_csv.py | 88 +++++- src/hecdss/paired_data.py | 24 +- tests/test_csv.py | 547 -------------------------------------- tests/test_pd_csv.py | 49 ++++ tests/test_ts_csv.py | 2 +- 5 files changed, 155 insertions(+), 555 deletions(-) delete mode 100644 tests/test_csv.py diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index 7788f6d..4cc8f40 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -1,3 +1,5 @@ +# THESE ARE ALL HELPER FUNCTIONS, MEANT TO BE TREATED AS PRIVATE. These functions are internally called in their respective classes. + import csv import re from datetime import datetime, timedelta @@ -9,6 +11,8 @@ METADATA_ROWS: list[str] = ["A", "B", "C", "D", "E", "F"] # Path of DSS object +ROUND_PRECISION: int = 4 +NAN: float = 0.0 def timeseries_to_csv( @@ -125,7 +129,7 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] value_str: str = row[2].strip() try: - value: float = float(value_str) if value_str else 0.0 + value: float = float(value_str) if value_str else NAN except ValueError: continue # Skip a malformed value @@ -188,12 +192,92 @@ def paired_data_to_csv(paired_data: PairedData, path: str, with_metadata: bool) x: float # x is the same as "ordinate" y_row: list[float] # Each y_row is one row of y_values, as paired_data.values is Row-Major for x, y_row in zip(paired_data.ordinates, paired_data.values): - full_row: list[float] = [x] + [y for y in y_row] + full_row: list[float] = [round(x, ROUND_PRECISION)] + \ + [round(y, ROUND_PRECISION) for y in y_row] writer.writerow([counter] + full_row) counter += 1 return +def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: + """ + Reads a .csv file and builds a new paired data of the given type. + + Parameters: + cls: the class to build - PairedData + path (str): File path to .csv that we are reading from + + Returns: + An instance of cls populated from the .csv file. + """ + if cls is not PairedData: + raise TypeError("cls must be PairedData") + + x_values: list[float] = [] + y_values: list[list[float]] = [] + labels: list[str] = [] + x_units: str = "" + x_type: str = "" + y_units: str = "" + y_type: str = "" + path_parts: dict[str, str] = {"A": "", "B": "", "C": "", "D": "", "E": "", "F": ""} + + with open(path, "r", newline="", encoding="utf-8") as f: + reader = csv.reader(f) + row: list[str] + for row in reader: + if not row: + continue + # first item in the row we grabbed, the first column's item + first_column_item: str = row[0].strip() + # If the first column item is a path component (['A', 'B', 'C', 'D', 'E', 'F']) + if first_column_item in path_parts: + path_parts[first_column_item] = row[-1].strip() # last cell in csv row (convention) + # TODO: Finish the read_csv function. After you do this and write many tests, consider looking into C code to incorporate text columns and switching y units and types to lists + elif first_column_item == "Labels": + for label in row[2:]: # First two elements of row are not labels + labels.append(label) + elif first_column_item == "Units": + x_units = row[1].strip() + y_units = row[2].strip() # If units change to a list, will have to update this + elif first_column_item == "Type": + x_type = row[1].strip() + y_type = row[2].strip() # If type changes to a list, will also have to update this + else: # Data row + if len(row) < 3: + continue # csv is malformed, something is missing + x_str: str = row[1].strip() + try: + x: float = float(x_str) if x_str else NAN + except ValueError: + continue + x_values.append(x) + + y_row: list[float] = [] + y_row_str: list[str] = row[2:] + y_str: str + for y_str in y_row_str: + try: + y: float = float(y_str) if y_str else NAN + except ValueError: + continue + y_row.append(y) + y_values.append(y_row) + + id_path: str = f"/{path_parts['A']}/{path_parts['B']}/{path_parts['C']}/{path_parts['D']}/{path_parts['E']}/{path_parts['F']}/" + + return cls.create( + x_values=x_values, + y_values=y_values, + labels=labels, + x_units=x_units, + x_type=x_type, + y_units=y_units, + y_type=y_type, + path=id_path, + ) + + def _needs_second_precision(series: RegularTimeSeries | IrregularTimeSeries) -> bool: """ Returns True if any datetime in the series has a non-zero seconds component diff --git a/src/hecdss/paired_data.py b/src/hecdss/paired_data.py index 705406d..6e5492f 100644 --- a/src/hecdss/paired_data.py +++ b/src/hecdss/paired_data.py @@ -12,11 +12,11 @@ def __init__(self): # ROW-MAJOR, each sublist represents the values for each ordinate self.values: np.ndarray = np.empty(0) self.labels: list[str] = [] - self.type_independent = "" - self.type_dependent = "" - self.units_independent = "" - self.units_dependent = "" - self.time_zone_name = "" + self.type_independent: str = "" + self.type_dependent: str = "" + self.units_independent: str = "" + self.units_dependent: str = "" + self.time_zone_name: str = "" self.location_info = None def curve_count(self): @@ -60,6 +60,20 @@ def to_csv(self, file_path: str, with_metadata: bool = True) -> None: # # return pd.DataFrame(data) + @staticmethod + def read_csv(file_path: str) -> "PairedData": + """ + Reads a .csv file and creates an PairedData instance from the data. + + Parameters: + file_path (str): The path to the .csv file to read + + Returns: + PairedData: A new instance of PairedData populated with the data from the .csv file. + """ + from .dss_csv import paired_data_read_csv + return paired_data_read_csv(PairedData, file_path) + @staticmethod def create(x_values, y_values, labels=[], x_units="", x_type="", y_units="", y_type="", time_zone_name="", path=None): """ diff --git a/tests/test_csv.py b/tests/test_csv.py deleted file mode 100644 index 4f783a9..0000000 --- a/tests/test_csv.py +++ /dev/null @@ -1,547 +0,0 @@ -import unittest -from datetime import datetime -from unittest.mock import mock_open, patch - -from file_manager import FileManager - -from hecdss import HecDss -from hecdss.irregular_timeseries import IrregularTimeSeries -from hecdss.regular_timeseries import RegularTimeSeries - - -class TestCSV(unittest.TestCase): - - def setUp(self) -> None: - self.test_files = FileManager() - - def tearDown(self) -> None: - self.test_files.cleanup() - - def test_to_csv_writes_correct_structure(self): - # Create a dummy RegularTimeSeries instance - rts = RegularTimeSeries.create( - values=[10.5, 20.0], - times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], - units="CFS", - data_type="INST-VAL", - path="/A/B/C/01Sep2021/6Hour/F/", - ) - - # Mock 'open' and capture written content - mock_file = mock_open() - with patch("builtins.open", mock_file): - rts.to_csv("fake_path.csv", with_metadata=True) - - # Assert that open was called with correct parameters - mock_file.assert_called_once_with( - "fake_path.csv", "w", newline="", encoding="utf-8" - ) - - # Extract all written data - handle = mock_file() - written_data = "".join(call.args[0] for call in handle.write.call_args_list) - - # Assertions on the CSV content structure - self.assertIn("A,,,A", written_data) - self.assertIn("B,,,B", written_data) - self.assertIn("C,,,C", written_data) - self.assertIn("Units,,,CFS", written_data) - self.assertIn("Type,Date/Time,INST-VAL", written_data) - self.assertIn("1,01Sep2021 0600,10.5", written_data) - self.assertIn("2,01Sep2021 1200,20.0", written_data) - - def test_to_csv_without_metadata(self): - """No metadata rows should be written; only data rows.""" - rts = RegularTimeSeries.create( - values=[1.0, 2.0], - times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//6Hour/F/", - ) - mock_file = mock_open() - with patch("builtins.open", mock_file): - rts.to_csv("fake.csv", with_metadata=False) - handle = mock_file() - written = "".join(call.args[0] for call in handle.write.call_args_list) - self.assertNotIn("Units", written) - self.assertNotIn("Type,Date/Time", written) - self.assertIn("1,01Sep2021 0600,1.0", written) - - def test_to_csv_empty_times(self): - rts = RegularTimeSeries.create( - values=[], - times=[], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//6Hour/F/", - ) - - mock_file = mock_open() - with patch("builtins.open", mock_file): - rts.to_csv("fake_path.csv", with_metadata=True) - - handle = mock_file() - written_data = "".join(call.args[0] for call in handle.write.call_args_list) - - self.assertIn("Units,,,CFS", written_data) - self.assertIn("Type,Date/Time,INST-VAL", written_data) - self.assertNotIn("1,", written_data) # No data rows should be present - - def test_to_csv_second_precision(self): - rts = RegularTimeSeries.create( - values=[i for i in range(10)], - times=[datetime(2021, 9, 1, 6, 0, i) for i in range(10)], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//1Second/F/", - ) - - mock_file = mock_open() - with patch("builtins.open", mock_file): - rts.to_csv("fake_path.csv", with_metadata=True) - - handle = mock_file() - written_data = "".join(call.args[0] for call in handle.write.call_args_list) - self.assertIn("Type,Date/Time,INST-VAL", written_data) - for i in range(10): - self.assertIn(f"{i + 1},01Sep2021 06000{i},", written_data) - - def test_to_csv_with_quality(self): - """When quality is present, header gets 'Quality' col and rows get flags.""" - rts = RegularTimeSeries.create( - values=[1.0, 2.0], - times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], - quality=[0, 5], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//6Hour/F/", - ) - mock_file = mock_open() - with patch("builtins.open", mock_file): - rts.to_csv("fake.csv", with_metadata=True) - handle = mock_file() - written = "".join(call.args[0] for call in handle.write.call_args_list) - self.assertIn("Type,Date/Time,INST-VAL,Quality", written) - self.assertIn("1,01Sep2021 0600,1.0,0", written) - self.assertIn("2,01Sep2021 1200,2.0,5", written) - - def read_rts_from_string(self, content): - """Helper to run read_csv against an in-memory CSV string.""" - m = mock_open(read_data=content) - with patch("builtins.open", m): - return RegularTimeSeries.read_csv("fake.csv") - - def test_read_csv_basic(self): - content = ( - "A,,,A\n" - "B,,,B\n" - "C,,,FLOW\n" - "E,,,6Hour\n" - "F,,,F\n" - "Units,,,CFS\n" - "Type,Date/Time,INST-VAL\n" - "1,01Sep2021 0600,10.5\n" - "2,01Sep2021 1200,20.0\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.units, "CFS") - self.assertEqual(rts.data_type, "INST-VAL") - self.assertEqual(rts.values.tolist(), [10.5, 20.0]) - self.assertEqual(rts.times[0], datetime(2021, 9, 1, 6, 0)) - - def test_read_csv_midnight_2400_rolls_to_next_day(self): - content = ( - "E,,,1Day\n" - "Type,Date/Time,INST-VAL\n" - "1,31Aug2021 2400,10.5\n" - "2,01Sep2021 2400,20.0\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual( - rts.times, [datetime(2021, 9, 1, 0, 0), datetime(2021, 9, 2, 0, 0)] - ) - self.assertEqual(rts.values.tolist(), [10.5, 20.0]) - - def test_read_csv_with_quality(self): - content = ( - "Type,Date/Time,INST-VAL,Quality\n" - "1,01Sep2021 0600,10.5,0\n" - "2,01Sep2021 1200,20.0,5\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [10.5, 20.0]) - self.assertEqual(rts.quality, [0, 5]) - - def test_read_csv_with_partial_quality(self): - content: tuple[str] = ( - "Type,Date/Time,INST-VAL,Quality\n" - "1,05Nov2004 0200,8,0\n" - "2,05Nov2004 0300,9\n" # missing quality! - "3,05Nov2004 0400,10,1\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist()[0], 8) - self.assertEqual(rts.values.tolist()[2], 10) - self.assertEqual(rts.quality[0], 0) - self.assertEqual(rts.quality[2], 1) - - def test_read_csv_skips_malformed_rows(self): - content = ( - "Type,Date/Time,INST-VAL\n" - "1,01Sep2021 0600,10.5\n" - "2,not-a-date,20.0\n" - "3,01Sep2021 1200,not-a-number\n" - "4,01Sep2021 1800,30.0\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [10.5, 30.0]) - self.assertEqual( - rts.times, [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 18, 0)] - ) - - def test_read_csv_seconds_precision_basic(self): - """Reading HHMMSS (seconds-precision) timestamps with no 2400 rollover involved. - Uses a 15-second gap (a standard DSS interval) so RegularTimeSeries can infer - the interval from the deltas without a metadata E row.""" - content = ( - "Type,Date/Time,INST-VAL\n" - "1,01Sep2021 060000,10.5\n" - "2,01Sep2021 060015,20.0\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [10.5, 20.0]) - self.assertEqual( - rts.times, - [datetime(2021, 9, 1, 6, 0, 0), datetime(2021, 9, 1, 6, 0, 15)], - ) - - def test_read_csv_skips_wrong_length_time(self): - """A time field that isn't 4 (minutes) or 6 (seconds) digits doesn't match - either DSS format, so the row should be skipped rather than raise.""" - content = ( - "Type,Date/Time,INST-VAL\n" - "1,01Sep2021 0600,10.5\n" - "2,01Sep2021 12345,20.0\n" # 5-digit clock -- not a valid DSS format - "3,01Sep2021 1800,30.0\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [10.5, 30.0]) - self.assertEqual( - rts.times, [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 18, 0)] - ) - - def test_read_csv_single_digit_day_skipped(self): - """The format regex requires a zero-padded 2-digit day (matching this - library's own writer output), so a single-digit day doesn't match - and the row is skipped.""" - content = ( - "Type,Date/Time,INST-VAL\n" - "1,1Sep2021 0600,10.5\n" # single-digit day - "2,01Sep2021 1200,20.0\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [20.0]) - self.assertEqual(rts.times, [datetime(2021, 9, 1, 12, 0)]) - - def test_read_csv_month_case_mismatch_skipped(self): - """The format regex requires a title-case month abbreviation (matching - this library's own writer output); other casings don't match and are - skipped, even though datetime.strptime itself would accept them.""" - content = ( - "Type,Date/Time,INST-VAL\n" - "1,01SEP2021 0600,10.5\n" # all-caps month - "2,01sep2021 1200,20.0\n" # all-lowercase month - "3,01Sep2021 1800,30.0\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [30.0]) - self.assertEqual(rts.times, [datetime(2021, 9, 1, 18, 0)]) - - def test_read_csv_2400_with_nonzero_seconds_not_treated_as_rollover(self): - """24:00:15 is not a valid DSS midnight-rollover (only 24:00:00 is) and - should be rejected as malformed rather than silently rolled to the - next day with the seconds preserved. Covers the same edge case as - test_year_is_2400, but for the RegularTimeSeries read path.""" - content = ( - "Type,Date/Time,INST-VAL\n" - "1,15Sep2021 240015,10.5\n" - ) - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), []) - self.assertEqual(rts.times, []) - - def test_read_csv_empty_file(self): - rts = self.read_rts_from_string("") - self.assertEqual(rts.values.tolist(), []) - self.assertEqual(rts.times, []) - - def test_read_csv_single_row_uses_path_interval(self): - content = "E,,,6Hour\n" "Type,Date/Time,INST-VAL\n" "1,01Sep2021 0600,10.5\n" - rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [10.5]) - self.assertEqual(rts.times, [datetime(2021, 9, 1, 6, 0)]) - - def test_read_csv_irregular_interval_raises(self): - """RegularTimeSeries.read_csv expects a genuinely regular interval; - data implying an irregular gap (with no usable E row) is not valid - input for this class (that's what IrregularTimeSeries is for).""" - content = ( - "Type,Date/Time,INST-VAL\n" - "1,01Sep2021 0600,10.5\n" - "2,01Sep2021 0637,20.0\n" - ) - with self.assertRaises(ValueError): - self.read_rts_from_string(content) - - def test_round_trip_basic(self): - path = self.test_files.create_test_file(".csv") - rts = RegularTimeSeries.create( - values=[10.5, 20.0], - times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], - units="CFS", - data_type="INST-VAL", - path="/A/B/C/01Sep2021/6Hour/F/", - ) - rts.to_csv(path, with_metadata=True) - result = RegularTimeSeries.read_csv(path) - - self.assertEqual(result.units, "CFS") - self.assertEqual(result.data_type, "INST-VAL") - self.assertEqual(result.interval, "6Hour") - self.assertEqual(result.id, "/A/B/C//6Hour/F/") - self.assertEqual(result.values.tolist(), [10.5, 20.0]) - self.assertEqual( - result.times, - [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], - ) - - def test_round_trip_with_quality(self): - path = self.test_files.create_test_file(".csv") - rts = RegularTimeSeries.create( - values=[1.0, 2.0, 3.0], - times=[ - datetime(2021, 9, 1, 6, 0), - datetime(2021, 9, 1, 12, 0), - datetime(2021, 9, 1, 18, 0), - ], - quality=[0, 5, 10], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//6Hour/F/", - ) - rts.to_csv(path, with_metadata=True) - result = RegularTimeSeries.read_csv(path) - - self.assertEqual(result.values.tolist(), [1.0, 2.0, 3.0]) - self.assertEqual(result.quality, [0, 5, 10]) - - def test_round_trip_without_metadata_infers_interval_from_times(self): - """With no metadata rows, units/data_type come back empty but values, - times, and the interval/id (inferred from the time deltas) are still - recovered correctly.""" - path = self.test_files.create_test_file(".csv") - rts = RegularTimeSeries.create( - values=[1.0, 2.0], - times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//6Hour/F/", - ) - rts.to_csv(path, with_metadata=False) - result = RegularTimeSeries.read_csv(path) - - self.assertEqual(result.units, "") - self.assertEqual(result.data_type, "") - self.assertEqual(result.interval, 21600) - self.assertEqual(result.id, "/////6Hour//") - self.assertEqual(result.values.tolist(), [1.0, 2.0]) - self.assertEqual( - result.times, - [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], - ) - - # IRREGULAR TIME SERIES TESTS: - - def test_basic_to_csv_irregular(self): - """ - Basic structure test for irregular time series to_csv - """ - # Create a dummy IrregularTimeSeries instance - its = IrregularTimeSeries.create( - values=[10.5, 20.0, 42.0], - times=[ - datetime(2021, 9, 1, 0, 0), - datetime(2021, 9, 2, 0, 0), - datetime(2021, 9, 4, 0, 0), - ], # inconsistent time interval - units="CFS", - data_type="INST-VAL", - path="/A/B/C/01Sep2021/E/F/", - ) - - # Mock 'open' and capture written content - mock_file = mock_open() - with patch("builtins.open", mock_file): - its.to_csv("fake_path.csv", with_metadata=True) - - # Assert that open was called with correct parameters - mock_file.assert_called_once_with( - "fake_path.csv", "w", newline="", encoding="utf-8" - ) - - # Extract all written data - handle = mock_file() - written_data = "".join(call.args[0] for call in handle.write.call_args_list) - - # Assertions on the CSV content structure - self.assertIn("A,,,A", written_data) - self.assertIn("B,,,B", written_data) - self.assertIn("C,,,C", written_data) - self.assertIn("Units,,,CFS", written_data) - self.assertIn("Type,Date/Time,INST-VAL", written_data) - self.assertIn("1,01Sep2021 0000,10.5", written_data) - self.assertIn("2,02Sep2021 0000,20.0", written_data) - self.assertIn("3,04Sep2021 0000,42.0", written_data) - - def test_irregular_to_csv_without_metadata(self): - """No metadata rows should be written; only data rows.""" - its = IrregularTimeSeries.create( - values=[10.5, 20.0, 42.0], - times=[ - datetime(2021, 9, 1, 0, 0), - datetime(2021, 9, 2, 0, 0), - datetime(2021, 9, 4, 0, 0), - ], # inconsistent time interval - units="CFS", - data_type="INST-VAL", - path="/A/B/C/01Sep2021/E/F/", - ) - mock_file = mock_open() - with patch("builtins.open", mock_file): - its.to_csv("fake.csv", with_metadata=False) - handle = mock_file() - written = "".join(call.args[0] for call in handle.write.call_args_list) - self.assertNotIn("Units", written) - self.assertNotIn("Type,Date/Time", written) - self.assertIn("1,01Sep2021 0000,10.5", written) - self.assertIn("2,02Sep2021 0000,20.0", written) - self.assertIn("3,04Sep2021 0000,42.0", written) - - def test_irregular_to_csv_empty_times(self): - its = IrregularTimeSeries.create( - values=[], - times=[], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//E/F/", - ) - - mock_file = mock_open() - with patch("builtins.open", mock_file): - its.to_csv("fake_path.csv", with_metadata=True) - - handle = mock_file() - written_data = "".join(call.args[0] for call in handle.write.call_args_list) - - self.assertIn("Units,,,CFS", written_data) - self.assertIn("Type,Date/Time,INST-VAL", written_data) - self.assertNotIn("1,", written_data) # No data rows should be present - - def test_irregular_to_csv_second_precision(self): - its = IrregularTimeSeries.create( - values=[i for i in range(3)], - times=[ - datetime(2021, 9, 1, 6, 0, 0), - datetime(2021, 9, 1, 6, 0, 1), - datetime(2021, 9, 1, 6, 0, 4), - ], - units="CFS", - data_type="INST-VAL", - path="/A/B/C//E/F/", - ) - - mock_file = mock_open() - with patch("builtins.open", mock_file): - its.to_csv("fake_path.csv", with_metadata=True) - - handle = mock_file() - written_data = "".join(call.args[0] for call in handle.write.call_args_list) - self.assertIn("Type,Date/Time,INST-VAL", written_data) - self.assertIn("1,01Sep2021 060000,0", written_data) - self.assertIn("2,01Sep2021 060001,1", written_data) - self.assertIn("3,01Sep2021 060004,2", written_data) - - def read_its_from_string(self, content): - """Helper to run read_csv against an in-memory CSV string.""" - m = mock_open(read_data=content) - with patch("builtins.open", m): - return IrregularTimeSeries.read_csv("fake.csv") - - def test_irregular_read_csv_basic(self): - content = ( - "A,,,A\n" - "B,,,B\n" - "C,,,FLOW\n" - "E,,,IR-Year\n" - "F,,,F\n" - "Units,,,CFS\n" - "Type,Date/Time,INST-VAL\n" - "1,01Sep2021 0000,10.5\n" - "2,02Sep2021 0000,20.0\n" - "3,04Sep2021 0000,20.0\n" - ) - its = self.read_its_from_string(content) - self.assertEqual(its.units, "CFS") - self.assertEqual(its.data_type, "INST-VAL") - self.assertEqual(its.values.tolist(), [10.5, 20.0, 20.0]) - self.assertEqual( - its.times, - [datetime(2021, 9, 1), datetime(2021, 9, 2), datetime(2021, 9, 4)], - ) - - # EDGE CASE TESTS: - - def test_roll_day_edge_case(self): - content = ( - "A,,,A\n" - "B,,,B\n" - "C,,,FLOW\n" - "E,,,IR-Day\n" - "F,,,F\n" - "Units,,,CFS\n" - "Type,Date/Time,INST-VAL\n" - "1,01Sep2021 002400,1\n" # 12:24 AM - "2,02Sep2021 024000,1\n" # 2:40 AM - "3,03Sep2021 240000,1\n" # 12:00 AM Next day - ) - its = self.read_its_from_string(content) - self.assertEqual(its.units, "CFS") - self.assertEqual(its.data_type, "INST-VAL") - self.assertEqual(its.values.tolist(), [1, 1, 1]) - self.assertEqual(its.times, [datetime(2021, 9, 1, 0, 24, 0), datetime( - 2021, 9, 2, 2, 40, 0), datetime(2021, 9, 4, 0, 0, 0)]) - - def test_year_is_2400(self): - content = ( - "A,,,A\n" - "B,,,B\n" - "C,,,FLOW\n" - "E,,,IR-Day\n" - "F,,,F\n" - "Units,,,CFS\n" - "Type,Date/Time,INST-VAL\n" - "1,01Sep2400 000000,1\n" - "2,01Sep2400 240000,1\n" - "3,01Sep2400 240015,1\n" # Should not be accepted - "4,01Sep2400 24000,1\n" # Should not be accepted - ) - its = self.read_its_from_string(content) - self.assertEqual(its.units, "CFS") - self.assertEqual(its.data_type, "INST-VAL") - self.assertEqual(its.values.tolist(), [1, 1]) - self.assertEqual(its.times, [datetime(2400, 9, 1, 0, 0, 0), datetime(2400, 9, 2, 0, 0, 0)]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pd_csv.py b/tests/test_pd_csv.py index b6ee39f..d376278 100644 --- a/tests/test_pd_csv.py +++ b/tests/test_pd_csv.py @@ -51,6 +51,55 @@ def test_multiple_curves_to_csv(self): self.assertIn("2,1.5,0.0,1.5,3.0", written_data) self.assertIn("3,4.0,0.0,4.0,8.0", written_data) + # def test_temp(self): + # file_path: str = "./tests/data/examples-all-data-types.dss" + # with HecDss(file_path) as dss: + # data_path: str = "/paired-data-multi-column/RIVERDALE/FREQ-FLOW/MAX ANALYTICAL//1969-01 H33(MAX)/" + # data: PairedData = dss.get(data_path) + # export_path: str = "./tests/csv_testing/paired_data_test.csv" + # data.to_csv(export_path) + + def read_pd_from_string(self, content: str) -> PairedData: + """ + Helper to run read_csv against an in-memory CSV string. + + Parameters: + content (str): PairedData CSV string + + Returns: + PairedData: PairedData object read from string CSV + """ + m = mock_open(read_data=content) + with patch("builtins.open", m): + return PairedData.read_csv("fake.csv") + + def test_basic_read_csv(self): + content: str = ( + "A,,paired-data-test\n" + "B,,berkeley\n" + "C,,FREQ-FLOW\n" + "E,,\n" + "F,,Source: I made it up\n" + "Labels,,IMAGINARY,X,Y,Z\n" + "Units,DAYS,PPG\n" + "Type,TIME,POINTS\n" + "1,0,1,5,6,8\n" + "2,1,3,5,6,9\n" + "3,5,1,2,3,4\n" + ) + pd: PairedData = self.read_pd_from_string(content) + self.assertEqual(pd.labels, ['IMAGINARY', 'X', 'Y', 'Z']) + self.assertEqual(pd.units_independent, "DAYS") + self.assertEqual(pd.units_dependent, "PPG") + self.assertEqual(pd.type_independent, 'TIME') + self.assertEqual(pd.type_dependent, 'POINTS') + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0, 5.0]) + self.assertEqual(pd.values.tolist(), [ + [1.0, 5.0, 6.0, 8.0], + [3.0, 5.0, 6.0, 9.0], + [1.0, 2.0, 3.0, 4.0] + ]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ts_csv.py b/tests/test_ts_csv.py index 4f783a9..98d220a 100644 --- a/tests/test_ts_csv.py +++ b/tests/test_ts_csv.py @@ -174,7 +174,7 @@ def test_read_csv_with_quality(self): self.assertEqual(rts.quality, [0, 5]) def test_read_csv_with_partial_quality(self): - content: tuple[str] = ( + content: str = ( "Type,Date/Time,INST-VAL,Quality\n" "1,05Nov2004 0200,8,0\n" "2,05Nov2004 0300,9\n" # missing quality! From 7a538d50734dc4d7d12f8837e4186b2478de0a69 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Tue, 14 Jul 2026 22:28:03 +0000 Subject: [PATCH 3/6] extensive testing for paired data csv --- src/hecdss/dss_csv.py | 109 +++++--- src/hecdss/paired_data.py | 5 +- tests/test_pd_csv.py | 561 +++++++++++++++++++++++++++++++++++++- tests/test_ts_csv.py | 3 - 4 files changed, 627 insertions(+), 51 deletions(-) diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index 4cc8f40..ca11d0d 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -10,9 +10,8 @@ from .regular_timeseries import RegularTimeSeries -METADATA_ROWS: list[str] = ["A", "B", "C", "D", "E", "F"] # Path of DSS object ROUND_PRECISION: int = 4 -NAN: float = 0.0 +DEFAULT_MISSING_VALUE: float = 0.0 def timeseries_to_csv( @@ -32,25 +31,20 @@ def timeseries_to_csv( with open(path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) if with_metadata: - if series.id: - id_components: list[str] = DssPath(series.id).path_to_list() - else: - id_components: list[str] = [""] * 6 - for i in range(len(METADATA_ROWS)): - row: str = METADATA_ROWS[i] - metadata_value: str = id_components[i] - if row == "D": # Skip D by convention # ts pattern - if metadata_value == "ts-pattern": + id_components: dict[str, str] = _id_to_path_parts(series.id) + for letter, value in id_components.items(): + if letter == 'D': # Skip D by convention # ts pattern + if value == "ts-pattern": print("Warning: ts-pattern skipped") continue - writer.writerow([row, "", "", metadata_value]) # Writing metadata rows + writer.writerow([letter, "", "", value]) # Writing metadata rows writer.writerow(["Units", "", "", series.units]) - if len(series.quality) > 0: - # Write column names with quality - writer.writerow(["Type", "Date/Time", series.data_type, "Quality"]) - else: - # Write column names without quality - writer.writerow(["Type", "Date/Time", series.data_type]) + if len(series.quality) > 0: + # Write column names with quality + writer.writerow(["Type", "Date/Time", series.data_type, "Quality"]) + else: + # Write column names without quality + writer.writerow(["Type", "Date/Time", series.data_type]) time_format: str = ("%d%b%Y %H%M%S" if _needs_second_precision(series) else "%d%b%Y %H%M") @@ -82,7 +76,7 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] raise TypeError("cls must be RegularTimeSeries or IrregularTimeSeries") times, values, quality, units, data_type = [], [], [], "", "" - path_parts: dict[str, str] = {"A": "", "B": "", "C": "", "D": "", "E": "", "F": ""} + path_parts: dict[str, str] = _empty_path_parts() has_quality: bool = False # flags with open(path, "r", newline="", encoding="utf-8") as f: @@ -129,7 +123,7 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] value_str: str = row[2].strip() try: - value: float = float(value_str) if value_str else NAN + value: float = float(value_str) if value_str else DEFAULT_MISSING_VALUE except ValueError: continue # Skip a malformed value @@ -137,9 +131,12 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] values.append(value) if has_quality: # Always keep quality index-aligned with values, defaulting a missing cell to 0 quality_str: str = row[3].strip() if len(row) >= 4 else "" - quality.append(int(quality_str) if quality_str else 0) + try: + quality.append(int(quality_str) if quality_str else 0) + except ValueError: + quality.append(0) - id_path: str = f"/{path_parts['A']}/{path_parts['B']}/{path_parts['C']}/{path_parts['D']}/{path_parts['E']}/{path_parts['F']}/" + id_path: str = _path_parts_to_id(path_parts) interval: str | int = path_parts["E"] return cls.create( @@ -171,12 +168,7 @@ def paired_data_to_csv(paired_data: PairedData, path: str, with_metadata: bool) with open(path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) if with_metadata: - if paired_data.id: - id_components: dict[str, str] = DssPath(paired_data.id).path_to_dict() - else: - id_components: dict[str, str] = { - 'A': '', 'B': '', 'C': '', 'D': '', 'E': '', 'F': '' - } + id_components: dict[str, str] = _id_to_path_parts(paired_data.id) for letter, value in id_components.items(): if letter == 'D': # Skipping D by convention continue @@ -186,7 +178,7 @@ def paired_data_to_csv(paired_data: PairedData, path: str, with_metadata: bool) labels.append(label) writer.writerow(labels) writer.writerow(["Units", paired_data.units_independent, paired_data.units_dependent]) - writer.writerow(["Type", paired_data.type_independent, paired_data.type_dependent]) + writer.writerow(["Type", paired_data.type_independent, paired_data.type_dependent]) counter: int = 1 # 1st column index x: float # x is the same as "ordinate" @@ -220,7 +212,7 @@ def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: x_type: str = "" y_units: str = "" y_type: str = "" - path_parts: dict[str, str] = {"A": "", "B": "", "C": "", "D": "", "E": "", "F": ""} + path_parts: dict[str, str] = _empty_path_parts() with open(path, "r", newline="", encoding="utf-8") as f: reader = csv.reader(f) @@ -238,19 +230,27 @@ def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: for label in row[2:]: # First two elements of row are not labels labels.append(label) elif first_column_item == "Units": + if len(row) < 2: + continue x_units = row[1].strip() + if len(row) < 3: + continue y_units = row[2].strip() # If units change to a list, will have to update this elif first_column_item == "Type": + if len(row) < 2: + continue x_type = row[1].strip() + if len(row) < 3: + continue y_type = row[2].strip() # If type changes to a list, will also have to update this else: # Data row if len(row) < 3: continue # csv is malformed, something is missing x_str: str = row[1].strip() try: - x: float = float(x_str) if x_str else NAN + x: float = float(x_str) if x_str else DEFAULT_MISSING_VALUE except ValueError: - continue + x: float = DEFAULT_MISSING_VALUE x_values.append(x) y_row: list[float] = [] @@ -258,13 +258,13 @@ def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: y_str: str for y_str in y_row_str: try: - y: float = float(y_str) if y_str else NAN + y: float = float(y_str) if y_str else DEFAULT_MISSING_VALUE except ValueError: - continue + y: float = DEFAULT_MISSING_VALUE y_row.append(y) y_values.append(y_row) - id_path: str = f"/{path_parts['A']}/{path_parts['B']}/{path_parts['C']}/{path_parts['D']}/{path_parts['E']}/{path_parts['F']}/" + id_path: str = _path_parts_to_id(path_parts) return cls.create( x_values=x_values, @@ -278,6 +278,44 @@ def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: ) +def _empty_path_parts() -> dict[str, str]: + """ + Returns a fresh dict of empty A-F DSS path components. + + Returns: + dict[str, str]: mapping of each path part letter to an empty string + """ + return {"A": "", "B": "", "C": "", "D": "", "E": "", "F": ""} + + +def _id_to_path_parts(dss_id: str | None) -> dict[str, str]: + """ + Splits a DSS id string into its A-F components, or returns empties if id is missing. + + Parameters: + dss_id (str | None): the DSS pathname to split, or None + + Returns: + dict[str, str]: mapping of each path part letter to its value + """ + if dss_id: + return DssPath(dss_id).path_to_dict() + return _empty_path_parts() + + +def _path_parts_to_id(path_parts: dict[str, str]) -> str: + """ + Rebuilds a '/A/B/C/D/E/F/' DSS id string from its component dict. + + Parameters: + path_parts (dict[str, str]): mapping of each path part letter to its value + + Returns: + str: the reconstructed DSS pathname + """ + return f"/{path_parts['A']}/{path_parts['B']}/{path_parts['C']}/{path_parts['D']}/{path_parts['E']}/{path_parts['F']}/" + + def _needs_second_precision(series: RegularTimeSeries | IrregularTimeSeries) -> bool: """ Returns True if any datetime in the series has a non-zero seconds component @@ -333,3 +371,4 @@ def _need_roll_day(time_format: str, raw_time: str) -> bool: return True return False + diff --git a/src/hecdss/paired_data.py b/src/hecdss/paired_data.py index 6e5492f..6f85ac1 100644 --- a/src/hecdss/paired_data.py +++ b/src/hecdss/paired_data.py @@ -26,7 +26,10 @@ def curve_count(self): Returns: int: The number of curves. """ - return len(self.values[0]) # values is ROW-MAJOR, each sublist represents a row of curve values for each ordinate + if len(self.values) == 0: + return 0 + # values is ROW-MAJOR, each sublist represents a row of curve values for each ordinate + return len(self.values[0]) def to_csv(self, file_path: str, with_metadata: bool = True) -> None: """ diff --git a/tests/test_pd_csv.py b/tests/test_pd_csv.py index d376278..cdf723b 100644 --- a/tests/test_pd_csv.py +++ b/tests/test_pd_csv.py @@ -1,3 +1,4 @@ +import math import unittest from datetime import datetime from unittest.mock import mock_open, patch @@ -5,7 +6,9 @@ from file_manager import FileManager from hecdss import HecDss +from hecdss.dss_csv import paired_data_read_csv, paired_data_to_csv from hecdss.paired_data import PairedData +from hecdss.regular_timeseries import RegularTimeSeries class TestCSV(unittest.TestCase): @@ -16,9 +19,64 @@ def setUp(self) -> None: def tearDown(self) -> None: self.test_files.cleanup() + # ------------------------------------------------------------------ # + # Helpers + # ------------------------------------------------------------------ # + + def read_pd_from_string(self, content: str) -> PairedData: + """ + Helper to run read_csv against an in-memory CSV string. + + Parameters: + content (str): PairedData CSV string + + Returns: + PairedData: PairedData object read from string CSV + """ + m = mock_open(read_data=content) + with patch("builtins.open", m): + return PairedData.read_csv("fake.csv") + + def write_pd_to_string(self, pd: PairedData, with_metadata: bool = True) -> str: + """ + Helper to run to_csv against a mocked file and return everything written. + + Parameters: + pd (PairedData): the object to export + with_metadata (bool): whether metadata rows are written + + Returns: + str: the full CSV text that to_csv wrote + """ + mock_file = mock_open() + with patch("builtins.open", mock_file): + pd.to_csv("fake_path.csv", with_metadata=with_metadata) + handle = mock_file() + return "".join(call.args[0] for call in handle.write.call_args_list) + + @staticmethod + def make_pd(**overrides) -> PairedData: + """A small, fully-populated PairedData with fields overridable per test.""" + params = dict( + x_values=[1.0, 2.0], + y_values=[[10.0, 100.0], [20.0, 200.0]], + labels=["L1", "L2"], + x_units="FEET", + x_type="UNT", + y_units="CFS", + y_type="LINEAR", + path="/A/B/C/DATE/E/F/", + ) + params.update(overrides) + return PairedData.create(**params) + + # ================================================================== # + # to_csv (write) + # ================================================================== # + def test_multiple_curves_to_csv(self): """ - This test ensures that the data values for a PairedData object with multiple curves are correct. + This test ensures that the data values for a PairedData object with multiple curves are correct. This has no checks related to units or labels. """ num_curves: int = 3 @@ -51,6 +109,40 @@ def test_multiple_curves_to_csv(self): self.assertIn("2,1.5,0.0,1.5,3.0", written_data) self.assertIn("3,4.0,0.0,4.0,8.0", written_data) + def test_to_csv_without_metadata(self): + num_curves: int = 3 + x_values: list[float] = [1.0, 1.5, 4.0] + y_values: list[list[float]] = [[x * i for i in range(num_curves)] for x in x_values] + labels: list[str] = [f"x times {i}" for i in range(num_curves)] + path: str = "/A/B/C///Source: I made it up/" + pd: PairedData = PairedData.create( + x_values=x_values, + y_values=y_values, + labels=labels, + path=path + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + pd.to_csv("fake_path.csv", with_metadata=False) + + mock_file.assert_called_once_with( + "fake_path.csv", "w", newline="", encoding="utf-8" + ) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + self.assertNotIn("A,,A", written_data) + self.assertNotIn("B,,B", written_data) + self.assertNotIn("C,,C", written_data) + self.assertNotIn("F,,Source: I made it up", written_data) + self.assertNotIn("Labels", written_data) + self.assertIn("Type", written_data) + self.assertIn("1,1.0,0.0,1.0,2.0", written_data) + self.assertIn("2,1.5,0.0,1.5,3.0", written_data) + self.assertIn("3,4.0,0.0,4.0,8.0", written_data) + # def test_temp(self): # file_path: str = "./tests/data/examples-all-data-types.dss" # with HecDss(file_path) as dss: @@ -59,19 +151,87 @@ def test_multiple_curves_to_csv(self): # export_path: str = "./tests/csv_testing/paired_data_test.csv" # data.to_csv(export_path) - def read_pd_from_string(self, content: str) -> PairedData: - """ - Helper to run read_csv against an in-memory CSV string. + def test_to_csv_metadata_rows_present(self): + """Every metadata row (path parts, labels, units, type) is written.""" + written = self.write_pd_to_string(self.make_pd(), with_metadata=True) + self.assertIn("A,,A", written) + self.assertIn("B,,B", written) + self.assertIn("C,,C", written) + self.assertIn("E,,E", written) + self.assertIn("F,,F", written) + self.assertIn("Labels,,L1,L2", written) + self.assertIn("Units,FEET,CFS", written) + self.assertIn("Type,UNT,LINEAR", written) - Parameters: - content (str): PairedData CSV string + def test_to_csv_skips_d_part(self): + """The D (date) path part is dropped by convention; E is still written.""" + pd = self.make_pd(path="/A/B/C/DPART/EPART/FPART/") + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertNotIn("DPART", written) + self.assertIn("E,,EPART", written) + self.assertIn("F,,FPART", written) - Returns: - PairedData: PairedData object read from string CSV - """ - m = mock_open(read_data=content) - with patch("builtins.open", m): - return PairedData.read_csv("fake.csv") + def test_to_csv_data_rows(self): + """Data rows are index, x, then the row-major y values.""" + written = self.write_pd_to_string(self.make_pd(), with_metadata=True) + self.assertIn("1,1.0,10.0,100.0", written) + self.assertIn("2,2.0,20.0,200.0", written) + + def test_to_csv_ordinate_counter_increments(self): + """The leading index column counts up from 1, one per ordinate.""" + pd = self.make_pd( + x_values=[5.0, 6.0, 7.0], + y_values=[[1.0], [2.0], [3.0]], + ) + written = self.write_pd_to_string(pd, with_metadata=False) + self.assertIn("1,5.0,1.0", written) + self.assertIn("2,6.0,2.0", written) + self.assertIn("3,7.0,3.0", written) + + def test_to_csv_single_curve(self): + """A single dependent column round-trips through the writer fine.""" + pd = self.make_pd(y_values=[[10.0], [20.0]], labels=["only"]) + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertIn("Labels,,only", written) + self.assertIn("1,1.0,10.0", written) + self.assertIn("2,2.0,20.0", written) + + def test_to_csv_rounds_to_four_decimals(self): + """x and y are rounded to ROUND_PRECISION (4) decimal places.""" + pd = self.make_pd(x_values=[1.23456], y_values=[[9.87654]], labels=["a"]) + written = self.write_pd_to_string(pd, with_metadata=False) + self.assertIn("1,1.2346,9.8765", written) + + def test_to_csv_negative_values(self): + """Negative x/y values are written verbatim.""" + pd = self.make_pd(x_values=[-1.0], y_values=[[-5.5, -0.25]], labels=["a", "b"]) + written = self.write_pd_to_string(pd, with_metadata=False) + self.assertIn("1,-1.0,-5.5,-0.25", written) + + def test_to_csv_empty_ordinates_writes_no_data_rows(self): + """No ordinates -> metadata is present but there are no numbered data rows.""" + # Labels avoid digits so the "no data row" check can't collide with a label. + pd = self.make_pd(x_values=[], y_values=[], labels=["aa", "bb"]) + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertIn("Units,FEET,CFS", written) + self.assertIn("Type,UNT,LINEAR", written) + self.assertNotIn("1,", written) # no data rows + + def test_to_csv_none_id_writes_empty_path_parts(self): + """A None id still emits empty A-F rows rather than raising.""" + pd = self.make_pd(path=None) + written = self.write_pd_to_string(pd, with_metadata=True) + for letter in ("A", "B", "C", "E", "F"): + self.assertIn(f"{letter},,", written) + + def test_to_csv_raises_on_non_paireddata(self): + """paired_data_to_csv rejects anything that isn't a PairedData.""" + with self.assertRaises(TypeError): + paired_data_to_csv("not a paired data", "fake.csv", True) + + # ================================================================== # + # read_csv (read) + # ================================================================== # def test_basic_read_csv(self): content: str = ( @@ -100,6 +260,383 @@ def test_basic_read_csv(self): [1.0, 2.0, 3.0, 4.0] ]) + def test_read_csv_single_curve(self): + content = ( + "Type,TIME,POINTS\n" + "1,0,10\n" + "2,1,20\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0], [20.0]]) + self.assertEqual(pd.curve_count(), 1) + + def test_read_csv_data_only_no_metadata(self): + """Data rows with no metadata rows still populate ordinates/values.""" + content = ( + "1,0,10,100\n" + "2,1,20,200\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + self.assertEqual(pd.labels, []) + self.assertEqual(pd.units_independent, "") + + def test_read_csv_builds_id_from_path_parts(self): + content = ( + "A,,SITE\n" + "B,,LOC\n" + "C,,STAGE-FLOW\n" + "E,,\n" + "F,,MADEUP\n" + "Type,STAGE,FLOW\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + # D is never present in a paired-data CSV, so both D and E are empty here. + self.assertEqual(pd.id, "/SITE/LOC/STAGE-FLOW///MADEUP/") + + def test_read_csv_curve_count(self): + content = ( + "Type,TIME,POINTS\n" + "1,0,1,2,3\n" + "2,1,4,5,6\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.curve_count(), 3) + + def test_read_csv_malformed_x_defaults_missing(self): + """A non-numeric x cell falls back to the missing-value default (0.0).""" + content = ( + "Type,TIME,POINTS\n" + "1,not-a-number,10,100\n" + "2,2,20,200\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 2.0]) + self.assertEqual(pd.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + + def test_read_csv_malformed_y_defaults_missing(self): + """A non-numeric y cell falls back to the default WITHOUT shortening the row, + so the value matrix stays rectangular.""" + content = ( + "Type,TIME,POINTS\n" + "1,1,bad,3\n" + "2,2,5,6\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.values.tolist(), [[0.0, 3.0], [5.0, 6.0]]) + + def test_read_csv_empty_cells_default_missing(self): + """Blank x and y cells become the missing-value default (0.0).""" + content = ( + "Type,TIME,POINTS\n" + "1,,\n" + "2,2,\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 2.0]) + self.assertEqual(pd.values.tolist(), [[0.0], [0.0]]) + + def test_read_csv_skips_short_rows(self): + """A data row with fewer than 3 columns is malformed and skipped.""" + content = ( + "Type,TIME,POINTS\n" + "1,5\n" # only 2 columns -> skipped + "2,2,9\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [2.0]) + self.assertEqual(pd.values.tolist(), [[9.0]]) + + def test_read_csv_strips_numeric_whitespace(self): + """Surrounding whitespace around numeric cells is stripped before parsing.""" + content = ( + "Type,TIME,POINTS\n" + "1, 1 , 2 , 3 \n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [1.0]) + self.assertEqual(pd.values.tolist(), [[2.0, 3.0]]) + + def test_read_csv_labels_preserve_whitespace(self): + """Labels are captured verbatim from row[2:] (not stripped).""" + content = ( + "Labels,, spacey ,X\n" + "Type,TIME,POINTS\n" + "1,0,1,2\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.labels, [" spacey ", "X"]) + + def test_read_csv_partial_units_row(self): + """A Units row missing the dependent column leaves y_units empty, no crash.""" + content = ( + "Units,DAYS\n" + "Type,TIME,POINTS\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "DAYS") + self.assertEqual(pd.units_dependent, "") + + def test_read_csv_partial_type_row(self): + """A Type row missing the dependent column leaves y_type empty, no crash.""" + content = ( + "Type,TIME\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.type_independent, "TIME") + self.assertEqual(pd.type_dependent, "") + + def test_read_csv_units_first_column_only(self): + """A bare 'Units' row (no values at all) is ignored gracefully.""" + content = ( + "Units\n" + "Type,TIME,POINTS\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "") + self.assertEqual(pd.units_dependent, "") + self.assertEqual(pd.ordinates.tolist(), [0.0]) + + def test_read_csv_empty_file(self): + pd = self.read_pd_from_string("") + self.assertEqual(pd.ordinates.tolist(), []) + self.assertEqual(pd.values.tolist(), []) + self.assertEqual(pd.labels, []) + + def test_read_csv_blank_lines_ignored(self): + content = ( + "Type,TIME,POINTS\n" + "\n" + "1,0,10\n" + "\n" + "2,1,20\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0], [20.0]]) + + def test_read_csv_ragged_rows_raise(self): + """Genuinely ragged data (rows with different column counts) can't form a + rectangular value matrix, so PairedData.create raises when numpy builds it.""" + content = ( + "Type,TIME,POINTS\n" + "1,0,1,2\n" # 2 y-values + "2,1,3\n" # 1 y-value + ) + with self.assertRaises(ValueError): + self.read_pd_from_string(content) + + def test_read_csv_wrong_cls_raises(self): + """paired_data_read_csv only builds PairedData; other classes are rejected.""" + with self.assertRaises(TypeError): + paired_data_read_csv(RegularTimeSeries, "fake.csv") + + # ================================================================== # + # round trips (write -> read on a real temp file) + # ================================================================== # + + def test_round_trip_basic(self): + path = self.test_files.create_test_file(".csv") + pd = self.make_pd() + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [1.0, 2.0]) + self.assertEqual(result.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + self.assertEqual(result.labels, ["L1", "L2"]) + self.assertEqual(result.units_independent, "FEET") + self.assertEqual(result.units_dependent, "CFS") + self.assertEqual(result.type_independent, "UNT") + self.assertEqual(result.type_dependent, "LINEAR") + # The D (date) part is dropped by the writer, so it comes back empty. + self.assertEqual(result.id, "/A/B/C//E/F/") + + def test_round_trip_multiple_curves(self): + path = self.test_files.create_test_file(".csv") + pd = self.make_pd( + x_values=[0.0, 1.0, 2.0], + y_values=[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], + labels=["a", "b", "c"], + ) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.curve_count(), 3) + self.assertEqual( + result.values.tolist(), + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], + ) + self.assertEqual(result.labels, ["a", "b", "c"]) + + def test_round_trip_without_metadata(self): + """Without metadata, units/labels/id are lost, but data and (because the + Type row is written unconditionally) the types survive.""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd() + pd.to_csv(path, with_metadata=False) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [1.0, 2.0]) + self.assertEqual(result.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + self.assertEqual(result.labels, []) + self.assertEqual(result.units_independent, "") + self.assertEqual(result.units_dependent, "") + self.assertEqual(result.id, "///////") + # Type row is always written, so type survives the no-metadata round trip. + self.assertEqual(result.type_independent, "UNT") + self.assertEqual(result.type_dependent, "LINEAR") + + def test_round_trip_rounding(self): + """Values are rounded to 4 decimals on write, and that rounded value is + exactly what is read back.""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd(x_values=[1.23456], y_values=[[2.34567]], labels=["a"]) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [1.2346]) + self.assertEqual(result.values.tolist(), [[2.3457]]) + + def test_round_trip_negative_values(self): + path = self.test_files.create_test_file(".csv") + pd = self.make_pd( + x_values=[-1.0, 0.0, 1.0], + y_values=[[-5.5], [0.0], [5.5]], + labels=["a"], + ) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [-1.0, 0.0, 1.0]) + self.assertEqual(result.values.tolist(), [[-5.5], [0.0], [5.5]]) + + + # ================================================================== # + # weird / adversarial edge cases + # + # These probe surprising inputs. Tests that DOCUMENT current behavior + # (even when that behavior is a footgun) pass; two known bugs are pinned + # with @unittest.expectedFailure so they stay visible until fixed. + # ================================================================== # + + def test_read_csv_quoted_label_with_comma_round_trips(self): + """A label containing a comma is CSV-quoted on write and comes back intact.""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd( + x_values=[1.0], y_values=[[2.0]], labels=["flow, cfs"], + ) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + self.assertEqual(result.labels, ["flow, cfs"]) + + def test_read_csv_metadata_after_data_still_applies(self): + """The reader is order-independent: a Units row placed AFTER the data rows + still populates the units (nothing forces metadata to come first).""" + content = ( + "1,0,10\n" + "Units,DAYS,PPG\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "DAYS") + self.assertEqual(pd.units_dependent, "PPG") + self.assertEqual(pd.values.tolist(), [[10.0]]) + + def test_read_csv_ignores_counter_column(self): + """The leading index column is never validated on read - non-sequential or + even non-numeric counters are ignored entirely (x comes from column 2).""" + content = ( + "Type,,\n" + "999,0,10\n" + "hello,1,20\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0], [20.0]]) + + def test_read_csv_duplicate_units_last_wins(self): + """When a keyword row appears twice, the last occurrence overwrites.""" + content = ( + "Units,AA,BB\n" + "Units,CC,DD\n" + "Type,,\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "CC") + self.assertEqual(pd.units_dependent, "DD") + + def test_read_csv_whitespace_only_cells_default_missing(self): + """Cells that are only whitespace strip to empty and become the default.""" + content = ( + "Type,,\n" + "1, , \n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0]) + self.assertEqual(pd.values.tolist(), [[0.0]]) + + def test_read_csv_inf_and_nan_parse_through(self): + """float() accepts 'inf'/'nan', so these land in the array as real inf/NaN + rather than being treated as missing. Surprising, given the missing-value + default is 0.0 - documented here so a future change is caught.""" + content = ( + "Type,,\n" + "1,inf,nan\n" + "2,-inf,5\n" + ) + pd = self.read_pd_from_string(content) + self.assertTrue(math.isinf(pd.ordinates[0])) + self.assertTrue(math.isnan(pd.values[0][0])) + self.assertTrue(math.isinf(pd.ordinates[1]) and pd.ordinates[1] < 0) + + def test_read_csv_miscased_keyword_becomes_phantom_data_row(self): + """FOOTGUN: keyword matching is case-sensitive, so a mis-cased 'units' row + is not recognized as metadata. It falls through to the data branch, its + text cells fail to parse, and it is silently injected as a (0.0, [0.0]) + data point instead of raising or being ignored.""" + content = ( + "units,DAYS,PPG\n" # lowercase -> NOT recognized as a Units row + "Type,TIME,POINTS\n" + "1,5,10\n" + ) + pd = self.read_pd_from_string(content) + # The real units were never captured... + self.assertEqual(pd.units_independent, "") + # ...and a bogus leading data point was manufactured from the keyword row. + self.assertEqual(pd.ordinates.tolist(), [0.0, 5.0]) + self.assertEqual(pd.values.tolist(), [[0.0], [10.0]]) + + def test_read_csv_extra_metadata_column_grabs_last_cell(self): + """FOOTGUN: path parts are read via the 'last cell' convention (row[-1]), + so a metadata row with an unexpected trailing column captures the trailing + value instead of the intended one.""" + content = ( + "A,,SITE,EXTRA\n" # intended A=SITE, but row[-1] is 'EXTRA' + "Type,,\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertIn("EXTRA", pd.id) + self.assertNotIn("SITE", pd.id) + + def test_curve_count_on_dataless_read_should_be_zero(self): + """BUG: reading a CSV with metadata but no data rows yields an empty + PairedData, and curve_count() then does len(self.values[0]) -> IndexError. + It should return 0. Pinned as expectedFailure until curve_count guards + against an empty value array.""" + content = ( + "Units,DAYS,PPG\n" + "Type,TIME,POINTS\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.curve_count(), 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ts_csv.py b/tests/test_ts_csv.py index 98d220a..0584b16 100644 --- a/tests/test_ts_csv.py +++ b/tests/test_ts_csv.py @@ -65,7 +65,6 @@ def test_to_csv_without_metadata(self): handle = mock_file() written = "".join(call.args[0] for call in handle.write.call_args_list) self.assertNotIn("Units", written) - self.assertNotIn("Type,Date/Time", written) self.assertIn("1,01Sep2021 0600,1.0", written) def test_to_csv_empty_times(self): @@ -352,7 +351,6 @@ def test_round_trip_without_metadata_infers_interval_from_times(self): result = RegularTimeSeries.read_csv(path) self.assertEqual(result.units, "") - self.assertEqual(result.data_type, "") self.assertEqual(result.interval, 21600) self.assertEqual(result.id, "/////6Hour//") self.assertEqual(result.values.tolist(), [1.0, 2.0]) @@ -423,7 +421,6 @@ def test_irregular_to_csv_without_metadata(self): handle = mock_file() written = "".join(call.args[0] for call in handle.write.call_args_list) self.assertNotIn("Units", written) - self.assertNotIn("Type,Date/Time", written) self.assertIn("1,01Sep2021 0000,10.5", written) self.assertIn("2,02Sep2021 0000,20.0", written) self.assertIn("3,04Sep2021 0000,42.0", written) From 6aa86ee3abd8eef377d44a677d679f0b11e02bde Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Tue, 14 Jul 2026 22:32:20 +0000 Subject: [PATCH 4/6] clean up TODO --- src/hecdss/dss_csv.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index ca11d0d..9ed0ce9 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -225,7 +225,6 @@ def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: # If the first column item is a path component (['A', 'B', 'C', 'D', 'E', 'F']) if first_column_item in path_parts: path_parts[first_column_item] = row[-1].strip() # last cell in csv row (convention) - # TODO: Finish the read_csv function. After you do this and write many tests, consider looking into C code to incorporate text columns and switching y units and types to lists elif first_column_item == "Labels": for label in row[2:]: # First two elements of row are not labels labels.append(label) @@ -334,7 +333,7 @@ def _get_time_format(raw_time: str) -> str | None: Given a raw DSS time string, detect and return the correct time format, whether it be minutes or seconds precision. Parameters: - raw_time (str): time in DSS string format (TODO: ISO) + raw_time (str): time in DSS string format Returns: str | None: time format to use to convert to datetime @@ -371,4 +370,3 @@ def _need_roll_day(time_format: str, raw_time: str) -> bool: return True return False - From 987ff4b85781d004fe1cc599e0da8473af01d9bd Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Wed, 15 Jul 2026 18:26:24 +0000 Subject: [PATCH 5/6] switch default missing value to None --- src/hecdss/dss_csv.py | 4 ++-- tests/test_pd_csv.py | 23 +++++++++++++---------- tests/test_ts_csv.py | 32 +++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index 9ed0ce9..7a4f37c 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -11,7 +11,7 @@ ROUND_PRECISION: int = 4 -DEFAULT_MISSING_VALUE: float = 0.0 +DEFAULT_MISSING_VALUE = None def timeseries_to_csv( @@ -125,7 +125,7 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] try: value: float = float(value_str) if value_str else DEFAULT_MISSING_VALUE except ValueError: - continue # Skip a malformed value + value: float = DEFAULT_MISSING_VALUE # If datetime is valid but not value, use DEFAULT_MISSING_VALUE times.append(time) values.append(value) diff --git a/tests/test_pd_csv.py b/tests/test_pd_csv.py index cdf723b..10c1a35 100644 --- a/tests/test_pd_csv.py +++ b/tests/test_pd_csv.py @@ -6,7 +6,11 @@ from file_manager import FileManager from hecdss import HecDss -from hecdss.dss_csv import paired_data_read_csv, paired_data_to_csv +from hecdss.dss_csv import ( + DEFAULT_MISSING_VALUE, + paired_data_read_csv, + paired_data_to_csv, +) from hecdss.paired_data import PairedData from hecdss.regular_timeseries import RegularTimeSeries @@ -314,7 +318,7 @@ def test_read_csv_malformed_x_defaults_missing(self): "2,2,20,200\n" ) pd = self.read_pd_from_string(content) - self.assertEqual(pd.ordinates.tolist(), [0.0, 2.0]) + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE, 2.0]) self.assertEqual(pd.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) def test_read_csv_malformed_y_defaults_missing(self): @@ -326,7 +330,7 @@ def test_read_csv_malformed_y_defaults_missing(self): "2,2,5,6\n" ) pd = self.read_pd_from_string(content) - self.assertEqual(pd.values.tolist(), [[0.0, 3.0], [5.0, 6.0]]) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE, 3.0], [5.0, 6.0]]) def test_read_csv_empty_cells_default_missing(self): """Blank x and y cells become the missing-value default (0.0).""" @@ -336,8 +340,8 @@ def test_read_csv_empty_cells_default_missing(self): "2,2,\n" ) pd = self.read_pd_from_string(content) - self.assertEqual(pd.ordinates.tolist(), [0.0, 2.0]) - self.assertEqual(pd.values.tolist(), [[0.0], [0.0]]) + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE, 2.0]) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE], [DEFAULT_MISSING_VALUE]]) def test_read_csv_skips_short_rows(self): """A data row with fewer than 3 columns is malformed and skipped.""" @@ -516,7 +520,6 @@ def test_round_trip_negative_values(self): self.assertEqual(result.ordinates.tolist(), [-1.0, 0.0, 1.0]) self.assertEqual(result.values.tolist(), [[-5.5], [0.0], [5.5]]) - # ================================================================== # # weird / adversarial edge cases # @@ -578,8 +581,8 @@ def test_read_csv_whitespace_only_cells_default_missing(self): "1, , \n" ) pd = self.read_pd_from_string(content) - self.assertEqual(pd.ordinates.tolist(), [0.0]) - self.assertEqual(pd.values.tolist(), [[0.0]]) + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE]) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE]]) def test_read_csv_inf_and_nan_parse_through(self): """float() accepts 'inf'/'nan', so these land in the array as real inf/NaN @@ -609,8 +612,8 @@ def test_read_csv_miscased_keyword_becomes_phantom_data_row(self): # The real units were never captured... self.assertEqual(pd.units_independent, "") # ...and a bogus leading data point was manufactured from the keyword row. - self.assertEqual(pd.ordinates.tolist(), [0.0, 5.0]) - self.assertEqual(pd.values.tolist(), [[0.0], [10.0]]) + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE, 5.0]) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE], [10.0]]) def test_read_csv_extra_metadata_column_grabs_last_cell(self): """FOOTGUN: path parts are read via the 'last cell' convention (row[-1]), diff --git a/tests/test_ts_csv.py b/tests/test_ts_csv.py index 0584b16..7053c3f 100644 --- a/tests/test_ts_csv.py +++ b/tests/test_ts_csv.py @@ -5,6 +5,7 @@ from file_manager import FileManager from hecdss import HecDss +from hecdss.dss_csv import DEFAULT_MISSING_VALUE from hecdss.irregular_timeseries import IrregularTimeSeries from hecdss.regular_timeseries import RegularTimeSeries @@ -185,7 +186,7 @@ def test_read_csv_with_partial_quality(self): self.assertEqual(rts.quality[0], 0) self.assertEqual(rts.quality[2], 1) - def test_read_csv_skips_malformed_rows(self): + def test_read_csv_skips_malformed_date_rows(self): content = ( "Type,Date/Time,INST-VAL\n" "1,01Sep2021 0600,10.5\n" @@ -194,9 +195,10 @@ def test_read_csv_skips_malformed_rows(self): "4,01Sep2021 1800,30.0\n" ) rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [10.5, 30.0]) + self.assertEqual(rts.values.tolist(), [10.5, DEFAULT_MISSING_VALUE, 30.0]) self.assertEqual( - rts.times, [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 18, 0)] + rts.times, [datetime(2021, 9, 1, 6, 0), datetime( + 2021, 9, 1, 12, 0), datetime(2021, 9, 1, 18, 0)] ) def test_read_csv_seconds_precision_basic(self): @@ -539,6 +541,30 @@ def test_year_is_2400(self): self.assertEqual(its.values.tolist(), [1, 1]) self.assertEqual(its.times, [datetime(2400, 9, 1, 0, 0, 0), datetime(2400, 9, 2, 0, 0, 0)]) + def test_read_write_with_missing(self): + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600,10.5\n" + "2,not-a-date,20.0\n" + "3,01Sep2021 1200,not-a-number\n" + "4,01Sep2021 1800,30.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, DEFAULT_MISSING_VALUE, 30.0]) + self.assertEqual( + rts.times, [datetime(2021, 9, 1, 6, 0), datetime( + 2021, 9, 1, 12, 0), datetime(2021, 9, 1, 18, 0)] + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake_path.csv", with_metadata=True) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + self.assertIn("2,01Sep2021 1200,\r\n", written_data) + if __name__ == "__main__": unittest.main() From adca707103574dc95fd04d910c5c3052a9f875e9 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Wed, 15 Jul 2026 19:54:55 +0000 Subject: [PATCH 6/6] more tests, rounding bug fix --- src/hecdss/dss_csv.py | 17 +++++++- tests/test_pd_csv.py | 22 ++++++++++ tests/test_ts_csv.py | 97 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index 7a4f37c..72ef558 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -184,8 +184,8 @@ def paired_data_to_csv(paired_data: PairedData, path: str, with_metadata: bool) x: float # x is the same as "ordinate" y_row: list[float] # Each y_row is one row of y_values, as paired_data.values is Row-Major for x, y_row in zip(paired_data.ordinates, paired_data.values): - full_row: list[float] = [round(x, ROUND_PRECISION)] + \ - [round(y, ROUND_PRECISION) for y in y_row] + full_row: list[float] = [_round_or_none(x)] + \ + [_round_or_none(y) for y in y_row] writer.writerow([counter] + full_row) counter += 1 return @@ -277,6 +277,19 @@ def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: ) +def _round_or_none(value: float | None) -> float | None: + """ + Rounds a numeric value to ROUND_PRECISION, passing None through unchanged. + + Parameters: + value (float | None): the value to round, or None if missing + + Returns: + float | None: the rounded value, or None if value was None + """ + return round(value, ROUND_PRECISION) if value is not None else None + + def _empty_path_parts() -> dict[str, str]: """ Returns a fresh dict of empty A-F DSS path components. diff --git a/tests/test_pd_csv.py b/tests/test_pd_csv.py index 10c1a35..d0654fa 100644 --- a/tests/test_pd_csv.py +++ b/tests/test_pd_csv.py @@ -640,6 +640,28 @@ def test_curve_count_on_dataless_read_should_be_zero(self): pd = self.read_pd_from_string(content) self.assertEqual(pd.curve_count(), 0) + def test_to_csv_missing_value_does_not_crash(self): + """A blank/malformed numeric cell is read back as DEFAULT_MISSING_VALUE + (None), producing an object-dtype array. The writer must tolerate a None + ordinate/value (write an empty cell) rather than crash trying to round it.""" + pd = self.read_pd_from_string( + "Type,TIME,POINTS\n" + "1,,10\n" # blank ordinate -> None + "2,2,20\n" + ) + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertIn("1,,10.0", written) + self.assertIn("2,2.0,20.0", written) + + def test_round_trip_nan_value_survives(self): + """A NaN dependent value is written as 'nan' and read back as a real NaN + (it is NOT collapsed into the missing-value default).""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd(x_values=[1.0], y_values=[[float("nan")]], labels=["a"]) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + self.assertTrue(math.isnan(result.values.tolist()[0][0])) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ts_csv.py b/tests/test_ts_csv.py index 7053c3f..29886d5 100644 --- a/tests/test_ts_csv.py +++ b/tests/test_ts_csv.py @@ -2,6 +2,8 @@ from datetime import datetime from unittest.mock import mock_open, patch +import numpy as np + from file_manager import FileManager from hecdss import HecDss @@ -565,6 +567,101 @@ def test_read_write_with_missing(self): self.assertIn("Type,Date/Time,INST-VAL", written_data) self.assertIn("2,01Sep2021 1200,\r\n", written_data) + def test_to_csv_writes_empty_cell_for_missing_value(self): + """A missing (None) value is written as an empty cell, never the literal + text 'None'. (Isolates the write path with an injected None, independent + of what the reader produces.)""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.values = np.array([1.0, None], dtype=object) # simulate a missing value + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("2,01Sep2021 1200,\r\n", written) + self.assertNotIn("None", written) + + def test_to_csv_quality_shorter_than_values_truncates(self): + """FOOTGUN: to_csv takes the with-quality branch whenever quality is + non-empty, then zips (times, values, quality). A quality list shorter than + values makes zip stop at the shortest input, so trailing data points are + silently dropped from the CSV.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0, 3.0], + times=[ + datetime(2021, 9, 1, 6, 0), + datetime(2021, 9, 1, 12, 0), + datetime(2021, 9, 1, 18, 0), + ], + quality=[0], # only one flag for three values + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("1,01Sep2021 0600,1.0,0", written) + self.assertNotIn("2,01Sep2021 1200", written) # silently dropped + self.assertNotIn("3,01Sep2021 1800", written) # silently dropped + + def test_read_csv_skips_short_data_row(self): + """A data row with fewer than 3 columns is malformed and skipped without + raising (parity with the paired-data reader's short-row handling).""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600\n" # only 2 columns -> skipped + "2,01Sep2021 1200,20.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [20.0]) + self.assertEqual(rts.times, [datetime(2021, 9, 1, 12, 0)]) + + def test_read_csv_metadata_only_no_data_rows(self): + """Metadata rows with zero data rows yield an empty series (no crash); + units and the E interval are still captured.""" + content = ( + "A,,,A\n" + "E,,,6Hour\n" + "Units,,,CFS\n" + "Type,Date/Time,INST-VAL\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), []) + self.assertEqual(rts.times, []) + self.assertEqual(rts.units, "CFS") + self.assertEqual(rts.interval, 21600) + + def test_round_trip_irregular(self): + """Full write->read round trip for IrregularTimeSeries on a real temp file: + irregular gaps, units, data_type and id all survive.""" + path = self.test_files.create_test_file(".csv") + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//IR-Year/F/", + ) + its.to_csv(path, with_metadata=True) + result = IrregularTimeSeries.read_csv(path) + self.assertEqual(result.values.tolist(), [10.5, 20.0, 42.0]) + self.assertEqual( + result.times, + [datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + ) + self.assertEqual(result.units, "CFS") + self.assertEqual(result.data_type, "INST-VAL") + self.assertEqual(result.id, "/A/B/C//IR-Year/F/") + if __name__ == "__main__": unittest.main()