diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index c6d8e79..72ef558 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -1,12 +1,19 @@ +# THESE ARE ALL HELPER FUNCTIONS, MEANT TO BE TREATED AS PRIVATE. These functions are internally called in their respective classes. + import csv import re from datetime import datetime, timedelta from .dsspath import DssPath from .irregular_timeseries import IrregularTimeSeries +from .paired_data import PairedData from .regular_timeseries import RegularTimeSeries +ROUND_PRECISION: int = 4 +DEFAULT_MISSING_VALUE = None + + def timeseries_to_csv( series: RegularTimeSeries | IrregularTimeSeries, path: str, with_metadata: bool ) -> None: @@ -21,29 +28,23 @@ def timeseries_to_csv( if not isinstance(series, (RegularTimeSeries, IrregularTimeSeries)): raise TypeError("series must be a RegularTimeSeries or IrregularTimeSeries") - metadata_rows: list[str] = ["A", "B", "C", "D", "E", "F"] with open(path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) if with_metadata: - if series.id: - id_components: list[str] = DssPath(series.id).path_to_list() - else: - id_components: list[str] = [""] * 6 - for i in range(len(metadata_rows)): - row: str = metadata_rows[i] - metadata_value: str = id_components[i] - if row == "D": # Skip D by convention # ts pattern - if metadata_value == "ts-pattern": + id_components: dict[str, str] = _id_to_path_parts(series.id) + for letter, value in id_components.items(): + if letter == 'D': # Skip D by convention # ts pattern + if value == "ts-pattern": print("Warning: ts-pattern skipped") continue - writer.writerow([row, "", "", metadata_value]) # Writing metadata rows + writer.writerow([letter, "", "", value]) # Writing metadata rows writer.writerow(["Units", "", "", series.units]) - if len(series.quality) > 0: - # Write column names with quality - writer.writerow(["Type", "Date/Time", series.data_type, "Quality"]) - else: - # Write column names without quality - writer.writerow(["Type", "Date/Time", series.data_type]) + if len(series.quality) > 0: + # Write column names with quality + writer.writerow(["Type", "Date/Time", series.data_type, "Quality"]) + else: + # Write column names without quality + writer.writerow(["Type", "Date/Time", series.data_type]) time_format: str = ("%d%b%Y %H%M%S" if _needs_second_precision(series) else "%d%b%Y %H%M") @@ -75,7 +76,7 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] raise TypeError("cls must be RegularTimeSeries or IrregularTimeSeries") times, values, quality, units, data_type = [], [], [], "", "" - path_parts: dict[str, str] = {"A": "", "B": "", "C": "", "D": "", "E": "", "F": ""} + path_parts: dict[str, str] = _empty_path_parts() has_quality: bool = False # flags with open(path, "r", newline="", encoding="utf-8") as f: @@ -122,17 +123,20 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] value_str: str = row[2].strip() try: - value: float = float(value_str) if value_str else 0.0 + value: float = float(value_str) if value_str else DEFAULT_MISSING_VALUE except ValueError: - continue # Skip a malformed value + value: float = DEFAULT_MISSING_VALUE # If datetime is valid but not value, use DEFAULT_MISSING_VALUE times.append(time) values.append(value) if has_quality: # Always keep quality index-aligned with values, defaulting a missing cell to 0 quality_str: str = row[3].strip() if len(row) >= 4 else "" - quality.append(int(quality_str) if quality_str else 0) + try: + quality.append(int(quality_str) if quality_str else 0) + except ValueError: + quality.append(0) - id_path: str = f"/{path_parts['A']}/{path_parts['B']}/{path_parts['C']}/{path_parts['D']}/{path_parts['E']}/{path_parts['F']}/" + id_path: str = _path_parts_to_id(path_parts) interval: str | int = path_parts["E"] return cls.create( @@ -146,6 +150,184 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] ) +def paired_data_to_csv(paired_data: PairedData, path: str, with_metadata: bool) -> None: + """ + Exports a PairedData object to a .csv file. + + Parameters: + paired_data (PairedData): Paired Data object to convert to csv. + path: (str): file path to export csv to. + with_metadata (bool): whether or not to include metadata in the csv file. + + Returns: + None + """ + if not isinstance(paired_data, PairedData): + raise TypeError("paired_data must be a PairedData!") + + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + if with_metadata: + id_components: dict[str, str] = _id_to_path_parts(paired_data.id) + for letter, value in id_components.items(): + if letter == 'D': # Skipping D by convention + continue + writer.writerow([letter, "", value]) # Writing metadata rows + labels: list[str] = ["Labels", ""] + for label in paired_data.labels: + labels.append(label) + writer.writerow(labels) + writer.writerow(["Units", paired_data.units_independent, paired_data.units_dependent]) + writer.writerow(["Type", paired_data.type_independent, paired_data.type_dependent]) + + counter: int = 1 # 1st column index + x: float # x is the same as "ordinate" + y_row: list[float] # Each y_row is one row of y_values, as paired_data.values is Row-Major + for x, y_row in zip(paired_data.ordinates, paired_data.values): + full_row: list[float] = [_round_or_none(x)] + \ + [_round_or_none(y) for y in y_row] + writer.writerow([counter] + full_row) + counter += 1 + return + + +def paired_data_read_csv(cls: type[PairedData], path: str) -> PairedData: + """ + Reads a .csv file and builds a new paired data of the given type. + + Parameters: + cls: the class to build - PairedData + path (str): File path to .csv that we are reading from + + Returns: + An instance of cls populated from the .csv file. + """ + if cls is not PairedData: + raise TypeError("cls must be PairedData") + + x_values: list[float] = [] + y_values: list[list[float]] = [] + labels: list[str] = [] + x_units: str = "" + x_type: str = "" + y_units: str = "" + y_type: str = "" + path_parts: dict[str, str] = _empty_path_parts() + + with open(path, "r", newline="", encoding="utf-8") as f: + reader = csv.reader(f) + row: list[str] + for row in reader: + if not row: + continue + # first item in the row we grabbed, the first column's item + first_column_item: str = row[0].strip() + # If the first column item is a path component (['A', 'B', 'C', 'D', 'E', 'F']) + if first_column_item in path_parts: + path_parts[first_column_item] = row[-1].strip() # last cell in csv row (convention) + elif first_column_item == "Labels": + for label in row[2:]: # First two elements of row are not labels + labels.append(label) + elif first_column_item == "Units": + if len(row) < 2: + continue + x_units = row[1].strip() + if len(row) < 3: + continue + y_units = row[2].strip() # If units change to a list, will have to update this + elif first_column_item == "Type": + if len(row) < 2: + continue + x_type = row[1].strip() + if len(row) < 3: + continue + y_type = row[2].strip() # If type changes to a list, will also have to update this + else: # Data row + if len(row) < 3: + continue # csv is malformed, something is missing + x_str: str = row[1].strip() + try: + x: float = float(x_str) if x_str else DEFAULT_MISSING_VALUE + except ValueError: + x: float = DEFAULT_MISSING_VALUE + x_values.append(x) + + y_row: list[float] = [] + y_row_str: list[str] = row[2:] + y_str: str + for y_str in y_row_str: + try: + y: float = float(y_str) if y_str else DEFAULT_MISSING_VALUE + except ValueError: + y: float = DEFAULT_MISSING_VALUE + y_row.append(y) + y_values.append(y_row) + + id_path: str = _path_parts_to_id(path_parts) + + return cls.create( + x_values=x_values, + y_values=y_values, + labels=labels, + x_units=x_units, + x_type=x_type, + y_units=y_units, + y_type=y_type, + path=id_path, + ) + + +def _round_or_none(value: float | None) -> float | None: + """ + Rounds a numeric value to ROUND_PRECISION, passing None through unchanged. + + Parameters: + value (float | None): the value to round, or None if missing + + Returns: + float | None: the rounded value, or None if value was None + """ + return round(value, ROUND_PRECISION) if value is not None else None + + +def _empty_path_parts() -> dict[str, str]: + """ + Returns a fresh dict of empty A-F DSS path components. + + Returns: + dict[str, str]: mapping of each path part letter to an empty string + """ + return {"A": "", "B": "", "C": "", "D": "", "E": "", "F": ""} + + +def _id_to_path_parts(dss_id: str | None) -> dict[str, str]: + """ + Splits a DSS id string into its A-F components, or returns empties if id is missing. + + Parameters: + dss_id (str | None): the DSS pathname to split, or None + + Returns: + dict[str, str]: mapping of each path part letter to its value + """ + if dss_id: + return DssPath(dss_id).path_to_dict() + return _empty_path_parts() + + +def _path_parts_to_id(path_parts: dict[str, str]) -> str: + """ + Rebuilds a '/A/B/C/D/E/F/' DSS id string from its component dict. + + Parameters: + path_parts (dict[str, str]): mapping of each path part letter to its value + + Returns: + str: the reconstructed DSS pathname + """ + return f"/{path_parts['A']}/{path_parts['B']}/{path_parts['C']}/{path_parts['D']}/{path_parts['E']}/{path_parts['F']}/" + + def _needs_second_precision(series: RegularTimeSeries | IrregularTimeSeries) -> bool: """ Returns True if any datetime in the series has a non-zero seconds component @@ -164,7 +346,7 @@ def _get_time_format(raw_time: str) -> str | None: Given a raw DSS time string, detect and return the correct time format, whether it be minutes or seconds precision. Parameters: - raw_time (str): time in DSS string format (TODO: ISO) + raw_time (str): time in DSS string format Returns: str | None: time format to use to convert to datetime diff --git a/src/hecdss/dsspath.py b/src/hecdss/dsspath.py index f58d954..b086f4b 100644 --- a/src/hecdss/dsspath.py +++ b/src/hecdss/dsspath.py @@ -2,6 +2,7 @@ from .dss_type import DssType from .record_type import RecordType + class DssPath: """ Manage parts of DSS path /A/B/C/D/E/F/ @@ -11,7 +12,7 @@ class DssPath: _timeSeriesFamily = [RecordType.IrregularTimeSeries, RecordType.RegularTimeSeries, RecordType.RegularTimeSeriesProfile] - def __init__(self, path: str, recType = 0): + def __init__(self, path: str, recType=0): """ Initialize a DssPath object. @@ -95,4 +96,13 @@ def path_to_list(self) -> list[str]: Returns: list: A list containing the components of the DSS path. """ - return [self.A, self.B, self.C, self.D, self.E, self.F] \ No newline at end of file + return [self.A, self.B, self.C, self.D, self.E, self.F] + + def path_to_dict(self) -> dict[str, str]: + """ + Convert the DSS path to a dictionary of its components. + + Returns: + dict[str, str]: A dictionary containing the components of the DSS path. + """ + return {'A': self.A, 'B': self.B, 'C': self.C, 'D': self.D, 'E': self.E, 'F': self.F} diff --git a/src/hecdss/paired_data.py b/src/hecdss/paired_data.py index 49da97a..6f85ac1 100644 --- a/src/hecdss/paired_data.py +++ b/src/hecdss/paired_data.py @@ -7,15 +7,16 @@ def __init__(self): """ Initialize a PairedData object with default values. """ - self.id = None - self.ordinates = np.empty(0) - self.values = np.empty(0) - self.labels = [] - self.type_independent = "" - self.type_dependent = "" - self.units_independent = "" - self.units_dependent = "" - self.time_zone_name = "" + self.id: str = None + self.ordinates: np.ndarray = np.empty(0) + # ROW-MAJOR, each sublist represents the values for each ordinate + self.values: np.ndarray = np.empty(0) + self.labels: list[str] = [] + self.type_independent: str = "" + self.type_dependent: str = "" + self.units_independent: str = "" + self.units_dependent: str = "" + self.time_zone_name: str = "" self.location_info = None def curve_count(self): @@ -25,7 +26,22 @@ def curve_count(self): Returns: int: The number of curves. """ - return len(self.values) + if len(self.values) == 0: + return 0 + # values is ROW-MAJOR, each sublist represents a row of curve values for each ordinate + return len(self.values[0]) + + def to_csv(self, file_path: str, with_metadata: bool = True) -> None: + """ + Exports the PairedData instance to a .csv file. + + Parameters: + file_path (str): The path to the .csv file where the data will be exported. + with_metadata (bool): Whether to include metadata in the exported file. + """ + from .dss_csv import paired_data_to_csv + paired_data_to_csv(self, file_path, with_metadata) + print(f"Wrote PairedData to .csv file at {file_path}.") # def to_data_frame(self, include_index=False): # """ @@ -47,6 +63,20 @@ def curve_count(self): # # return pd.DataFrame(data) + @staticmethod + def read_csv(file_path: str) -> "PairedData": + """ + Reads a .csv file and creates an PairedData instance from the data. + + Parameters: + file_path (str): The path to the .csv file to read + + Returns: + PairedData: A new instance of PairedData populated with the data from the .csv file. + """ + from .dss_csv import paired_data_read_csv + return paired_data_read_csv(PairedData, file_path) + @staticmethod def create(x_values, y_values, labels=[], x_units="", x_type="", y_units="", y_type="", time_zone_name="", path=None): """ @@ -75,4 +105,4 @@ def create(x_values, y_values, labels=[], x_units="", x_type="", y_units="", y_t pd.type_independent = x_type pd.type_dependent = y_type pd.time_zone_name = time_zone_name - return pd \ No newline at end of file + return pd diff --git a/tests/test_paired_data.py b/tests/test_paired_data.py index b3c9121..79d0c29 100644 --- a/tests/test_paired_data.py +++ b/tests/test_paired_data.py @@ -63,6 +63,7 @@ def test_paired_data_create(self): assert (pd.values[4][2] == 46), f"pd.values[4][2] should be 46. is {pd.values[4][2]}" assert (len(pd.ordinates) == 5), f"len(pd.ordinates) should be 5. is {len(pd.ordinates)}" assert (pd.labels[1] == "x plus 1"), f"pd.labels[1] should be 'x plus 1'. is {pd.labels[1]}" + assert (pd.curve_count() == 3) def test_paired_data_create_store(self): """ diff --git a/tests/test_pd_csv.py b/tests/test_pd_csv.py new file mode 100644 index 0000000..d0654fa --- /dev/null +++ b/tests/test_pd_csv.py @@ -0,0 +1,667 @@ +import math +import unittest +from datetime import datetime +from unittest.mock import mock_open, patch + +from file_manager import FileManager + +from hecdss import HecDss +from hecdss.dss_csv import ( + DEFAULT_MISSING_VALUE, + paired_data_read_csv, + paired_data_to_csv, +) +from hecdss.paired_data import PairedData +from hecdss.regular_timeseries import RegularTimeSeries + + +class TestCSV(unittest.TestCase): + + def setUp(self) -> None: + self.test_files = FileManager() + + def tearDown(self) -> None: + self.test_files.cleanup() + + # ------------------------------------------------------------------ # + # Helpers + # ------------------------------------------------------------------ # + + def read_pd_from_string(self, content: str) -> PairedData: + """ + Helper to run read_csv against an in-memory CSV string. + + Parameters: + content (str): PairedData CSV string + + Returns: + PairedData: PairedData object read from string CSV + """ + m = mock_open(read_data=content) + with patch("builtins.open", m): + return PairedData.read_csv("fake.csv") + + def write_pd_to_string(self, pd: PairedData, with_metadata: bool = True) -> str: + """ + Helper to run to_csv against a mocked file and return everything written. + + Parameters: + pd (PairedData): the object to export + with_metadata (bool): whether metadata rows are written + + Returns: + str: the full CSV text that to_csv wrote + """ + mock_file = mock_open() + with patch("builtins.open", mock_file): + pd.to_csv("fake_path.csv", with_metadata=with_metadata) + handle = mock_file() + return "".join(call.args[0] for call in handle.write.call_args_list) + + @staticmethod + def make_pd(**overrides) -> PairedData: + """A small, fully-populated PairedData with fields overridable per test.""" + params = dict( + x_values=[1.0, 2.0], + y_values=[[10.0, 100.0], [20.0, 200.0]], + labels=["L1", "L2"], + x_units="FEET", + x_type="UNT", + y_units="CFS", + y_type="LINEAR", + path="/A/B/C/DATE/E/F/", + ) + params.update(overrides) + return PairedData.create(**params) + + # ================================================================== # + # to_csv (write) + # ================================================================== # + + def test_multiple_curves_to_csv(self): + """ + This test ensures that the data values for a PairedData object with multiple curves are correct. + This has no checks related to units or labels. + """ + num_curves: int = 3 + x_values: list[float] = [1.0, 1.5, 4.0] + y_values: list[list[float]] = [[x * i for i in range(num_curves)] for x in x_values] + labels: list[str] = [f"x times {i}" for i in range(num_curves)] + path: str = "/A/B/C///Source: I made it up/" + pd: PairedData = PairedData.create( + x_values=x_values, + y_values=y_values, + labels=labels, + path=path + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + pd.to_csv("fake_path.csv", with_metadata=True) + + mock_file.assert_called_once_with( + "fake_path.csv", "w", newline="", encoding="utf-8" + ) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + self.assertIn("A,,A", written_data) + self.assertIn("B,,B", written_data) + self.assertIn("C,,C", written_data) + self.assertIn("1,1.0,0.0,1.0,2.0", written_data) + self.assertIn("2,1.5,0.0,1.5,3.0", written_data) + self.assertIn("3,4.0,0.0,4.0,8.0", written_data) + + def test_to_csv_without_metadata(self): + num_curves: int = 3 + x_values: list[float] = [1.0, 1.5, 4.0] + y_values: list[list[float]] = [[x * i for i in range(num_curves)] for x in x_values] + labels: list[str] = [f"x times {i}" for i in range(num_curves)] + path: str = "/A/B/C///Source: I made it up/" + pd: PairedData = PairedData.create( + x_values=x_values, + y_values=y_values, + labels=labels, + path=path + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + pd.to_csv("fake_path.csv", with_metadata=False) + + mock_file.assert_called_once_with( + "fake_path.csv", "w", newline="", encoding="utf-8" + ) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + self.assertNotIn("A,,A", written_data) + self.assertNotIn("B,,B", written_data) + self.assertNotIn("C,,C", written_data) + self.assertNotIn("F,,Source: I made it up", written_data) + self.assertNotIn("Labels", written_data) + self.assertIn("Type", written_data) + self.assertIn("1,1.0,0.0,1.0,2.0", written_data) + self.assertIn("2,1.5,0.0,1.5,3.0", written_data) + self.assertIn("3,4.0,0.0,4.0,8.0", written_data) + + # def test_temp(self): + # file_path: str = "./tests/data/examples-all-data-types.dss" + # with HecDss(file_path) as dss: + # data_path: str = "/paired-data-multi-column/RIVERDALE/FREQ-FLOW/MAX ANALYTICAL//1969-01 H33(MAX)/" + # data: PairedData = dss.get(data_path) + # export_path: str = "./tests/csv_testing/paired_data_test.csv" + # data.to_csv(export_path) + + def test_to_csv_metadata_rows_present(self): + """Every metadata row (path parts, labels, units, type) is written.""" + written = self.write_pd_to_string(self.make_pd(), with_metadata=True) + self.assertIn("A,,A", written) + self.assertIn("B,,B", written) + self.assertIn("C,,C", written) + self.assertIn("E,,E", written) + self.assertIn("F,,F", written) + self.assertIn("Labels,,L1,L2", written) + self.assertIn("Units,FEET,CFS", written) + self.assertIn("Type,UNT,LINEAR", written) + + def test_to_csv_skips_d_part(self): + """The D (date) path part is dropped by convention; E is still written.""" + pd = self.make_pd(path="/A/B/C/DPART/EPART/FPART/") + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertNotIn("DPART", written) + self.assertIn("E,,EPART", written) + self.assertIn("F,,FPART", written) + + def test_to_csv_data_rows(self): + """Data rows are index, x, then the row-major y values.""" + written = self.write_pd_to_string(self.make_pd(), with_metadata=True) + self.assertIn("1,1.0,10.0,100.0", written) + self.assertIn("2,2.0,20.0,200.0", written) + + def test_to_csv_ordinate_counter_increments(self): + """The leading index column counts up from 1, one per ordinate.""" + pd = self.make_pd( + x_values=[5.0, 6.0, 7.0], + y_values=[[1.0], [2.0], [3.0]], + ) + written = self.write_pd_to_string(pd, with_metadata=False) + self.assertIn("1,5.0,1.0", written) + self.assertIn("2,6.0,2.0", written) + self.assertIn("3,7.0,3.0", written) + + def test_to_csv_single_curve(self): + """A single dependent column round-trips through the writer fine.""" + pd = self.make_pd(y_values=[[10.0], [20.0]], labels=["only"]) + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertIn("Labels,,only", written) + self.assertIn("1,1.0,10.0", written) + self.assertIn("2,2.0,20.0", written) + + def test_to_csv_rounds_to_four_decimals(self): + """x and y are rounded to ROUND_PRECISION (4) decimal places.""" + pd = self.make_pd(x_values=[1.23456], y_values=[[9.87654]], labels=["a"]) + written = self.write_pd_to_string(pd, with_metadata=False) + self.assertIn("1,1.2346,9.8765", written) + + def test_to_csv_negative_values(self): + """Negative x/y values are written verbatim.""" + pd = self.make_pd(x_values=[-1.0], y_values=[[-5.5, -0.25]], labels=["a", "b"]) + written = self.write_pd_to_string(pd, with_metadata=False) + self.assertIn("1,-1.0,-5.5,-0.25", written) + + def test_to_csv_empty_ordinates_writes_no_data_rows(self): + """No ordinates -> metadata is present but there are no numbered data rows.""" + # Labels avoid digits so the "no data row" check can't collide with a label. + pd = self.make_pd(x_values=[], y_values=[], labels=["aa", "bb"]) + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertIn("Units,FEET,CFS", written) + self.assertIn("Type,UNT,LINEAR", written) + self.assertNotIn("1,", written) # no data rows + + def test_to_csv_none_id_writes_empty_path_parts(self): + """A None id still emits empty A-F rows rather than raising.""" + pd = self.make_pd(path=None) + written = self.write_pd_to_string(pd, with_metadata=True) + for letter in ("A", "B", "C", "E", "F"): + self.assertIn(f"{letter},,", written) + + def test_to_csv_raises_on_non_paireddata(self): + """paired_data_to_csv rejects anything that isn't a PairedData.""" + with self.assertRaises(TypeError): + paired_data_to_csv("not a paired data", "fake.csv", True) + + # ================================================================== # + # read_csv (read) + # ================================================================== # + + def test_basic_read_csv(self): + content: str = ( + "A,,paired-data-test\n" + "B,,berkeley\n" + "C,,FREQ-FLOW\n" + "E,,\n" + "F,,Source: I made it up\n" + "Labels,,IMAGINARY,X,Y,Z\n" + "Units,DAYS,PPG\n" + "Type,TIME,POINTS\n" + "1,0,1,5,6,8\n" + "2,1,3,5,6,9\n" + "3,5,1,2,3,4\n" + ) + pd: PairedData = self.read_pd_from_string(content) + self.assertEqual(pd.labels, ['IMAGINARY', 'X', 'Y', 'Z']) + self.assertEqual(pd.units_independent, "DAYS") + self.assertEqual(pd.units_dependent, "PPG") + self.assertEqual(pd.type_independent, 'TIME') + self.assertEqual(pd.type_dependent, 'POINTS') + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0, 5.0]) + self.assertEqual(pd.values.tolist(), [ + [1.0, 5.0, 6.0, 8.0], + [3.0, 5.0, 6.0, 9.0], + [1.0, 2.0, 3.0, 4.0] + ]) + + def test_read_csv_single_curve(self): + content = ( + "Type,TIME,POINTS\n" + "1,0,10\n" + "2,1,20\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0], [20.0]]) + self.assertEqual(pd.curve_count(), 1) + + def test_read_csv_data_only_no_metadata(self): + """Data rows with no metadata rows still populate ordinates/values.""" + content = ( + "1,0,10,100\n" + "2,1,20,200\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + self.assertEqual(pd.labels, []) + self.assertEqual(pd.units_independent, "") + + def test_read_csv_builds_id_from_path_parts(self): + content = ( + "A,,SITE\n" + "B,,LOC\n" + "C,,STAGE-FLOW\n" + "E,,\n" + "F,,MADEUP\n" + "Type,STAGE,FLOW\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + # D is never present in a paired-data CSV, so both D and E are empty here. + self.assertEqual(pd.id, "/SITE/LOC/STAGE-FLOW///MADEUP/") + + def test_read_csv_curve_count(self): + content = ( + "Type,TIME,POINTS\n" + "1,0,1,2,3\n" + "2,1,4,5,6\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.curve_count(), 3) + + def test_read_csv_malformed_x_defaults_missing(self): + """A non-numeric x cell falls back to the missing-value default (0.0).""" + content = ( + "Type,TIME,POINTS\n" + "1,not-a-number,10,100\n" + "2,2,20,200\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE, 2.0]) + self.assertEqual(pd.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + + def test_read_csv_malformed_y_defaults_missing(self): + """A non-numeric y cell falls back to the default WITHOUT shortening the row, + so the value matrix stays rectangular.""" + content = ( + "Type,TIME,POINTS\n" + "1,1,bad,3\n" + "2,2,5,6\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE, 3.0], [5.0, 6.0]]) + + def test_read_csv_empty_cells_default_missing(self): + """Blank x and y cells become the missing-value default (0.0).""" + content = ( + "Type,TIME,POINTS\n" + "1,,\n" + "2,2,\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE, 2.0]) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE], [DEFAULT_MISSING_VALUE]]) + + def test_read_csv_skips_short_rows(self): + """A data row with fewer than 3 columns is malformed and skipped.""" + content = ( + "Type,TIME,POINTS\n" + "1,5\n" # only 2 columns -> skipped + "2,2,9\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [2.0]) + self.assertEqual(pd.values.tolist(), [[9.0]]) + + def test_read_csv_strips_numeric_whitespace(self): + """Surrounding whitespace around numeric cells is stripped before parsing.""" + content = ( + "Type,TIME,POINTS\n" + "1, 1 , 2 , 3 \n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [1.0]) + self.assertEqual(pd.values.tolist(), [[2.0, 3.0]]) + + def test_read_csv_labels_preserve_whitespace(self): + """Labels are captured verbatim from row[2:] (not stripped).""" + content = ( + "Labels,, spacey ,X\n" + "Type,TIME,POINTS\n" + "1,0,1,2\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.labels, [" spacey ", "X"]) + + def test_read_csv_partial_units_row(self): + """A Units row missing the dependent column leaves y_units empty, no crash.""" + content = ( + "Units,DAYS\n" + "Type,TIME,POINTS\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "DAYS") + self.assertEqual(pd.units_dependent, "") + + def test_read_csv_partial_type_row(self): + """A Type row missing the dependent column leaves y_type empty, no crash.""" + content = ( + "Type,TIME\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.type_independent, "TIME") + self.assertEqual(pd.type_dependent, "") + + def test_read_csv_units_first_column_only(self): + """A bare 'Units' row (no values at all) is ignored gracefully.""" + content = ( + "Units\n" + "Type,TIME,POINTS\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "") + self.assertEqual(pd.units_dependent, "") + self.assertEqual(pd.ordinates.tolist(), [0.0]) + + def test_read_csv_empty_file(self): + pd = self.read_pd_from_string("") + self.assertEqual(pd.ordinates.tolist(), []) + self.assertEqual(pd.values.tolist(), []) + self.assertEqual(pd.labels, []) + + def test_read_csv_blank_lines_ignored(self): + content = ( + "Type,TIME,POINTS\n" + "\n" + "1,0,10\n" + "\n" + "2,1,20\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0], [20.0]]) + + def test_read_csv_ragged_rows_raise(self): + """Genuinely ragged data (rows with different column counts) can't form a + rectangular value matrix, so PairedData.create raises when numpy builds it.""" + content = ( + "Type,TIME,POINTS\n" + "1,0,1,2\n" # 2 y-values + "2,1,3\n" # 1 y-value + ) + with self.assertRaises(ValueError): + self.read_pd_from_string(content) + + def test_read_csv_wrong_cls_raises(self): + """paired_data_read_csv only builds PairedData; other classes are rejected.""" + with self.assertRaises(TypeError): + paired_data_read_csv(RegularTimeSeries, "fake.csv") + + # ================================================================== # + # round trips (write -> read on a real temp file) + # ================================================================== # + + def test_round_trip_basic(self): + path = self.test_files.create_test_file(".csv") + pd = self.make_pd() + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [1.0, 2.0]) + self.assertEqual(result.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + self.assertEqual(result.labels, ["L1", "L2"]) + self.assertEqual(result.units_independent, "FEET") + self.assertEqual(result.units_dependent, "CFS") + self.assertEqual(result.type_independent, "UNT") + self.assertEqual(result.type_dependent, "LINEAR") + # The D (date) part is dropped by the writer, so it comes back empty. + self.assertEqual(result.id, "/A/B/C//E/F/") + + def test_round_trip_multiple_curves(self): + path = self.test_files.create_test_file(".csv") + pd = self.make_pd( + x_values=[0.0, 1.0, 2.0], + y_values=[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], + labels=["a", "b", "c"], + ) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.curve_count(), 3) + self.assertEqual( + result.values.tolist(), + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], + ) + self.assertEqual(result.labels, ["a", "b", "c"]) + + def test_round_trip_without_metadata(self): + """Without metadata, units/labels/id are lost, but data and (because the + Type row is written unconditionally) the types survive.""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd() + pd.to_csv(path, with_metadata=False) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [1.0, 2.0]) + self.assertEqual(result.values.tolist(), [[10.0, 100.0], [20.0, 200.0]]) + self.assertEqual(result.labels, []) + self.assertEqual(result.units_independent, "") + self.assertEqual(result.units_dependent, "") + self.assertEqual(result.id, "///////") + # Type row is always written, so type survives the no-metadata round trip. + self.assertEqual(result.type_independent, "UNT") + self.assertEqual(result.type_dependent, "LINEAR") + + def test_round_trip_rounding(self): + """Values are rounded to 4 decimals on write, and that rounded value is + exactly what is read back.""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd(x_values=[1.23456], y_values=[[2.34567]], labels=["a"]) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [1.2346]) + self.assertEqual(result.values.tolist(), [[2.3457]]) + + def test_round_trip_negative_values(self): + path = self.test_files.create_test_file(".csv") + pd = self.make_pd( + x_values=[-1.0, 0.0, 1.0], + y_values=[[-5.5], [0.0], [5.5]], + labels=["a"], + ) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + + self.assertEqual(result.ordinates.tolist(), [-1.0, 0.0, 1.0]) + self.assertEqual(result.values.tolist(), [[-5.5], [0.0], [5.5]]) + + # ================================================================== # + # weird / adversarial edge cases + # + # These probe surprising inputs. Tests that DOCUMENT current behavior + # (even when that behavior is a footgun) pass; two known bugs are pinned + # with @unittest.expectedFailure so they stay visible until fixed. + # ================================================================== # + + def test_read_csv_quoted_label_with_comma_round_trips(self): + """A label containing a comma is CSV-quoted on write and comes back intact.""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd( + x_values=[1.0], y_values=[[2.0]], labels=["flow, cfs"], + ) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + self.assertEqual(result.labels, ["flow, cfs"]) + + def test_read_csv_metadata_after_data_still_applies(self): + """The reader is order-independent: a Units row placed AFTER the data rows + still populates the units (nothing forces metadata to come first).""" + content = ( + "1,0,10\n" + "Units,DAYS,PPG\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "DAYS") + self.assertEqual(pd.units_dependent, "PPG") + self.assertEqual(pd.values.tolist(), [[10.0]]) + + def test_read_csv_ignores_counter_column(self): + """The leading index column is never validated on read - non-sequential or + even non-numeric counters are ignored entirely (x comes from column 2).""" + content = ( + "Type,,\n" + "999,0,10\n" + "hello,1,20\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [0.0, 1.0]) + self.assertEqual(pd.values.tolist(), [[10.0], [20.0]]) + + def test_read_csv_duplicate_units_last_wins(self): + """When a keyword row appears twice, the last occurrence overwrites.""" + content = ( + "Units,AA,BB\n" + "Units,CC,DD\n" + "Type,,\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.units_independent, "CC") + self.assertEqual(pd.units_dependent, "DD") + + def test_read_csv_whitespace_only_cells_default_missing(self): + """Cells that are only whitespace strip to empty and become the default.""" + content = ( + "Type,,\n" + "1, , \n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE]) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE]]) + + def test_read_csv_inf_and_nan_parse_through(self): + """float() accepts 'inf'/'nan', so these land in the array as real inf/NaN + rather than being treated as missing. Surprising, given the missing-value + default is 0.0 - documented here so a future change is caught.""" + content = ( + "Type,,\n" + "1,inf,nan\n" + "2,-inf,5\n" + ) + pd = self.read_pd_from_string(content) + self.assertTrue(math.isinf(pd.ordinates[0])) + self.assertTrue(math.isnan(pd.values[0][0])) + self.assertTrue(math.isinf(pd.ordinates[1]) and pd.ordinates[1] < 0) + + def test_read_csv_miscased_keyword_becomes_phantom_data_row(self): + """FOOTGUN: keyword matching is case-sensitive, so a mis-cased 'units' row + is not recognized as metadata. It falls through to the data branch, its + text cells fail to parse, and it is silently injected as a (0.0, [0.0]) + data point instead of raising or being ignored.""" + content = ( + "units,DAYS,PPG\n" # lowercase -> NOT recognized as a Units row + "Type,TIME,POINTS\n" + "1,5,10\n" + ) + pd = self.read_pd_from_string(content) + # The real units were never captured... + self.assertEqual(pd.units_independent, "") + # ...and a bogus leading data point was manufactured from the keyword row. + self.assertEqual(pd.ordinates.tolist(), [DEFAULT_MISSING_VALUE, 5.0]) + self.assertEqual(pd.values.tolist(), [[DEFAULT_MISSING_VALUE], [10.0]]) + + def test_read_csv_extra_metadata_column_grabs_last_cell(self): + """FOOTGUN: path parts are read via the 'last cell' convention (row[-1]), + so a metadata row with an unexpected trailing column captures the trailing + value instead of the intended one.""" + content = ( + "A,,SITE,EXTRA\n" # intended A=SITE, but row[-1] is 'EXTRA' + "Type,,\n" + "1,0,10\n" + ) + pd = self.read_pd_from_string(content) + self.assertIn("EXTRA", pd.id) + self.assertNotIn("SITE", pd.id) + + def test_curve_count_on_dataless_read_should_be_zero(self): + """BUG: reading a CSV with metadata but no data rows yields an empty + PairedData, and curve_count() then does len(self.values[0]) -> IndexError. + It should return 0. Pinned as expectedFailure until curve_count guards + against an empty value array.""" + content = ( + "Units,DAYS,PPG\n" + "Type,TIME,POINTS\n" + ) + pd = self.read_pd_from_string(content) + self.assertEqual(pd.curve_count(), 0) + + def test_to_csv_missing_value_does_not_crash(self): + """A blank/malformed numeric cell is read back as DEFAULT_MISSING_VALUE + (None), producing an object-dtype array. The writer must tolerate a None + ordinate/value (write an empty cell) rather than crash trying to round it.""" + pd = self.read_pd_from_string( + "Type,TIME,POINTS\n" + "1,,10\n" # blank ordinate -> None + "2,2,20\n" + ) + written = self.write_pd_to_string(pd, with_metadata=True) + self.assertIn("1,,10.0", written) + self.assertIn("2,2.0,20.0", written) + + def test_round_trip_nan_value_survives(self): + """A NaN dependent value is written as 'nan' and read back as a real NaN + (it is NOT collapsed into the missing-value default).""" + path = self.test_files.create_test_file(".csv") + pd = self.make_pd(x_values=[1.0], y_values=[[float("nan")]], labels=["a"]) + pd.to_csv(path, with_metadata=True) + result = PairedData.read_csv(path) + self.assertTrue(math.isnan(result.values.tolist()[0][0])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_csv.py b/tests/test_ts_csv.py similarity index 78% rename from tests/test_csv.py rename to tests/test_ts_csv.py index 4f783a9..29886d5 100644 --- a/tests/test_csv.py +++ b/tests/test_ts_csv.py @@ -2,9 +2,12 @@ from datetime import datetime from unittest.mock import mock_open, patch +import numpy as np + from file_manager import FileManager from hecdss import HecDss +from hecdss.dss_csv import DEFAULT_MISSING_VALUE from hecdss.irregular_timeseries import IrregularTimeSeries from hecdss.regular_timeseries import RegularTimeSeries @@ -65,7 +68,6 @@ def test_to_csv_without_metadata(self): handle = mock_file() written = "".join(call.args[0] for call in handle.write.call_args_list) self.assertNotIn("Units", written) - self.assertNotIn("Type,Date/Time", written) self.assertIn("1,01Sep2021 0600,1.0", written) def test_to_csv_empty_times(self): @@ -174,7 +176,7 @@ def test_read_csv_with_quality(self): self.assertEqual(rts.quality, [0, 5]) def test_read_csv_with_partial_quality(self): - content: tuple[str] = ( + content: str = ( "Type,Date/Time,INST-VAL,Quality\n" "1,05Nov2004 0200,8,0\n" "2,05Nov2004 0300,9\n" # missing quality! @@ -186,7 +188,7 @@ def test_read_csv_with_partial_quality(self): self.assertEqual(rts.quality[0], 0) self.assertEqual(rts.quality[2], 1) - def test_read_csv_skips_malformed_rows(self): + def test_read_csv_skips_malformed_date_rows(self): content = ( "Type,Date/Time,INST-VAL\n" "1,01Sep2021 0600,10.5\n" @@ -195,9 +197,10 @@ def test_read_csv_skips_malformed_rows(self): "4,01Sep2021 1800,30.0\n" ) rts = self.read_rts_from_string(content) - self.assertEqual(rts.values.tolist(), [10.5, 30.0]) + self.assertEqual(rts.values.tolist(), [10.5, DEFAULT_MISSING_VALUE, 30.0]) self.assertEqual( - rts.times, [datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 18, 0)] + rts.times, [datetime(2021, 9, 1, 6, 0), datetime( + 2021, 9, 1, 12, 0), datetime(2021, 9, 1, 18, 0)] ) def test_read_csv_seconds_precision_basic(self): @@ -352,7 +355,6 @@ def test_round_trip_without_metadata_infers_interval_from_times(self): result = RegularTimeSeries.read_csv(path) self.assertEqual(result.units, "") - self.assertEqual(result.data_type, "") self.assertEqual(result.interval, 21600) self.assertEqual(result.id, "/////6Hour//") self.assertEqual(result.values.tolist(), [1.0, 2.0]) @@ -423,7 +425,6 @@ def test_irregular_to_csv_without_metadata(self): handle = mock_file() written = "".join(call.args[0] for call in handle.write.call_args_list) self.assertNotIn("Units", written) - self.assertNotIn("Type,Date/Time", written) self.assertIn("1,01Sep2021 0000,10.5", written) self.assertIn("2,02Sep2021 0000,20.0", written) self.assertIn("3,04Sep2021 0000,42.0", written) @@ -542,6 +543,125 @@ def test_year_is_2400(self): self.assertEqual(its.values.tolist(), [1, 1]) self.assertEqual(its.times, [datetime(2400, 9, 1, 0, 0, 0), datetime(2400, 9, 2, 0, 0, 0)]) + def test_read_write_with_missing(self): + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600,10.5\n" + "2,not-a-date,20.0\n" + "3,01Sep2021 1200,not-a-number\n" + "4,01Sep2021 1800,30.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, DEFAULT_MISSING_VALUE, 30.0]) + self.assertEqual( + rts.times, [datetime(2021, 9, 1, 6, 0), datetime( + 2021, 9, 1, 12, 0), datetime(2021, 9, 1, 18, 0)] + ) + + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake_path.csv", with_metadata=True) + + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL", written_data) + self.assertIn("2,01Sep2021 1200,\r\n", written_data) + + def test_to_csv_writes_empty_cell_for_missing_value(self): + """A missing (None) value is written as an empty cell, never the literal + text 'None'. (Isolates the write path with an injected None, independent + of what the reader produces.)""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.values = np.array([1.0, None], dtype=object) # simulate a missing value + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("2,01Sep2021 1200,\r\n", written) + self.assertNotIn("None", written) + + def test_to_csv_quality_shorter_than_values_truncates(self): + """FOOTGUN: to_csv takes the with-quality branch whenever quality is + non-empty, then zips (times, values, quality). A quality list shorter than + values makes zip stop at the shortest input, so trailing data points are + silently dropped from the CSV.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0, 3.0], + times=[ + datetime(2021, 9, 1, 6, 0), + datetime(2021, 9, 1, 12, 0), + datetime(2021, 9, 1, 18, 0), + ], + quality=[0], # only one flag for three values + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("1,01Sep2021 0600,1.0,0", written) + self.assertNotIn("2,01Sep2021 1200", written) # silently dropped + self.assertNotIn("3,01Sep2021 1800", written) # silently dropped + + def test_read_csv_skips_short_data_row(self): + """A data row with fewer than 3 columns is malformed and skipped without + raising (parity with the paired-data reader's short-row handling).""" + content = ( + "Type,Date/Time,INST-VAL\n" + "1,01Sep2021 0600\n" # only 2 columns -> skipped + "2,01Sep2021 1200,20.0\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [20.0]) + self.assertEqual(rts.times, [datetime(2021, 9, 1, 12, 0)]) + + def test_read_csv_metadata_only_no_data_rows(self): + """Metadata rows with zero data rows yield an empty series (no crash); + units and the E interval are still captured.""" + content = ( + "A,,,A\n" + "E,,,6Hour\n" + "Units,,,CFS\n" + "Type,Date/Time,INST-VAL\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), []) + self.assertEqual(rts.times, []) + self.assertEqual(rts.units, "CFS") + self.assertEqual(rts.interval, 21600) + + def test_round_trip_irregular(self): + """Full write->read round trip for IrregularTimeSeries on a real temp file: + irregular gaps, units, data_type and id all survive.""" + path = self.test_files.create_test_file(".csv") + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//IR-Year/F/", + ) + its.to_csv(path, with_metadata=True) + result = IrregularTimeSeries.read_csv(path) + self.assertEqual(result.values.tolist(), [10.5, 20.0, 42.0]) + self.assertEqual( + result.times, + [datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + ) + self.assertEqual(result.units, "CFS") + self.assertEqual(result.data_type, "INST-VAL") + self.assertEqual(result.id, "/A/B/C//IR-Year/F/") + if __name__ == "__main__": unittest.main()