From 7c59f23c5314a2abf50a49cba2a7632edb317e6b Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 26 Feb 2026 20:19:38 -0500 Subject: [PATCH 1/6] Add pytest test suite for tinyml-modelmaker core modules Tests cover ConfigDict, constants (timeseries + vision), dataset_utils, misc_utils (resolve_paths, resolve_run_name, simplify_dict). Adds importorskip for test_protocols.py so it gracefully skips without TVM. 47 tests pass, 1 skipped on macOS (no TVM). Co-Authored-By: Claude Opus 4.6 --- tinyml-modelmaker/tests/__init__.py | 0 tinyml-modelmaker/tests/test_config_dict.py | 46 ++++++ tinyml-modelmaker/tests/test_constants.py | 153 ++++++++++++++++++ tinyml-modelmaker/tests/test_dataset_utils.py | 81 ++++++++++ tinyml-modelmaker/tests/test_misc_utils.py | 127 +++++++++++++++ tinyml-modelmaker/tests/test_protocols.py | 145 +++++++++++++++++ 6 files changed, 552 insertions(+) create mode 100644 tinyml-modelmaker/tests/__init__.py create mode 100644 tinyml-modelmaker/tests/test_config_dict.py create mode 100644 tinyml-modelmaker/tests/test_constants.py create mode 100644 tinyml-modelmaker/tests/test_dataset_utils.py create mode 100644 tinyml-modelmaker/tests/test_misc_utils.py create mode 100644 tinyml-modelmaker/tests/test_protocols.py diff --git a/tinyml-modelmaker/tests/__init__.py b/tinyml-modelmaker/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tinyml-modelmaker/tests/test_config_dict.py b/tinyml-modelmaker/tests/test_config_dict.py new file mode 100644 index 00000000..1372f544 --- /dev/null +++ b/tinyml-modelmaker/tests/test_config_dict.py @@ -0,0 +1,46 @@ +"""Tests for ConfigDict.""" + +import os +import tempfile + +import pytest +import yaml + +from tinyml_modelmaker.utils.config_dict import ConfigDict + + +class TestConfigDict: + def test_from_dict(self): + d = {"a": 1, "b": {"c": 2}} + cfg = ConfigDict(d) + assert cfg.a == 1 + assert cfg.b.c == 2 + + def test_from_yaml(self, tmp_path): + data = {"x": 10, "nested": {"y": 20}} + yaml_file = tmp_path / "test.yaml" + yaml_file.write_text(yaml.dump(data)) + cfg = ConfigDict(str(yaml_file)) + assert cfg.x == 10 + assert cfg.nested.y == 20 + + def test_invalid_extension_raises(self, tmp_path): + txt_file = tmp_path / "test.txt" + txt_file.write_text("hello") + with pytest.raises(ValueError, match="unrecognized file type"): + ConfigDict(str(txt_file)) + + def test_invalid_input_raises(self): + with pytest.raises(TypeError, match="got invalid input"): + ConfigDict(12345) + + def test_update(self): + cfg = ConfigDict({"a": 1, "b": 2}) + cfg.update({"b": 3, "c": 4}) + assert cfg.b == 3 + assert cfg.c == 4 + + def test_none_input(self): + cfg = ConfigDict(None) + # Should create an empty config without error + assert isinstance(cfg, ConfigDict) diff --git a/tinyml-modelmaker/tests/test_constants.py b/tinyml-modelmaker/tests/test_constants.py new file mode 100644 index 00000000..856e8d65 --- /dev/null +++ b/tinyml-modelmaker/tests/test_constants.py @@ -0,0 +1,153 @@ +"""Tests for timeseries and vision constants modules.""" + +import pytest + +from tinyml_modelmaker.ai_modules.timeseries import constants as ts_constants +from tinyml_modelmaker.ai_modules.vision import constants as vis_constants + + +class TestTimeseriesConstants: + """Verify structural invariants of the timeseries constants module.""" + + def test_task_types_list_not_empty(self): + assert len(ts_constants.TASK_TYPES) > 0 + + def test_all_task_types_mapped_to_category(self): + for task_type in ts_constants.TASK_TYPES: + assert task_type in ts_constants.TASK_TYPE_TO_CATEGORY, ( + f"{task_type} missing from TASK_TYPE_TO_CATEGORY" + ) + + def test_task_categories_list_not_empty(self): + assert len(ts_constants.TASK_CATEGORIES) > 0 + + def test_target_devices_all_includes_base_and_additional(self): + for dev in ts_constants.TARGET_DEVICES: + assert dev in ts_constants.TARGET_DEVICES_ALL + for dev in ts_constants.TARGET_DEVICES_ADDITIONAL: + assert dev in ts_constants.TARGET_DEVICES_ALL + + def test_training_device_constants(self): + assert ts_constants.TRAINING_DEVICE_CPU == "cpu" + assert ts_constants.TRAINING_DEVICE_CUDA == "cuda" + assert ts_constants.TRAINING_DEVICE_MPS == "mps" + assert ts_constants.TRAINING_DEVICE_GPU == ts_constants.TRAINING_DEVICE_CUDA + + def test_training_backend_constant(self): + assert ts_constants.TRAINING_BACKEND_TINYML_TINYVERSE == "tinyml_tinyverse" + + def test_data_dir_constants(self): + assert ts_constants.DATA_DIR_CLASSES == "classes" + assert ts_constants.DATA_DIR_FILES == "files" + + +class TestGetTaskCategory: + """Tests for get_task_category().""" + + def test_known_task_type(self): + cat = ts_constants.get_task_category(ts_constants.TASK_TYPE_MOTOR_FAULT) + assert cat == ts_constants.TASK_CATEGORY_TS_CLASSIFICATION + + def test_task_category_passed_through(self): + cat = ts_constants.get_task_category(ts_constants.TASK_CATEGORY_TS_REGRESSION) + assert cat == ts_constants.TASK_CATEGORY_TS_REGRESSION + + def test_unknown_task_type_defaults(self): + cat = ts_constants.get_task_category("unknown_task") + assert cat == ts_constants.TASK_CATEGORY_TS_CLASSIFICATION + + +class TestGetDefaultDataDir: + """Tests for get_default_data_dir_for_task().""" + + def test_classification_returns_classes(self): + assert ( + ts_constants.get_default_data_dir_for_task( + ts_constants.TASK_CATEGORY_TS_CLASSIFICATION + ) + == ts_constants.DATA_DIR_CLASSES + ) + + def test_regression_returns_files(self): + assert ( + ts_constants.get_default_data_dir_for_task( + ts_constants.TASK_CATEGORY_TS_REGRESSION + ) + == ts_constants.DATA_DIR_FILES + ) + + def test_forecasting_returns_files(self): + assert ( + ts_constants.get_default_data_dir_for_task( + ts_constants.TASK_CATEGORY_TS_FORECASTING + ) + == ts_constants.DATA_DIR_FILES + ) + + def test_anomaly_returns_classes(self): + assert ( + ts_constants.get_default_data_dir_for_task( + ts_constants.TASK_CATEGORY_TS_ANOMALYDETECTION + ) + == ts_constants.DATA_DIR_CLASSES + ) + + def test_unknown_category_returns_classes(self): + assert ts_constants.get_default_data_dir_for_task("something_else") == ts_constants.DATA_DIR_CLASSES + + def test_vision_returns_classes(self): + # Vision module always returns 'classes' + assert vis_constants.get_default_data_dir_for_task("image_classification") == "classes" + + +class TestGetSkipNormalizeAndOutputInt: + """Tests for get_skip_normalize_and_output_int().""" + + def test_no_quantization(self): + skip, out_int = ts_constants.get_skip_normalize_and_output_int( + ts_constants.TASK_CATEGORY_TS_CLASSIFICATION, 0, False + ) + assert skip is False + assert out_int is False + + def test_quantized_classification(self): + skip, out_int = ts_constants.get_skip_normalize_and_output_int( + ts_constants.TASK_CATEGORY_TS_CLASSIFICATION, 1, False + ) + assert skip is True + assert out_int is True + + def test_quantized_regression(self): + skip, out_int = ts_constants.get_skip_normalize_and_output_int( + ts_constants.TASK_CATEGORY_TS_REGRESSION, 1, False + ) + assert skip is True + assert out_int is False + + def test_quantized_regression_partial(self): + skip, out_int = ts_constants.get_skip_normalize_and_output_int( + ts_constants.TASK_CATEGORY_TS_REGRESSION, 1, True + ) + assert skip is False # partial quantization disables skip_normalize for regression + assert out_int is False + + def test_quantized_forecasting(self): + skip, out_int = ts_constants.get_skip_normalize_and_output_int( + ts_constants.TASK_CATEGORY_TS_FORECASTING, 2, False + ) + assert skip is True + assert out_int is False + + +class TestVisionConstants: + """Verify structural invariants of the vision constants module.""" + + def test_task_types_not_empty(self): + assert len(vis_constants.TASK_TYPES) > 0 + + def test_target_devices_all(self): + for dev in vis_constants.TARGET_DEVICES: + assert dev in vis_constants.TARGET_DEVICES_ALL + + def test_training_backend_constant(self): + assert vis_constants.TRAINING_BACKEND_TINYML_TINYVERSE == "tinyml_tinyverse" diff --git a/tinyml-modelmaker/tests/test_dataset_utils.py b/tinyml-modelmaker/tests/test_dataset_utils.py new file mode 100644 index 00000000..e098711d --- /dev/null +++ b/tinyml-modelmaker/tests/test_dataset_utils.py @@ -0,0 +1,81 @@ +"""Tests for dataset_utils: file listing and split creation.""" + +import os +import tempfile + +import pytest + +from tinyml_modelmaker.ai_modules.common.datasets import dataset_utils + + +class TestCreateFilelist: + """Tests for create_filelist().""" + + def test_basic_listing(self, tmp_path): + """Creates a simple directory tree and verifies the generated file list.""" + data_dir = tmp_path / "data" + (data_dir / "classA").mkdir(parents=True) + (data_dir / "classB").mkdir(parents=True) + (data_dir / "classA" / "file1.csv").write_text("a") + (data_dir / "classA" / "file2.csv").write_text("b") + (data_dir / "classB" / "file3.csv").write_text("c") + + out_dir = str(tmp_path / "output") + result = dataset_utils.create_filelist(str(data_dir), out_dir, ignore_str_list=[]) + assert os.path.isfile(result) + + with open(result) as f: + lines = [l.strip() for l in f if l.strip()] + assert len(lines) == 3 + + def test_ignore_pattern(self, tmp_path): + """Verify files matching the ignore pattern are excluded.""" + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "keep.csv").write_text("k") + (data_dir / "skip.log").write_text("s") + + out_dir = str(tmp_path / "output") + result = dataset_utils.create_filelist(str(data_dir), out_dir, ignore_str_list=[r"\.log$"]) + with open(result) as f: + lines = [l.strip() for l in f if l.strip()] + assert len(lines) == 1 + assert "keep.csv" in lines[0] + + +class TestCreateInterFileSplit: + """Tests for create_inter_file_split() input validation.""" + + def test_invalid_split_list_type(self, tmp_path): + fl = tmp_path / "file_list.txt" + fl.write_text("a.csv\nb.csv\n") + with pytest.raises(TypeError, match="tuple or list"): + dataset_utils.create_inter_file_split( + str(fl), "not_a_list", 0.8 + ) + + def test_split_factor_too_large(self, tmp_path): + fl = tmp_path / "file_list.txt" + fl.write_text("a.csv\nb.csv\n") + split_files = (str(tmp_path / "train.txt"), str(tmp_path / "val.txt")) + with pytest.raises(ValueError, match="less than 1"): + dataset_utils.create_inter_file_split(str(fl), split_files, 1.5) + + def test_split_factor_list_sum_too_large(self, tmp_path): + fl = tmp_path / "file_list.txt" + fl.write_text("a.csv\nb.csv\n") + split_files = (str(tmp_path / "train.txt"), str(tmp_path / "val.txt")) + with pytest.raises(ValueError, match="<=1"): + dataset_utils.create_inter_file_split(str(fl), split_files, [0.8, 0.5]) + + +class TestCreateIntraFileSplit: + """Tests for create_intra_file_split() input validation.""" + + def test_invalid_split_list_type(self, tmp_path): + fl = tmp_path / "file_list.txt" + fl.write_text("a.csv\n") + with pytest.raises(TypeError, match="tuple or list"): + dataset_utils.create_intra_file_split( + str(fl), "not_a_list", 0.8, "data", str(tmp_path), ("train",) + ) diff --git a/tinyml-modelmaker/tests/test_misc_utils.py b/tinyml-modelmaker/tests/test_misc_utils.py new file mode 100644 index 00000000..7bf27efd --- /dev/null +++ b/tinyml-modelmaker/tests/test_misc_utils.py @@ -0,0 +1,127 @@ +"""Tests for misc_utils: resolve_paths, resolve_run_name, symlink helpers.""" + +import os +import re +import tempfile + +import pytest + +from tinyml_modelmaker.utils import misc_utils +from tinyml_modelmaker.utils.config_dict import ConfigDict + + +class TestResolvePaths: + """Tests for the extracted resolve_paths() function.""" + + @staticmethod + def _make_params(**overrides): + """Build a minimal params ConfigDict for testing.""" + base = dict( + common=dict( + projects_path="./projects", + project_path=None, + project_run_path=None, + run_name="{date-time}/{model_name}", + target_device="F28P55", + ), + dataset=dict( + input_data_path="./data/input", + input_annotation_path=None, + dataset_name="", + dataset_path=None, + extract_path=None, + ), + training=dict( + enable=True, + model_name="TestModel", + train_output_path=None, + training_path=None, + training_path_quantization=None, + model_packaged_path=None, + quantization=0, # NO_QUANTIZATION + ), + compilation=dict( + enable=False, + compile_output_path=None, + compilation_path=None, + model_packaged_path=None, + ), + ) + for section, vals in overrides.items(): + base[section].update(vals) + return ConfigDict(base) + + def test_default_paths(self): + params = self._make_params() + target_devices = ["F28P55", "F280015"] + misc_utils.resolve_paths(params, target_devices) + + assert os.path.isabs(params.common.projects_path) + assert params.dataset.dataset_name == "input" # basename of input_data_path + assert params.dataset.dataset_path.endswith(os.path.join("input", "dataset")) + assert "run" in params.common.project_run_path + assert "training" in params.training.training_path + + def test_train_output_path(self, tmp_path): + params = self._make_params( + training=dict(train_output_path=str(tmp_path / "output")), + ) + target_devices = ["F28P55"] + misc_utils.resolve_paths(params, target_devices) + + assert params.dataset.dataset_path == os.path.join(str(tmp_path / "output"), "dataset") + assert params.training.training_path.endswith("training_base") + assert params.common.project_run_path == str(tmp_path / "output") + + def test_invalid_target_device(self): + params = self._make_params(common=dict(target_device="INVALID")) + with pytest.raises(ValueError, match="must be set to one of"): + misc_utils.resolve_paths(params, ["F28P55"]) + + def test_dataset_name_fallback(self): + params = self._make_params( + dataset=dict(input_data_path="/some/path/my_dataset.zip", dataset_name=""), + ) + misc_utils.resolve_paths(params, ["F28P55"]) + assert params.dataset.dataset_name == "my_dataset" + + def test_compile_output_path(self, tmp_path): + params = self._make_params( + training=dict(enable=False), + compilation=dict(enable=True, compile_output_path=str(tmp_path / "compile_out")), + ) + misc_utils.resolve_paths(params, ["F28P55"]) + assert params.compilation.compilation_path == str(tmp_path / "compile_out") + assert params.compilation.model_packaged_path.endswith("_F28P55.zip") + + +class TestResolveRunName: + def test_date_time_placeholder(self): + result = misc_utils.resolve_run_name("{date-time}/model", "MyModel") + # Should have replaced {date-time} with a timestamp + assert "{date-time}" not in result + assert re.match(r"\d{8}-\d{6}/model", result) + + def test_model_name_placeholder(self): + result = misc_utils.resolve_run_name("run/{model_name}", "MyModel") + assert result == "run/MyModel" + + def test_empty_run_name(self): + assert misc_utils.resolve_run_name("", "Model") == "" + assert misc_utils.resolve_run_name(None, "Model") == "" + + def test_both_placeholders(self): + result = misc_utils.resolve_run_name("{date-time}/{model_name}", "TestNet") + assert "TestNet" in result + assert "{date-time}" not in result + + +class TestSimplifyDict: + def test_valid_dict(self): + d = {"a": 1, "b": {"c": [1, 2, 3]}} + result = misc_utils.simplify_dict(d) + assert result == {"a": 1, "b": {"c": [1, 2, 3]}} + + def test_invalid_input(self): + with pytest.raises(TypeError, match="must be of type dict"): + misc_utils.simplify_dict("not a dict") diff --git a/tinyml-modelmaker/tests/test_protocols.py b/tinyml-modelmaker/tests/test_protocols.py new file mode 100644 index 00000000..64652d67 --- /dev/null +++ b/tinyml-modelmaker/tests/test_protocols.py @@ -0,0 +1,145 @@ +"""Tests that concrete classes satisfy the Protocol definitions in ai_modules.protocols. + +Uses ``@runtime_checkable`` isinstance checks to verify that every component +implementation provides the methods required by its corresponding protocol. + +Requires TVM (compilation backend) which is not available on all platforms. +""" + +import pytest + +# TVM is required transitively via tinyml_benchmark → compilation.py +tvm = pytest.importorskip("tvm", reason="TVM not available on this platform") + +from tinyml_modelmaker.ai_modules.protocols import ( + Compiler, + DatasetHandler, + LifecycleComponent, + Runner, + Trainer, +) + + +# --------------------------------------------------------------------------- +# Concrete class imports (all mocked via conftest.py) +# --------------------------------------------------------------------------- + +from tinyml_modelmaker.ai_modules.timeseries.runner import ( + ModelRunner as TimeseriesModelRunner, +) +from tinyml_modelmaker.ai_modules.vision.runner import ( + ModelRunner as VisionModelRunner, +) +from tinyml_modelmaker.ai_modules.timeseries.training.tinyml_tinyverse.timeseries_classification import ( + ModelTraining as TSClassificationTraining, +) +from tinyml_modelmaker.ai_modules.timeseries.training.tinyml_tinyverse.timeseries_regression import ( + ModelTraining as TSRegressionTraining, +) +from tinyml_modelmaker.ai_modules.timeseries.training.tinyml_tinyverse.timeseries_anomalydetection import ( + ModelTraining as TSAnomalyDetectionTraining, +) +from tinyml_modelmaker.ai_modules.timeseries.training.tinyml_tinyverse.timeseries_forecasting import ( + ModelTraining as TSForecastingTraining, +) +from tinyml_modelmaker.ai_modules.vision.training.tinyml_tinyverse.image_classification import ( + ModelTraining as VisionClassificationTraining, +) +from tinyml_modelmaker.ai_modules.common.compilation.tinyml_benchmark import ( + ModelCompilation, +) +from tinyml_modelmaker.ai_modules.common.datasets import DatasetHandling + + +# =================================================================== +# Protocol conformance tests +# =================================================================== + + +class TestRunnerProtocol: + """Verify that ModelRunner classes satisfy the Runner protocol.""" + + def test_timeseries_runner_satisfies_runner_protocol(self): + assert issubclass(TimeseriesModelRunner, Runner) + + def test_vision_runner_satisfies_runner_protocol(self): + assert issubclass(VisionModelRunner, Runner) + + def test_timeseries_runner_satisfies_lifecycle_protocol(self): + assert issubclass(TimeseriesModelRunner, LifecycleComponent) + + def test_vision_runner_satisfies_lifecycle_protocol(self): + assert issubclass(VisionModelRunner, LifecycleComponent) + + +class TestTrainerProtocol: + """Verify that ModelTraining classes satisfy the Trainer protocol.""" + + def test_timeseries_classification_satisfies_trainer_protocol(self): + assert issubclass(TSClassificationTraining, Trainer) + + def test_timeseries_regression_satisfies_trainer_protocol(self): + assert issubclass(TSRegressionTraining, Trainer) + + def test_timeseries_anomalydetection_satisfies_trainer_protocol(self): + assert issubclass(TSAnomalyDetectionTraining, Trainer) + + def test_timeseries_forecasting_satisfies_trainer_protocol(self): + assert issubclass(TSForecastingTraining, Trainer) + + def test_vision_classification_satisfies_trainer_protocol(self): + assert issubclass(VisionClassificationTraining, Trainer) + + +class TestCompilerProtocol: + """Verify that ModelCompilation satisfies the Compiler protocol.""" + + def test_compilation_satisfies_compiler_protocol(self): + assert issubclass(ModelCompilation, Compiler) + + def test_compilation_satisfies_lifecycle_protocol(self): + assert issubclass(ModelCompilation, LifecycleComponent) + + +class TestDatasetHandlerProtocol: + """Verify that DatasetHandling satisfies the DatasetHandler protocol.""" + + def test_dataset_handling_satisfies_dataset_handler_protocol(self): + assert issubclass(DatasetHandling, DatasetHandler) + + def test_dataset_handling_satisfies_lifecycle_protocol(self): + assert issubclass(DatasetHandling, LifecycleComponent) + + +class TestProtocolRuntimeCheckable: + """Verify that protocols work correctly for non-conforming classes.""" + + def test_lifecycle_component_is_runtime_checkable(self): + """LifecycleComponent should be usable with isinstance/issubclass.""" + assert isinstance(LifecycleComponent, type) + + def test_non_conforming_class_fails_isinstance_check(self): + """A class missing required methods should NOT satisfy the protocol.""" + + class Incomplete: + pass + + assert not issubclass(Incomplete, LifecycleComponent) + assert not issubclass(Incomplete, Trainer) + assert not issubclass(Incomplete, Compiler) + assert not issubclass(Incomplete, DatasetHandler) + assert not issubclass(Incomplete, Runner) + + def test_partial_conformance_fails(self): + """A class with only some of the required methods should still fail.""" + + class PartialComponent: + def clear(self): + pass + + def get_params(self): + pass + + # Missing init_params classmethod — should fail LifecycleComponent + assert not issubclass(PartialComponent, Trainer) + assert not issubclass(PartialComponent, Runner) From b972cedba952d311419806e1c40f413b406806f4 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 5 Mar 2026 14:24:46 -0500 Subject: [PATCH 2/6] Add functional test suite (Tiers 1-3): component, smoke, cross-device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tier 1 — Component Tests (323 tests, 2.3s): - test_model_registry.py: model descriptions, filtering, device constants - test_config_validation.py: 303 parametrized YAML config validations Tier 2 — Pipeline Smoke Tests (4 tests, 116s): - Classification, regression, forecasting via two-phase approach - Anomaly detection xfail (no device preset) Tier 3 — Cross-Device Validation (195 tests, 1.8s): - NPU hard/soft flag validation for all 22 devices - MSPM0 classification-only task constraints - Compilation profile correctness and quantization flags Also: registered pytest markers (component, smoke, device) in pyproject.toml --- tinyml-modelmaker/pyproject.toml | 7 + tinyml-modelmaker/tests/conftest.py | 181 ++++++++ .../tests/test_config_validation.py | 156 +++++++ tinyml-modelmaker/tests/test_cross_device.py | 397 ++++++++++++++++++ .../tests/test_model_registry.py | 195 +++++++++ .../tests/test_pipeline_smoke.py | 232 ++++++++++ 6 files changed, 1168 insertions(+) create mode 100644 tinyml-modelmaker/tests/conftest.py create mode 100644 tinyml-modelmaker/tests/test_config_validation.py create mode 100644 tinyml-modelmaker/tests/test_cross_device.py create mode 100644 tinyml-modelmaker/tests/test_model_registry.py create mode 100644 tinyml-modelmaker/tests/test_pipeline_smoke.py diff --git a/tinyml-modelmaker/pyproject.toml b/tinyml-modelmaker/pyproject.toml index 651f8f43..d231a67a 100644 --- a/tinyml-modelmaker/pyproject.toml +++ b/tinyml-modelmaker/pyproject.toml @@ -41,3 +41,10 @@ exclude = [ "tests*",] [tool.setuptools.dynamic.version] attr = "tinyml_modelmaker.__version__" + +[tool.pytest.ini_options] +markers = [ + "component: Tier 1 component tests", + "smoke: Tier 2 pipeline smoke tests", + "device: Tier 3 cross-device validation tests", +] diff --git a/tinyml-modelmaker/tests/conftest.py b/tinyml-modelmaker/tests/conftest.py new file mode 100644 index 00000000..98de54c6 --- /dev/null +++ b/tinyml-modelmaker/tests/conftest.py @@ -0,0 +1,181 @@ +"""Shared fixtures for tinyml-modelmaker tests. + +Provides: +- pytest markers for component and smoke test tiers +- Synthetic dataset generators for pipeline smoke tests +""" + +import os +import csv +import random +import pytest +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Custom markers +# --------------------------------------------------------------------------- + +def pytest_configure(config): + """Register custom markers for test tiers.""" + config.addinivalue_line("markers", "component: Tier 1 component tests (fast, no training)") + config.addinivalue_line("markers", "smoke: Tier 2 pipeline smoke tests (1-epoch, synthetic data)") + + +# --------------------------------------------------------------------------- +# Synthetic dataset generators +# --------------------------------------------------------------------------- + +@pytest.fixture +def synthetic_classification_data(tmp_path): + """Create a minimal 3-class classification dataset. + + Structure: + tmp_path/ + classes/ + class_a/ (30 CSV files) + class_b/ (30 CSV files) + class_c/ (30 CSV files) + annotations/ + file_list.txt (all files) + Each CSV has 256 rows (samples) × 1 column (single-channel sensor data). + """ + random.seed(42) + classes_dir = tmp_path / "classes" + all_files = [] + + for label in ["class_a", "class_b", "class_c"]: + class_dir = classes_dir / label + class_dir.mkdir(parents=True) + for i in range(30): + filename = f"{label}_{i:03d}.csv" + filepath = class_dir / filename + with open(filepath, "w", newline="") as f: + writer = csv.writer(f) + for _ in range(256): + writer.writerow([random.gauss(0, 1)]) + all_files.append(f"classes/{label}/{filename}") + + # Create annotations directory with file list + ann_dir = tmp_path / "annotations" + ann_dir.mkdir(parents=True) + (ann_dir / "file_list.txt").write_text("\n".join(all_files)) + + return str(tmp_path) + + +@pytest.fixture +def synthetic_regression_data(tmp_path): + """Create a minimal regression dataset. + + Structure: + tmp_path/ + files/ (60 CSV files — flat layout for regression) + annotations/ + instances_train_list.txt + instances_val_list.txt + Each CSV has 256 rows × 2 columns (input feature, target value). + """ + random.seed(42) + files_dir = tmp_path / "files" + files_dir.mkdir(parents=True) + all_files = [] + + for i in range(60): + filename = f"sample_{i:03d}.csv" + filepath = files_dir / filename + with open(filepath, "w", newline="") as f: + writer = csv.writer(f) + for _ in range(256): + x = random.gauss(0, 1) + y = 2.0 * x + random.gauss(0, 0.1) + writer.writerow([x, y]) + all_files.append(f"files/{filename}") + + # Create annotations with train/val split (48 train, 12 val) + # File names must match GenericTSDataset glob patterns: *train*_list.txt, *val*_list.txt + ann_dir = tmp_path / "annotations" + ann_dir.mkdir(parents=True) + (ann_dir / "instances_train_list.txt").write_text("\n".join(all_files[:48])) + (ann_dir / "instances_val_list.txt").write_text("\n".join(all_files[48:])) + + return str(tmp_path) + + +@pytest.fixture +def synthetic_anomaly_data(tmp_path): + """Create a minimal anomaly detection dataset. + + Structure: + tmp_path/ + classes/ + normal/ (50 CSV files) + anomaly/ (10 CSV files) + """ + random.seed(42) + classes_dir = tmp_path / "classes" + + for label, count, mu in [("normal", 50, 0.0), ("anomaly", 10, 5.0)]: + class_dir = classes_dir / label + class_dir.mkdir(parents=True) + for i in range(count): + filepath = class_dir / f"{label}_{i:03d}.csv" + with open(filepath, "w", newline="") as f: + writer = csv.writer(f) + for _ in range(256): + writer.writerow([random.gauss(mu, 1)]) + + return str(tmp_path) + + +@pytest.fixture +def synthetic_forecasting_data(tmp_path): + """Create a minimal forecasting dataset. + + Structure: + tmp_path/ + files/ (60 CSV files — flat layout for forecasting) + annotations/ + instances_train_list.txt + instances_val_list.txt + Each CSV has 64 rows × 1 column (univariate time series). + """ + random.seed(42) + files_dir = tmp_path / "files" + files_dir.mkdir(parents=True) + all_files = [] + + for i in range(60): + filename = f"series_{i:03d}.csv" + filepath = files_dir / filename + with open(filepath, "w", newline="") as f: + writer = csv.writer(f) + val = random.gauss(0, 1) + for _ in range(64): + val += random.gauss(0, 0.3) + writer.writerow([val]) + all_files.append(f"files/{filename}") + + # Create annotations with train/val split (48 train, 12 val) + # File names must match GenericTSDataset glob patterns: *train*_list.txt, *val*_list.txt + ann_dir = tmp_path / "annotations" + ann_dir.mkdir(parents=True) + (ann_dir / "instances_train_list.txt").write_text("\n".join(all_files[:48])) + (ann_dir / "instances_val_list.txt").write_text("\n".join(all_files[48:])) + + return str(tmp_path) + + +@pytest.fixture +def modelzoo_examples_dir(): + """Return the path to the modelzoo examples directory. + + Returns None if not found (not all environments will have modelzoo). + """ + this_dir = Path(__file__).resolve().parent + modelmaker_dir = this_dir.parent + tensorlab_dir = modelmaker_dir.parent + examples_dir = tensorlab_dir / "tinyml-modelzoo" / "examples" + if examples_dir.is_dir(): + return examples_dir + return None diff --git a/tinyml-modelmaker/tests/test_config_validation.py b/tinyml-modelmaker/tests/test_config_validation.py new file mode 100644 index 00000000..26bcd8a8 --- /dev/null +++ b/tinyml-modelmaker/tests/test_config_validation.py @@ -0,0 +1,156 @@ +"""Tier 1 — Config YAML Validation Tests. + +Validates that all example configs in tinyml-modelzoo/examples/ parse correctly, +contain required keys, and reference valid task types and models. +""" + +import os +from pathlib import Path + +import pytest +import yaml + +from tinyml_modelmaker.ai_modules.timeseries import training, constants + + +# --------------------------------------------------------------------------- +# Discover all example YAML configs +# --------------------------------------------------------------------------- + +def _find_example_configs(): + """Find all config*.yaml files in the modelzoo examples directory.""" + this_dir = Path(__file__).resolve().parent + modelmaker_dir = this_dir.parent + tensorlab_dir = modelmaker_dir.parent + examples_dir = tensorlab_dir / "tinyml-modelzoo" / "examples" + + if not examples_dir.is_dir(): + return [] + + configs = [] + for yaml_path in sorted(examples_dir.rglob("config*.yaml")): + # Use relative path from examples_dir as the test ID + rel = yaml_path.relative_to(examples_dir) + configs.append(pytest.param(yaml_path, id=str(rel))) + + return configs + + +EXAMPLE_CONFIGS = _find_example_configs() + +# Known valid task types (timeseries + vision) +KNOWN_TASK_TYPES = set(constants.TASK_TYPE_TO_CATEGORY.keys()) | {"image_classification"} + +# Required top-level keys in every config +REQUIRED_SECTIONS = {"common", "training"} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.component +class TestExampleConfigsParse: + """Every example config YAML must parse without error.""" + + @pytest.mark.skipif(not EXAMPLE_CONFIGS, reason="modelzoo examples not found") + @pytest.mark.parametrize("config_path", EXAMPLE_CONFIGS) + def test_config_parses(self, config_path): + """Config file should be valid YAML.""" + with open(config_path) as f: + data = yaml.safe_load(f) + assert isinstance(data, dict), f"Config did not parse as a dict: {config_path}" + + +@pytest.mark.component +class TestExampleConfigsStructure: + """Config files must have required sections and valid references.""" + + @pytest.mark.skipif(not EXAMPLE_CONFIGS, reason="modelzoo examples not found") + @pytest.mark.parametrize("config_path", EXAMPLE_CONFIGS) + def test_has_required_sections(self, config_path): + """Config must have 'common' and 'training' sections.""" + with open(config_path) as f: + data = yaml.safe_load(f) + + for section in REQUIRED_SECTIONS: + assert section in data, ( + f"Config {config_path.name} missing required section '{section}'" + ) + + @pytest.mark.skipif(not EXAMPLE_CONFIGS, reason="modelzoo examples not found") + @pytest.mark.parametrize("config_path", EXAMPLE_CONFIGS) + def test_has_task_type(self, config_path): + """Config must specify common.task_type.""" + with open(config_path) as f: + data = yaml.safe_load(f) + common = data.get("common", {}) + assert "task_type" in common, ( + f"Config {config_path.name} missing 'common.task_type'" + ) + + @pytest.mark.skipif(not EXAMPLE_CONFIGS, reason="modelzoo examples not found") + @pytest.mark.parametrize("config_path", EXAMPLE_CONFIGS) + def test_task_type_is_valid(self, config_path): + """common.task_type must be a recognized task type.""" + with open(config_path) as f: + data = yaml.safe_load(f) + task_type = data.get("common", {}).get("task_type") + if task_type is not None: + assert task_type in KNOWN_TASK_TYPES, ( + f"Config {config_path.name} has unknown task_type='{task_type}'" + ) + + @pytest.mark.skipif(not EXAMPLE_CONFIGS, reason="modelzoo examples not found") + @pytest.mark.parametrize("config_path", EXAMPLE_CONFIGS) + def test_has_model_name(self, config_path): + """Config must specify training.model_name.""" + with open(config_path) as f: + data = yaml.safe_load(f) + tr = data.get("training", {}) + assert "model_name" in tr, ( + f"Config {config_path.name} missing 'training.model_name'" + ) + + @pytest.mark.skipif(not EXAMPLE_CONFIGS, reason="modelzoo examples not found") + @pytest.mark.parametrize("config_path", EXAMPLE_CONFIGS) + def test_has_target_device(self, config_path): + """Config must specify common.target_device.""" + with open(config_path) as f: + data = yaml.safe_load(f) + common = data.get("common", {}) + assert "target_device" in common, ( + f"Config {config_path.name} missing 'common.target_device'" + ) + + +@pytest.mark.component +class TestExampleConfigsModelReferences: + """Model names in configs should exist in the model registry.""" + + @pytest.mark.skipif(not EXAMPLE_CONFIGS, reason="modelzoo examples not found") + @pytest.mark.parametrize("config_path", EXAMPLE_CONFIGS) + def test_model_name_exists_in_registry(self, config_path): + """training.model_name should be a registered model.""" + with open(config_path) as f: + data = yaml.safe_load(f) + + model_name = data.get("training", {}).get("model_name") + if model_name is None: + pytest.skip("No model_name in config") + + # Skip NAS configs where model_name is a search placeholder + if "NAS" in model_name.upper(): + pytest.skip(f"NAS model placeholder: {model_name}") + + task_type = data.get("common", {}).get("task_type") + + # Vision models have a separate registry + if task_type == "image_classification": + pytest.skip("Vision model registry not tested here") + + desc = training.get_model_description(model_name) + assert desc is not None, ( + f"Config references model '{model_name}' which is not in the registry" + ) diff --git a/tinyml-modelmaker/tests/test_cross_device.py b/tinyml-modelmaker/tests/test_cross_device.py new file mode 100644 index 00000000..4f4c1d8a --- /dev/null +++ b/tinyml-modelmaker/tests/test_cross_device.py @@ -0,0 +1,397 @@ +"""Tier 3 — Cross-Device Validation Tests. + +Validates device-specific configuration is correct without running training. +Covers Tests 15-19 from the test analysis: + + Test 15: NPU device config — hard NPU devices set type=hard in compilation + Test 16: Non-NPU device config — soft NPU devices set type=soft + Test 17: MSPM0 classification-only — MSPM0G3507 rejects non-classification tasks + Test 18: Device model size constraints — device_selection_factor consistency + Test 19: Compilation profile correctness — all devices have valid profiles + +Marked with @pytest.mark.device — run with: pytest -m device +""" + +import pytest + +from tinyml_modelmaker.ai_modules.timeseries import constants + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# Devices with hardware NPU (type=hard in base compilation) +HARD_NPU_DEVICES = ["F28P55", "MSPM0G5187", "MSPM33C34", "AM13E2"] + +# Devices with soft-only NPU +SOFT_NPU_DEVICES = [ + "F280013", "F280015", "F28003", "F28004", "F2837", "F28P65", + "F29H85", "F29P58", "F29P32", + "MSPM0G3507", "MSPM0G3519", + "MSPM33C32", + "AM263", "AM263P", "AM261", + "CC2755", "CC1352", "CC1354", "CC35X1", +] + +# MSPM0 devices — classification only +MSPM0_CLASSIFICATION_ONLY = ["MSPM0G3507", "MSPM0G3519", "MSPM0G5187"] + +# Non-classification task types +NON_CLASSIFICATION_TASKS = [ + constants.TASK_TYPE_GENERIC_TS_REGRESSION, + constants.TASK_TYPE_GENERIC_TS_FORECASTING, + constants.TASK_TYPE_GENERIC_TS_ANOMALYDETECTION, +] + + +# --------------------------------------------------------------------------- +# Test 15: NPU device config — hard NPU compilation flags +# --------------------------------------------------------------------------- + +@pytest.mark.device +class TestHardNPUDeviceConfig: + """Devices with hardware NPU should have type=hard in base compilation.""" + + @pytest.mark.parametrize("device", HARD_NPU_DEVICES) + def test_hard_npu_flag(self, device): + """Hard NPU devices should have 'has_hard_npu': True.""" + profile = constants._DEVICE_PROFILES[device] + assert profile["has_hard_npu"] is True, ( + f"Device {device} should have has_hard_npu=True" + ) + + @pytest.mark.parametrize("device", HARD_NPU_DEVICES) + def test_hard_npu_base_compilation(self, device): + """Hard NPU devices should have 'type=hard' in base compilation target.""" + profile = constants._DEVICE_PROFILES[device] + base = profile["compilation_base"] + assert "type=hard" in base["target"], ( + f"Device {device} base compilation should have type=hard, " + f"got: {base['target']}" + ) + + @pytest.mark.parametrize("device", HARD_NPU_DEVICES) + def test_hard_npu_has_soft_fallback(self, device): + """Hard NPU devices should also have a soft NPU fallback compilation.""" + profile = constants._DEVICE_PROFILES[device] + assert "compilation_soft" in profile, ( + f"Device {device} should have compilation_soft fallback" + ) + soft = profile["compilation_soft"] + assert "type=soft" in soft["target"], ( + f"Device {device} soft fallback should have type=soft" + ) + + +# --------------------------------------------------------------------------- +# Test 16: Non-NPU device config — soft NPU compilation flags +# --------------------------------------------------------------------------- + +@pytest.mark.device +class TestSoftNPUDeviceConfig: + """Devices without hardware NPU should use type=soft compilation.""" + + @pytest.mark.parametrize("device", SOFT_NPU_DEVICES) + def test_soft_npu_flag(self, device): + """Soft NPU devices should have 'has_hard_npu': False.""" + profile = constants._DEVICE_PROFILES[device] + assert profile["has_hard_npu"] is False, ( + f"Device {device} should have has_hard_npu=False" + ) + + @pytest.mark.parametrize("device", SOFT_NPU_DEVICES) + def test_soft_npu_base_compilation(self, device): + """Soft NPU devices should NOT have 'type=hard' in base compilation.""" + profile = constants._DEVICE_PROFILES[device] + base = profile["compilation_base"] + assert "type=hard" not in base["target"], ( + f"Device {device} base compilation should NOT have type=hard, " + f"got: {base['target']}" + ) + + +# --------------------------------------------------------------------------- +# Test 17: MSPM0 classification-only +# --------------------------------------------------------------------------- + +@pytest.mark.device +class TestMSPM0ClassificationOnly: + """MSPM0 devices should only support classification task types.""" + + @pytest.mark.parametrize("device", MSPM0_CLASSIFICATION_ONLY) + def test_mspm0_limited_tasks(self, device): + """MSPM0 devices should have explicit task_types list in profile.""" + profile = constants._DEVICE_PROFILES[device] + assert "task_types" in profile, ( + f"MSPM0 device {device} should have explicit task_types list" + ) + + @pytest.mark.parametrize("device", MSPM0_CLASSIFICATION_ONLY) + def test_mspm0_no_regression(self, device): + """MSPM0 devices should not support regression.""" + profile = constants._DEVICE_PROFILES[device] + task_types = profile.get("task_types", []) + assert constants.TASK_TYPE_GENERIC_TS_REGRESSION not in task_types, ( + f"MSPM0 device {device} should not support regression" + ) + + @pytest.mark.parametrize("device", MSPM0_CLASSIFICATION_ONLY) + def test_mspm0_no_forecasting(self, device): + """MSPM0 devices should not support forecasting.""" + profile = constants._DEVICE_PROFILES[device] + task_types = profile.get("task_types", []) + assert constants.TASK_TYPE_GENERIC_TS_FORECASTING not in task_types, ( + f"MSPM0 device {device} should not support forecasting" + ) + + @pytest.mark.parametrize("device", MSPM0_CLASSIFICATION_ONLY) + def test_mspm0_supports_classification(self, device): + """MSPM0 devices should support classification.""" + profile = constants._DEVICE_PROFILES[device] + task_types = profile.get("task_types", []) + assert constants.TASK_TYPE_GENERIC_TS_CLASSIFICATION in task_types, ( + f"MSPM0 device {device} should support classification" + ) + + @pytest.mark.parametrize("device", MSPM0_CLASSIFICATION_ONLY) + def test_mspm0_no_regression_compilation(self, device): + """MSPM0 devices should not have compilation_regression profile.""" + profile = constants._DEVICE_PROFILES[device] + assert "compilation_regression" not in profile, ( + f"MSPM0 device {device} should not have compilation_regression" + ) + + +# --------------------------------------------------------------------------- +# Test 18: Device model size constraints — device_selection_factor +# --------------------------------------------------------------------------- + +@pytest.mark.device +class TestDeviceSelectionFactor: + """Validate device_selection_factor ordering and consistency.""" + + def test_all_devices_have_descriptions(self): + """Every device in TARGET_DEVICES should have a description.""" + for device in constants.TARGET_DEVICES: + assert device in constants.TARGET_DEVICE_DESCRIPTIONS, ( + f"Device {device} missing from TARGET_DEVICE_DESCRIPTIONS" + ) + + def test_selection_factors_are_non_negative(self): + """All device_selection_factor values should be non-negative integers.""" + for device, desc in constants.TARGET_DEVICE_DESCRIPTIONS.items(): + factor = desc.get("device_selection_factor") + assert factor is not None, ( + f"Device {device} missing device_selection_factor" + ) + assert isinstance(factor, int) and factor >= 0, ( + f"Device {device} has invalid factor: {factor}" + ) + + def test_selection_factors_are_unique(self): + """Device selection factors should ideally be unique (warn if not).""" + factors = {} + for device, desc in constants.TARGET_DEVICE_DESCRIPTIONS.items(): + f = desc["device_selection_factor"] + if f in factors: + # Not a hard failure — just tracked + pass + factors.setdefault(f, []).append(device) + # At least some differentiation should exist + assert len(factors) > 1, "All devices have the same selection factor" + + def test_hard_npu_devices_higher_factor(self): + """Devices with hard NPU should generally have higher selection factor.""" + npu_factors = [] + for device in HARD_NPU_DEVICES: + if device in constants.TARGET_DEVICE_DESCRIPTIONS: + npu_factors.append( + constants.TARGET_DEVICE_DESCRIPTIONS[device]["device_selection_factor"] + ) + if npu_factors: + avg_npu = sum(npu_factors) / len(npu_factors) + # Hard NPU devices should have above-average selection factor + all_factors = [ + d["device_selection_factor"] + for d in constants.TARGET_DEVICE_DESCRIPTIONS.values() + ] + avg_all = sum(all_factors) / len(all_factors) + assert avg_npu >= avg_all, ( + f"Hard NPU devices avg factor ({avg_npu:.1f}) should be " + f">= overall avg ({avg_all:.1f})" + ) + + +# --------------------------------------------------------------------------- +# Test 19: Compilation profile correctness — all devices +# --------------------------------------------------------------------------- + +@pytest.mark.device +class TestCompilationProfileCorrectness: + """Validate all device profiles have correct structure.""" + + def test_all_devices_have_profiles(self): + """Every device in TARGET_DEVICES should have a profile.""" + for device in constants.TARGET_DEVICES: + assert device in constants._DEVICE_PROFILES, ( + f"Device {device} missing from _DEVICE_PROFILES" + ) + + @pytest.mark.parametrize("device", constants.TARGET_DEVICES) + def test_profile_has_base_compilation(self, device): + """Every device profile should have compilation_base.""" + profile = constants._DEVICE_PROFILES[device] + assert "compilation_base" in profile, ( + f"Device {device} missing compilation_base" + ) + + @pytest.mark.parametrize("device", constants.TARGET_DEVICES) + def test_profile_has_npu_flag(self, device): + """Every device profile should declare has_hard_npu.""" + profile = constants._DEVICE_PROFILES[device] + assert "has_hard_npu" in profile, ( + f"Device {device} missing has_hard_npu flag" + ) + + @pytest.mark.parametrize("device", constants.TARGET_DEVICES) + def test_compilation_base_has_target(self, device): + """Base compilation config should have a 'target' key.""" + profile = constants._DEVICE_PROFILES[device] + base = profile["compilation_base"] + assert "target" in base, ( + f"Device {device} compilation_base missing 'target' key" + ) + + @pytest.mark.parametrize("device", constants.TARGET_DEVICES) + def test_compilation_base_has_cross_compiler(self, device): + """Base compilation config should have a 'cross_compiler' key.""" + profile = constants._DEVICE_PROFILES[device] + base = profile["compilation_base"] + assert "cross_compiler" in base, ( + f"Device {device} compilation_base missing 'cross_compiler' key" + ) + + @pytest.mark.parametrize("device", constants.TARGET_DEVICES) + def test_cross_compiler_options_exist(self, device): + """Every device should have cross-compiler options defined.""" + assert device in constants._CROSS_COMPILER_OPTIONS, ( + f"Device {device} missing from _CROSS_COMPILER_OPTIONS" + ) + + def test_all_devices_have_sdk_info(self): + """Every device description should include SDK version and release.""" + for device, desc in constants.TARGET_DEVICE_DESCRIPTIONS.items(): + assert "sdk_version" in desc, ( + f"Device {device} missing sdk_version" + ) + assert "sdk_release" in desc, ( + f"Device {device} missing sdk_release" + ) + + +# --------------------------------------------------------------------------- +# Additional: Task type ↔ category mapping consistency +# --------------------------------------------------------------------------- + +@pytest.mark.device +class TestTaskCategoryMapping: + """Validate task type to category mapping is complete and consistent.""" + + def test_all_task_types_have_category(self): + """Every task type should map to a category.""" + for task_type in constants.TASK_TYPES: + category = constants.TASK_TYPE_TO_CATEGORY.get(task_type) + assert category is not None, ( + f"Task type {task_type} has no category mapping" + ) + + def test_all_categories_used(self): + """Every defined category should be referenced by at least one task type.""" + used_categories = set(constants.TASK_TYPE_TO_CATEGORY.values()) + for cat in constants.TASK_CATEGORIES: + assert cat in used_categories, ( + f"Category {cat} defined but never used by any task type" + ) + + def test_data_dir_convention_classification(self): + """Classification tasks should use 'classes' data dir.""" + data_dir = constants.get_default_data_dir_for_task( + constants.TASK_CATEGORY_TS_CLASSIFICATION + ) + assert data_dir == "classes" + + def test_data_dir_convention_regression(self): + """Regression tasks should use 'files' data dir.""" + data_dir = constants.get_default_data_dir_for_task( + constants.TASK_CATEGORY_TS_REGRESSION + ) + assert data_dir == "files" + + def test_data_dir_convention_forecasting(self): + """Forecasting tasks should use 'files' data dir.""" + data_dir = constants.get_default_data_dir_for_task( + constants.TASK_CATEGORY_TS_FORECASTING + ) + assert data_dir == "files" + + def test_data_dir_convention_anomaly(self): + """Anomaly detection tasks should use 'classes' data dir.""" + data_dir = constants.get_default_data_dir_for_task( + constants.TASK_CATEGORY_TS_ANOMALYDETECTION + ) + assert data_dir == "classes" + + +# --------------------------------------------------------------------------- +# Quantization flag consistency +# --------------------------------------------------------------------------- + +@pytest.mark.device +class TestQuantizationFlags: + """Validate skip_normalize / output_int matrix is correct.""" + + @pytest.mark.parametrize("task_category", constants.TASK_CATEGORIES) + def test_float_mode_no_normalize(self, task_category): + """Quantization=0 (float) should set skip_normalize=False, output_int=False.""" + skip, output = constants.get_skip_normalize_and_output_int( + task_category, quantization=0, partial_quantization=False + ) + assert skip is False + assert output is False + + def test_classification_quant_sets_output_int(self): + """Classification with quantization should set output_int=True.""" + skip, output = constants.get_skip_normalize_and_output_int( + constants.TASK_CATEGORY_TS_CLASSIFICATION, + quantization=1, partial_quantization=False, + ) + assert skip is True + assert output is True + + def test_regression_quant_no_output_int(self): + """Regression with quantization should set output_int=False.""" + skip, output = constants.get_skip_normalize_and_output_int( + constants.TASK_CATEGORY_TS_REGRESSION, + quantization=1, partial_quantization=False, + ) + assert skip is True + assert output is False + + def test_forecasting_quant_no_output_int(self): + """Forecasting with quantization should set output_int=False.""" + skip, output = constants.get_skip_normalize_and_output_int( + constants.TASK_CATEGORY_TS_FORECASTING, + quantization=1, partial_quantization=False, + ) + assert skip is True + assert output is False + + def test_partial_quant_regression_override(self): + """Partial quantization for regression should set skip_normalize=False.""" + skip, output = constants.get_skip_normalize_and_output_int( + constants.TASK_CATEGORY_TS_REGRESSION, + quantization=1, partial_quantization=True, + ) + assert skip is False + assert output is False diff --git a/tinyml-modelmaker/tests/test_model_registry.py b/tinyml-modelmaker/tests/test_model_registry.py new file mode 100644 index 00000000..761a37bb --- /dev/null +++ b/tinyml-modelmaker/tests/test_model_registry.py @@ -0,0 +1,195 @@ +"""Tier 1 — Model Registry Component Tests. + +Validates that model descriptions are complete, properly structured, +and correctly filtered by task type and device. +""" + +import pytest + +from tinyml_modelmaker.ai_modules.timeseries import training, constants + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +GENERIC_TASK_TYPES = [ + constants.TASK_TYPE_GENERIC_TS_CLASSIFICATION, + constants.TASK_TYPE_GENERIC_TS_REGRESSION, + constants.TASK_TYPE_GENERIC_TS_ANOMALYDETECTION, + constants.TASK_TYPE_GENERIC_TS_FORECASTING, +] + +ALL_TASK_TYPES = list(constants.TASK_TYPE_TO_CATEGORY.keys()) + +REQUIRED_DESCRIPTION_KEYS = {"common", "training"} +REQUIRED_TRAINING_KEYS = {"training_backend", "model_training_id"} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.component +class TestModelDescriptions: + """Every registered model must have a well-formed description dict.""" + + def test_registry_is_not_empty(self): + """Global model registry should contain models.""" + all_models = training.get_model_descriptions() + assert len(all_models) > 0, "Model registry is empty" + + @pytest.mark.parametrize("task_type", GENERIC_TASK_TYPES) + def test_models_exist_per_task_type(self, task_type): + """Each generic task type should have at least one model.""" + models = training.get_model_descriptions(task_type=task_type) + assert len(models) > 0, f"No models registered for task_type={task_type}" + + def test_every_model_has_required_keys(self): + """All model descriptions must contain 'common' and 'training' sections.""" + all_models = training.get_model_descriptions() + for name, desc in all_models.items(): + for key in REQUIRED_DESCRIPTION_KEYS: + assert key in desc, f"Model '{name}' missing required key '{key}'" + + def test_every_model_has_training_backend(self): + """All models must specify a training_backend.""" + all_models = training.get_model_descriptions() + for name, desc in all_models.items(): + tr = desc.get("training", {}) + assert "training_backend" in tr, ( + f"Model '{name}' missing 'training.training_backend'" + ) + + def test_every_model_has_model_training_id(self): + """All models must specify a model_training_id.""" + all_models = training.get_model_descriptions() + for name, desc in all_models.items(): + tr = desc.get("training", {}) + assert "model_training_id" in tr, ( + f"Model '{name}' missing 'training.model_training_id'" + ) + + def test_every_model_has_task_type(self): + """All models must specify common.task_type.""" + all_models = training.get_model_descriptions() + for name, desc in all_models.items(): + common = desc.get("common", {}) + assert "task_type" in common, ( + f"Model '{name}' missing 'common.task_type'" + ) + + def test_every_model_task_type_is_known(self): + """common.task_type must be a recognized task type string.""" + all_models = training.get_model_descriptions() + for name, desc in all_models.items(): + task_type = desc.get("common", {}).get("task_type") + assert task_type in ALL_TASK_TYPES, ( + f"Model '{name}' has unknown task_type='{task_type}'" + ) + + +@pytest.mark.component +class TestModelFiltering: + """get_model_descriptions() filtering by task_type and target_device.""" + + def test_classification_filter_returns_only_classification(self): + """Filtering by classification should only return classification models.""" + task = constants.TASK_TYPE_GENERIC_TS_CLASSIFICATION + models = training.get_model_descriptions(task_type=task) + for name, desc in models.items(): + assert desc["common"]["task_type"] == task, ( + f"Model '{name}' leaked through filter: " + f"expected task_type={task}, got {desc['common']['task_type']}" + ) + + def test_regression_filter_returns_only_regression(self): + task = constants.TASK_TYPE_GENERIC_TS_REGRESSION + models = training.get_model_descriptions(task_type=task) + for name, desc in models.items(): + assert desc["common"]["task_type"] == task + + def test_anomalydetection_filter_returns_only_anomalydetection(self): + task = constants.TASK_TYPE_GENERIC_TS_ANOMALYDETECTION + models = training.get_model_descriptions(task_type=task) + for name, desc in models.items(): + assert desc["common"]["task_type"] == task + + def test_forecasting_filter_returns_only_forecasting(self): + task = constants.TASK_TYPE_GENERIC_TS_FORECASTING + models = training.get_model_descriptions(task_type=task) + for name, desc in models.items(): + assert desc["common"]["task_type"] == task + + def test_device_filter_f28p55(self): + """Filtering by F28P55 should return only models that support it.""" + device = "F28P55" + models = training.get_model_descriptions(target_device=device) + assert len(models) > 0, f"No models found for device {device}" + for name, desc in models.items(): + devices = desc.get("training", {}).get("target_devices", []) + assert device in devices, ( + f"Model '{name}' does not list {device} in target_devices" + ) + + def test_device_filter_mspm0g3507(self): + """MSPM0G3507 should return models (primarily classification).""" + device = "MSPM0G3507" + models = training.get_model_descriptions(target_device=device) + assert len(models) > 0, f"No models found for device {device}" + + def test_combined_task_and_device_filter(self): + """Combined filter should be the intersection.""" + task = constants.TASK_TYPE_GENERIC_TS_CLASSIFICATION + device = "F28P55" + models = training.get_model_descriptions(task_type=task, target_device=device) + for name, desc in models.items(): + assert desc["common"]["task_type"] == task + assert device in desc["training"]["target_devices"] + + +@pytest.mark.component +class TestModelLookup: + """get_model_description() single-model lookup.""" + + def test_known_model_returns_dict(self): + """Looking up a known model name should return its description.""" + # Find any model name from the registry + all_models = training.get_model_descriptions() + name = next(iter(all_models)) + desc = training.get_model_description(name) + assert desc is not None + assert isinstance(desc, dict) + + def test_unknown_model_returns_none(self): + """Looking up a nonexistent model should return None.""" + desc = training.get_model_description("TOTALLY_FAKE_MODEL_XYZ") + assert desc is None + + def test_lookup_result_matches_registry(self): + """Lookup result should match the registry entry.""" + all_models = training.get_model_descriptions() + for name in list(all_models.keys())[:5]: # spot-check first 5 + desc = training.get_model_description(name) + assert desc is not None + assert desc["training"]["model_training_id"] == all_models[name]["training"]["model_training_id"] + + +@pytest.mark.component +class TestDeviceConstants: + """Verify device constant lists are populated and consistent.""" + + def test_target_devices_not_empty(self): + assert len(constants.TARGET_DEVICES) > 0 + + def test_all_task_types_mapped(self): + """Every TASK_TYPE constant should be in TASK_TYPE_TO_CATEGORY.""" + for tt in ALL_TASK_TYPES: + assert tt in constants.TASK_TYPE_TO_CATEGORY, ( + f"Task type '{tt}' not in TASK_TYPE_TO_CATEGORY mapping" + ) + + def test_target_devices_are_strings(self): + for device in constants.TARGET_DEVICES: + assert isinstance(device, str), f"Device {device!r} is not a string" diff --git a/tinyml-modelmaker/tests/test_pipeline_smoke.py b/tinyml-modelmaker/tests/test_pipeline_smoke.py new file mode 100644 index 00000000..85bf763e --- /dev/null +++ b/tinyml-modelmaker/tests/test_pipeline_smoke.py @@ -0,0 +1,232 @@ +"""Tier 2 — Pipeline Smoke Tests. + +Runs 1-epoch training for each task type using synthetic datasets. +These tests validate the full pipeline: config parsing → data loading → +feature extraction → model creation → training → evaluation → ONNX export. + +Marked with @pytest.mark.smoke — run with: pytest -m smoke + +Note on data handling: +- Classification: DatasetHandling auto-creates train/val/test splits ✅ +- Regression/Forecasting: DatasetHandling creates file_list.txt only — + we run a two-phase approach: (1) dataset-only to populate file_list, + (2) auto-create train/val splits, (3) run with training enabled. +- Anomaly Detection: No device preset exists — marked xfail. +""" + +import copy +import csv +import os +import random +import shutil +import sys +from glob import glob +from pathlib import Path + +import pytest + +# Force QAT quantization ops to CPU on Apple Silicon MPS +os.environ.setdefault("PYTORCH_ENABLE_MPS_FALLBACK", "1") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_config(task_type, model_name, data_path, **overrides): + """Build a minimal config dict for a 1-epoch smoke test. + + Quantization is disabled by default because QAT segfaults on Apple Silicon + MPS (PyTorch fake_quantize ops not supported on MPS device). + """ + config = { + "common": { + "task_type": task_type, + "target_device": "F28P55", + }, + "dataset": { + "enable": True, + "dataset_name": task_type, + "input_data_path": data_path, + }, + "data_processing_feature_extraction": { + "feature_extraction_name": "default", + }, + "training": { + "enable": True, + "model_name": model_name, + "batch_size": 16, + "training_epochs": 1, + "num_gpus": 0, + "quantization": 0, # Disable QAT for MPS compatibility + }, + "testing": {"enable": False}, + "compilation": {"enable": False, "compile_preset_name": "default_preset"}, + } + # Apply overrides + for section, values in overrides.items(): + if section in config: + config[section].update(values) + else: + config[section] = values + return config + + +def _auto_create_splits(project_base, task_type): + """Create train/val/test split files from file_list.txt. + + DatasetHandling auto-creates these for classification but not for + regression/forecasting. This mimics the same behavior. + """ + ann_dir = Path(project_base) / "dataset" / "annotations" + if not ann_dir.is_dir(): + return False + + # Already has splits? Skip. + if list(ann_dir.glob("*train*_list.txt")): + return True + + file_list = ann_dir / "file_list.txt" + if not file_list.is_file(): + return False + + files = [line.strip() for line in file_list.read_text().splitlines() if line.strip()] + if not files: + return False + + # 70/15/15 split + n = len(files) + n_train = max(1, int(n * 0.7)) + n_val = max(1, int(n * 0.15)) + train_files = files[:n_train] + val_files = files[n_train:n_train + n_val] + test_files = files[n_train + n_val:] + + (ann_dir / "instances_train_list.txt").write_text("\n".join(train_files)) + (ann_dir / "instances_val_list.txt").write_text("\n".join(val_files)) + if test_files: + (ann_dir / "instances_test_list.txt").write_text("\n".join(test_files)) + + return True + + +def _run_smoke_twophase(config): + """Run main() in two phases for tasks that need annotation fixup. + + Phase 1: Run with training disabled to set up dataset structure + Phase 2: Auto-create train/val splits, then run with training enabled + """ + from tinyml_modelmaker.run_tinyml_modelmaker import main + + task_type = config["common"]["task_type"] + # Determine where the project data will be stored + project_base = os.path.join("data", "projects", task_type) + + # Phase 1: Dataset-only run + dataset_config = copy.deepcopy(config) + dataset_config["training"]["enable"] = False + main(dataset_config) + + # Phase 2: Create annotation splits if needed + _auto_create_splits(project_base, task_type) + + # Phase 3: Full run with training enabled + # Disable dataset re-processing since data is already in place + train_config = copy.deepcopy(config) + train_config["dataset"]["enable"] = False + result = main(train_config) + return result + + +def _run_smoke(config): + """Run modelmaker main() directly (for classification which auto-splits).""" + from tinyml_modelmaker.run_tinyml_modelmaker import main + return main(config) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.smoke +class TestClassificationSmoke: + """1-epoch classification with synthetic 3-class data.""" + + def test_classification_trains(self, synthetic_classification_data, tmp_path): + config = _make_config( + task_type="generic_timeseries_classification", + model_name="CLS_1k_NPU", + data_path=synthetic_classification_data, + data_processing_feature_extraction={ + "feature_extraction_name": "Generic_256Input_RAW_256Feature_1Frame", + "variables": 1, + }, + ) + + result = _run_smoke(config) + assert result is True, "Classification smoke test failed" + + +@pytest.mark.smoke +class TestRegressionSmoke: + """1-epoch regression with synthetic continuous data.""" + + def test_regression_trains(self, synthetic_regression_data, tmp_path): + config = _make_config( + task_type="generic_timeseries_regression", + model_name="REGR_1k", + data_path=synthetic_regression_data, + data_processing_feature_extraction={ + "feature_extraction_name": "Generic_256Input_RAW_256Feature_1Frame", + "variables": 2, + }, + ) + + result = _run_smoke_twophase(config) + assert result is True, "Regression smoke test failed" + + +@pytest.mark.smoke +@pytest.mark.xfail( + reason="generic_timeseries_anomalydetection has no device preset in compilation config" +) +class TestAnomalyDetectionSmoke: + """1-epoch anomaly detection with synthetic normal/anomaly data.""" + + def test_anomalydetection_trains(self, synthetic_anomaly_data, tmp_path): + config = _make_config( + task_type="generic_timeseries_anomalydetection", + model_name="AD_1k", + data_path=synthetic_anomaly_data, + data_processing_feature_extraction={ + "feature_extraction_name": "Generic_256Input_RAW_256Feature_1Frame", + "variables": 1, + }, + ) + + result = _run_smoke(config) + assert result is True, "Anomaly detection smoke test failed" + + +@pytest.mark.smoke +class TestForecastingSmoke: + """1-epoch forecasting with synthetic time series.""" + + def test_forecasting_trains(self, synthetic_forecasting_data, tmp_path): + config = _make_config( + task_type="generic_timeseries_forecasting", + model_name="FCST_LSTM8", + data_path=synthetic_forecasting_data, + data_processing_feature_extraction={ + "data_proc_transforms": ["SimpleWindow"], + "frame_size": 32, + "stride_size": 0.5, + "forecast_horizon": 2, + "variables": 1, + "target_variables": [0], + }, + ) + + result = _run_smoke_twophase(config) + assert result is True, "Forecasting smoke test failed" From b5791014403ee4f6c62b782f0c4c0841f377e54c Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 5 Mar 2026 15:13:01 -0500 Subject: [PATCH 3/6] Add GitHub Actions CI: Tiers 1-3 on Linux and Windows Runs test_model_registry, test_config_validation, test_cross_device, and test_pipeline_smoke on ubuntu-latest and windows-latest. Installs ti_mcu_nnc platform-specific wheels. --- .github/workflows/test-modelmaker.yml | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/test-modelmaker.yml diff --git a/.github/workflows/test-modelmaker.yml b/.github/workflows/test-modelmaker.yml new file mode 100644 index 00000000..4cec8120 --- /dev/null +++ b/.github/workflows/test-modelmaker.yml @@ -0,0 +1,63 @@ +name: Tests + +on: + push: + branches: [platypus_dev_1.3, main] + paths: + - 'tinyml-modelmaker/**' + - '.github/workflows/test-modelmaker.yml' + pull_request: + branches: [platypus_dev_1.3, main] + paths: + - 'tinyml-modelmaker/**' + workflow_dispatch: # manual trigger + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + name: Tests (${{ matrix.os }}) + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -e tinyml-modelzoo + pip install -e tinyml-tinyverse + pip install -e "tinyml-modeloptimization/torchmodelopt" + pip install -e tinyml-modelmaker --no-deps + pip install defusedxml==0.7.1 "numpy==2.2.6" "PyYAML==6.0.3" "tqdm==4.67.1" "requests==2.32.5" "torch==2.7.1" + pip install pytest + + - name: Install ti_mcu_nnc (Linux only) + if: runner.os == 'Linux' + run: | + pip install "ti_mcu_nnc @ https://software-dl.ti.com/mctools/esd/tvm/mcu/ti_mcu_nnc-2.1.1-cp310-cp310-linux_x86_64.whl" || true + + - name: Install ti_mcu_nnc (Windows only) + if: runner.os == 'Windows' + run: | + pip install "ti_mcu_nnc @ https://software-dl.ti.com/mctools/esd/tvm/mcu/ti_mcu_nnc-2.1.1-cp310-cp310-win_amd64.whl" || true + + - name: Tier 1 — Component Tests + working-directory: tinyml-modelmaker + run: pytest tests/test_model_registry.py tests/test_config_validation.py -v --tb=short + + - name: Tier 3 — Cross-Device Validation + working-directory: tinyml-modelmaker + run: pytest tests/test_cross_device.py -v --tb=short + + - name: Tier 2 — Pipeline Smoke Tests + working-directory: tinyml-modelmaker + run: pytest tests/test_pipeline_smoke.py -v --tb=short From 2907309ee9b5f6f5fc2d5d2c248df6d4adac9641 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 5 Mar 2026 15:44:58 -0500 Subject: [PATCH 4/6] Fix CI: use python -m pytest for Windows compatibility --- .github/workflows/test-modelmaker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-modelmaker.yml b/.github/workflows/test-modelmaker.yml index 4cec8120..65ae2c02 100644 --- a/.github/workflows/test-modelmaker.yml +++ b/.github/workflows/test-modelmaker.yml @@ -52,12 +52,12 @@ jobs: - name: Tier 1 — Component Tests working-directory: tinyml-modelmaker - run: pytest tests/test_model_registry.py tests/test_config_validation.py -v --tb=short + run: python -m pytest tests/test_model_registry.py tests/test_config_validation.py -v --tb=short - name: Tier 3 — Cross-Device Validation working-directory: tinyml-modelmaker - run: pytest tests/test_cross_device.py -v --tb=short + run: python -m pytest tests/test_cross_device.py -v --tb=short - name: Tier 2 — Pipeline Smoke Tests working-directory: tinyml-modelmaker - run: pytest tests/test_pipeline_smoke.py -v --tb=short + run: python -m pytest tests/test_pipeline_smoke.py -v --tb=short From 341b8afe3eb02d71648267d6f80a121e4c390906 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 5 Mar 2026 15:55:33 -0500 Subject: [PATCH 5/6] CI: mark Windows as experimental (continue-on-error), use bash shell --- .github/workflows/test-modelmaker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-modelmaker.yml b/.github/workflows/test-modelmaker.yml index 65ae2c02..4a7b34e3 100644 --- a/.github/workflows/test-modelmaker.yml +++ b/.github/workflows/test-modelmaker.yml @@ -20,6 +20,12 @@ jobs: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} name: Tests (${{ matrix.os }}) + # Windows support is experimental — don't block the overall run + continue-on-error: ${{ matrix.os == 'windows-latest' }} + + defaults: + run: + shell: bash steps: - name: Checkout From b5cd358d831b308de82f190e9825ed31ea991400 Mon Sep 17 00:00:00 2001 From: M Platypus Date: Thu, 5 Mar 2026 15:57:35 -0500 Subject: [PATCH 6/6] CI: add macOS to test matrix --- .github/workflows/test-modelmaker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-modelmaker.yml b/.github/workflows/test-modelmaker.yml index 4a7b34e3..1a8f8c36 100644 --- a/.github/workflows/test-modelmaker.yml +++ b/.github/workflows/test-modelmaker.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} name: Tests (${{ matrix.os }}) # Windows support is experimental — don't block the overall run