From ccf6954e3eabbbc364b67ee5fa8fa2e824263872 Mon Sep 17 00:00:00 2001 From: Jochen Hoenle Date: Mon, 27 Jul 2026 14:17:38 +0200 Subject: [PATCH] [rules score] add test case coverage --- MODULE.bazel | 3 +- .../docs/requirements/tool_requirements.trlc | 101 ++++ .../docs/requirements/use_cases.trlc | 47 ++ .../rules_score/docs/tool_reference/index.rst | 1 + .../tool_reference/test_case_coverage.rst | 137 ++++++ .../rules_score/docs/user_guide/index.rst | 1 + .../docs/user_guide/tutorial/validation.rst | 12 +- .../docs/user_guide/validation.rst | 91 ++++ .../examples/minimal/test/my_unit_test.cpp | 5 + bazel/rules/rules_score/examples/seooc/BUILD | 3 + .../component_requirements_extra.trlc | 2 +- .../seooc/test_case_coverage.lock.yaml | 88 ++++ .../examples/seooc/unit_1/foo_test.cpp | 48 ++ .../rules_score/examples/seooc/unit_2/bar.cpp | 6 + .../rules_score/examples/seooc/unit_2/bar.h | 3 + .../examples/seooc/unit_2/bar_test.cpp | 17 + .../lobster/config/lobster_de.conf.tpl | 1 + bazel/rules/rules_score/private/component.bzl | 149 +++++- .../private/dependable_element.bzl | 118 ++++- bazel/rules/rules_score/providers.bzl | 17 +- .../rules_score/src/test_case_coverage/BUILD | 185 +++++++ .../src/test_case_coverage/__init__.py | 20 + .../assets/component_diagram.puml | 81 ++++ .../src/test_case_coverage/check_lock.py | 120 +++++ .../src/test_case_coverage/check_lock_test.py | 157 ++++++ .../src/test_case_coverage/compute_lock.py | 211 ++++++++ .../test_case_coverage/compute_lock_test.py | 211 ++++++++ .../src/test_case_coverage/conftest.py | 79 +++ .../test_case_coverage/lobster_generator.py | 167 +++++++ .../lobster_generator_test.py | 236 +++++++++ .../test_case_coverage/read_gtest_lobster.py | 306 ++++++++++++ .../read_gtest_lobster_test.py | 454 ++++++++++++++++++ .../src/test_case_coverage/runner_common.py | 74 +++ .../src/test_case_coverage/test_runner.py | 115 +++++ .../test_case_coverage/test_runner_test.py | 171 +++++++ .../src/test_case_coverage/update_runner.py | 115 +++++ .../test_case_coverage/update_runner_test.py | 120 +++++ bazel/rules/rules_score/test/BUILD | 67 +++ .../test_case_coverage.lock.yaml | 14 + .../rules_score/test/unit_component_test.bzl | 106 ++++ bazel/rules/rules_score/trlc/config/BUILD | 4 +- .../rules/rules_score/trlc/config/tools.trlc | 5 + 42 files changed, 3851 insertions(+), 17 deletions(-) create mode 100644 bazel/rules/rules_score/docs/tool_reference/test_case_coverage.rst create mode 100644 bazel/rules/rules_score/docs/user_guide/validation.rst create mode 100644 bazel/rules/rules_score/examples/seooc/test_case_coverage.lock.yaml create mode 100644 bazel/rules/rules_score/src/test_case_coverage/BUILD create mode 100644 bazel/rules/rules_score/src/test_case_coverage/__init__.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/assets/component_diagram.puml create mode 100644 bazel/rules/rules_score/src/test_case_coverage/check_lock.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/check_lock_test.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/compute_lock.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/compute_lock_test.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/conftest.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/lobster_generator.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/lobster_generator_test.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster_test.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/runner_common.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/test_runner.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/test_runner_test.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/update_runner.py create mode 100644 bazel/rules/rules_score/src/test_case_coverage/update_runner_test.py create mode 100644 bazel/rules/rules_score/test/fixtures/test_case_coverage_lock_test/test_case_coverage.lock.yaml diff --git a/MODULE.bazel b/MODULE.bazel index 8d349a22..04c6fd1e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -42,7 +42,8 @@ bazel_dep(name = "trlc", version = "3.0.0") bazel_dep(name = "lobster", version = "1.0.3") git_override( module_name = "lobster", - commit = "07382c33f95e22992e0f8788c0537ec06877e51f", + # Bumped for req_coverage's use of lobster.common.{io,items,errors,location}. + commit = "247927a438647f68c70a61f9fdde155f5aee145d", remote = "https://github.com/bmw-software-engineering/lobster.git", ) diff --git a/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc b/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc index 0af48c4c..a47952f4 100644 --- a/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc +++ b/bazel/rules/rules_score/docs/requirements/tool_requirements.trlc @@ -176,4 +176,105 @@ section "Tool Requirements" { } + section "Test Case Coverage" { + + ToolQualification.ToolRequirement Lock_File_As_Spec_And_Approval { + description = ''' + The tool shall use one YAML lock file as both the declaration of + which requirements are claimed to be completely covered and as the + committed approval artifact. + ''' + derived_from = [Declare_Complete_Coverage] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement Requirement_IDs_From_TRLC { + description = ''' + The tool shall extract requirement identifiers automatically from + the TrlcProviderInfo provider of the requirements target attribute. + ''' + derived_from = [Semi_Automated_Lock_Update, Declare_Complete_Coverage] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement Parse_GWT_Annotations { + description = ''' + The tool shall parse GTest XML result files, extracting + RecordProperty values for the keys lobster-tracing, given, when, + and then on a per-TEST, per-TEST_F, and per-TEST_P basis. A test case + without a given, when, or then property shall be included with an + empty string for each missing field. + ''' + derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement Multi_Requirement_Linking { + description = ''' + A test case that carries lobster-tracing values matching more than + one requirement identifier shall appear in the lock file entry of + every matched requirement. + ''' + derived_from = [Declare_Complete_Coverage] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement Lock_File_Content { + description = ''' + The lock file shall contain, per requirement: the requirement + identifier and an ordered list of linked test cases. Each test + case entry shall contain: a uid in the format + "//bazel_package/SuiteName:TestName" (package-scoped gtest tag) + and the individual given, when, and then strings as separate + fields. + ''' + derived_from = [Detect_Specification_Drift, Declare_Complete_Coverage] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement Lock_File_Ordering { + description = ''' + Test cases within a requirement entry shall be ordered + lexicographically by uid to produce a deterministic, + diff-friendly lock file. + ''' + derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement Update_Command { + description = ''' + The .update Bazel run target shall extract requirement identifiers + from the TrlcProviderInfo target, parse GTest XML result files for + lobster-tracing-linked test cases, build the spec from + given/when/then, and overwrite the committed lock file. + ''' + derived_from = [Semi_Automated_Lock_Update] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement Test_Command_Verification { + description = ''' + The Bazel test target shall parse GTest XML result files, rebuild + the spec for each linked test, and compare uid and spec against + the committed lock file, failing if they differ. On failure it + shall print a human-readable diff showing which test cases changed + and the exact bazel run command needed to refresh the lock file. + ''' + derived_from = [Detect_Specification_Drift, Detect_Coverage_Set_Changes] + satisfied_by = Tools.TestCaseCoverage + } + + ToolQualification.ToolRequirement LOBSTER_Covered_Attribute { + description = ''' + The test target shall emit a lobster-act-trace JSON artifact that + marks each linked test case as "ok" when its uid exists in the + committed lock file and its spec matches, and "fail" otherwise. + ''' + derived_from = [Generate_Covered_Traceability_Artifact] + satisfied_by = Tools.TestCaseCoverage + } + + } + } diff --git a/bazel/rules/rules_score/docs/requirements/use_cases.trlc b/bazel/rules/rules_score/docs/requirements/use_cases.trlc index 96f310ed..05e7c957 100644 --- a/bazel/rules/rules_score/docs/requirements/use_cases.trlc +++ b/bazel/rules/rules_score/docs/requirements/use_cases.trlc @@ -173,4 +173,51 @@ section "Use Cases" { affected_tools = [Tools.Verifier, Tools.PlantumlParser, Tools.LibClang, Tools.Bazel] } + ToolQualification.UseCase Declare_Complete_Coverage { + description = ''' + As a safety engineer I want to declare that a set of requirements + is completely covered by automated test cases so that the coverage + claim is explicit, auditable, and version-controlled. + ''' + affected_tools = [Tools.TestCaseCoverage] + } + + ToolQualification.UseCase Detect_Specification_Drift { + description = ''' + In the CI pipeline I want to detect automatically when a test's + given/when/then specification has changed without re-approval so + that stale coverage claims are caught before release. + ''' + affected_tools = [Tools.TestCaseCoverage] + } + + ToolQualification.UseCase Detect_Coverage_Set_Changes { + description = ''' + In the CI pipeline I want to detect when a test case is added to or + removed from the set of tests linked to a requirement so that + undeclared coverage changes cannot pass silently. + ''' + affected_tools = [Tools.TestCaseCoverage] + } + + ToolQualification.UseCase Generate_Covered_Traceability_Artifact { + description = ''' + As a project integrator I want the tool to emit a LOBSTER-compatible + artifact that marks each linked test case as covered or not so that + complete requirement coverage is visible in the LOBSTER dashboard. + ''' + affected_tools = [Tools.TestCaseCoverage, Tools.Lobster] + } + + ToolQualification.UseCase Semi_Automated_Lock_Update { + description = ''' + As an engineer I want a single bazel run command that reads + requirement identifiers from the TRLC requirements target, + discovers all linked test cases in the GTest XML results, and + rewrites the lock file, so that I only need to review the diff + before committing my approval. + ''' + affected_tools = [Tools.TestCaseCoverage, Tools.Bazel] + } + } diff --git a/bazel/rules/rules_score/docs/tool_reference/index.rst b/bazel/rules/rules_score/docs/tool_reference/index.rst index 13e4b10f..69fac59c 100644 --- a/bazel/rules/rules_score/docs/tool_reference/index.rst +++ b/bazel/rules/rules_score/docs/tool_reference/index.rst @@ -48,3 +48,4 @@ of truth. lobster_bazel lobster + test_case_coverage diff --git a/bazel/rules/rules_score/docs/tool_reference/test_case_coverage.rst b/bazel/rules/rules_score/docs/tool_reference/test_case_coverage.rst new file mode 100644 index 00000000..a164db1d --- /dev/null +++ b/bazel/rules/rules_score/docs/tool_reference/test_case_coverage.rst @@ -0,0 +1,137 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Test Case Coverage (test_case_coverage) +======================================= + +``test_case_coverage`` verifies that a declared set of requirements is **completely** +covered by automated test cases. Coverage is locked and confirmed by a committed YAML file; + +Requirement IDs are extracted automatically and test records are read from the ``gtest.lobster`` file. + +.. note:: + + For how to annotate tests with ``lobster-tracing`` see + :doc:`../user_guide/validation`. + +The ``component`` rule integrates test case coverage when ``test_case_coverage_lock`` +is provided: + +.. code-block:: starlark + + component( + name = "my_component", + requirements = [":my_component_requirements"], + components = [":unit_a", ":unit_b"], + test_case_coverage_lock = "test_case_coverage.lock.yaml", + ) + +When the build reports a test_case_coverage drift failure, run (here on my_component): + + .. code-block:: bash + + bazel run //:.update + +Two phases +---------- + +``test_case_coverage`` has two distinct operational phases: + +**Test phase** — ``bazel build //...`` / ``bazel test //...`` + + ``test_runner`` is invoked as a **build action** inside the + ``dependable_element`` rule for each component that has ``test_case_coverage_lock`` + set. The effective enforcement stringency is controlled by the enclosing + ``dependable_element``'s ``maturity`` attribute: + + * ``maturity = "development"`` — ``--allow-check-failures`` is passed so + the ``.lobster`` artifact is always produced even when coverage drifts. + Both lock-drift failures **and** missing-GWT-annotation errors are + downgraded to warnings; + * ``maturity = "release"`` — no ``--allow-check-failures``; a drift or + missing GWT annotation fails the Bazel build action directly. The + ``.lobster`` artifact is physically written before the action exits, but + Bazel marks the action as failed. + +**Run phase** — ``bazel run //:.update`` + + Computes the lock from the current ``gtest.lobster`` and **overwrites** + the committed ``test_case_coverage.lock.yaml`` in the workspace. Committing the + result constitutes approval of test case coverage. + +Architecture +------------ + +.. code-block:: text + + bazel/rules/rules_score/src/test_case_coverage/ + ├── read_gtest_lobster.py # reads gtest.lobster; also resolve_path, scan, req_ids + ├── compute_lock.py # uid/spec serialisation + YAML lock + ├── check_lock.py # committed vs computed comparison + ├── lobster_generator.py # emits LOBSTER activity artifact + ├── update_runner.py # entry point: bazel run .update + └── test_runner.py # entry point: bazel test //... + +Data flow +--------- + +.. code-block:: text + + Requirements targets (.lobster) ──► extract requirement IDs + subrule_lobster_gtest ───────────► gtest.lobster + │ + read_gtest_lobster + (uid, req refs, spec text) + │ + compute_lock + ┌─────────┴──────────────┐ + RUN phase TEST phase + (bazel run .update) (bazel test) + │ │ + rewrite lock compare uid+spec + file in WS vs committed lock + │ + lobster_generator + (ok / fail per test) + +Lock file format +---------------- + +Auto-generated by the ``.update`` target. The engineer commits it after +reviewing ``git diff``. + +.. code-block:: yaml + + schema_version: 3 + requirements: + - id: MessagePassing.OsIpcFaultHandling + test_cases: + - uid: "//score/message_passing/ConnectionSuite:OsIpcFaultHandlingTest" + given: a connected client + when: the OS IPC call fails + then: the client receives an error + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Field + - Notes + * - ``requirements[].id`` + - Matches ``lobster-tracing`` values; extracted from requirements targets + * - ``test_cases[].uid`` + - ``//bazel_package/SuiteName:TestName`` — package-scoped gtest tag + * - ``test_cases[].given`` / ``when`` / ``then`` + - Individual GWT fields from ``RecordProperty`` annotations; any change + makes the lock stale diff --git a/bazel/rules/rules_score/docs/user_guide/index.rst b/bazel/rules/rules_score/docs/user_guide/index.rst index ee930a2d..5e4dc76c 100644 --- a/bazel/rules/rules_score/docs/user_guide/index.rst +++ b/bazel/rules/rules_score/docs/user_guide/index.rst @@ -36,3 +36,4 @@ User Guide architectural_design unit_design dependability_analysis + validation diff --git a/bazel/rules/rules_score/docs/user_guide/tutorial/validation.rst b/bazel/rules/rules_score/docs/user_guide/tutorial/validation.rst index f7f872e8..c522a885 100644 --- a/bazel/rules/rules_score/docs/user_guide/tutorial/validation.rst +++ b/bazel/rules/rules_score/docs/user_guide/tutorial/validation.rst @@ -33,7 +33,17 @@ test/my_unit_test.cpp :lines: 14- Each ``RecordProperty("lobster-tracing", "...")`` call names the requirement -identifiers covered by that test case. +identifiers covered by that test case. The optional ``given``/``when``/``then`` +properties capture the Given-When-Then specification for that test case: + +* ``given`` — the initial state or precondition +* ``when`` — the action or event under test +* ``then`` — the expected outcome + +They are picked up the same way as ``lobster-tracing`` — as plain +``RecordProperty`` calls — and are rendered alongside the test result in the +traceability report. Test cases without them are still traced, but show up +without a Given-When-Then specification. BUILD ------ diff --git a/bazel/rules/rules_score/docs/user_guide/validation.rst b/bazel/rules/rules_score/docs/user_guide/validation.rst new file mode 100644 index 00000000..f92c6e23 --- /dev/null +++ b/bazel/rules/rules_score/docs/user_guide/validation.rst @@ -0,0 +1,91 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Validation +========== + +Unit Tests +---------- + +Unit tests in ``rules_score`` components are written with **GoogleTest** and built +with ``cc_test``. Each test case that covers a component requirement must carry +a ``lobster-tracing`` annotation so that the build can link the test back to the +requirement. + +Annotating Tests +~~~~~~~~~~~~~~~~ + +Call ``RecordProperty`` inside the test body next to the respective code blocks: + +.. code-block:: cpp + + TEST_F(MyFixture, DoesXWhenY) { + ::testing::Test::RecordProperty("lobster-tracing", "MessagePassing.OsIpcFaultHandling"); + ::testing::Test::RecordProperty("given", "a connected client"); + ::testing::Test::RecordProperty("when", "the OS IPC call fails"); + ::testing::Test::RecordProperty("then", "the client receives an error"); + } + +.. list-table:: + :header-rows: 1 + :widths: 20 15 65 + + * - Property + - Required + - Description + * - ``lobster-tracing`` + - yes + - Comma-separated requirement IDs; links the test to one or more ``CompReq`` records + * - ``given`` + - no + - Initial state / precondition + * - ``when`` + - no + - Action or event under test + * - ``then`` + - no + - Expected outcome + +A test without ``lobster-tracing`` has no traceability and is not included in +coverage tracking. + +Stating Coverage for test cases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Coverage is declared through a committed ``test_case_coverage.lock.yaml`` file that lists, +per requirement, every test case (uid + given/when/then) that covers it. +Committing the file is stating the coverage claim. + +In Bazel the yaml file can be linked to the ``component`` macro via the ``test_case_coverage_lock`` attribute: + +.. code-block:: starlark + + component( + name = "my_component", + requirements = [":my_component_requirements"], + components = [":unit_a", ":unit_b"], + test_case_coverage_lock = "test_case_coverage.lock.yaml", + ) + +Two complementary workflows keep the lock file in sync: + +* **``bazel run …update``** — reads the current test results and **rewrites** + ``test_case_coverage.lock.yaml`` in the source tree. +* **``bazel test //…``** — the build action recomputes coverage from the same + test results and **compares** it against the committed lock. Any drift (new + test, removed test, changed GWT text, version bump) fails the build until the + lock is refreshed and re-committed. + +For the full tool description — lock file format, update workflow, design +decisions — see :doc:`../tool_reference/test_case_coverage`. diff --git a/bazel/rules/rules_score/examples/minimal/test/my_unit_test.cpp b/bazel/rules/rules_score/examples/minimal/test/my_unit_test.cpp index 378489a9..881c2156 100644 --- a/bazel/rules/rules_score/examples/minimal/test/my_unit_test.cpp +++ b/bazel/rules/rules_score/examples/minimal/test/my_unit_test.cpp @@ -18,6 +18,11 @@ TEST(MyUnitTest, ConfigureAndGet) { ::testing::Test::RecordProperty( "lobster-tracing", "MinimalExample.FEAT_001 MinimalExample.FEAT_002"); + ::testing::Test::RecordProperty("given", + "a default-constructed MyUnit instance"); + ::testing::Test::RecordProperty("when", + "configure is called with a known key"); + ::testing::Test::RecordProperty("then", "get returns the configured value"); MyUnit unit; unit.configure("mode", "fast"); diff --git a/bazel/rules/rules_score/examples/seooc/BUILD b/bazel/rules/rules_score/examples/seooc/BUILD index 817383dd..c42ad63b 100644 --- a/bazel/rules/rules_score/examples/seooc/BUILD +++ b/bazel/rules/rules_score/examples/seooc/BUILD @@ -47,6 +47,7 @@ component( "//docs/requirements:component_requirements_sub", ], tags = ["manual"], + test_case_coverage_lock = "test_case_coverage.lock.yaml", tests = [], ) @@ -56,6 +57,7 @@ component( name = "sub_component_example", requirements = [ "//docs/requirements:component_requirements_sub", + "//docs/requirements:feature_requirements", ], tags = ["manual"], tests = [], @@ -80,6 +82,7 @@ dependable_element( ], glossary = ["//docs:glossary"], integrity_level = "B", + maturity = "development", requirements = [ "//docs/requirements:feature_requirements", ], diff --git a/bazel/rules/rules_score/examples/seooc/docs/requirements/component_requirements_extra.trlc b/bazel/rules/rules_score/examples/seooc/docs/requirements/component_requirements_extra.trlc index 17a4ff54..91475004 100644 --- a/bazel/rules/rules_score/examples/seooc/docs/requirements/component_requirements_extra.trlc +++ b/bazel/rules/rules_score/examples/seooc/docs/requirements/component_requirements_extra.trlc @@ -16,7 +16,7 @@ import ScoreReq import SampleSEooC ScoreReq.CompReq REQ_COMP_EXTRA_001 { - description = "The numeric value management interface shall initialize its internal value to zero upon construction" + description = "The numeric value management interface shall return a fixed, well-defined value immediately upon construction, without requiring any prior initialization calls" safety = ScoreReq.Asil.B derived_from = [SampleSEooC.FEAT_001@1] version = 1 diff --git a/bazel/rules/rules_score/examples/seooc/test_case_coverage.lock.yaml b/bazel/rules/rules_score/examples/seooc/test_case_coverage.lock.yaml new file mode 100644 index 00000000..956cb743 --- /dev/null +++ b/bazel/rules/rules_score/examples/seooc/test_case_coverage.lock.yaml @@ -0,0 +1,88 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# test_case_coverage.lock.yaml — auto-generated by test_case_coverage +# Run `bazel run @@//:component_example.update` to refresh. +schema_version: 3 +requirements: + - id: SampleComponent.REQ_COMP_001 + version: '1' + description: The numeric value management interface shall provide a read operation that returns a uint8_t value + test_cases: + - uid: //Foo:GetNumber + given: a default-constructed Foo instance + when: GetNumber is called + then: it returns 42 + - uid: //Foo:GetNumberViaConstInstance + given: a const default-constructed Foo instance + when: GetNumber is called through a const reference + then: it still returns 42 + - id: SampleComponent.REQ_COMP_002 + version: '1' + description: The numeric value management interface shall not be extensible by external code + test_cases: + - uid: //Foo:IsFinal + given: the Foo class definition + when: checking whether the class is extensible + then: it is declared final, preventing any subclassing + - uid: //Foo:IsFinalStaticAssert + given: the Foo class definition + when: querying the type trait std::is_final for Foo + then: the trait reports true, confirming Foo cannot be subclassed + - id: SampleComponent.REQ_COMP_003 + version: '1' + description: The numeric value validator shall return a boolean result indicating whether the managed numeric value equals 42 + test_cases: + - uid: //Bar:AssertNumber + given: a Bar instance owning a Foo with value 42 + when: AssertNumber is called + then: it returns true + - id: SampleComponent.REQ_COMP_004 + version: '1' + description: The numeric value validator shall accept a numeric value manager instance as its sole constructor argument + test_cases: + - uid: //Bar:AssertNumber + given: a Bar instance owning a Foo with value 42 + when: AssertNumber is called + then: it returns true + - id: SampleComponent.REQ_COMP_005 + version: '1' + description: The numeric value validator shall take exclusive ownership of the received numeric value manager instance + test_cases: + - uid: //Bar:AssertNumber + given: a Bar instance owning a Foo with value 42 + when: AssertNumber is called + then: it returns true + - id: SampleComponent.REQ_COMP_006 + version: '1' + description: The numeric value formatter shall convert a validated numeric value manager into a human-readable string representation + test_cases: + - uid: //Bar:Format + given: a Bar instance owning a Foo with value 42 + when: Format is called + then: it returns the human-readable string "42" + - id: SampleComponentExtra.REQ_COMP_EXTRA_001 + version: '1' + description: The numeric value management interface shall return a fixed, well-defined value immediately upon construction, without requiring any prior initialization calls + test_cases: + - uid: //Foo:InitializesToKnownValue + given: a default-constructed Foo instance + when: GetNumber is called for the first time + then: it returns 42 + - id: SampleComponentExtra.REQ_COMP_EXTRA_002 + version: '1' + description: The numeric value management interface shall guarantee that the stored value is never modified after construction + test_cases: + - uid: //Foo:ValueConsistentAcrossReads + given: a const Foo instance + when: GetNumber is called multiple times + then: the same value is returned on each call diff --git a/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp b/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp index 6cc8e1cb..bed64dc0 100644 --- a/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp +++ b/bazel/rules/rules_score/examples/seooc/unit_1/foo_test.cpp @@ -15,9 +15,15 @@ #include +#include + TEST(Foo, GetNumber) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponent.REQ_COMP_001"); + ::testing::Test::RecordProperty("given", + "a default-constructed Foo instance"); + ::testing::Test::RecordProperty("when", "GetNumber is called"); + ::testing::Test::RecordProperty("then", "it returns 42"); unit_1::Foo unit{}; @@ -27,13 +33,51 @@ TEST(Foo, GetNumber) { TEST(Foo, IsFinal) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponent.REQ_COMP_002"); + + ::testing::Test::RecordProperty("given", "the Foo class definition"); + ::testing::Test::RecordProperty("when", + "checking whether the class is extensible"); + ::testing::Test::RecordProperty( + "then", "it is declared final, preventing any subclassing"); // Foo is declared final; extensibility is enforced at compile time. SUCCEED(); } +TEST(Foo, IsFinalStaticAssert) { + ::testing::Test::RecordProperty("lobster-tracing", + "SampleComponent.REQ_COMP_002"); + + ::testing::Test::RecordProperty("given", "the Foo class definition"); + ::testing::Test::RecordProperty( + "when", "querying the type trait std::is_final for Foo"); + ::testing::Test::RecordProperty( + "then", "the trait reports true, confirming Foo cannot be subclassed"); + static_assert(std::is_final::value, + "Foo must remain final"); + SUCCEED(); +} + +TEST(Foo, GetNumberViaConstInstance) { + ::testing::Test::RecordProperty("lobster-tracing", + "SampleComponent.REQ_COMP_001"); + ::testing::Test::RecordProperty("given", + "a const default-constructed Foo instance"); + ::testing::Test::RecordProperty("when", "GetNumber is called through a const reference"); + ::testing::Test::RecordProperty("then", "it still returns 42"); + + const unit_1::Foo unit{}; + + EXPECT_EQ(unit.GetNumber(), 42u); +} + TEST(Foo, InitializesToKnownValue) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponentExtra.REQ_COMP_EXTRA_001"); + ::testing::Test::RecordProperty("given", + "a default-constructed Foo instance"); + ::testing::Test::RecordProperty("when", + "GetNumber is called for the first time"); + ::testing::Test::RecordProperty("then", "it returns 42"); unit_1::Foo unit{}; EXPECT_EQ(unit.GetNumber(), 42u); @@ -42,6 +86,10 @@ TEST(Foo, InitializesToKnownValue) { TEST(Foo, ValueConsistentAcrossReads) { ::testing::Test::RecordProperty("lobster-tracing", "SampleComponentExtra.REQ_COMP_EXTRA_002"); + ::testing::Test::RecordProperty("given", "a const Foo instance"); + ::testing::Test::RecordProperty("when", "GetNumber is called multiple times"); + ::testing::Test::RecordProperty("then", + "the same value is returned on each call"); const unit_1::Foo unit{}; EXPECT_EQ(unit.GetNumber(), unit.GetNumber()); diff --git a/bazel/rules/rules_score/examples/seooc/unit_2/bar.cpp b/bazel/rules/rules_score/examples/seooc/unit_2/bar.cpp index d4c3c584..cb71383e 100644 --- a/bazel/rules/rules_score/examples/seooc/unit_2/bar.cpp +++ b/bazel/rules/rules_score/examples/seooc/unit_2/bar.cpp @@ -29,4 +29,10 @@ bool Bar::AssertNumber() const { assert(foo_ != nullptr); return foo_->GetNumber() == kExpectedNumber; } + +// trace: SampleComponent.REQ_COMP_006 +std::string Bar::Format() const { + assert(foo_ != nullptr); + return std::to_string(foo_->GetNumber()); +} } // namespace unit_2 diff --git a/bazel/rules/rules_score/examples/seooc/unit_2/bar.h b/bazel/rules/rules_score/examples/seooc/unit_2/bar.h index 6f24c0ee..b8fe91a6 100644 --- a/bazel/rules/rules_score/examples/seooc/unit_2/bar.h +++ b/bazel/rules/rules_score/examples/seooc/unit_2/bar.h @@ -18,6 +18,7 @@ #include #include +#include namespace unit_2 { @@ -27,6 +28,8 @@ class Bar final { public: explicit Bar(std::unique_ptr foo); bool AssertNumber() const; + // trace: SampleComponent.REQ_COMP_006 + std::string Format() const; private: std::unique_ptr foo_; diff --git a/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp b/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp index ad633943..6e7b3145 100644 --- a/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp +++ b/bazel/rules/rules_score/examples/seooc/unit_2/bar_test.cpp @@ -19,7 +19,24 @@ TEST(Bar, AssertNumber) { ::testing::Test::RecordProperty( "lobster-tracing", "SampleComponent.REQ_COMP_003,SampleComponent.REQ_" "COMP_004,SampleComponent.REQ_COMP_005"); + ::testing::Test::RecordProperty("given", + "a Bar instance owning a Foo with value 42"); + ::testing::Test::RecordProperty("when", "AssertNumber is called"); + ::testing::Test::RecordProperty("then", "it returns true"); unit_2::Bar unit{std::make_unique()}; EXPECT_TRUE(unit.AssertNumber()); } + +TEST(Bar, Format) { + ::testing::Test::RecordProperty("lobster-tracing", + "SampleComponent.REQ_COMP_006"); + ::testing::Test::RecordProperty("given", + "a Bar instance owning a Foo with value 42"); + ::testing::Test::RecordProperty("when", "Format is called"); + ::testing::Test::RecordProperty( + "then", "it returns the human-readable string \"42\""); + unit_2::Bar unit{std::make_unique()}; + + EXPECT_EQ(unit.Format(), "42"); +} diff --git a/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl b/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl index b4ccfd6b..fae6335e 100644 --- a/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl +++ b/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl @@ -13,6 +13,7 @@ activity "Unit Test" { {UNIT_TEST_SOURCES} } +{COVERAGE_BLOCK} implementation "Architecture" { {ARCH_SOURCES} trace to: "Component Requirements"; diff --git a/bazel/rules/rules_score/private/component.bzl b/bazel/rules/rules_score/private/component.bzl index 1d38e02f..650e2dd1 100644 --- a/bazel/rules/rules_score/private/component.bzl +++ b/bazel/rules/rules_score/private/component.bzl @@ -20,11 +20,125 @@ with associated requirements and tests. """ load("@lobster//:lobster.bzl", "subrule_lobster_gtest", "subrule_lobster_html_report", "subrule_lobster_report") -load("//bazel/rules/rules_score:providers.bzl", "AssumedSystemRequirementsInfo", "CertifiedScope", "ComponentInfo", "ComponentRequirementsInfo", "FeatureRequirementsInfo", "SphinxSourcesInfo", "UnitInfo") +load("//bazel/rules/rules_score:providers.bzl", "AssumedSystemRequirementsInfo", "CertifiedScope", "ComponentInfo", "ComponentRequirementsInfo", "ComponentTestCaseCoverageInfo", "FeatureRequirementsInfo", "SphinxSourcesInfo", "UnitInfo") load("//bazel/rules/rules_score/private:lobster_config.bzl", "format_lobster_sources") # ============================================================================ -# Private Rule Implementation +# Private Rule Implementation: Component .update target +# ============================================================================ + +def _collect_req_lobster_files(ctx): + """Collect CompReq .lobster files from requirements targets. + + Only ComponentRequirementsInfo targets contribute files; FeatReq and + AssumedSystemReq targets are intentionally excluded because the coverage + runner filters to ``kind == CompReq`` anyway. + """ + req_lobster_files = [] + for req_target in ctx.attr.requirements: + if ComponentRequirementsInfo in req_target: + req_lobster_files.extend(req_target[ComponentRequirementsInfo].srcs.to_list()) + return req_lobster_files + +def _component_update_impl(ctx): + """Implementation for component.update — rewrites test_case_coverage.lock.yaml. + + Runs ``update_runner`` as an executable, passing the gtest.lobster file + and the req lobster manifest via environment variables. The gtest.lobster + file is *not* recomputed here: it is read off ``ComponentTestCaseCoverageInfo`` + from the sibling main component target (``ctx.attr.component``), the exact + same File the coverage check consumes. Recomputing it independently would + risk the two runs diverging and ``.update`` writing a lock file that the + check then rejects. + """ + + # Collect req lobster files from requirements targets + req_lobster_files = _collect_req_lobster_files(ctx) + + # Reuse the gtest.lobster already produced by the main component target, + # instead of running subrule_lobster_gtest a second time. + gtest_lobster_file = ctx.attr.component[ComponentTestCaseCoverageInfo].gtest_lobster_file + + # Write a runfiles manifest (short paths) — used by the runner at bazel run time. + runfiles_manifest = ctx.actions.declare_file("{}_req_lobster_runfiles_manifest.txt".format(ctx.label.name)) + ctx.actions.write( + output = runfiles_manifest, + content = "\n".join([f.short_path for f in req_lobster_files]) + "\n", + ) + + # The lock file is written back to the source tree by update_runner using + # $BUILD_WORKSPACE_DIRECTORY. Pass only the package-relative path; no + # symlink into runfiles is needed (the runner does not read the lock). + lock_file = ctx.file.test_case_coverage_lock + + # Symlink the update_runner binary as the rule's executable. + executable = ctx.actions.declare_file("{}_update_runner_bin".format(ctx.label.name)) + ctx.actions.symlink( + output = executable, + target_file = ctx.executable._update_runner, + is_executable = True, + ) + + all_inputs = req_lobster_files + [gtest_lobster_file, runfiles_manifest] + runfiles = ctx.runfiles(files = all_inputs) + runfiles = runfiles.merge(ctx.attr._update_runner[DefaultInfo].default_runfiles) + + return [ + DefaultInfo( + executable = executable, + runfiles = runfiles, + ), + RunEnvironmentInfo(environment = { + "TEST_CASE_COVERAGE_LOBSTER_MANIFEST": runfiles_manifest.short_path, + "TEST_CASE_COVERAGE_GTEST_LOBSTER": gtest_lobster_file.short_path, + # Package-relative path; update_runner prepends $BUILD_WORKSPACE_DIRECTORY. + # NOTE: ctx.label.package is "" for targets defined at the repo + # root — do not unconditionally prepend "/", or the path becomes + # "/test_case_coverage.lock.yaml" which Python's pathlib treats as an + # absolute filesystem-root path when joined with the workspace + # dir (silently discarding the workspace dir and failing with a + # permission error instead of writing into the source tree). + # Named distinctly from TEST_CASE_COVERAGE_LOCK_FILE (used by the + # check action below) because the two runners require different + # kinds of paths: this one is workspace-relative, that one is a + # directly resolvable/exec path. + "TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH": (ctx.label.package + "/" + lock_file.basename) if ctx.label.package else lock_file.basename, + "TEST_CASE_COVERAGE_LABEL": str(ctx.label), + "TEST_CASE_COVERAGE_PACKAGE": "//" + ctx.label.package, + }), + ] + +_component_update = rule( + implementation = _component_update_impl, + doc = "Refreshes component test_case_coverage.lock.yaml with current test coverage.", + attrs = { + "requirements": attr.label_list( + default = [], + providers = [[ComponentRequirementsInfo], [FeatureRequirementsInfo], [AssumedSystemRequirementsInfo]], + doc = "Requirements targets forwarded from the component() macro. Only ComponentRequirementsInfo targets contribute files; FeatReq and AssumedSystemReq labels are accepted so the macro can pass its full requirements list through without filtering.", + ), + "component": attr.label( + mandatory = True, + providers = [ComponentTestCaseCoverageInfo], + doc = "The sibling main component() target (self). Supplies the shared gtest.lobster File via ComponentTestCaseCoverageInfo so .update reuses the exact same artifact as the coverage check, instead of recomputing it.", + ), + "test_case_coverage_lock": attr.label( + allow_single_file = True, + mandatory = True, + doc = "Committed test_case_coverage.lock.yaml file to overwrite.", + ), + "_update_runner": attr.label( + doc = "test_case_coverage update runner executable.", + default = Label("//bazel/rules/rules_score/src/test_case_coverage:update_runner"), + executable = True, + cfg = "exec", + ), + }, + executable = True, +) + +# ============================================================================ +# Private Rule Implementation: Component main rule # ============================================================================ def _component_impl(ctx): @@ -211,6 +325,12 @@ def _component_impl(ctx): ), dependent_labels = depset(transitive = collected_dependencies), ), + # ComponentTestCaseCoverageInfo: test-case-coverage-lock data consumed by dependable_element + # (only emitted when test_case_coverage_lock is set; presence signals the DE to run the check). + ComponentTestCaseCoverageInfo( + gtest_lobster_file = gtest_lobster_file, + test_case_coverage_lock_file = ctx.file.test_case_coverage_lock, + ), # SphinxSourcesInfo: RST sources from component requirements + transitive sources from sub-components/units SphinxSourcesInfo( srcs = req_sphinx_depset, @@ -240,6 +360,10 @@ _component_test = rule( default = [], doc = "Component-level integration test targets", ), + "test_case_coverage_lock": attr.label( + allow_single_file = True, + doc = "Optional committed test_case_coverage.lock.yaml file for coverage validation", + ), "_lobster_ci_report": attr.label( default = "@lobster//:lobster-ci-report", executable = True, @@ -270,6 +394,7 @@ def component( tests = [], requirements = None, components = [], + test_case_coverage_lock = None, testonly = True, **kwargs): """Define a software component following S-CORE process guidelines. @@ -279,6 +404,7 @@ def component( - Component requirements: Requirements specification for the component - Components: Nested components (for hierarchical structures) - Tests: Integration tests that verify the component as a whole + - Coverage: Optional test case coverage traceability (if test_case_coverage_lock provided) Args: name: The name of the component. Used as the target name. @@ -294,6 +420,10 @@ def component( component structures). tests: List of labels to Bazel test targets that verify the component integration. + test_case_coverage_lock: Optional label to committed test_case_coverage.lock.yaml file. + If provided, the component rule generates test_case_coverage.lobster by comparing + computed test coverage against the committed lock. Enables `.update` + target for refreshing coverage. testonly: If true, only testonly targets can depend on this component. visibility: Bazel visibility specification for the component target. @@ -304,6 +434,7 @@ def component( requirements = [":kvs_component_requirements"], components = [":kvs_unit1", ":kvs_unit2"], tests = ["//persistency/kvs/tests:score_kvs_component_integration_tests"], + test_case_coverage_lock = "test_case_coverage.lock.yaml", visibility = ["//visibility:public"], ) ``` @@ -314,6 +445,20 @@ def component( requirements = requirements, components = components, tests = tests, + test_case_coverage_lock = test_case_coverage_lock, testonly = testonly, **kwargs ) + + # Create .update target if test_case_coverage_lock is provided + if test_case_coverage_lock: + # Only forward visibility — _component_update is an executable rule + # and does not accept the generic build attrs (tags, deprecation, etc.). + _component_update( + name = name + ".update", + requirements = requirements or [], + component = ":" + name, + test_case_coverage_lock = test_case_coverage_lock, + testonly = testonly, + visibility = kwargs.get("visibility"), + ) diff --git a/bazel/rules/rules_score/private/dependable_element.bzl b/bazel/rules/rules_score/private/dependable_element.bzl index 811d450a..3605a32b 100644 --- a/bazel/rules/rules_score/private/dependable_element.bzl +++ b/bazel/rules/rules_score/private/dependable_element.bzl @@ -33,6 +33,7 @@ load( "AssumptionsOfUseInfo", "CertifiedScope", "ComponentInfo", + "ComponentTestCaseCoverageInfo", "DependabilityAnalysisInfo", "DependableElementInfo", "DependableElementLobsterInfo", @@ -977,12 +978,20 @@ def _dependable_element_index_impl(ctx): if type(node) == type([]): if certified_scope.name in node: - fail("The same scope is covered twice: {}".format(certified_scope)) + msg = "The same scope is covered twice: {}".format(certified_scope) + if ctx.attr.maturity == "development": + print("WARNING: " + msg) + continue + fail(msg) node.append(certified_scope.name) else: inserted_element = node.setdefault(last_element, default = []) if certified_scope.name in inserted_element: - fail("The same scope is covered twice: {}".format(certified_scope)) + msg = "The same scope is covered twice: {}".format(certified_scope) + if ctx.attr.maturity == "development": + print("WARNING: " + msg) + continue + fail(msg) inserted_element.append(certified_scope.name) dependencies = depset(transitive = collected_dependent_labels).to_list() @@ -1021,14 +1030,14 @@ def _dependable_element_index_impl(ctx): dep_info = dep[DependableElementInfo] dep_rank = _INTEGRITY_LEVEL_RANK[dep_info.integrity_level] if dep_rank < own_rank: - fail( + msg = ( "Integrity level violation: '{}' (level {}) depends on '{}' (level {}). " + - "A dependable element must not depend on elements with a lower integrity level.", - ctx.label, - ctx.attr.integrity_level, - dep_info.label, - dep_info.integrity_level, - ) + "A dependable element must not depend on elements with a lower integrity level." + ).format(ctx.label, ctx.attr.integrity_level, dep_info.label, dep_info.integrity_level) + if ctx.attr.maturity == "development": + print("WARNING: " + msg) + else: + fail(msg) # ========================================================================= # Lobster Traceability: Dependable Element Level @@ -1065,6 +1074,74 @@ def _dependable_element_index_impl(ctx): comp_test_lobster_depset = depset(transitive = comp_test_lobster_files) comp_arch_lobster_depset = depset(transitive = comp_arch_lobster_files) + # Test-case-coverage lock check: run test_runner once per component that has a + # test_case_coverage_lock_file. Maturity controls whether drift is an error or a + # warning — matching the existing certified-scope / validation pattern. + # ComponentTestCaseCoverageInfo is only present on components that declared test_case_coverage_lock. + # ComponentInfo.requirements is CompReq-only (FeatReq/AssumedSystemReq are + # kept in a separate depset by _component_impl and not included here). + coverage_lobster_files = [] + for comp_target in ctx.attr.components: + if ComponentTestCaseCoverageInfo not in comp_target: + continue + cov_info = comp_target[ComponentTestCaseCoverageInfo] + if not cov_info.test_case_coverage_lock_file: + continue + + comp_info = comp_target[ComponentInfo] + req_files = comp_info.requirements.to_list() + gtest_lobster = cov_info.gtest_lobster_file + if not gtest_lobster: + # ComponentTestCaseCoverageInfo always carries a gtest_lobster_file + # from _component_impl; a missing one here means the provider + # contract was violated (e.g. a hand-rolled provider elsewhere). + # Fail with a clear message instead of passing None into + # ctx.actions.run, which would produce an opaque analysis error. + fail("ComponentTestCaseCoverageInfo.gtest_lobster_file is missing for {}".format(comp_target.label)) + lock_file = cov_info.test_case_coverage_lock_file + comp_name = comp_info.name + + # Package-qualify the discriminator used in declared output paths: + # two components in different packages can share the same target + # name (e.g. both named "component_example"), and comp_name alone + # would collide. + comp_unique = "{}_{}".format(comp_target.label.package.replace("/", "_"), comp_name) if comp_target.label.package else comp_name + + # Write req lobster manifest (exec paths — used inside the build action sandbox) + cov_req_manifest = ctx.actions.declare_file( + "{}/test_case_coverage_req_manifest_{}.txt".format(ctx.label.name, comp_unique), + ) + ctx.actions.write( + output = cov_req_manifest, + content = "\n".join([f.path for f in req_files]) + "\n", + ) + + cov_lobster = ctx.actions.declare_file( + "{}/test_case_coverage_{}.lobster".format(ctx.label.name, comp_unique), + ) + + cov_args = [] + if ctx.attr.maturity == "development": + cov_args.append("--allow-check-failures") + + ctx.actions.run( + executable = ctx.executable._test_runner, + inputs = depset(req_files + [gtest_lobster, cov_req_manifest, lock_file]), + outputs = [cov_lobster], + arguments = cov_args, + env = { + "TEST_CASE_COVERAGE_LOBSTER_MANIFEST": cov_req_manifest.path, + "TEST_CASE_COVERAGE_GTEST_LOBSTER": gtest_lobster.path, + "TEST_CASE_COVERAGE_LOCK_FILE": lock_file.path, + "TEST_CASE_COVERAGE_LABEL": str(comp_target.label), + "TEST_CASE_COVERAGE_PACKAGE": "//" + comp_target.label.package, + "TEST_CASE_COVERAGE_LOBSTER_OUTPUT": cov_lobster.path, + }, + mnemonic = "ComponentTestCaseCoverageLockCheck", + progress_message = "Checking test case coverage lock for {}".format(comp_target.label), + ) + coverage_lobster_files.append(cov_lobster) + # Collect safety analysis lobster files from dependability_analysis targets sa_lobster_files = {} # canonical name -> File, merged from all DA targets for da_target in ctx.attr.dependability_analysis: @@ -1149,6 +1226,19 @@ def _dependable_element_index_impl(ctx): comp_req_trace_lines = " trace to: \"Feature Requirements\";\n" lobster_config = ctx.actions.declare_file(ctx.label.name + "/de_traceability_config") + + # Only emit the "Test Case Coverage" LOBSTER layer when there is + # actual coverage data; an empty layer with `trace to` would cause + # LOBSTER to report every component requirement as missing a reference. + if coverage_lobster_files: + coverage_block = ( + "activity \"Test Case Coverage\" {\n" + + format_lobster_sources(coverage_lobster_files) + "\n" + + " trace to: \"Component Requirements\";\n}" + ) + else: + coverage_block = "" + ctx.actions.expand_template( template = ctx.file._lobster_de_template, output = lobster_config, @@ -1159,6 +1249,7 @@ def _dependable_element_index_impl(ctx): "{COMP_REQ_TRACE}": comp_req_trace_lines, "{ARCH_SOURCES}": format_lobster_sources(comp_arch_list), "{UNIT_TEST_SOURCES}": format_lobster_sources(comp_test_list), + "{COVERAGE_BLOCK}": coverage_block, "{PUBLIC_API_SOURCES}": format_lobster_sources(interface_req_list), "{FM_SOURCES}": format_lobster_sources(fm_list), "{CM_SOURCES}": format_lobster_sources(cm_list), @@ -1166,7 +1257,7 @@ def _dependable_element_index_impl(ctx): }, ) - all_lobster_inputs = feat_req_list + comp_req_list + comp_arch_list + comp_test_list + interface_req_list + fm_list + cm_list + rc_list + received_aou_list + all_lobster_inputs = feat_req_list + comp_req_list + comp_arch_list + comp_test_list + interface_req_list + fm_list + cm_list + rc_list + received_aou_list + coverage_lobster_files lobster_report_file = subrule_lobster_report(all_lobster_inputs, lobster_config) lobster_html_report = subrule_lobster_html_report(lobster_report_file) @@ -1308,6 +1399,12 @@ def _dependable_element_index_attrs(): cfg = "exec", doc = "Tool for filtering received AoU lobster entries based on chain-forwarding YAML.", ), + "_test_runner": attr.label( + default = Label("//bazel/rules/rules_score/src/test_case_coverage:test_runner"), + executable = True, + cfg = "exec", + doc = "test_case_coverage test runner — checks test_case_coverage.lock.yaml for each component; maturity controls error vs. warning.", + ), } attrs.update(VALIDATION_ATTRS) attrs.update(VERBOSITY_ATTR) @@ -1545,6 +1642,7 @@ def dependable_element( name = name + "_rst", srcs = [":" + name + "_index"], prefix = docs_prefix, + tags = kwargs.get("tags"), testonly = testonly, visibility = ["//visibility:public"], ) diff --git a/bazel/rules/rules_score/providers.bzl b/bazel/rules/rules_score/providers.bzl index 735f7ad4..d9d4554b 100644 --- a/bazel/rules/rules_score/providers.bzl +++ b/bazel/rules/rules_score/providers.bzl @@ -145,7 +145,7 @@ ComponentInfo = provider( doc = "Provider for component artifacts.", fields = { "name": "Name of the component target.", - "requirements": "Depset of requirement traceability files (.lobster) collected from requirements targets, including transitive files from nested components.", + "requirements": "Depset of component requirement traceability files (.lobster) collected from ComponentRequirementsInfo targets only (CompReq kind). Does not include feature or assumed-system requirement files.", "components": "Depset of nested component and/or unit Targets that comprise this component.", "tests": "Depset of test traceability files (.lobster) generated from unit test results, collected transitively from all nested components and units.", "architecture": "Depset of architecture traceability files (.lobster) generated from unit architectural designs, collected transitively from all nested components and units.", @@ -153,6 +153,21 @@ ComponentInfo = provider( }, ) +ComponentTestCaseCoverageInfo = provider( + doc = """Internal provider carrying test-case-coverage-lock data for a component. + + Returned alongside ComponentInfo when ``test_case_coverage_lock`` is set on a + ``component()`` target. Consumed by ``dependable_element`` to run the + per-component ``ComponentTestCaseCoverageLockCheck`` build action. This is an + implementation detail of the ``component`` / ``dependable_element`` + interaction; do not depend on it from external rules. + """, + fields = { + "gtest_lobster_file": "The component's own gtest.lobster File produced by lobster-gtest. Used by the dependable_element to run the coverage lock check.", + "test_case_coverage_lock_file": "The committed test_case_coverage.lock.yaml File, or None if no test_case_coverage_lock is declared on the component.", + }, +) + CcDependencyInfo = provider( doc = """Provider for collecting transitive dependencies from C/C++ targets. diff --git a/bazel/rules/rules_score/src/test_case_coverage/BUILD b/bazel/rules/rules_score/src/test_case_coverage/BUILD new file mode 100644 index 00000000..9b264926 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/BUILD @@ -0,0 +1,185 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@pip_lobster_bazel//:requirements.bzl", lobster_requirement = "requirement") +load("@pip_tooling//:requirements.bzl", tooling_requirement = "requirement") +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +# --------------------------------------------------------------------------- +# Core libraries +# --------------------------------------------------------------------------- + +py_library( + name = "read_gtest_lobster", + srcs = ["read_gtest_lobster.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":compute_lock", + "@lobster//lobster/common", + ], +) + +py_library( + name = "compute_lock", + srcs = ["compute_lock.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [lobster_requirement("pyyaml")], +) + +py_library( + name = "check_lock", + srcs = ["check_lock.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [":compute_lock"], +) + +py_library( + name = "lobster_generator", + srcs = ["lobster_generator.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":compute_lock", + "@lobster//lobster/common", + ], +) + +py_library( + name = "runner_common", + srcs = ["runner_common.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":compute_lock", + ":read_gtest_lobster", + ], +) + +py_library( + name = "update_runner_lib", + srcs = ["update_runner.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":check_lock", + ":compute_lock", + ":read_gtest_lobster", + ":runner_common", + ], +) + +py_binary( + name = "update_runner", + srcs = ["update_runner.py"], + imports = [".."], + main = "update_runner.py", + visibility = ["//visibility:public"], + deps = [":update_runner_lib"], +) + +py_library( + name = "test_runner_lib", + srcs = ["test_runner.py"], + imports = [".."], + visibility = ["//visibility:public"], + deps = [ + ":check_lock", + ":compute_lock", + ":lobster_generator", + ":read_gtest_lobster", + ":runner_common", + ], +) + +py_binary( + name = "test_runner", + srcs = ["test_runner.py"], + imports = [".."], + main = "test_runner.py", + visibility = ["//visibility:public"], + deps = [":test_runner_lib"], +) + +# --------------------------------------------------------------------------- +# Unit tests +# --------------------------------------------------------------------------- + +py_test( + name = "read_gtest_lobster_test", + srcs = ["read_gtest_lobster_test.py"], + imports = [".."], + deps = [ + ":read_gtest_lobster", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "compute_lock_test", + srcs = ["compute_lock_test.py"], + imports = [".."], + deps = [ + ":compute_lock", + ":read_gtest_lobster", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "check_lock_test", + srcs = ["check_lock_test.py"], + imports = [".."], + deps = [ + ":check_lock", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "lobster_generator_test", + srcs = ["lobster_generator_test.py"], + imports = [".."], + deps = [ + ":lobster_generator", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "update_runner_test", + srcs = [ + "conftest.py", + "update_runner_test.py", + ], + imports = [".."], + deps = [ + ":update_runner_lib", + tooling_requirement("pytest"), + ], +) + +py_test( + name = "test_runner_test", + srcs = [ + "conftest.py", + "test_runner_test.py", + ], + imports = [".."], + deps = [ + ":test_runner_lib", + tooling_requirement("pytest"), + ], +) diff --git a/bazel/rules/rules_score/src/test_case_coverage/__init__.py b/bazel/rules/rules_score/src/test_case_coverage/__init__.py new file mode 100644 index 00000000..411d0bb4 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/__init__.py @@ -0,0 +1,20 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Test case coverage lock mechanism and traceability.""" + +__all__ = [ + "compute_lock", + "read_gtest_lobster", + "check_lock", + "lobster_generator", +] diff --git a/bazel/rules/rules_score/src/test_case_coverage/assets/component_diagram.puml b/bazel/rules/rules_score/src/test_case_coverage/assets/component_diagram.puml new file mode 100644 index 00000000..0bb54fbe --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/assets/component_diagram.puml @@ -0,0 +1,81 @@ +' ******************************************************************************* +' Copyright (c) 2026 Contributors to the Eclipse Foundation +' +' See the NOTICE file(s) distributed with this work for additional +' information regarding copyright ownership. +' +' This program and the accompanying materials are made available under the +' terms of the Apache License Version 2.0 which is available at +' https://www.apache.org/licenses/LICENSE-2.0 +' +' SPDX-License-Identifier: Apache-2.0 +' ******************************************************************************* + +@startuml test_case_coverage Architecture +skinparam componentStyle rectangle +skinparam linetype ortho +left to right direction + +package "Inputs" { + [TRLC .lobster] as trlc + [GTest binary] as gtest_bin + [test_case_coverage.lock.yaml\n(committed)] as lock_committed +} + +package "component rule\n(component.bzl)" { + [subrule_lobster_gtest] as subrule + [_component_test\n(test_case_coverage_lock attr)] as rule_test + [_component_update\n(.update target)] as rule_update +} + +gtest_bin --> subrule : XML results +subrule --> rule_test : gtest.lobster +subrule --> rule_update : gtest.lobster +trlc --> rule_test : action manifest\n(exec paths) +trlc --> rule_update : runfiles manifest\n(short paths) + +package "Python modules\n(src/test_case_coverage/)" { + [read_gtest_lobster\n+ scan + resolve + req_ids] as reader + [compute_lock] as compute + [check_lock] as check + [lobster_generator] as gen +} + +package "Test phase (bazel test //...)" #LightBlue { + [test_runner] as test_r +} + +package "Run phase (bazel run .update)" #LightYellow { + [update_runner] as update_r +} + +rule_test ..> test_r : ctx.actions.run\n(--allow-check-failures) +rule_update ..> update_r : bazel run + +lock_committed --> check : committed lock +reader --> compute +compute --> check +compute --> gen +check --> test_r +gen --> test_r +reader --> update_r +compute --> update_r + +package "Outputs" { + [test_case_coverage.lobster\nartifact] as lobster_out + [test_case_coverage.lock.yaml\n(refreshed)] as lock_out +} + +test_r --> lobster_out : always emitted +update_r --> lock_out : overwrites committed + +note bottom of rule_test + User API: + component( + name = "my_comp", + test_case_coverage_lock = "test_case_coverage.lock.yaml", + ... + ) +end note + +@enduml diff --git a/bazel/rules/rules_score/src/test_case_coverage/check_lock.py b/bazel/rules/rules_score/src/test_case_coverage/check_lock.py new file mode 100644 index 00000000..96427a6e --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/check_lock.py @@ -0,0 +1,120 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Compare a committed lock file against a freshly computed one.""" + +from __future__ import annotations + +from test_case_coverage.compute_lock import LockFile, TestCase + + +def _req_map_by_uid(lock: LockFile) -> dict[str, dict[str, TestCase]]: + """Index a LockFile as {req_id: {uid: TestCase}}.""" + return {req.id: {tc.uid: tc for tc in req.test_cases} for req in lock.requirements} + + +def _req_version_map(lock: LockFile) -> dict[str, str]: + """Index a LockFile as {req_id: version}.""" + return {req.id: req.version for req in lock.requirements} + + +def compare_lock_files( + committed: LockFile, + computed: LockFile, +) -> tuple[bool, list[str]]: + """Return (ok, diff_lines) comparing committed vs computed lock. + + Checks (in order per requirement): + 1. Requirement added / removed. + 2. Version changed (triggers re-approval). + 3. Test cases added / removed / GWT changed. + + Description is intentionally excluded from comparison (display only). + diff_lines is empty when ok is True. + """ + diff: list[str] = [] + + committed_map = _req_map_by_uid(committed) + computed_map = _req_map_by_uid(computed) + committed_ver = _req_version_map(committed) + computed_ver = _req_version_map(computed) + + all_req_ids = sorted(set(committed_map) | set(computed_map)) + + for req_id in all_req_ids: + if req_id not in committed_map: + diff.append(f" + requirement added: {req_id}") + continue + if req_id not in computed_map: + diff.append(f" - requirement removed: {req_id}") + continue + + # Version check — a version bump needs explicit re-approval. + comm_ver = committed_ver.get(req_id, "") + comp_ver = computed_ver.get(req_id, "") + if comm_ver != comp_ver: + diff.append(f" ~ [{req_id}] version changed: {comm_ver!r} → {comp_ver!r}") + + comm_tests = committed_map[req_id] + comp_tests = computed_map[req_id] + all_uids = sorted(set(comm_tests) | set(comp_tests)) + + for uid in all_uids: + if uid not in comm_tests: + diff.append(f" + [{req_id}] test added: {uid}") + elif uid not in comp_tests: + diff.append(f" - [{req_id}] test removed: {uid}") + else: + comm_tc = comm_tests[uid] + comp_tc = comp_tests[uid] + for field_name, comm_val, comp_val in ( + ("given", comm_tc.given, comp_tc.given), + ("when", comm_tc.when, comp_tc.when), + ("then", comm_tc.then, comp_tc.then), + ): + if comm_val != comp_val: + diff.append( + f" ~ [{req_id}] {field_name} changed: {uid}\n" + f" lockfile: {comm_val!r}\n" + f" testcase: {comp_val!r}" + ) + + return len(diff) == 0, diff + + +def validate_specs(computed: LockFile) -> tuple[bool, list[str]]: + """Fail if any test case in *computed* is missing a GWT spec. + + A missing spec means the developer did not add ``given``/``when``/``then`` + ``RecordProperty`` annotations to the test. Coverage without a spec + cannot be meaningfully reviewed or approved. + + Returns (ok, issue_lines). + """ + issues: list[str] = [] + for req in computed.requirements: + for tc in req.test_cases: + missing = [ + name + for name, val in ( + ("given", tc.given), + ("when", tc.when), + ("then", tc.then), + ) + if not val.strip() + ] + if missing: + issues.append( + f" ! [{req.id}] test missing GWT fields ({', '.join(missing)}): {tc.uid}\n" + f" Add RecordProperty('given'/'when'/'then') annotations." + ) + return len(issues) == 0, issues diff --git a/bazel/rules/rules_score/src/test_case_coverage/check_lock_test.py b/bazel/rules/rules_score/src/test_case_coverage/check_lock_test.py new file mode 100644 index 00000000..354c2ad3 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/check_lock_test.py @@ -0,0 +1,157 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from test_case_coverage.check_lock import compare_lock_files, validate_specs +from test_case_coverage.compute_lock import SCHEMA_VERSION, LockFile, RequirementEntry, TestCase + + +def _lock(*reqs): + return LockFile(schema_version=SCHEMA_VERSION, requirements=list(reqs)) + + +def _req(req_id, *test_cases, version="1"): + return RequirementEntry(id=req_id, version=version, test_cases=list(test_cases)) + + +def _tc(uid="Suite:TestName", given="g", when="w", then="t"): + return TestCase(uid=uid, given=given, when=when, then=then) + + +def test_identical_locks_pass(): + lock = _lock(_req("P.R", _tc())) + ok, diff = compare_lock_files(lock, lock) + assert ok + assert diff == [] + + +def test_test_case_added_fails(): + committed = _lock(_req("P.R", _tc(uid="Suite:A"))) + computed = _lock(_req("P.R", _tc(uid="Suite:A"), _tc(uid="Suite:B"))) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("Suite:B" in line and "added" in line for line in diff) + + +def test_test_case_removed_fails(): + committed = _lock(_req("P.R", _tc(uid="Suite:A"), _tc(uid="Suite:B"))) + computed = _lock(_req("P.R", _tc(uid="Suite:A"))) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("Suite:B" in line and "removed" in line for line in diff) + + +def test_spec_changed_fails(): + tc_old = TestCase(uid="Suite:A", given="g", when="w", then="old") + tc_new = TestCase(uid="Suite:A", given="g", when="w", then="new") + committed = _lock(_req("P.R", tc_old)) + computed = _lock(_req("P.R", tc_new)) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("Suite:A" in line and "changed" in line for line in diff) + + +def test_requirement_added_fails(): + committed = _lock(_req("P.A")) + computed = _lock(_req("P.A"), _req("P.B")) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("P.B" in line and "added" in line for line in diff) + + +def test_requirement_removed_fails(): + committed = _lock(_req("P.A"), _req("P.B")) + computed = _lock(_req("P.A")) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("P.B" in line and "removed" in line for line in diff) + + +def test_version_changed_fails(): + committed = _lock(_req("P.R", _tc(), version="1")) + computed = _lock(_req("P.R", _tc(), version="2")) + ok, diff = compare_lock_files(committed, computed) + assert not ok + assert any("version changed" in line and "P.R" in line for line in diff) + + +def test_version_unchanged_passes(): + committed = _lock(_req("P.R", _tc(), version="3")) + computed = _lock(_req("P.R", _tc(), version="3")) + ok, diff = compare_lock_files(committed, computed) + assert ok + + +def test_description_change_does_not_fail(): + """Description is display-only — a change must not affect the drift verdict.""" + committed = _lock( + RequirementEntry(id="P.R", version="1", description="old", test_cases=[_tc()]) + ) + computed = _lock( + RequirementEntry(id="P.R", version="1", description="new", test_cases=[_tc()]) + ) + ok, diff = compare_lock_files(committed, computed) + assert ok + + +def test_empty_vs_empty_passes(): + ok, diff = compare_lock_files(_lock(), _lock()) + assert ok + assert diff == [] + + +# --------------------------------------------------------------------------- +# validate_specs tests +# --------------------------------------------------------------------------- + + +def test_validate_specs_passes_with_spec(): + lock = _lock(_req("P.R", _tc())) + ok, issues = validate_specs(lock) + assert ok + assert issues == [] + + +def test_validate_specs_fails_with_empty_spec(): + lock = _lock(_req("P.R", _tc(given="", when="", then=""))) + ok, issues = validate_specs(lock) + assert not ok + assert any("Suite:TestName" in line for line in issues) + assert any("P.R" in line for line in issues) + + +def test_validate_specs_fails_with_whitespace_only_spec(): + lock = _lock(_req("P.R", _tc(given=" ", when=" ", then=" "))) + ok, issues = validate_specs(lock) + assert not ok + + +def test_validate_specs_reports_all_missing(): + lock = _lock( + _req("P.A", _tc(uid="Suite:A", given="", when="", then="")), + _req("P.B", _tc(uid="Suite:B", given="", when="", then="")), + ) + ok, issues = validate_specs(lock) + assert not ok + assert len(issues) == 2 + + +def test_validate_specs_empty_lock_passes(): + ok, issues = validate_specs(_lock()) + assert ok + assert issues == [] + + +if __name__ == "__main__": + import sys + import pytest + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/test_case_coverage/compute_lock.py b/bazel/rules/rules_score/src/test_case_coverage/compute_lock.py new file mode 100644 index 00000000..0f5e768e --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/compute_lock.py @@ -0,0 +1,211 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""YAML lock-file serialization/deserialization and lock computation.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import yaml + +if TYPE_CHECKING: + from test_case_coverage.read_gtest_lobster import TestRecord + +# Version of the lock-file schema produced by this module. +SCHEMA_VERSION: int = 3 + + +@dataclass +class TestCase: + uid: str # "Suite:TestName" — matches gtest.lobster tag (without namespace) + given: str = "" # :Given: text from RecordProperty annotation + when: str = "" # :When: text + then: str = "" # :Then: text + # Populated only on the freshly *computed* side (never read from / written + # to the committed lock YAML): it's the raw, unprefixed gtest tag, used by + # lobster_generator.py to add a native cross-reference to the underlying + # gtest lobster item for each test case. + gtest_tag: str = "" # raw, unprefixed "Suite:TestName" tag + + +@dataclass +class RequirementMeta: + """Metadata for a single requirement read from a lobster-req-trace file.""" + + id: str # noqa: A003 e.g. "SampleComponent.REQ_COMP_001" + version: str = "" # from TRLC tag @version suffix, e.g. "1" + description: str = "" # from TRLC text field; stored for display only + + +@dataclass +class RequirementEntry: + id: str # noqa: A003 + version: str = "" # checked on drift — version bump requires re-approval + description: str = "" # display only; not compared in lock checks + test_cases: list[TestCase] = field(default_factory=list) + + +@dataclass +class LockFile: + schema_version: int + requirements: list[RequirementEntry] = field(default_factory=list) + + def to_yaml(self) -> str: + """Serialize to YAML string.""" + return serialize(self) + + @staticmethod + def from_yaml(data: dict[str, Any]) -> "LockFile": + """Deserialize from YAML dict.""" + return _from_dict(data) + + +# --------------------------------------------------------------------------- +# Serialization +# --------------------------------------------------------------------------- + + +def _tc_to_dict(tc: TestCase) -> dict[str, Any]: + return { + "uid": tc.uid, + "given": tc.given, + "when": tc.when, + "then": tc.then, + } + + +def _req_to_dict(req: RequirementEntry) -> dict[str, Any]: + d: dict[str, Any] = {"id": req.id} + if req.version: + d["version"] = req.version + if req.description: + d["description"] = req.description + d["test_cases"] = [_tc_to_dict(tc) for tc in req.test_cases] + return d + + +def serialize(lock: LockFile) -> str: + """Serialize a LockFile to a YAML string.""" + data: dict[str, Any] = { + "schema_version": lock.schema_version, + "requirements": [_req_to_dict(r) for r in lock.requirements], + } + return yaml.dump( + data, default_flow_style=False, allow_unicode=True, sort_keys=False + ) + + +# --------------------------------------------------------------------------- +# Deserialization +# --------------------------------------------------------------------------- + + +def _tc_from_dict(d: dict[str, Any]) -> TestCase: + return TestCase( + uid=str(d.get("uid", "")), + given=str(d.get("given", "")), + when=str(d.get("when", "")), + then=str(d.get("then", "")), + ) + + +def _req_from_dict(d: dict[str, Any]) -> RequirementEntry: + if "id" not in d: + raise ValueError(f"Lock file requirement entry missing required 'id' field: {d!r}") + return RequirementEntry( + id=str(d["id"]), + version=str(d.get("version", "")), + description=str(d.get("description", "")), + test_cases=[_tc_from_dict(tc) for tc in d.get("test_cases", [])], + ) + + +def _from_dict(data: dict[str, Any]) -> LockFile: + """Deserialize from YAML dict.""" + if not isinstance(data, dict): + raise ValueError("Lock file must be a YAML mapping") + + schema_version = int(data.get("schema_version", 0)) + if schema_version != SCHEMA_VERSION: + raise ValueError( + f"Lock file schema_version is {schema_version}, but this tool expects " + f"{SCHEMA_VERSION}. This is a lock *format* mismatch, not a coverage " + "drift -- run the update runner to regenerate it in the current " + "format, then review and commit the diff like any other lock file " + "change." + ) + requirements = [_req_from_dict(r) for r in data.get("requirements", [])] + return LockFile(schema_version=schema_version, requirements=requirements) + + +def deserialize(yaml_str: str) -> LockFile: + """Deserialize a YAML string into a LockFile.""" + try: + data = yaml.safe_load(yaml_str) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid lock file YAML: {exc}") from exc + + return _from_dict(data) + + +def load_lock_file(path: Path) -> LockFile: + """Read and deserialize a lock file from disk.""" + try: + return deserialize(path.read_text(encoding="utf-8")) + except OSError as exc: + raise ValueError(f"Cannot read lock file {path}: {exc}") from exc + + +# --------------------------------------------------------------------------- +# Computation from scan results +# --------------------------------------------------------------------------- + + +def compute_lock( + req_metadata: list[RequirementMeta], + test_cases_by_req: dict[str, list["TestRecord"]], +) -> LockFile: + """Build a LockFile from scanned test records, sorted deterministically. + + Each RequirementEntry stores version (checked on drift) and description + (display only). Each TestCase stores uid and GWT fields. + """ + requirements: list[RequirementEntry] = [] + for meta in req_metadata: + records = test_cases_by_req.get(meta.id, []) + test_cases: list[TestCase] = [ + TestCase( + uid=rec.uid, + given=rec.given, + when=rec.when, + then=rec.then, + gtest_tag=rec.gtest_tag, + ) + for rec in records + ] + # Sort lexicographically by uid for determinism + test_cases.sort(key=lambda tc: tc.uid) + requirements.append( + RequirementEntry( + id=meta.id, + version=meta.version, + description=meta.description, + test_cases=test_cases, + ) + ) + + # Sort requirements by ID for determinism + requirements.sort(key=lambda r: r.id) + return LockFile(schema_version=SCHEMA_VERSION, requirements=requirements) diff --git a/bazel/rules/rules_score/src/test_case_coverage/compute_lock_test.py b/bazel/rules/rules_score/src/test_case_coverage/compute_lock_test.py new file mode 100644 index 00000000..dc9b7076 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/compute_lock_test.py @@ -0,0 +1,211 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from test_case_coverage.compute_lock import ( + SCHEMA_VERSION, + LockFile, + RequirementEntry, + RequirementMeta, + TestCase, + compute_lock, + deserialize, + serialize, +) +from test_case_coverage.read_gtest_lobster import TestRecord + + +def _meta(req_id: str, version: str = "1", description: str = "") -> RequirementMeta: + return RequirementMeta(id=req_id, version=version, description=description) + + +def test_testcase_uid_and_spec(): + """Test that TestCase stores uid and given/when/then correctly.""" + tc = TestCase( + uid="Suite:TestName", + given="g", + when="w", + then="t", + ) + assert tc.uid == "Suite:TestName" + assert tc.given == "g" + assert tc.when == "w" + assert tc.then == "t" + + +def test_serialize_roundtrip(): + lock = LockFile( + schema_version=3, + requirements=[ + RequirementEntry( + id="Pkg.Req", + version="2", + description="Some description", + test_cases=[ + TestCase( + uid="Suite:Case", + given="g", + when="w", + then="t", + ) + ], + ) + ], + ) + yaml_str = serialize(lock) + restored = deserialize(yaml_str) + assert restored.schema_version == 3 + assert len(restored.requirements) == 1 + req = restored.requirements[0] + assert req.id == "Pkg.Req" + assert req.version == "2" + assert req.description == "Some description" + assert len(req.test_cases) == 1 + tc = req.test_cases[0] + assert tc.uid == "Suite:Case" + assert tc.given == "g" + assert tc.when == "w" + assert tc.then == "t" + + +def test_deserialize_missing_spec_defaults_to_empty(): + yaml_str = "schema_version: 3\nrequirements:\n - id: P.R\n test_cases:\n - uid: Suite:Test\n given: ''\n when: ''\n then: ''\n" + lock = deserialize(yaml_str) + tc = lock.requirements[0].test_cases[0] + assert tc.given == "" + assert tc.when == "" + assert tc.then == "" + + +def test_deserialize_invalid_yaml_raises(): + import pytest + + with pytest.raises(ValueError, match="Invalid lock file YAML"): + deserialize("{bad: [yaml") + + +def test_deserialize_wrong_schema_version_raises(): + import pytest + + yaml_str = "schema_version: 999\nrequirements: []\n" + with pytest.raises(ValueError, match="schema_version"): + deserialize(yaml_str) + + +def test_deserialize_matching_schema_version_ok(): + yaml_str = f"schema_version: {SCHEMA_VERSION}\nrequirements: []\n" + lock = deserialize(yaml_str) + assert lock.schema_version == SCHEMA_VERSION + assert lock.requirements == [] + + +def test_deserialize_requirement_missing_id_raises(): + import pytest + + yaml_str = ( + f"schema_version: {SCHEMA_VERSION}\n" + "requirements:\n" + " - version: '1'\n" + " test_cases: []\n" + ) + with pytest.raises(ValueError, match="missing required 'id'"): + deserialize(yaml_str) + + +def test_compute_lock_sorts_test_cases_by_uid(): + records = { + "P.R": [ + TestRecord(uid="Suite:B", lobster_traces=["P.R"]), + TestRecord(uid="Suite:A", lobster_traces=["P.R"]), + ] + } + lock = compute_lock([_meta("P.R")], records) + uids = [tc.uid for tc in lock.requirements[0].test_cases] + assert uids == ["Suite:A", "Suite:B"] + + +def test_compute_lock_sorts_requirements(): + lock = compute_lock([_meta("P.B"), _meta("P.A")], {"P.A": [], "P.B": []}) + ids = [r.id for r in lock.requirements] + assert ids == ["P.A", "P.B"] + + +def test_compute_lock_missing_req_yields_empty_test_cases(): + lock = compute_lock([_meta("P.R")], {}) + assert lock.requirements[0].test_cases == [] + + +def test_compute_lock_builds_gwt_from_record(): + records = { + "P.R": [ + TestRecord( + uid="Suite:T", lobster_traces=["P.R"], given="g", when="w", then="t" + ) + ] + } + lock = compute_lock([_meta("P.R")], records) + tc = lock.requirements[0].test_cases[0] + assert tc.given == "g" + assert tc.when == "w" + assert tc.then == "t" + + +def test_compute_lock_stores_version_and_description(): + lock = compute_lock([_meta("P.R", version="3", description="desc")], {}) + req = lock.requirements[0] + assert req.version == "3" + assert req.description == "desc" + + +def test_compute_lock_threads_gtest_tag(): + """gtest_tag from the scanned record must reach the TestCase, so + lobster_generator.py can build a native cross-reference to the + underlying gtest lobster item.""" + records = { + "P.R": [ + TestRecord( + uid="//pkg/Suite:T", + lobster_traces=["P.R"], + gtest_tag="Suite:T", + ) + ] + } + lock = compute_lock([_meta("P.R")], records) + tc = lock.requirements[0].test_cases[0] + assert tc.gtest_tag == "Suite:T" + + +def test_gtest_tag_excluded_from_serialized_yaml(): + """This field is a raw gtest tag, sandbox/build-derived, and must never + leak into the committed lock file (that would cause spurious lock drift). + """ + lock = LockFile( + schema_version=3, + requirements=[ + RequirementEntry( + id="P.R", + test_cases=[TestCase(uid="Suite:T", gtest_tag="Suite:T")], + ) + ], + ) + yaml_str = serialize(lock) + assert "gtest_tag" not in yaml_str + restored = deserialize(yaml_str) + tc = restored.requirements[0].test_cases[0] + assert tc.gtest_tag == "" + + + +if __name__ == "__main__": + import sys + import pytest + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/test_case_coverage/conftest.py b/bazel/rules/rules_score/src/test_case_coverage/conftest.py new file mode 100644 index 00000000..4ae435db --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/conftest.py @@ -0,0 +1,79 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Shared pytest helpers for test_case_coverage tests.""" + +from __future__ import annotations + +from pathlib import Path + +from lobster.common.io import lobster_write +from lobster.common.items import Activity, Requirement, Tracing_Tag +from lobster.common.location import Void_Reference + +collect_ignore = ["test_runner.py", "update_runner.py"] + +_DEFAULT_SPEC = ":Given: g\n:When: w\n:Then: t" + + +def write_req_lobster( + path: Path, + req_ids: list[str], + *, + version: str = "1", + kind: str = "CompReq", + text: str = "", +) -> None: + """Write a minimal lobster-req-trace JSON file to *path*. + + Args: + req_ids: Requirement identifiers. + version: Version suffix appended to every tag (``@version``). + kind: TRLC requirement kind written to the ``kind`` field. + Defaults to ``"CompReq"``; use ``"FeatReq"`` to write + feature requirements that should be filtered out. + text: Description text written to the ``text`` field. + """ + items = [ + Requirement( + tag=Tracing_Tag("req", r, version=version), + location=Void_Reference(), + framework="TRLC", + kind=kind, + name=r, + text=text or None, + ) + for r in req_ids + ] + with path.open("w", encoding="utf-8") as fd: + lobster_write(fd, Requirement, "t", items) + + +def write_gtest_lobster( + path: Path, + uid: str, + req_id: str, + spec: str = _DEFAULT_SPEC, +) -> None: + """Write a minimal lobster-act-trace JSON file to *path*.""" + item = Activity( + tag=Tracing_Tag("gtest", uid), + location=Void_Reference(), + framework="GoogleTest", + kind="test", + text=spec or None, + status="ok", + ) + item.add_tracing_target(Tracing_Tag("req", req_id)) + with path.open("w", encoding="utf-8") as fd: + lobster_write(fd, Activity, "lobster_gtest", [item]) diff --git a/bazel/rules/rules_score/src/test_case_coverage/lobster_generator.py b/bazel/rules/rules_score/src/test_case_coverage/lobster_generator.py new file mode 100644 index 00000000..1551ae03 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/lobster_generator.py @@ -0,0 +1,167 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Emit a lobster-act-trace artifact using the lobster library. + +Each *requirement* in the lock file becomes one ``Activity`` item, grouping +all test cases that cover that requirement. The ``status`` field carries the +coverage verdict: + + - ``"ok"`` — every test case for this requirement matches the committed lock + - ``"fail"`` — at least one test case is new, removed, or its GWT has changed + +The artifact is always written, even when verification fails, so that the +LOBSTER dashboard remains current during CI failures (D-6). + +Reuses ``Activity``, ``Tracing_Tag``, ``Void_Reference``, and +``lobster_write`` from the lobster library (no manual JSON construction). +""" + +from __future__ import annotations + +from pathlib import Path + +from lobster.common.io import lobster_write +from lobster.common.items import Activity, Tracing_Tag +from lobster.common.location import Void_Reference + +from test_case_coverage.compute_lock import LockFile, TestCase + +GENERATOR = "test_case_coverage" + + +def _committed_index( + committed: LockFile | None, +) -> dict[str, tuple[str, dict[str, tuple[str, str, str]]]]: + """Build {req_id: (version, {uid: (given, when, then)})} from committed lock.""" + if committed is None: + return {} + return { + req.id: ( + req.version, + {tc.uid: (tc.given, tc.when, tc.then) for tc in req.test_cases}, + ) + for req in committed.requirements + } + + +def _make_activity( + req_id: str, + req_version: str, + test_cases: list[TestCase], + committed_version: str, + committed_map: dict[str, tuple[str, str, str]], + label: str, +) -> Activity: + """Build one Activity representing coverage of *req_id* by all its test cases. + + ``covered`` is True only when the committed version matches, every test + case matches the committed GWT, and no committed test has been removed. + Text concatenates the GWT block of each test case, separated by a blank line. + + The item's own location is always void: this Activity aggregates + multiple test cases (with potentially different source locations), so + there is no single canonical source to point at. Each test case instead + gets a native cross-reference to its underlying gtest lobster item + (which carries its own real location), so the rendered report links + straight through to the actual source of every individual test case. + """ + tag = Tracing_Tag("test_case_coverage", req_id) + + blocks = [] + for tc in test_cases: + parts = [ + f":{k.capitalize()}: {v}" + for k, v in (("given", tc.given), ("when", tc.when), ("then", tc.then)) + if v + ] + if parts: + # Prefix each block with the test case uid as a bold heading so that + # multiple test cases for one requirement stay visually distinct in + # the rendered report, instead of blurring into one GWT list. + blocks.append(f"**{tc.uid}**\n\n" + "\n".join(parts)) + # Separate multiple test case blocks with a horizontal rule; a single block + # needs no separator, so the join is a no-op in that case. + separator = "\n\n.. raw:: html\n\n
\n\n" + text = separator.join(blocks) if blocks else None + + computed_uids = {tc.uid for tc in test_cases} + covered = ( + # version must match the committed approval + req_version == committed_version + # every computed test matches the committed GWT + and all( + committed_map.get(tc.uid) == (tc.given, tc.when, tc.then) + for tc in test_cases + ) + # AND no committed test has been removed from computed + and set(committed_map.keys()) <= computed_uids + ) + + item = Activity( + tag=tag, + location=Void_Reference(), + framework=GENERATOR, + kind="test", + text=text, + status="ok" if covered else "fail", + ) + item.add_tracing_target(Tracing_Tag("req", req_id)) + for tc in test_cases: + if tc.gtest_tag: + item.add_tracing_target(Tracing_Tag("gtest", tc.gtest_tag)) + + if not covered: + item.messages.append(f"test case coverage not confirmed for {label}") + if req_version != committed_version: + item.messages.append( + f"Requirement Version updated: {committed_version!r} → {req_version!r}" + ) + + return item + + +def generate_lobster( + computed: LockFile, + committed: LockFile | None, + label: str, + output_path: Path, +) -> None: + """Write a lobster-act-trace file for *computed*, validated against *committed*. + + Args: + computed: Freshly computed lock (from current test XML scan). + committed: Committed lock file content, or None if missing. + label: Bazel label of the test_case_coverage target (for context). + output_path: Where to write the .lobster JSON file. + """ + committed_index = _committed_index(committed) + items: list[Activity] = [] + + for req in computed.requirements: + if not req.test_cases: + continue + committed_version, committed_map = committed_index.get(req.id, ("", {})) + items.append( + _make_activity( + req_id=req.id, + req_version=req.version, + test_cases=req.test_cases, + committed_version=committed_version, + committed_map=committed_map, + label=label, + ) + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as fd: + lobster_write(fd, Activity, GENERATOR, items) diff --git a/bazel/rules/rules_score/src/test_case_coverage/lobster_generator_test.py b/bazel/rules/rules_score/src/test_case_coverage/lobster_generator_test.py new file mode 100644 index 00000000..45156010 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/lobster_generator_test.py @@ -0,0 +1,236 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +from test_case_coverage.compute_lock import LockFile, RequirementEntry, TestCase +from test_case_coverage.lobster_generator import generate_lobster + + +def _make_lock(*reqs): + return LockFile(schema_version=3, requirements=list(reqs)) + + +def _req(req_id, *tcs, version="1"): + return RequirementEntry(id=req_id, version=version, test_cases=list(tcs)) + + +def _tc(uid="Suite:A", given="g", when="w", then="t", gtest_tag=""): + return TestCase(uid=uid, given=given, when=when, then=then, gtest_tag=gtest_tag) + + +def test_generates_valid_lobster_json(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + committed = _make_lock(_req("P.R", _tc())) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + data = json.loads(out.read_text()) + assert data["schema"] == "lobster-act-trace" + assert data["version"] == 3 + assert data["generator"] == "test_case_coverage" + assert len(data["data"]) == 1 + + +def test_one_activity_per_requirement(tmp_path): + """Each requirement produces exactly one Activity, even if the same test covers both.""" + tc = _tc(uid="S:T") + computed = _make_lock(_req("P.A", tc), _req("P.B", tc)) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + data = json.loads(out.read_text())["data"] + tags = [item["tag"] for item in data] + assert len(tags) == 2 + assert set(tags) == {"test_case_coverage P.A", "test_case_coverage P.B"} + + +def test_covered_test_has_status_ok(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + committed = _make_lock(_req("P.R", _tc())) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "ok" + + +def test_uncovered_test_has_drift_message(tmp_path): + tc_new = TestCase(uid="Suite:A", given="g", when="w", then="new") + tc_old = TestCase(uid="Suite:A", given="g", when="w", then="old") + computed = _make_lock(_req("P.R", tc_new)) + committed = _make_lock(_req("P.R", tc_old)) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + assert any("test case coverage not confirmed" in m for m in item["messages"]) + + +def test_new_test_not_in_committed_has_status_fail(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + committed = _make_lock(_req("P.R")) # no test cases + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + + +def test_emitted_even_with_none_committed(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + assert out.exists() + data = json.loads(out.read_text()) + assert data["data"][0]["status"] == "fail" + + +def test_tracing_ref_points_to_requirement(tmp_path): + computed = _make_lock(_req("P.R", _tc("S.A"))) + committed = _make_lock(_req("P.R", _tc("S.A"))) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert "req P.R" in item["refs"] + + +def test_tracing_ref_points_to_each_gtest_item(tmp_path): + """Every test case with a gtest_tag gets a native cross-reference to its + underlying gtest lobster item, so the rendered report can link to it. + """ + tc_a = _tc(uid="//pkg/Suite:A", gtest_tag="Suite:A") + tc_b = _tc(uid="//pkg/Suite:B", gtest_tag="Suite:B") + computed = _make_lock(_req("P.R", tc_a, tc_b)) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert "gtest Suite:A" in item["refs"] + assert "gtest Suite:B" in item["refs"] + + +def test_no_gtest_ref_when_gtest_tag_missing(tmp_path): + computed = _make_lock(_req("P.R", _tc(gtest_tag=""))) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert not any(ref.startswith("gtest ") for ref in item["refs"]) + + +def test_item_location_is_always_void(tmp_path): + """The aggregate Activity has no single canonical source once it groups + multiple test cases, so its own location always stays void; the real + per-test source is reachable via the gtest cross-references instead. + """ + computed = _make_lock(_req("P.R", _tc(uid="S:A", gtest_tag="S:A"), _tc(uid="S:B", gtest_tag="S:B"))) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["location"]["kind"] == "void" + + +def test_text_contains_gwt(tmp_path): + computed = _make_lock(_req("P.R", _tc(given="g1", when="w1", then="t1"))) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert "g1" in item.get("text", "") + assert "w1" in item.get("text", "") + assert "t1" in item.get("text", "") + + +def test_empty_gwt_omits_text(tmp_path): + computed = _make_lock(_req("P.R", _tc(given="", when="", then=""))) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item.get("text") is None + + +def test_requirement_with_no_test_cases_produces_no_activity(tmp_path): + computed = _make_lock(_req("P.R")) # no test cases + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + assert json.loads(out.read_text())["data"] == [] + + +def test_text_contains_all_test_gwt_blocks(tmp_path): + computed = _make_lock( + _req( + "P.R", + _tc(uid="S:A", given="g1", when="w1", then="t1"), + _tc(uid="S:B", given="g2", when="w2", then="t2"), + ) + ) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + text = json.loads(out.read_text())["data"][0].get("text", "") + assert "g1" in text and "g2" in text + + +def test_removed_test_case_causes_fail(tmp_path): + """Committed lock has 2 tests; computed has only 1 (tracing tag removed) → PARTIAL.""" + tc_a = _tc(uid="S:A") + tc_b = _tc(uid="S:B") + computed = _make_lock(_req("P.R", tc_a)) # S:B disappeared + committed = _make_lock(_req("P.R", tc_a, tc_b)) # S:B was committed + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + assert any("test case coverage not confirmed" in m for m in item["messages"]) + + +def test_version_changed_causes_fail(tmp_path): + """Requirement version bump in computed but committed lock still has old version → fail.""" + tc = _tc() + computed = _make_lock(_req("P.R", tc, version="2")) + committed = _make_lock(_req("P.R", tc, version="1")) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + assert any("test case coverage not confirmed" in m for m in item["messages"]) + assert any("Requirement Version updated" in m for m in item["messages"]) + + +def test_same_version_and_gwt_is_ok(tmp_path): + tc = _tc() + lock = _make_lock(_req("P.R", tc, version="3")) + out = tmp_path / "out.lobster" + generate_lobster(lock, lock, "//pkg:cov", out) + assert json.loads(out.read_text())["data"][0]["status"] == "ok" + + +def test_partial_drift_in_one_test_case_fails_whole_requirement(tmp_path): + """Activity is ok only when every test case in the req matches committed.""" + tc_ok = _tc(uid="S:A", given="g", when="w", then="t") + tc_drift = TestCase(uid="S:B", given="g", when="w", then="new") + tc_old = TestCase(uid="S:B", given="g", when="w", then="old") + computed = _make_lock(_req("P.R", tc_ok, tc_drift)) + committed = _make_lock(_req("P.R", tc_ok, tc_old)) + out = tmp_path / "out.lobster" + generate_lobster(computed, committed, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert item["status"] == "fail" + + +def test_not_covered_message_includes_label(tmp_path): + computed = _make_lock(_req("P.R", _tc())) + out = tmp_path / "out.lobster" + generate_lobster(computed, None, "//pkg:cov", out) + item = json.loads(out.read_text())["data"][0] + assert any("//pkg:cov" in m for m in item["messages"]) + + +if __name__ == "__main__": + import sys + import pytest + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster.py b/bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster.py new file mode 100644 index 00000000..18e7cadd --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster.py @@ -0,0 +1,306 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Read test records from lobster-gtest JSON artifacts. + +Provides all lobster-file reading utilities used by both runners: + +* :func:`read_gtest_lobster` — parse a single gtest.lobster file +* :func:`scan_gtest_lobster` — group test records by requirement ID +* :func:`read_req_metadata_from_lobster_files` — extract CompReq metadata from a manifest +* :func:`resolve_path` — resolve Bazel-relative path in action/runfiles contexts +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field, replace +from pathlib import Path + +from lobster.common.errors import LOBSTER_Error, Message_Handler +from lobster.common.io import lobster_read +from lobster.common.items import Activity, Requirement + +from test_case_coverage.compute_lock import RequirementMeta + +logger = logging.getLogger(__name__) + + +@dataclass +class TestRecord: + """Metadata extracted from a single gtest.lobster item.""" + + uid: str # "Suite:TestName" (gtest tag without "gtest " prefix) + lobster_traces: list[str] = field(default_factory=list) # requirement IDs + given: str = "" # :Given: text from RecordProperty annotation + when: str = "" # :When: text + then: str = "" # :Then: text + gtest_tag: str = "" # raw, unprefixed "Suite:TestName" tag — survives the + # package-prefixing done to `uid` in scan_gtest_lobster, so it always + # matches the Tracing_Tag("gtest", ...) of the underlying gtest item. + + +_GWT_KEYS: tuple[tuple[str, str], ...] = ( + ("given", ":Given:"), + ("when", ":When:"), + ("then", ":Then:"), +) + + +def _parse_gwt(text: str) -> tuple[str, str, str]: + """Parse a ``:Given:/:When:/:Then:`` text field into its three components. + + lobster-gtest capitalises RecordProperty keys, so the expected format is:: + + :Given: value + :When: value + :Then: value + + Handles a few real-world variations robustly: + + * A missing trailing space after the ``:`` (e.g. ``:Given:value`` or an + empty value ``:Given:``) is accepted. + * A value that wraps onto following lines is joined (space-separated) + until the next recognised key or the end of the text. + * If a key appears more than once, the last occurrence wins. + """ + values = {"given": "", "when": "", "then": ""} + current: str | None = None + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line: + continue + matched_key = None + for key, prefix in _GWT_KEYS: + if line.startswith(prefix): + values[key] = line[len(prefix) :].strip() + matched_key = key + break + if matched_key is not None: + current = matched_key + elif current is not None: + values[current] = f"{values[current]} {line}".strip() + return values["given"], values["when"], values["then"] + + +def read_gtest_lobster(lobster_path: Path) -> list[TestRecord]: + """Parse a single gtest.lobster JSON file and return test records. + + Only items with at least one ``req`` reference are returned; items that + carry no tracing annotation are silently skipped. + + Args: + lobster_path: Path to a ``lobster-act-trace`` JSON file produced by + ``lobster-gtest``. + + Raises: + ValueError: if the file cannot be read or parsed. + """ + mh = Message_Handler() + items: dict = {} + try: + lobster_read(mh, str(lobster_path), "act", items) + except (OSError, LOBSTER_Error) as exc: + raise ValueError(f"Cannot parse gtest.lobster {lobster_path}: {exc}") from exc + + records: list[TestRecord] = [] + for item in items.values(): + if not isinstance(item, Activity): + continue + if item.tag.namespace != "gtest": + continue + + refs = [ + ref.tag + for ref in item.unresolved_references + if ref.namespace == "req" + ] + if not refs: + continue + + gwt = _parse_gwt(item.text or "") + records.append( + TestRecord( + uid=item.tag.tag, + lobster_traces=refs, + given=gwt[0], + when=gwt[1], + then=gwt[2], + gtest_tag=item.tag.tag, + ) + ) + + return records + + +# --------------------------------------------------------------------------- +# Path resolution (action-sandbox vs runfiles contexts) +# --------------------------------------------------------------------------- + + +def resolve_path(raw: str) -> Path: + """Resolve a Bazel-relative path to an absolute filesystem path. + + Resolution order: + + 1. Absolute path that exists — returned as-is. + 2. Relative path that exists from CWD — works in Bazel action sandboxes + where CWD is the execroot and short paths are available. + 3. ``$RUNFILES_DIR/`` — used by ``bazel run`` executables whose CWD + is ``$BUILD_WORKSPACE_DIRECTORY``, not the execroot. + 4. Raw path returned unchanged as a last resort (caller will get a clear + ``FileNotFoundError`` rather than a silent wrong-path write). + """ + candidate = Path(raw) + if candidate.is_absolute() and candidate.exists(): + return candidate + if not candidate.is_absolute() and candidate.exists(): + return candidate.resolve() + runfiles_dir = os.environ.get("RUNFILES_DIR") + if runfiles_dir and not candidate.is_absolute(): + via_runfiles = Path(runfiles_dir) / raw + if via_runfiles.exists(): + return via_runfiles + return candidate + + +# --------------------------------------------------------------------------- +# Req-ID extraction from lobster-req-trace manifests +# --------------------------------------------------------------------------- + +# Only these TRLC requirement kinds belong in the coverage lock file. +# Feature requirements and assumed-system requirements are excluded — they +# are traceability targets, not directly testable component-level items. +_COMP_REQ_KINDS: frozenset[str] = frozenset({"CompReq"}) + + +def read_req_metadata_from_lobster_files( + lobster_manifest_path: Path, +) -> list[RequirementMeta]: + """Parse a single-column manifest of lobster paths and return CompReq metadata. + + Each line is a path to a ``lobster-req-trace`` JSON file. Only items + with ``kind == "CompReq"`` are included; FeatReq and AssumedSystemReq + items are silently skipped. + + Returns a deduplicated list sorted by requirement ID. Each entry carries: + + * ``id`` — requirement identifier (tag without ``@version``) + * ``version`` — version string from the ``@version`` tag suffix (e.g. ``"1"``) + * ``description`` — requirement text from the ``text`` field + """ + metadata: list[RequirementMeta] = [] + seen: dict[str, RequirementMeta] = {} + + for line in lobster_manifest_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + lobster_path = resolve_path(line) + mh = Message_Handler() + items: dict = {} + try: + lobster_read(mh, str(lobster_path), "req", items) + except (OSError, LOBSTER_Error) as exc: + raise ValueError(f"Cannot read lobster file {lobster_path}: {exc}") from exc + + for item in items.values(): + if not isinstance(item, Requirement): + continue + if item.kind not in _COMP_REQ_KINDS: + continue # skip FeatReq, AssumedSystemReq, etc. + if item.tag.namespace != "req": + continue + req_id = item.tag.tag + meta = RequirementMeta( + id=req_id, + version=str(item.tag.version) if item.tag.version is not None else "", + description=str(item.text or ""), + ) + existing = seen.get(req_id) + if existing is not None: + # Duplicate requirement ID across lobster files: keep the first + # occurrence, but only warn when the duplicate actually + # disagrees (version/description) with it — an identical + # repeat (e.g. the same requirement reachable via two paths) + # is expected and not worth flagging. + if (existing.version, existing.description) != (meta.version, meta.description): + logger.warning( + "Requirement %r found more than once with differing " + "metadata (version %r vs %r); keeping the first occurrence.", + req_id, + existing.version, + meta.version, + ) + continue + seen[req_id] = meta + metadata.append(meta) + + return sorted(metadata, key=lambda m: m.id) + + +# --------------------------------------------------------------------------- +# Gtest scan — group records by requirement ID +# --------------------------------------------------------------------------- + + +def scan_gtest_lobster( + gtest_lobster_path: Path, + req_ids: list[str], + package: str = "", + label: str = "", +) -> dict[str, list[TestRecord]]: + """Read a gtest.lobster file and group test records by requirement ID. + + Only records whose ``lobster_traces`` intersect with *req_ids* are kept. + A record that covers multiple requirements appears in all their lists. + Any traced ID that does *not* match a known CompReq (typo, or a trace to + a FeatReq/AssumedSystemReq instead of a CompReq) is dropped and reported + via a ``WARNING`` on stderr, instead of silently disappearing. + + The ``package`` (e.g. ``//score/message_passing``) is prepended to each + uid to make it globally unique across components. Pass + ``"//" + ctx.label.package`` from the Bazel rule via the + ``TEST_CASE_COVERAGE_PACKAGE`` environment variable. + + Args: + label: Bazel label of the calling test_case_coverage target, used + only to prefix warning messages. + """ + all_records = read_gtest_lobster(gtest_lobster_path) + req_id_set = set(req_ids) + by_req: dict[str, list[TestRecord]] = {rid: [] for rid in req_ids} + unmatched_traces: set[str] = set() + for record in all_records: + if package: + # ``package`` is already "//" for a root-package target (no + # trailing package name), so don't add a second "/" or the uid + # ends up as "///Foo:GetNumber" instead of "//Foo:GetNumber". + sep = "" if package.endswith("/") else "/" + record = replace(record, uid=f"{package}{sep}{record.uid}") + for trace in record.lobster_traces: + if trace in req_id_set: + by_req[trace].append(record) + else: + unmatched_traces.add(trace) + + for trace in sorted(unmatched_traces): + logger.warning( + "[%s] test case(s) trace to %r, which is not a known CompReq " + "requirement ID for this component (typo, or tracing to a " + "non-CompReq requirement) \u2014 this coverage is not recorded.", + label, + trace, + ) + + return by_req diff --git a/bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster_test.py b/bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster_test.py new file mode 100644 index 00000000..1f6c3d83 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/read_gtest_lobster_test.py @@ -0,0 +1,454 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +import pytest + +from test_case_coverage.read_gtest_lobster import ( + TestRecord, + _parse_gwt, + read_gtest_lobster, + read_req_metadata_from_lobster_files, + resolve_path, + scan_gtest_lobster, +) + + +def _write_gtest_lobster(path: Path, items: list[tuple[str, list[str], str]]) -> Path: + """Write a minimal gtest.lobster JSON file. + + Args: + path: Directory to write the file in. + items: List of (uid, req_ids, text) tuples. + """ + data = { + "schema": "lobster-act-trace", + "version": 3, + "generator": "lobster_gtest", + "data": [ + { + "tag": f"gtest {uid}", + "location": {"kind": "void"}, + "name": uid, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [f"req {r}" for r in req_ids], + "framework": "GoogleTest", + "kind": "test", + "text": text or None, + "status": "ok", + } + for uid, req_ids, text in items + ], + } + p = path / "gtest.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + return p + + +def test_single_item_basic(tmp_path): + p = _write_gtest_lobster( + tmp_path, [("Suite:TestName", ["Req.A"], ":Given: g\n:When: w\n:Then: t")] + ) + records = read_gtest_lobster(p) + assert len(records) == 1 + r = records[0] + assert r.uid == "Suite:TestName" + assert r.lobster_traces == ["Req.A"] + assert r.given == "g" + assert r.when == "w" + assert r.then == "t" + + +def test_item_captures_raw_gtest_tag(tmp_path): + """gtest_tag preserves the raw "Suite:TestName" tag so lobster_generator.py + can build a Tracing_Tag("gtest", ...) cross-reference to the source item. + """ + p = _write_gtest_lobster(tmp_path, [("Suite:TestName", ["Req.A"], "")]) + r = read_gtest_lobster(p)[0] + assert r.gtest_tag == "Suite:TestName" + + +def test_item_with_no_req_refs_is_skipped(tmp_path): + p = _write_gtest_lobster(tmp_path, [("Suite:TestName", [], "")]) + records = read_gtest_lobster(p) + assert records == [] + + +def test_non_gtest_tags_are_skipped(tmp_path): + data = { + "schema": "lobster-act-trace", + "version": 3, + "generator": "test", + "data": [ + { + "tag": "req SomeReq", + "location": {"kind": "void"}, + "name": "SomeReq", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": [], + "framework": "trlc", + "kind": "requirement", + "text": None, + "status": "ok", + } + ], + } + p = tmp_path / "mixed.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + records = read_gtest_lobster(p) + assert records == [] + + +def test_multiple_req_refs(tmp_path): + p = _write_gtest_lobster(tmp_path, [("S:T", ["Req.A", "Req.B"], "")]) + records = read_gtest_lobster(p) + assert len(records) == 1 + assert records[0].lobster_traces == ["Req.A", "Req.B"] + + +def test_empty_spec_stored_as_empty_string(tmp_path): + p = _write_gtest_lobster(tmp_path, [("S:T", ["Req.A"], "")]) + records = read_gtest_lobster(p) + assert records[0].given == "" + assert records[0].when == "" + assert records[0].then == "" + + +def test_none_text_stored_as_empty_string(tmp_path): + """Items with text=null (no GWT annotation) store given/when/then as empty strings.""" + data = { + "schema": "lobster-act-trace", + "version": 3, + "generator": "test", + "data": [ + { + "tag": "gtest S:T", + "location": {"kind": "void"}, + "name": "S:T", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "refs": ["req Req.A"], + "framework": "GoogleTest", + "kind": "test", + "text": None, + "status": "ok", + } + ], + } + p = tmp_path / "g.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + records = read_gtest_lobster(p) + assert records[0].given == "" + assert records[0].when == "" + assert records[0].then == "" + + +def test_invalid_json_raises_value_error(tmp_path): + p = tmp_path / "bad.lobster" + p.write_text("{invalid json", encoding="utf-8") + with pytest.raises(ValueError, match="Cannot parse gtest.lobster"): + read_gtest_lobster(p) + + +# --------------------------------------------------------------------------- +# resolve_path +# --------------------------------------------------------------------------- + + +def test_resolve_path_absolute_existing(tmp_path): + f = tmp_path / "x.txt" + f.write_text("hi") + assert resolve_path(str(f)) == f + + +def test_resolve_path_relative_existing(tmp_path, monkeypatch): + f = tmp_path / "x.txt" + f.write_text("hi") + monkeypatch.chdir(tmp_path) + result = resolve_path("x.txt") + assert result.exists() + + +def test_resolve_path_nonexistent_returns_path(): + p = resolve_path("/no/such/file.lobster") + assert str(p) == "/no/such/file.lobster" + + +def test_resolve_path_via_runfiles_dir(tmp_path, monkeypatch): + runfiles = tmp_path / "runfiles" + (runfiles / "pkg").mkdir(parents=True) + target = runfiles / "pkg" / "x.txt" + target.write_text("hi") + monkeypatch.chdir(tmp_path) # so the plain-CWD lookup does not match first + monkeypatch.setenv("RUNFILES_DIR", str(runfiles)) + result = resolve_path("pkg/x.txt") + assert result == target + + +def test_resolve_path_runfiles_dir_set_but_file_missing_falls_through(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("RUNFILES_DIR", str(tmp_path / "nonexistent_runfiles")) + result = resolve_path("missing/x.txt") + assert str(result) == "missing/x.txt" + + +# --------------------------------------------------------------------------- +# _parse_gwt +# --------------------------------------------------------------------------- + + +def test_parse_gwt_basic(): + assert _parse_gwt(":Given: g\n:When: w\n:Then: t") == ("g", "w", "t") + + +def test_parse_gwt_missing_trailing_space(): + """A key with no space before the value (or no value at all) must not crash.""" + assert _parse_gwt(":Given:g\n:When:\n:Then: t") == ("g", "", "t") + + +def test_parse_gwt_multiline_value_is_joined(): + text = ":Given: first line\nsecond line\n:When: w\n:Then: t" + given, when, then = _parse_gwt(text) + assert given == "first line second line" + assert when == "w" + assert then == "t" + + +def test_parse_gwt_duplicate_key_last_wins(): + text = ":Given: first\n:Given: second\n:When: w\n:Then: t" + given, _when, _then = _parse_gwt(text) + assert given == "second" + + +def test_parse_gwt_blank_lines_ignored(): + text = ":Given: g\n\n:When: w\n\n:Then: t" + assert _parse_gwt(text) == ("g", "w", "t") + + +def test_parse_gwt_empty_text(): + assert _parse_gwt("") == ("", "", "") + + +def test_parse_gwt_text_before_any_key_ignored(): + text = "preamble noise\n:Given: g\n:When: w\n:Then: t" + assert _parse_gwt(text) == ("g", "w", "t") + + +# --------------------------------------------------------------------------- +# read_req_ids_from_lobster_files +# --------------------------------------------------------------------------- + + +def _write_req_lobster( + path: Path, + req_ids: list[str], + kind: str = "CompReq", + version: str = "1", + text: str = "", +) -> Path: + data = { + "schema": "lobster-req-trace", + "version": 4, + "generator": "test", + "data": [ + { + "tag": f"req {rid}@{version}", + "name": rid, + "location": {"kind": "void"}, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "TRLC", + "kind": kind, + "text": text or None, + "status": None, + } + for rid in req_ids + ], + } + p = path / "reqs.lobster" + p.write_text(json.dumps(data), encoding="utf-8") + return p + + +def _write_manifest(tmp_path: Path, lobster_path: Path) -> Path: + m = tmp_path / "manifest.txt" + m.write_text(str(lobster_path) + "\n", encoding="utf-8") + return m + + +def test_read_req_ids_basic(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.A", "P.B"]) + manifest = _write_manifest(tmp_path, lobster) + metadata = read_req_metadata_from_lobster_files(manifest) + assert [m.id for m in metadata] == ["P.A", "P.B"] + + +def test_read_req_ids_sorted(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.Z", "P.A"]) + manifest = _write_manifest(tmp_path, lobster) + assert [m.id for m in read_req_metadata_from_lobster_files(manifest)] == [ + "P.A", + "P.Z", + ] + + +def test_read_req_ids_deduplicated(tmp_path): + # A real lobster-req-trace file can't contain the same tag twice (lobster + # itself rejects that as a duplicate definition); dedup only matters when + # the same requirement is reachable via two different manifest entries + # (e.g. two lobster files listing the same requirement). + dir_a = tmp_path / "a" + dir_a.mkdir() + dir_b = tmp_path / "b" + dir_b.mkdir() + lobster_a = _write_req_lobster(dir_a, ["P.A"]) + lobster_b = _write_req_lobster(dir_b, ["P.A"]) + manifest = tmp_path / "manifest.txt" + manifest.write_text(f"{lobster_a}\n{lobster_b}\n", encoding="utf-8") + assert len(read_req_metadata_from_lobster_files(manifest)) == 1 + + +def test_read_req_version_extracted(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.A"], version="3") + manifest = _write_manifest(tmp_path, lobster) + meta = read_req_metadata_from_lobster_files(manifest) + assert meta[0].version == "3" + + +def test_read_req_description_extracted(tmp_path): + lobster = _write_req_lobster(tmp_path, ["P.A"], text="The system shall do X.") + manifest = _write_manifest(tmp_path, lobster) + meta = read_req_metadata_from_lobster_files(manifest) + assert meta[0].description == "The system shall do X." + + +def test_feat_req_items_excluded(tmp_path): + """FeatReq items in the lobster file must not appear in the output.""" + lobster = _write_req_lobster(tmp_path, ["Feat.001"], kind="FeatReq") + manifest = _write_manifest(tmp_path, lobster) + assert read_req_metadata_from_lobster_files(manifest) == [] + + +def test_mixed_kinds_only_comp_req_returned(tmp_path): + """CompReq returned, FeatReq silently skipped.""" + import json as _json + + data = { + "schema": "lobster-req-trace", + "version": 4, + "generator": "test", + "data": [ + { + "tag": "req C.001@1", + "name": "C.001", + "location": {"kind": "void"}, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "TRLC", + "kind": "CompReq", + "text": None, + "status": None, + }, + { + "tag": "req F.001@1", + "name": "F.001", + "location": {"kind": "void"}, + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "framework": "TRLC", + "kind": "FeatReq", + "text": None, + "status": None, + }, + ], + } + p = tmp_path / "mixed.lobster" + p.write_text(_json.dumps(data), encoding="utf-8") + manifest = tmp_path / "manifest.txt" + manifest.write_text(str(p) + "\n", encoding="utf-8") + meta = read_req_metadata_from_lobster_files(manifest) + assert [m.id for m in meta] == ["C.001"] + + +# --------------------------------------------------------------------------- +# scan_gtest_lobster +# --------------------------------------------------------------------------- + + +def test_scan_groups_by_req(tmp_path): + gtest = _write_gtest_lobster( + tmp_path, [("Suite:T", ["P.R"], ":Given: g\n:When: w\n:Then: t")] + ) + by_req = scan_gtest_lobster(gtest, ["P.R"]) + assert len(by_req["P.R"]) == 1 + rec = by_req["P.R"][0] + assert rec.given == "g" + assert rec.when == "w" + assert rec.then == "t" + + +def test_scan_uid_no_package(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.R"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.R"]) + assert by_req["P.R"][0].uid == "Suite:T" + + +def test_scan_uid_with_package(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.R"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.R"], package="//score/msg") + assert by_req["P.R"][0].uid == "//score/msg/Suite:T" + + +def test_scan_gtest_tag_survives_package_prefix(tmp_path): + """gtest_tag must stay the raw, unprefixed tag even once uid is prefixed, + so it keeps matching the underlying gtest item's Tracing_Tag. + """ + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.R"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.R"], package="//score/msg") + rec = by_req["P.R"][0] + assert rec.uid == "//score/msg/Suite:T" + assert rec.gtest_tag == "Suite:T" + + +def test_scan_skips_unknown_reqs(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.Unknown"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.R"]) + assert by_req["P.R"] == [] + + +def test_scan_multi_trace_appears_in_each(tmp_path): + gtest = _write_gtest_lobster(tmp_path, [("Suite:T", ["P.A", "P.B"], "")]) + by_req = scan_gtest_lobster(gtest, ["P.A", "P.B"]) + assert len(by_req["P.A"]) == 1 + assert len(by_req["P.B"]) == 1 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/test_case_coverage/runner_common.py b/bazel/rules/rules_score/src/test_case_coverage/runner_common.py new file mode 100644 index 00000000..a1fe4167 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/runner_common.py @@ -0,0 +1,74 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Shared CLI helpers used by both ``test_runner`` and ``update_runner``. + +Both entry points read the same set of environment variables, scan the +gtest.lobster / req-lobster manifest to compute a fresh :class:`LockFile`, and +warn about requirements with zero linked test cases. Keeping that pipeline in +one place avoids the two runners drifting apart. +""" + +from __future__ import annotations + +import logging +import os +import sys +from pathlib import Path + +from test_case_coverage.compute_lock import LockFile, RequirementMeta, compute_lock +from test_case_coverage.read_gtest_lobster import ( + read_req_metadata_from_lobster_files, + scan_gtest_lobster, +) + +logger = logging.getLogger(__name__) + + +def require_env(name: str) -> str: + """Return the value of environment variable *name*, or exit(1) if unset/empty.""" + value = os.environ.get(name) + if not value: + logger.error("%s is required", name) + sys.exit(1) + return value + + +def scan_and_compute( + lobster_manifest: Path, + gtest_lobster_path: Path, + package: str, + label: str, +) -> tuple[list[RequirementMeta], LockFile]: + """Extract requirement metadata, scan gtest.lobster, and compute the lock. + + Emits a ``WARNING`` to stderr for every requirement with zero linked test + cases found in the scanned XML files. + + Returns: + (req_metadata, computed_lock) + """ + req_metadata = read_req_metadata_from_lobster_files(lobster_manifest) + req_ids = [m.id for m in req_metadata] + + by_req = scan_gtest_lobster(gtest_lobster_path, req_ids, package=package, label=label) + + for req_id, records in by_req.items(): + if not records: + logger.warning( + "[%s] Requirement %r has no linked test cases in the scanned XML files.", + label, + req_id, + ) + + computed = compute_lock(req_metadata, by_req) + return req_metadata, computed diff --git a/bazel/rules/rules_score/src/test_case_coverage/test_runner.py b/bazel/rules/rules_score/src/test_case_coverage/test_runner.py new file mode 100644 index 00000000..ecd772d1 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/test_runner.py @@ -0,0 +1,115 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Test runner: verify test_case_coverage.lock.yaml is current and emit LOBSTER artifact. + +Environment variables +--------------------- +TEST_CASE_COVERAGE_LOBSTER_MANIFEST + Path to a newline-delimited file listing lobster-req-trace JSON paths. +TEST_CASE_COVERAGE_GTEST_LOBSTER + Path to the gtest.lobster file produced by subrule_lobster_gtest. +TEST_CASE_COVERAGE_LOCK_FILE + Path to the committed lock YAML (short_path in runfiles). +TEST_CASE_COVERAGE_LABEL + Bazel label of the test_case_coverage target. +TEST_CASE_COVERAGE_LOBSTER_OUTPUT + Path where the .lobster artifact should be written. +""" + +from __future__ import annotations + +import logging +import os +import sys +from pathlib import Path + +from test_case_coverage.check_lock import compare_lock_files, validate_specs +from test_case_coverage.compute_lock import load_lock_file +from test_case_coverage.lobster_generator import generate_lobster +from test_case_coverage.read_gtest_lobster import resolve_path +from test_case_coverage.runner_common import require_env, scan_and_compute + +logger = logging.getLogger(__name__) + + +def main(argv: list[str] | None = None) -> None: + import argparse + + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s", force=True) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--allow-check-failures", + action="store_true", + help="Return success even when lock checks fail (used for LOBSTER build action).", + ) + args = parser.parse_args(argv) + + lobster_manifest = resolve_path(require_env("TEST_CASE_COVERAGE_LOBSTER_MANIFEST")) + gtest_lobster_path = resolve_path(require_env("TEST_CASE_COVERAGE_GTEST_LOBSTER")) + committed_lock_path = resolve_path(require_env("TEST_CASE_COVERAGE_LOCK_FILE")) + label = os.environ.get("TEST_CASE_COVERAGE_LABEL", "") + package = os.environ.get("TEST_CASE_COVERAGE_PACKAGE", "") + lobster_output_raw = os.environ.get("TEST_CASE_COVERAGE_LOBSTER_OUTPUT") + + # Steps 1-3: extract requirement metadata, scan gtest.lobster, compute lock + _req_metadata, computed = scan_and_compute( + lobster_manifest, gtest_lobster_path, package=package, label=label + ) + + # Step 3b: validate GWT specs (collect, don't exit yet — artifact must be written first) + spec_ok, spec_issues = validate_specs(computed) + if not spec_ok: + logger.error("One or more test cases are missing GWT spec annotations.") + for line in spec_issues: + logger.error(line) + + # Step 4: load committed lock once (may not exist / may fail to parse) — + # reused for both the LOBSTER artifact (Step 5) and the drift check + # (Step 6) so the file is never read twice. + try: + committed = load_lock_file(committed_lock_path) + lock_load_error: str | None = None + except ValueError as exc: + committed = None + lock_load_error = str(exc) + + # Step 5: emit LOBSTER artifact — ALWAYS, before any sys.exit, so Bazel's + # declared output is produced even when the lock check fails (D-6). + if lobster_output_raw: + generate_lobster( + computed=computed, + committed=committed, + label=label, + output_path=Path(lobster_output_raw), + ) + + # Step 6: compare against the already-loaded committed lock + if lock_load_error is not None: + lock_ok, diff_lines = False, [lock_load_error] + else: + lock_ok, diff_lines = compare_lock_files(committed, computed) + if not lock_ok: + logger.error("Coverage lock file is out of date.") + for line in diff_lines: + logger.error(line) + logger.error("Run `bazel run %s.update` to refresh the lock file.", label) + + # Step 7: exit if any check failed (after artifact is written) + if not args.allow_check_failures: + if not spec_ok or not lock_ok: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/bazel/rules/rules_score/src/test_case_coverage/test_runner_test.py b/bazel/rules/rules_score/src/test_case_coverage/test_runner_test.py new file mode 100644 index 00000000..3ba2e25d --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/test_runner_test.py @@ -0,0 +1,171 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +import pytest + +from test_case_coverage.compute_lock import LockFile, RequirementEntry, TestCase, serialize +from test_case_coverage.test_runner import main as runner_main +from test_case_coverage.conftest import write_req_lobster, write_gtest_lobster + +_SPEC = ":Given: g\n:When: w\n:Then: t" +_UID = "Suite:T" # raw uid in gtest.lobster tag +_LABEL = "//pkg:cov" # Bazel label set in all tests +_PACKAGE = "//pkg" # package extracted from _LABEL + + +def _setup(tmp_path: Path, req_ids, uid=_UID): + req_lobster = tmp_path / "reqs.lobster" + write_req_lobster(req_lobster, req_ids) + lm = tmp_path / "lobster_manifest.txt" + lm.write_text(str(req_lobster) + "\n") + + gtest = tmp_path / "gtest.lobster" + write_gtest_lobster(gtest, uid, req_ids[0] if req_ids else "X") + + # uid in lock must include package prefix (as scan_gtest_lobster will produce) + scoped_uid = f"{_PACKAGE}/{uid}" + lock_content = serialize( + LockFile( + schema_version=3, + requirements=[ + RequirementEntry( + id=req_ids[0] if req_ids else "X", + version="1", + test_cases=[ + TestCase(uid=scoped_uid, given="g", when="w", then="t") + ], + ) + ], + ) + ) + lock = tmp_path / "test_case_coverage.lock.yaml" + lock.write_text(lock_content) + + lobster_out = tmp_path / "out.lobster" + return lm, gtest, lock, lobster_out + + +def test_clean_pass(tmp_path, monkeypatch): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + runner_main([]) # must not raise + + +def test_drift_fails(tmp_path, monkeypatch): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + # Tamper with the gtest.lobster so spec differs from committed + write_gtest_lobster(gtest, _UID, "P.R", spec=":Given: CHANGED\n:When: w\n:Then: t") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + with pytest.raises(SystemExit) as exc_info: + runner_main([]) + assert exc_info.value.code == 1 + + +def test_drift_still_emits_lobster(tmp_path, monkeypatch): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + write_gtest_lobster(gtest, _UID, "P.R", spec=":Given: CHANGED\n:When: w\n:Then: t") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + try: + runner_main([]) + except SystemExit: + pass + assert lobster_out.exists() + data = json.loads(lobster_out.read_text()) + assert data["data"][0]["status"] == "fail" + + +def test_failure_prints_remediation(tmp_path, monkeypatch, capsys): + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + write_gtest_lobster(gtest, _UID, "P.R", spec=":Given: CHANGED\n:When: w\n:Then: t") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.delenv("TEST_CASE_COVERAGE_LOBSTER_OUTPUT", raising=False) + try: + runner_main([]) + except SystemExit: + pass + err = capsys.readouterr().err + assert "bazel run" in err + assert ".update" in err + + +def test_missing_committed_lock_file_fails(tmp_path, monkeypatch): + """No committed lock file at all (e.g. first run) must be reported as a failure.""" + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + lock.unlink() + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + with pytest.raises(SystemExit) as exc_info: + runner_main([]) + assert exc_info.value.code == 1 + assert lobster_out.exists() # artifact still written (D-6) + + +def test_malformed_committed_lock_file_fails_with_message(tmp_path, monkeypatch, capsys): + """A lock file with a bad/mismatched schema_version must fail with a clear message.""" + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + lock.write_text("schema_version: 999\nrequirements: []\n") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + with pytest.raises(SystemExit) as exc_info: + runner_main([]) + assert exc_info.value.code == 1 + err = capsys.readouterr().err + assert "schema_version" in err + + +def test_allow_check_failures_suppresses_exit(tmp_path, monkeypatch): + """--allow-check-failures must return success even when the lock check fails.""" + lm, gtest, lock, lobster_out = _setup(tmp_path, ["P.R"]) + write_gtest_lobster(gtest, _UID, "P.R", spec=":Given: CHANGED\n:When: w\n:Then: t") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", _LABEL) + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", _PACKAGE) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_OUTPUT", str(lobster_out)) + runner_main(["--allow-check-failures"]) # must not raise + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/src/test_case_coverage/update_runner.py b/bazel/rules/rules_score/src/test_case_coverage/update_runner.py new file mode 100644 index 00000000..c4058f70 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/update_runner.py @@ -0,0 +1,115 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Update runner: refresh test_case_coverage.lock.yaml from current gtest.lobster output. + +Environment variables +--------------------- +TEST_CASE_COVERAGE_LOBSTER_MANIFEST + Path to a newline-delimited file listing lobster-req-trace JSON paths + (one per line). Requirement IDs are extracted from these files. +TEST_CASE_COVERAGE_GTEST_LOBSTER + Path to the gtest.lobster file produced by subrule_lobster_gtest. +TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH + Workspace-relative path of the committed lock YAML file to overwrite. + Distinct from ``TEST_CASE_COVERAGE_LOCK_FILE`` (used by test_runner), + which points to a resolvable file rather than a workspace-relative one. +TEST_CASE_COVERAGE_LABEL + Bazel label of the test_case_coverage target (used in log output only). +""" + +from __future__ import annotations + +import logging +import os +import sys +from pathlib import Path + +from test_case_coverage.check_lock import validate_specs +from test_case_coverage.compute_lock import serialize +from test_case_coverage.read_gtest_lobster import resolve_path +from test_case_coverage.runner_common import require_env, scan_and_compute + +logger = logging.getLogger(__name__) + + +def main(argv: list[str] | None = None) -> None: + import argparse + + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s", force=True) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--allow-check-failures", + action="store_true", + help="Write the lock file even when GWT spec validation fails " + "(mirrors test_runner's flag of the same name).", + ) + args = parser.parse_args(argv) + + lobster_manifest = resolve_path(require_env("TEST_CASE_COVERAGE_LOBSTER_MANIFEST")) + gtest_lobster_path = resolve_path(require_env("TEST_CASE_COVERAGE_GTEST_LOBSTER")) + # The lock file must be resolved to the source tree so that ``bazel run`` + # overwrites the committed file. ``$BUILD_WORKSPACE_DIRECTORY`` is set + # by Bazel for all ``bazel run`` invocations; the env var carries the + # package-relative path (e.g. ``score/message_passing/test_case_coverage.lock.yaml``). + _raw_lock = require_env("TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH") + build_workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + if build_workspace: + lock_file_path = Path(build_workspace) / _raw_lock + else: + lock_file_path = resolve_path(_raw_lock) + label = os.environ.get("TEST_CASE_COVERAGE_LABEL", "") + package = os.environ.get("TEST_CASE_COVERAGE_PACKAGE", "") + + # Steps 1-3: extract requirement metadata, scan gtest.lobster, compute lock + req_metadata, lock = scan_and_compute( + lobster_manifest, gtest_lobster_path, package=package, label=label + ) + if not req_metadata: + logger.warning("[%s] No CompReq requirement IDs found in lobster manifest.", label) + + # Step 3b: validate GWT specs -- same severity as test_runner (ERROR + exit) + # so `.update` cannot silently write a lock file that the check step + # (test_runner) would then reject for missing specs. --allow-check-failures + # writes the lock anyway, mirroring test_runner's flag of the same name. + spec_ok, spec_issues = validate_specs(lock) + if not spec_ok: + logger.error("One or more test cases are missing GWT spec annotations:") + for line in spec_issues: + logger.error(line) + if not args.allow_check_failures: + logger.error( + "Lock file NOT written. Add RecordProperty given/when/then " + "annotations, or pass --allow-check-failures to write anyway." + ) + sys.exit(1) + + # Step 4: write lock file + yaml_content = serialize(lock) + # `label` is TEST_CASE_COVERAGE_LABEL, which is already this `.update` + # target's own label (see component.bzl's _component_update_impl) — do + # not append another ".update" suffix here, or the generated + # instructions end up as ".update.update". + header = ( + f"# test_case_coverage.lock.yaml — auto-generated by test_case_coverage\n" + f"# Run `bazel run {label}` to refresh.\n" + ) + lock_file_path.write_text(header + yaml_content, encoding="utf-8") + + logger.info("[%s] Lock file updated: %s", label, lock_file_path) + for req in lock.requirements: + logger.info(" %s: %d test case(s)", req.id, len(req.test_cases)) + + +if __name__ == "__main__": + main() diff --git a/bazel/rules/rules_score/src/test_case_coverage/update_runner_test.py b/bazel/rules/rules_score/src/test_case_coverage/update_runner_test.py new file mode 100644 index 00000000..e6149e20 --- /dev/null +++ b/bazel/rules/rules_score/src/test_case_coverage/update_runner_test.py @@ -0,0 +1,120 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from pathlib import Path + +import pytest + +from test_case_coverage.compute_lock import deserialize +from test_case_coverage.update_runner import main as update_main +from test_case_coverage.conftest import write_req_lobster, write_gtest_lobster + +_SPEC = ":Given: g\n:When: w\n:Then: t" +_UID = "Suite:T" + + +def _setup(tmp_path: Path, req_ids, uid=_UID, trace=None): + req_lobster = tmp_path / "reqs.lobster" + write_req_lobster(req_lobster, req_ids) + + lm = tmp_path / "lobster_manifest.txt" + lm.write_text(str(req_lobster) + "\n", encoding="utf-8") + + gtest = tmp_path / "gtest.lobster" + traced_req = trace or (req_ids[0] if req_ids else "X") + write_gtest_lobster(gtest, uid, traced_req) + + lock = tmp_path / "test_case_coverage.lock.yaml" + return lm, gtest, lock + + +def test_update_writes_lock_file(tmp_path, monkeypatch): + lm, gtest, lock = _setup(tmp_path, ["P.R"]) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", "//pkg") + update_main([]) + assert lock.exists() + parsed = deserialize(lock.read_text()) + assert parsed.requirements[0].id == "P.R" + assert parsed.requirements[0].version == "1" + assert parsed.requirements[0].test_cases[0].uid == f"//pkg/{_UID}" + + +def test_update_warns_on_zero_linked_tests(tmp_path, monkeypatch, capsys): + lm, gtest, lock = _setup(tmp_path, ["P.R", "P.NoTest"], trace="P.R") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", "//pkg") + update_main([]) # must not raise + captured = capsys.readouterr() + assert "P.NoTest" in captured.err + assert "WARNING" in captured.err + + +def test_update_exits_0_on_zero_linked_tests(tmp_path, monkeypatch): + lm, gtest, lock = _setup(tmp_path, ["P.NoTest"], trace="NOMATCH") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", "//pkg") + update_main([]) # should not raise SystemExit + + +def test_update_fails_on_missing_gwt_spec(tmp_path, monkeypatch): + """Missing GWT specs are an ERROR that blocks the write, same as test_runner.""" + lm, gtest, lock = _setup(tmp_path, ["P.R"]) + write_gtest_lobster(gtest, _UID, "P.R", spec="") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", "//pkg") + with pytest.raises(SystemExit) as exc_info: + update_main([]) + assert exc_info.value.code == 1 + assert not lock.exists() + + +def test_update_allow_check_failures_writes_despite_missing_spec(tmp_path, monkeypatch): + lm, gtest, lock = _setup(tmp_path, ["P.R"]) + write_gtest_lobster(gtest, _UID, "P.R", spec="") + monkeypatch.setenv("TEST_CASE_COVERAGE_LOBSTER_MANIFEST", str(lm)) + monkeypatch.setenv("TEST_CASE_COVERAGE_GTEST_LOBSTER", str(gtest)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH", str(lock)) + monkeypatch.setenv("TEST_CASE_COVERAGE_LABEL", "//pkg:cov") + monkeypatch.setenv("TEST_CASE_COVERAGE_PACKAGE", "//pkg") + update_main(["--allow-check-failures"]) # must not raise + assert lock.exists() + + +def test_update_missing_env_exits_1(monkeypatch): + for var in ( + "TEST_CASE_COVERAGE_LOBSTER_MANIFEST", + "TEST_CASE_COVERAGE_GTEST_LOBSTER", + "TEST_CASE_COVERAGE_LOCK_FILE_WORKSPACE_PATH", + ): + monkeypatch.delenv(var, raising=False) + with pytest.raises(SystemExit) as exc_info: + update_main([]) + assert exc_info.value.code == 1 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/bazel/rules/rules_score/test/BUILD b/bazel/rules/rules_score/test/BUILD index e7b7d955..7684381b 100644 --- a/bazel/rules/rules_score/test/BUILD +++ b/bazel/rules/rules_score/test/BUILD @@ -92,6 +92,8 @@ load( "component_excludes_feature_req_docs_test", "component_provider_test", "component_sphinx_sources_test", + "component_test_case_coverage_lock_test", + "dependable_element_test_case_coverage_lock_check_action_test", "unit_component_test_suite", "unit_provider_test", "unit_sphinx_sources_test", @@ -390,6 +392,37 @@ dependable_element( tests = [], # Empty for testing ) +# Dependable elements wrapping a test_case_coverage_lock component — used to verify the +# ComponentTestCaseCoverageLockCheck action wiring (mnemonic/inputs/argv) at both +# maturity levels via test_case_coverage_lock_check_action_{release,development}_test. +dependable_element( + name = "test_dependable_element_coverage_release", + testonly = True, + architectural_design = [":arch_design_elem"], + assumptions_of_use = [":aous"], + components = [":test_component_with_test_case_coverage_lock"], + dependability_analysis = [":dependability_analysis_target"], + integrity_level = "B", + maturity = "release", + requirements = [":feat_req"], + tags = ["manual"], + tests = [], # Empty for testing +) + +dependable_element( + name = "test_dependable_element_coverage_development", + testonly = True, + architectural_design = [":arch_design_elem"], + assumptions_of_use = [":aous"], + components = [":test_component_with_test_case_coverage_lock"], + dependability_analysis = [":dependability_analysis_target"], + integrity_level = "B", + maturity = "development", + requirements = [":feat_req"], + tags = ["manual"], + tests = [], # Empty for testing +) + # ============================================================================ # Test Fixtures - Nested Components for Recursive Testing # ============================================================================ @@ -668,6 +701,18 @@ unit_sphinx_sources_test( target_under_test = ":test_unit", ) +# Test Component with test case coverage lock — verifies analysis phase succeeds when +# test_case_coverage_lock is provided. +component( + name = "test_component_with_test_case_coverage_lock", + testonly = True, + components = [":test_unit"], + requirements = [":comp_req"], + tags = ["manual"], + test_case_coverage_lock = "fixtures/test_case_coverage_lock_test/test_case_coverage.lock.yaml", + tests = [], +) + # Component tests component_provider_test( name = "component_provider_test", @@ -684,6 +729,28 @@ component_excludes_feature_req_docs_test( target_under_test = ":test_component", ) +component_test_case_coverage_lock_test( + name = "component_test_case_coverage_lock_test", + target_under_test = ":test_component_with_test_case_coverage_lock", +) + +# Verifies the ComponentTestCaseCoverageLockCheck action is wired up end-to-end through +# dependable_element (not just component() in isolation), and that +# --allow-check-failures is only passed for maturity == "development". +dependable_element_test_case_coverage_lock_check_action_test( + name = "test_case_coverage_lock_check_action_release_test", + expect_allow_check_failures = False, + maturity_under_test = "release", + target_under_test = ":test_dependable_element_coverage_release_index", +) + +dependable_element_test_case_coverage_lock_check_action_test( + name = "test_case_coverage_lock_check_action_development_test", + expect_allow_check_failures = True, + maturity_under_test = "development", + target_under_test = ":test_dependable_element_coverage_development_index", +) + # Unit, Component, and Dependable Element test suite unit_component_test_suite(name = "unit_component_tests") diff --git a/bazel/rules/rules_score/test/fixtures/test_case_coverage_lock_test/test_case_coverage.lock.yaml b/bazel/rules/rules_score/test/fixtures/test_case_coverage_lock_test/test_case_coverage.lock.yaml new file mode 100644 index 00000000..efd58f06 --- /dev/null +++ b/bazel/rules/rules_score/test/fixtures/test_case_coverage_lock_test/test_case_coverage.lock.yaml @@ -0,0 +1,14 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +schema_version: 3 +requirements: [] diff --git a/bazel/rules/rules_score/test/unit_component_test.bzl b/bazel/rules/rules_score/test/unit_component_test.bzl index b51c0141..7ad2b46a 100644 --- a/bazel/rules/rules_score/test/unit_component_test.bzl +++ b/bazel/rules/rules_score/test/unit_component_test.bzl @@ -178,6 +178,109 @@ def _component_excludes_feature_req_docs_test_impl(ctx): component_excludes_feature_req_docs_test = analysistest.make(_component_excludes_feature_req_docs_test_impl) +# ============================================================================ +# Test Case Coverage Lock Tests +# ============================================================================ + +def _component_test_case_coverage_lock_test_impl(ctx): + """Test that a component with test_case_coverage_lock still provides ComponentInfo and + DefaultInfo with output files (analysis phase must succeed).""" + env = analysistest.begin(ctx) + target_under_test = analysistest.target_under_test(env) + + asserts.true( + env, + ComponentInfo in target_under_test, + "Component with test_case_coverage_lock should provide ComponentInfo", + ) + + # DefaultInfo.files must contain the lobster report (non-empty). + output_files = target_under_test[DefaultInfo].files.to_list() + asserts.true( + env, + len(output_files) > 0, + "Component with test_case_coverage_lock should declare output files (expected lobster report); got: %s" % output_files, + ) + + # At least one output file should be named like the lobster report. + lobster_report_present = any([f.basename.endswith(".lobster_report") or f.extension == "html" or f.basename.endswith(".lobster") for f in output_files]) + asserts.true( + env, + lobster_report_present, + "Component with test_case_coverage_lock should declare a lobster report output; got: %s" % [f.basename for f in output_files], + ) + + return analysistest.end(env) + +component_test_case_coverage_lock_test = analysistest.make(_component_test_case_coverage_lock_test_impl) + +def _dependable_element_test_case_coverage_lock_check_action_test_impl(ctx): + """ + Given a dependable_element wrapping a component with test_case_coverage_lock set, + When the target is analyzed, + Then a ComponentTestCaseCoverageLockCheck action must be registered that consumes + the committed test_case_coverage.lock.yaml as an input, and it must carry + ``--allow-check-failures`` in its argv only when the + dependable_element's maturity is "development" (never for "release"). + + This exercises the actual component -> ComponentTestCaseCoverageInfo -> + dependable_element wiring (unlike component_test_case_coverage_lock_test, which only + inspects the component() rule in isolation). + """ + env = analysistest.begin(ctx) + + actions = analysistest.target_actions(env) + check_action = None + for action in actions: + if action.mnemonic == "ComponentTestCaseCoverageLockCheck": + check_action = action + break + + asserts.true( + env, + check_action != None, + "Expected a ComponentTestCaseCoverageLockCheck action to be registered for a " + + "dependable_element wrapping a component with test_case_coverage_lock set", + ) + + if check_action == None: + return analysistest.end(env) + + input_names = [f.basename for f in check_action.inputs.to_list()] + asserts.true( + env, + "test_case_coverage.lock.yaml" in input_names, + "ComponentTestCaseCoverageLockCheck action must take the committed " + + "test_case_coverage.lock.yaml as an input; got inputs: %s" % input_names, + ) + + has_allow_flag = "--allow-check-failures" in check_action.argv + asserts.equals( + env, + ctx.attr.expect_allow_check_failures, + has_allow_flag, + "expected --allow-check-failures presence=%s (maturity=%s) but argv was: %s" % ( + ctx.attr.expect_allow_check_failures, + ctx.attr.maturity_under_test, + check_action.argv, + ), + ) + + return analysistest.end(env) + +dependable_element_test_case_coverage_lock_check_action_test = analysistest.make( + impl = _dependable_element_test_case_coverage_lock_check_action_test_impl, + attrs = { + "expect_allow_check_failures": attr.bool( + mandatory = True, + doc = "Whether --allow-check-failures is expected in the check action's argv.", + ), + "maturity_under_test": attr.string( + doc = "The maturity value of the target under test (for assertion messages only).", + ), + }, +) + # ============================================================================ # Dependable Element Tests # ============================================================================ @@ -202,5 +305,8 @@ def unit_component_test_suite(name): ":component_provider_test", ":component_sphinx_sources_test", ":component_excludes_feature_req_docs_test", + ":component_test_case_coverage_lock_test", + ":test_case_coverage_lock_check_action_release_test", + ":test_case_coverage_lock_check_action_development_test", ], ) diff --git a/bazel/rules/rules_score/trlc/config/BUILD b/bazel/rules/rules_score/trlc/config/BUILD index 4bbdbb0c..7b9e3853 100644 --- a/bazel/rules/rules_score/trlc/config/BUILD +++ b/bazel/rules/rules_score/trlc/config/BUILD @@ -45,7 +45,9 @@ trlc_requirements_test( trlc_requirements( name = "tools", - srcs = ["tools.trlc"], + srcs = [ + "tools.trlc", + ], spec = [":qualification_model"], visibility = ["//visibility:public"], ) diff --git a/bazel/rules/rules_score/trlc/config/tools.trlc b/bazel/rules/rules_score/trlc/config/tools.trlc index c3bf19c3..d0b56c67 100644 --- a/bazel/rules/rules_score/trlc/config/tools.trlc +++ b/bazel/rules/rules_score/trlc/config/tools.trlc @@ -79,4 +79,9 @@ section "Tools" { description = "Manual Analysis Framework" classification = ToolQualification.Classification.None } + + ToolQualification.Tool TestCaseCoverage { + description = "Test Case Coverage Lock — verifies complete test coverage of requirements via lobster-tracing annotations" + classification = ToolQualification.Classification.None + } }