From 78fd9a59f08dde71c2dadcf07659a131642b3ad2 Mon Sep 17 00:00:00 2001 From: epi13 Date: Fri, 14 Aug 2026 17:23:05 -0800 Subject: [PATCH 1/3] Import canonical MNCDS 0.1 material and a standalone validator. Specification text, schemas, examples, RFC 0004, and the mncds CLI now live here, extracted from MNCS commit f0088c4d46dec84f289d9b4417eec32b0ac028e6. MNCDS is independently versioned and no longer a nested chapter of MNCS. --- .gitignore | 13 + CHANGELOG.md | 8 +- MIGRATION.md | 8 +- README.md | 76 +- .../mncds-conformance-corpus/corpus.json | 184 +++ .../MNCS-v0.3-MNCDS-v0.1-decisions.md | 34 + docs/mncds-decisions.md | 114 ++ docs/mncds-evidence-plan.md | 99 ++ docs/mncds-specification.md | 30 + docs/mncds.md | 74 ++ examples/mncds-0.1-rc/development-record.json | 288 +++++ examples/mncds-d4/README.md | 20 + examples/mncds-d4/development-record.json | 209 ++++ migration/inventory.json | 4 +- pyproject.toml | 57 + ...achine-native-development-specification.md | 273 +++++ schemas/README.md | 8 +- .../mncds-development-record-0.1.schema.json | 386 ++++++ schemas/mncds-development-record.schema.json | 344 ++++++ spec/MNCDS-v0.1-draft.md | 340 ++++++ spec/MNCDS-v0.1-rc.1.md | 173 +++ spec/MNCDS-v0.1-records-and-decisions.md | 180 +++ spec/README.md | 17 +- src/mncds_validator/__init__.py | 9 + src/mncds_validator/cli.py | 86 ++ src/mncds_validator/errors.py | 19 + src/mncds_validator/mncds.py | 1087 +++++++++++++++++ src/mncds_validator/resources/__init__.py | 1 + .../resources/schemas/.gitkeep | 0 .../mncds-development-record-0.1.schema.json | 386 ++++++ .../mncds-development-record.schema.json | 344 ++++++ src/mncds_validator/schemas.py | 62 + src/mncds_validator/validation.py | 46 + tests/test_mncds.py | 136 +++ 34 files changed, 5080 insertions(+), 35 deletions(-) create mode 100644 .gitignore create mode 100644 conformance/mncds-conformance-corpus/corpus.json create mode 100644 docs/interoperability/MNCS-v0.3-MNCDS-v0.1-decisions.md create mode 100644 docs/mncds-decisions.md create mode 100644 docs/mncds-evidence-plan.md create mode 100644 docs/mncds-specification.md create mode 100644 docs/mncds.md create mode 100644 examples/mncds-0.1-rc/development-record.json create mode 100644 examples/mncds-d4/README.md create mode 100644 examples/mncds-d4/development-record.json create mode 100644 pyproject.toml create mode 100644 rfcs/0004-machine-native-development-specification.md create mode 100644 schemas/mncds-development-record-0.1.schema.json create mode 100644 schemas/mncds-development-record.schema.json create mode 100644 spec/MNCDS-v0.1-draft.md create mode 100644 spec/MNCDS-v0.1-rc.1.md create mode 100644 spec/MNCDS-v0.1-records-and-decisions.md create mode 100644 src/mncds_validator/__init__.py create mode 100644 src/mncds_validator/cli.py create mode 100644 src/mncds_validator/errors.py create mode 100644 src/mncds_validator/mncds.py create mode 100644 src/mncds_validator/resources/__init__.py create mode 100644 src/mncds_validator/resources/schemas/.gitkeep create mode 100644 src/mncds_validator/resources/schemas/mncds-development-record-0.1.schema.json create mode 100644 src/mncds_validator/resources/schemas/mncds-development-record.schema.json create mode 100644 src/mncds_validator/schemas.py create mode 100644 src/mncds_validator/validation.py create mode 100644 tests/test_mncds.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f7ac4b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +__pycache__/ +*.py[cod] +.venv/ +venv/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ +build/ +dist/ +*.egg-info/ +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 9700ebc..c89b6f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,4 +12,10 @@ All notable MNCDS specification and reference-tooling changes should be recorded - Added a provenance-preserving extraction plan and machine-readable migration inventory. - Added specification, schema, conformance, RFC, documentation, and CI scaffolding. -No normative MNCDS meaning is changed by this repository bootstrap. MNCDS 0.1-rc.1 remains sourced from the historical combined MNCS repository until the migration is completed and validated. +- Migrated MNCDS 0.1-draft, 0.1-rc.1, records-and-decisions, schemas, examples, + RFC 0004, and a standalone `mncds` validator from MNCS commit + `f0088c4d46dec84f289d9b4417eec32b0ac028e6`. +- This repository is now the canonical home of MNCDS meaning. MNCS retains a + consumer and consumed schema copies only. + +No normative MNCDS meaning is changed by the extraction itself. diff --git a/MIGRATION.md b/MIGRATION.md index 7ba79c9..e99bd2c 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,6 +1,12 @@ # MNCDS Extraction from the Historical MNCS Repository -This repository is being established before the normative MNCDS material is extracted from `epi13/machine-native-complexity-standard`. +This repository now holds the canonical MNCDS 0.1 specification, schemas, +examples, RFC 0004, and a standalone `mncds` validator extracted from +`epi13/machine-native-complexity-standard` commit +`f0088c4d46dec84f289d9b4417eec32b0ac028e6`. + +The remaining work is to keep the MNCS-side consumer compatible and to avoid +reintroducing a second authoritative copy of MNCDS meaning. The goal is **not** to fork MNCS. The goal is to make the existing conceptual separation between MNCS and MNCDS real at the repository, release, governance, validator, and conformance layers. diff --git a/README.md b/README.md index 3bdab00..64e2b88 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,22 @@ MNCDS governs how machine-native implementations are created, evaluated, selected, released, monitored, regenerated, replaced, and retired. It is a companion to the [Machine-Native Complexity Standard (MNCS)](https://github.com/epi13/machine-native-complexity-standard), but it is **independently versioned, governed, and released**. -> **Repository bootstrap in progress.** The normative MNCDS 0.1 release-candidate material currently lives in the MNCS repository and will be migrated here under an explicit provenance-preserving plan. Until that migration is completed and validated, this repository does not supersede the existing MNCDS source material. +This repository is the canonical home of MNCDS 0.1-draft, MNCDS 0.1-rc.1, the development-record schemas, and the MNCDS-owned reference validator. + +## What belongs here + +- development-process semantics and profiles (D1–D4) +- development records, authority, lineage, partitions, selection, release, monitoring, regeneration, and retirement +- MNCDS schemas, conformance vectors, and the `mncds` validator +- RFCs that change MNCDS meaning + +## What does not belong here + +- MNCS implementation-evidence acceptance rules +- operator infrastructure such as MNCS Harness, MNCS Control MCP, MNCS Fabric, or MNCS Commons +- Forge workflows, RAVEL/MNEL research, or reference-study implementations + +Those projects may *produce* MNCDS records. They do not own MNCDS meaning. ## Core boundary @@ -14,41 +29,52 @@ MNCDS governs how machine-native implementations are created, evaluated, selecte Those responsibilities interact through explicit, versioned bindings. Neither project silently owns or rewrites the other's normative meaning. -## Design principles - -1. **Independent normative authority.** MNCDS changes are reviewed and released here. -2. **Explicit interoperability.** Dependencies on MNCS use versioned contracts and identities rather than source duplication. -3. **Process evidence is first-class.** Candidate lineage, partitions, authority, selection, reproducibility, release, and lifecycle state remain inspectable artifacts. -4. **Unknown stays unknown.** Missing, inaccessible, unsupported, crashed, or timed-out evidence does not become `PASS`. -5. **History is immutable.** Corrections create new records and preserve superseded identities. -6. **Tool neutrality.** Models, generators, analyzers, compilers, providers, benchmarks, languages, Forge, RAVEL, and case studies are implementations or research mechanisms, not normative authorities. -7. **No self-promotion.** Generators, evaluators, orchestration systems, and recursive agents cannot silently broaden authority or promote their own results. - -## Repository map - -- `spec/` — normative and release-candidate MNCDS specification text. -- `schemas/` — versioned machine-readable MNCDS schemas. -- `conformance/` — valid/invalid vectors and release-candidate conformance corpora. -- `rfcs/` — proposals that change normative meaning, governance, or interoperability. -- `docs/` — architecture, rationale, migration, and non-normative guidance. -- `migration/` — provenance and transfer planning for material currently housed in the MNCS repository. -- `scripts/` — repository and conformance support tooling; tools are non-normative unless a specification explicitly says otherwise. +A reader can understand MNCS without first understanding this repository. MNCDS may require an MNCS result only when a record declares an MNCS binding. ## Status - Project: **Machine-Native Complexity Development Specification** - Acronym: **MNCDS** - Current specification line: **0.1** -- Current source release candidate: **0.1-rc.1**, pending migration from the MNCS repository +- Current release candidate: **0.1-rc.1** - Maturity: **experimental / pre-1.0** - License: **Apache-2.0** +- Historical source: extracted from [`epi13/machine-native-complexity-standard`](https://github.com/epi13/machine-native-complexity-standard) commit `f0088c4d46dec84f289d9b4417eec32b0ac028e6` + +See [`MIGRATION.md`](MIGRATION.md) and [`INTEROPERABILITY.md`](INTEROPERABILITY.md). + +## Validate a record + +```bash +python3 -m pip install -e '.[dev]' +mncds validate examples/mncds-0.1-rc/development-record.json --json +mncds validate examples/mncds-d4/development-record.json --require-pass +``` + +The validator checks the declared record and invariants. It does not launch generators, evaluators, models, Forge, Fabric, or Harness. + +## Repository map + +- `spec/` — normative MNCDS 0.1-draft and 0.1-rc.1 text +- `schemas/` — versioned machine-readable MNCDS schemas +- `examples/` — development-record examples +- `conformance/` — MNCDS-owned corpora +- `rfcs/` — proposals that change MNCDS meaning +- `docs/` — architecture, rationale, and non-normative guidance +- `src/mncds_validator/` — reference `mncds` CLI -The current migration source is [`epi13/machine-native-complexity-standard`](https://github.com/epi13/machine-native-complexity-standard). See [`MIGRATION.md`](MIGRATION.md) for the transfer boundary and [`INTEROPERABILITY.md`](INTEROPERABILITY.md) for the long-term MNCS ↔ MNCDS relationship. +## Family relationships -## Contributing +Family orientation lives in [MNCS Atlas](https://github.com/epi13/mncs-atlas). Atlas is descriptive. -Read [`CONTRIBUTING.md`](CONTRIBUTING.md), [`GOVERNANCE.md`](GOVERNANCE.md), and [`AGENTS.md`](AGENTS.md) before changing normative material. Normative changes require explicit review and must not be smuggled in as migration cleanup. +| Project | Relationship to MNCDS | +|---|---| +| MNCS | sibling standard for implementation-evidence acceptance | +| MNCS Forge | may produce or evaluate development evidence; not an authority | +| MNCS Validator (in MNCS / Rust) | may consume MNCDS records as a shared interface | +| MNCS Harness, Control, Fabric, Commons | operator/implementation ecosystem; not required by MNCDS | +| RAVEL / MNEL / Reference Studies | research and empirical work that may emit MNCDS records | ## Non-claims -MNCDS is experimental. It is not accredited certification, a security warranty, organizational independence, protected custody, or proof that a development process is safe or correct. A validator can check declared records and invariants; it cannot manufacture missing real-world evidence or authority. +MNCDS is experimental. It is not accredited certification, a security warranty, organizational independence, protected custody, or proof that a development process is safe or correct. diff --git a/conformance/mncds-conformance-corpus/corpus.json b/conformance/mncds-conformance-corpus/corpus.json new file mode 100644 index 0000000..34c5475 --- /dev/null +++ b/conformance/mncds-conformance-corpus/corpus.json @@ -0,0 +1,184 @@ +{ + "schema_version": "0.1", + "mncds_version": "0.1-draft", + "base_record": "examples/mncds-d4/development-record.json", + "cases": [ + { + "id": "valid/d4-pass", + "mutations": [], + "expected": { + "valid": true, + "computed_status": "PASS", + "issue_codes": [] + } + }, + { + "id": "invalid/generator-modifies-evaluator", + "mutations": [ + { + "path": "/generator/permissions/modify_evaluators", + "value": true + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "generator-authority-violation" + ] + } + }, + { + "id": "invalid/generator-modifies-threshold", + "mutations": [ + { + "path": "/generator/permissions/modify_thresholds", + "value": true + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "generator-authority-violation" + ] + } + }, + { + "id": "invalid/candidate-lineage-cycle", + "mutations": [ + { + "path": "/candidates/0/parent_ids", + "value": [ + "candidate-b" + ] + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "lineage-cycle" + ] + } + }, + { + "id": "invalid/unknown-promoted", + "mutations": [ + { + "path": "/candidates/1/evaluator_results/0/status", + "value": "UNKNOWN" + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "unknown-promoted" + ] + } + }, + { + "id": "invalid/holdout-contaminated", + "mutations": [ + { + "path": "/partitions/holdout_contaminated", + "value": true + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "holdout-contaminated" + ] + } + }, + { + "id": "invalid/selection-rule-post-hoc", + "mutations": [ + { + "path": "/selection/rule_recorded_before_holdout", + "value": false + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "selection-rule-post-hoc" + ] + } + }, + { + "id": "invalid/independence-conflict", + "mutations": [ + { + "path": "/evaluators/1/authority_id", + "value": "authority-generation-team" + }, + { + "path": "/evaluators/1/executable_id", + "value": "generator-runner-v2" + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "independence-authority-conflict", + "independence-executable-conflict", + "independent-role-mismatch" + ] + } + }, + { + "id": "invalid/mncs-binding-mismatch", + "mutations": [ + { + "path": "/mncs_binding/candidate_id", + "value": "candidate-a" + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "mncs-binding-mismatch" + ] + } + }, + { + "id": "invalid/rollback-not-tested", + "mutations": [ + { + "path": "/release_controls/rollback_test_status", + "value": "UNKNOWN" + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "rollback-not-tested" + ] + } + }, + { + "id": "invalid/regeneration-drill-failed", + "mutations": [ + { + "path": "/release_controls/regeneration_drill/status", + "value": "FAIL" + } + ], + "expected": { + "valid": false, + "computed_status": "FAIL", + "issue_codes": [ + "regeneration-drill-failed" + ] + } + } + ] +} diff --git a/docs/interoperability/MNCS-v0.3-MNCDS-v0.1-decisions.md b/docs/interoperability/MNCS-v0.3-MNCDS-v0.1-decisions.md new file mode 100644 index 0000000..f74854e --- /dev/null +++ b/docs/interoperability/MNCS-v0.3-MNCDS-v0.1-decisions.md @@ -0,0 +1,34 @@ + + +# MNCS 0.3 / MNCDS 0.1 release-candidate decisions + +Status: decision-ready evidence for Draft RFCs 0004 and 0005. External review and +governance approval remain required. + +| Issue | Alternatives | Selected rule | Compatibility, security, migration, tests | External requirement | +|---|---|---|---|---| +| MNCS core | Every Wave; extend 0.2 manifest; small family | Contract, assurance/lifecycle, threat, measurement | 0.2 frozen; RC corpus and two consumers | RFC 0005 approval | +| MNCDS core | Many subrecords; guidance; aggregate | One offline-resolvable aggregate | Draft preserved; migration creates identity | RFC 0004 approval | +| Experimental scope | Promote language/provider/Waves | Keep language profiles, MNEA, providers, Waves experimental | Tools/cases stay non-normative | Review | +| Contract adequacy | Optional; manifest fields; required | Required for new 0.3 claims | Historical claims unchanged; focused fixtures | Domain review per claim | +| Ambiguity | Always FAIL; always UNKNOWN | Contradiction/violation FAIL; unresolved material ambiguity UNKNOWN | Preserves uncertainty without accepting circularity | RFC approval | +| Contract change | Reuse ID; always new | Changed bytes new content ID; material semantics new logical ID and impact | No historical rewrite | RFC approval | +| Composition | Flat; adopter-only; graph | Acyclic required/optional claim graph | Tests propagation/cycles/shared evidence | Domain policy may be stricter | +| Correlation | Free text; universal taxonomy | Identified group/source/members | Concealment regression fixtures | RFC approval | +| Mixed versions | Normalize | Preserve exact level/profile/version | Prevents downgrade/promotion | RFC approval | +| Revalidation | Always full; unrestricted partial | Partial only over complete impact closure/fresh evidence | Insufficient scope UNKNOWN | Domain freshness policy | +| Invalidation | Artifact only; external | Ten identity dimensions plus custody | Material/non-material and stale fixtures | RFC approval | +| Freshness | Universal duration; none | Explicit expiry or identified no-default policy | Avoids universal unevidenced duration | Domain defaults | +| Lifecycle | Mutate claim; immutable events | Supersession/replacement/rollback/retirement; replacement new ID | Retired claim cannot support current PASS | RFC approval | +| Combined presentation | Score; separate files | One case, separate MNCS/MNCDS, no score | Collapse attempt invalid | RFC approval | +| Disposition | Third score; omit | Separate policy decision with no status | Review-required cannot become PASS | RFC approval | +| Criticality | Universal mapping; prose | Portable facts, external identified mapping | Avoids unsafe universal policy | Adoption evidence | +| Independence | Language difference; assertion | Separate implementation/executable/operator/organization facts | Code proves first two only | External actors | +| Unsupported | FAIL; omit; PASS | `UNSUPPORTED` boundary, required `UNKNOWN` | Both consumers test it | RFC approval | +| Migration | Auto-upgrade; invalidate | Wrap or reevaluate; missing facts UNKNOWN | 0.1/0.1.1/0.2/draft frozen | RFC approval | +| D3 | Organization required; identity only | Authority/executable separation, reviewer, fresh/protected evidence | Technical boundary locally testable | Organization remains external | +| Candidate retention | All bytes; aggregates | Material candidates individual; pre-material audited aggregate | Cycles/parents/disposition/aggregate tests | RFC approval | +| Recursive improvement | Forbid; unrestricted | Versioned epochs, fresh final partition, retained regressions | Two-epoch study | Independent review | + +Every selected rule has a local schema, semantic, corpus, migration, or documentation +obligation. External actor and governance obligations remain open. diff --git a/docs/mncds-decisions.md b/docs/mncds-decisions.md new file mode 100644 index 0000000..e7fe6de --- /dev/null +++ b/docs/mncds-decisions.md @@ -0,0 +1,114 @@ + + +# MNCDS initial design decisions + +RFC 0004 identified seven questions that blocked a stable experimental implementation. +The following decisions define the 0.1-draft implementation baseline. They remain subject +to RFC review and may change before acceptance. + +## Companion status + +MNCDS remains a separately versioned companion specification throughout the MNCS 0.x +series. Combining it with MNCS may be reconsidered only for a future MNCS major version, +after adoption experience shows that the two conformance families cannot remain cleanly +separable. + +This avoids making historical MNCS bundles retroactively nonconforming because their +development history was not recorded. + +## Minimum D1 record + +D1 requires one aggregate development record containing: + +- charter, objective, constraints, contract, baseline, environment, and threat-model identities; +- all six logical roles and disclosed executable identities where applicable; +- generator identity, configuration, permissions, and resource limits; +- development and selection partition identities; +- evaluator identities; +- every materially evaluated candidate's identity, lineage, outcome, objective value, and disposition; +- the selected candidate and selection rationale; +- explicit PASS, FAIL, and UNKNOWN treatment; +- a declared reproducibility class, which may be `NONE` at D1. + +Separate ledgers and role records may be introduced later, but the aggregate record is the +minimum interoperable unit for 0.1. + +## Stochastic reproducibility + +MNCDS uses five declared classes: + +- `EXACT`: byte-identical regeneration under the declared environment; +- `SEEDED`: the same algorithm, effective configuration, and seeds reproduce the run; +- `STATISTICAL`: repeated runs reproduce predeclared summary statistics within bounds; +- `DISTRIBUTIONAL`: runs reproduce a declared candidate family or outcome distribution; +- `NONE`: no credible regeneration or distributional claim. + +D1 may truthfully declare `NONE`. D2 and above require one of the first four classes. +`EXACT` and `SEEDED` require preserved seeds where randomness exists. A stronger label +must not be inferred from a weaker one. + +## Minimum evaluator independence + +For D3, the final evaluator must have: + +- an authority identity different from the generator authority; +- an executable identity different from the generator executable; +- immutable configuration identity; +- protected holdout or fresh challenge access unavailable to ordinary generation and ranking; +- an explicit independent-reviewer role binding; +- a result recorded against the selected candidate. + +Organizational separation, separate infrastructure, multiple reviewers, and threshold +signatures can support higher assurance, but they are not mandatory for baseline D3. +Identity difference alone does not prove honesty; it proves only that the required role +and executable boundaries were declared and checked. + +## Retaining rejected candidates + +Every materially evaluated candidate is retained individually by identity, lineage, +measurements, gate outcomes, and disposition. + +Candidates rejected before material evaluation may be summarized when the charter +predeclares the summarization boundary. A summary should preserve at least counts, +rejection stage, generator/configuration identity, time or sequence range, and a stable +digest or reproducible query over the omitted set. Search scale does not permit selective +omission of materially evaluated failures. + +This rule scales to millions of candidates while preserving evidence about selection +pressure and evaluator exploitation. + +## Privacy proofs and transparency logs + +Privacy-preserving proofs, confidential-computing attestations, and transparency logs are +permitted as namespaced experimental extensions. They are not required by D1-D4 and do +not replace required evidence unless a future RFC defines their predicate, trust, and +failure semantics and at least two implementations interoperate. + +Redaction without an accepted proof remains UNKNOWN when the hidden material is required +for the claimed profile. + +## CLI result separation + +MNCS and MNCDS results remain separate commands and separate result objects: + +```text +mncs validate ... +mncds validate ... +``` + +A future summary command may display both, but it must preserve two independent statuses, +issue sets, versions, identities, and scopes. It must not collapse them into a single +boolean or imply that one result establishes the other. + +## Deferred non-core research questions + +The following questions still require evidence rather than an editorial decision: + +- What aggregation structure best preserves million-candidate search history without + exposing proprietary candidate bodies? +- Which privacy-preserving proof systems are practical for restricted prompts, datasets, + and model configurations? +- What evidence threshold should define a higher-assurance evaluator-independence profile? + +The independent consumer and normalized RC issue set are now implemented in Rust. That +local fact does not establish independent operation or organizational independence. diff --git a/docs/mncds-evidence-plan.md b/docs/mncds-evidence-plan.md new file mode 100644 index 0000000..c68484f --- /dev/null +++ b/docs/mncds-evidence-plan.md @@ -0,0 +1,99 @@ + + +# MNCDS test and evidence plan + +RFC 0004 requires executable evidence before MNCDS can move from draft to accepted +normative status. This page tracks that evidence without treating implementation work as +proof that the proposal is already accepted. + +## Acceptance matrix + +| Required demonstration | Repository artifact | Current state | +|---|---|---| +| D1 multiple-candidate ledger | D4 reference record reduced to D1 in `tests/test_mncds.py` | Implemented | +| Reject generator evaluator/threshold mutation | Unit tests and deterministic corpus mutations | Implemented | +| Reject omitted or promoted UNKNOWN | Unit tests and `invalid/unknown-promoted` corpus case | Implemented | +| D2 reproducible generation and repeated measurement | Seeded profile test and D4 reference record | Implemented | +| D3 protected holdout and independent evaluator | D3 profile test and D4 reference record | Implemented | +| Recursive analyzer or harness improvement across epochs | `studies/recursive-analyzer` and RC development record | Internally reproducible; external custody UNKNOWN | +| D4 rollback, regeneration drill, retirement | D4 reference record and rejection tests | Implemented | +| Independent validator agreement | `independent/rc-consumer` and combined RC corpus | 74/74 local agreement; operator/organization UNKNOWN | + +## Deterministic corpus + +Run: + +```bash +PYTHONPATH=src python scripts/run-mncds-corpus +``` + +The corpus starts from one valid cumulative D4 record and applies declared JSON Pointer +mutations. Every case states its expected validity, computed status, and required issue +codes. This lets another implementation consume the same vectors without copying Python +validator behavior. + +Current corpus coverage includes: + +- forbidden evaluator and threshold mutation; +- candidate-lineage cycles; +- UNKNOWN promotion; +- holdout contamination; +- post-hoc selection rules; +- evaluator authority and executable conflicts; +- mismatched MNCS candidate binding; +- untested rollback; and +- failed regeneration drills. + +## Independent implementation evidence + +RFC 0004 must not be accepted solely because the reference Python validator agrees with +its own tests. Before acceptance, at least one independently implemented consumer should: + +1. parse `mncds-conformance-corpus/corpus.json` without importing the Python validator; +2. apply the declared mutations; +3. produce normalized validity, status, and issue-class outcomes; +4. publish an agreement report including every disagreement and unsupported rule; and +5. preserve unsupported behavior as UNKNOWN rather than PASS. + +The checked-in Rust implementation satisfies the source and executable diversity +requirements for the bounded RC corpus. It does not establish independent operation or +organizational independence; those remain external evidence gates. + +## Recursive analyzer and harness study + +The checked-in bounded study freezes two analyzer identities, converts epoch-one blind +spots into regression fixtures, and uses a fresh developer-withheld final partition. +Joern is not a normative dependency. + +The study does: + +1. freeze the original analyzer or harness, corpus, environment, and evaluation policy as + epoch one; +2. evaluate at least two competing implementation or analysis ideas; +3. record false positives, false negatives, incorrect PASS, UNKNOWN, crashes, timeouts, + runtime, memory, determinism, and diagnostic utility; +4. convert discovered disagreements and blind spots into classified regression fixtures; +5. create a newly identified epoch-two analyzer or harness; +6. rerun the analyzer regression corpus; +7. evaluate the frozen epoch-two candidate using fresh developer-withheld inputs; +8. compare detection quality, evidence quality, resource cost, and reproducibility across + epochs; and +9. retain unresolved disagreement cases as UNKNOWN. + +The experimental Machine-Native Evidence Analyzer described in +`docs/machine-native-evidence-analyzer.md` is one possible epoch-two implementation. It is +not required for conformance and must be evaluated as an untrusted provider. + +The study produces an MNCDS development record and an offline MNCS assurance resolution +set. Internal selection is PASS, while MNCS and MNCDS remain UNKNOWN because external +independence and protected custody are absent. + +## Acceptance gate + +MNCDS 0.1 should remain Draft until all of the following are true: + +- the Python implementation and corpus pass CI; +- an independent corpus consumer publishes normalized agreement; +- the recursive two-epoch analyzer or harness study is reproducible; +- security and privacy review finds no unresolved claim-broadening issue; and +- the RFC receives the independent approvals required by governance. diff --git a/docs/mncds-specification.md b/docs/mncds-specification.md new file mode 100644 index 0000000..c2cc577 --- /dev/null +++ b/docs/mncds-specification.md @@ -0,0 +1,30 @@ + + +# MNCDS 0.1 specifications + +The proposed normative text is maintained in two linked modules: + +- [`spec/MNCDS-v0.1-draft.md`](https://github.com/epi13/machine-native-complexity-standard/blob/main/spec/MNCDS-v0.1-draft.md) — lifecycle and cumulative profile requirements; +- [`spec/MNCDS-v0.1-records-and-decisions.md`](https://github.com/epi13/machine-native-complexity-standard/blob/main/spec/MNCDS-v0.1-records-and-decisions.md) — aggregate record, stochastic reproducibility, evaluator independence, candidate-retention, privacy-extension, and result-separation semantics. +- `spec/MNCDS-v0.1-rc.1.md` — complete release-candidate text and migration + rules; the historical draft is preserved. + +MNCDS defines cumulative development-process profiles for controlled generation, +reproducible experimentation, independent selection, and operational regeneration. It +standardizes identities, authority boundaries, evidence partitions, candidate lineage, +selection controls, evaluator independence, release binding, rollback, regeneration, +and retirement. + +The machine-readable implementation currently consists of: + +- `schemas/mncds-development-record.schema.json`; +- the packaged schema exposed through `mncs schema mncds-development-record`; +- the offline `mncds validate` command; +- the cumulative D4 example; +- unit tests and the deterministic MNCDS conformance corpus. +- `schemas/mncds-development-record-0.1.schema.json`, the combined RC corpus, + and independent Rust consumer. + +MNCDS 0.1-rc.1 remains under Draft RFC 0004. The implementation makes the proposal testable but +does not bypass the repository's review, independent-approval, or interoperability +requirements. diff --git a/docs/mncds.md b/docs/mncds.md new file mode 100644 index 0000000..6510e5e --- /dev/null +++ b/docs/mncds.md @@ -0,0 +1,74 @@ + + +# Machine-Native Complexity Development Specification + +MNCDS is the experimental development-process companion to MNCS. + +MNCS asks whether a selected implementation is supported by adequate correctness, safety, +resource, structural, performance, provenance, and regeneration evidence. MNCDS asks +whether the process that generated, compared, selected, released, and later replaces that +implementation remained controlled and auditable. + +The two claims remain separate: + +```text +MNCDS-D3 / MNCS-L4 +``` + +The first result describes development-process assurance. The second describes candidate +implementation conformance. Neither implies the other. + +## Profiles + +| Profile | Required control surface | +|---|---| +| D1 | Charter, immutable baseline, bounded generator, candidate identities, lineage, ledger, explicit PASS/FAIL/UNKNOWN | +| D2 | Pinned environment, evidence partitions, reproducibility class, repeated measurement, evaluator regression corpus | +| D3 | Predeclared selection, protected holdout, independent final evaluator, role-conflict checks, MNCS binding | +| D4 | Release identity, monitoring thresholds, tested rollback, regeneration drill, retirement triggers | + +Profiles are cumulative. + +## Offline validation + +Install the repository and validate the reference record: + +```bash +python -m pip install -e '.[dev]' +mncds validate examples/mncds-d4/development-record.json --require-pass +mncds validate examples/mncds-0.1-rc/development-record.json --json +``` + +Machine-readable output is available with `--json`. The validator performs schema and +cross-record semantic checks. It never launches or imports a generator, candidate, +evaluator, analyzer, benchmark, or evidence binary. + +The release-candidate validator checks: + +- required roles and role uniqueness; +- forbidden generator authority; +- evidence-partition identity overlap and holdout contamination; +- candidate identity uniqueness, parent existence, and lineage cycles; +- selected-candidate presence and disposition; +- required FAIL and UNKNOWN treatment; +- cumulative D2 reproducibility requirements; +- D3 holdout, predeclared-selection, authority, executable, and evidence independence; +- candidate/contract/environment agreement with an MNCS binding; +- D4 rollback and regeneration-drill outcomes. + +The independent Rust consumer applies the same bounded RC semantics directly to the +72-case golden corpus. It agrees with Python on all vectors without importing or +executing Python. + +## Recursive improvement + +Evidence from epoch `n` may improve a Joern harness, evaluator, generator, or search +strategy in epoch `n+1`. The changed toolchain receives a new identity, preserves the +failure cases that motivated the update, reruns its regression corpus, and must not reuse +a contaminated protected holdout for the same acceptance claim. + +## Status + +MNCDS 0.1-rc.1 remains a release-candidate proposal under Draft RFC 0004. The +implementation is ready for independent review but does not make the proposal +accredited, Accepted, Final, organizationally independent, or governance approved. diff --git a/examples/mncds-0.1-rc/development-record.json b/examples/mncds-0.1-rc/development-record.json new file mode 100644 index 0000000..11868e1 --- /dev/null +++ b/examples/mncds-0.1-rc/development-record.json @@ -0,0 +1,288 @@ +{ + "schema_version": "0.1-rc.1", + "mncds_version": "0.1-rc.1", + "record_id": "development.recursive-study-v1", + "profile": "MNCDS-D4", + "epoch_id": "epoch.two", + "created_at": "2026-07-28T00:00:00Z", + "supersedes_record_id": null, + "charter": { + "charter_id": "charter.recursive-study-v1", + "problem_statement": "Reduce incorrect PASS classifications without weakening detection.", + "intended_use": "Develop and evaluate a bounded structural analyzer over a frozen corpus.", + "exclusions": ["External organizational independence is not claimed."], + "contract_id": "contract.example-v1", + "baseline_id": "baseline.epoch-one", + "environment_id": "environment.study-v1", + "threat_model_id": "threat.result-collapse-v1", + "objective": { + "objective_id": "objective.reduce-incorrect-pass-v1", + "metric": "incorrect PASS count", + "unit": "cases", + "direction": "minimize", + "minimum_useful_benefit": 1, + "operational_rationale": "Unsupported analysis must remain UNKNOWN rather than becoming an incorrect PASS." + }, + "selection_policy_id": "selection.policy-v1", + "planned_mncs_level": "MNCS-L2", + "hard_rejection_gates": ["gate.no-false-negative-regression", "gate.no-crash", "gate.no-timeout"], + "release_owner_id": "authority.release", + "rollback_owner_id": "authority.rollback", + "retirement_owner_id": "authority.retirement" + }, + "baseline": { + "baseline_id": "baseline.epoch-one", + "artifact_id": "analyzer.epoch-one", + "source_id": "source.epoch-one", + "build_id": "build.epoch-one", + "dependency_ids": ["dependency.python-runtime"], + "environment_id": "environment.study-v1", + "evaluator_ids": ["evaluator.development"], + "results": [ + { + "evaluator_id": "evaluator.development", + "gate_id": "gate.baseline-captured", + "partition_id": "partition.development", + "required": true, + "status": "PASS", + "evidence_id": "evidence.baseline-v1" + } + ], + "captured_at": "2026-07-27T00:00:00Z", + "immutable": true + }, + "environment_lock": { + "environment_id": "environment.study-v1", + "toolchain_id": "toolchain.python-v1", + "dependency_ids": ["dependency.python-runtime"], + "hardware_id": "hardware.local-v1", + "configuration_id": "configuration.study-v1", + "permitted_variance": ["Wall-clock and resident-memory measurements may vary and are reported per run."], + "locked": true + }, + "roles": [ + {"role": "contract_authority", "authority_id": "authority.contract", "executable_id": null}, + {"role": "generator_authority", "authority_id": "authority.generator", "executable_id": "generator.repair-v1"}, + {"role": "evaluator_authority", "authority_id": "authority.evaluator", "executable_id": "evaluator.development-exec-v1"}, + {"role": "selection_authority", "authority_id": "authority.selection", "executable_id": null}, + {"role": "release_authority", "authority_id": "authority.release", "executable_id": null}, + {"role": "independent_reviewer", "authority_id": "authority.withheld-evaluator", "executable_id": "evaluator.final-exec-v1"} + ], + "authority_overlaps": [], + "generator": { + "generator_id": "generator.repair-v1", + "configuration_id": "generator.configuration-v1", + "authority_id": "authority.generator", + "executable_id": "generator.repair-exec-v1", + "permissions": { + "modify_contract": false, + "modify_baseline": false, + "modify_evaluators": false, + "modify_selection_policy": false, + "modify_thresholds": false, + "access_protected_holdout": false, + "network_access": false, + "filesystem_scope": ["study working directory"], + "process_scope": ["no subprocess execution"], + "tool_ids": ["analyzer.epoch-one", "analyzer.epoch-two"], + "mutation_scope": ["classification of unsupported alias constructs"] + }, + "resource_limits": { + "max_candidates": 2, + "max_wall_seconds": 60, + "max_memory_bytes": 268435456, + "max_processes": 1 + } + }, + "partitions": { + "development_id": "partition.development", + "selection_id": "partition.selection", + "final_evaluation_id": "partition.final", + "holdout_contaminated": false, + "access_policy_ids": ["policy.partition-access-v1"] + }, + "protected_evidence": [ + { + "evidence_id": "evidence.final-withheld-v1", + "partition_id": "partition.final", + "commitment_id": "commitment.final-v1", + "custodian_id": "authority.withheld-evaluator", + "custody_class": "developer_withheld", + "disclosed_at": "2026-07-28T00:00:00Z", + "generator_access": false, + "reuse_claim_ids": ["result.analyzer-epoch-two"], + "contaminated": false, + "status": "UNKNOWN" + } + ], + "evaluators": [ + { + "evaluator_id": "evaluator.development", + "purpose": "development", + "authority_id": "authority.evaluator", + "executable_id": "evaluator.development-exec-v1", + "configuration_id": "evaluator.configuration-v1", + "environment_id": "environment.study-v1", + "independent": false, + "operator_independence": "FAIL", + "organizational_independence": "FAIL", + "regression_corpus_id": "corpus.development-v1" + }, + { + "evaluator_id": "evaluator.final", + "purpose": "independent", + "authority_id": "authority.withheld-evaluator", + "executable_id": "evaluator.final-exec-v1", + "configuration_id": "evaluator.final-configuration-v1", + "environment_id": "environment.study-v1", + "independent": true, + "operator_independence": "UNKNOWN", + "organizational_independence": "UNKNOWN", + "regression_corpus_id": "corpus.final-v1" + } + ], + "candidates": [ + { + "candidate_id": "candidate.epoch-one", + "parent_ids": [], + "epoch_id": "epoch.one", + "generator_id": "generator.repair-v1", + "generation_sequence": 0, + "materially_evaluated": true, + "retained": true, + "build_status": "PASS", + "disposition": "retained", + "objective_value": 2, + "evaluator_results": [ + { + "evaluator_id": "evaluator.development", + "gate_id": "gate.objective", + "partition_id": "partition.development", + "required": true, + "status": "PASS", + "evidence_id": "evidence.epoch-one-development-v1" + } + ] + }, + { + "candidate_id": "candidate.epoch-two", + "parent_ids": ["candidate.epoch-one"], + "epoch_id": "epoch.two", + "generator_id": "generator.repair-v1", + "generation_sequence": 1, + "materially_evaluated": true, + "retained": true, + "build_status": "PASS", + "disposition": "selected", + "objective_value": 0, + "evaluator_results": [ + { + "evaluator_id": "evaluator.development", + "gate_id": "gate.objective", + "partition_id": "partition.selection", + "required": true, + "status": "PASS", + "evidence_id": "evidence.epoch-two-selection-v1" + }, + { + "evaluator_id": "evaluator.final", + "gate_id": "gate.final-evaluation", + "partition_id": "partition.final", + "required": true, + "status": "PASS", + "evidence_id": "evidence.final-withheld-v1" + } + ] + } + ], + "candidate_aggregates": [], + "selection": { + "policy_id": "selection.policy-v1", + "selection_epoch_id": "epoch.two", + "selected_candidate_id": "candidate.epoch-two", + "rule_recorded_before_final_evaluation": true, + "unknown_policy": "reject", + "minimum_useful_benefit_met": true, + "hard_gates_passed": true, + "rationale": "The selected candidate reduces incorrect PASS without regressing hard gates.", + "human_review": null + }, + "reproducibility": { + "class": "EXACT", + "seeds_preserved": true, + "protocol": "Run the frozen offline evaluator over lexically ordered fixtures.", + "measurement_repetitions": 3, + "comparison_statistic": "Exact per-case outcome equality and median runtime.", + "acceptance_bounds": "All classifications must reproduce exactly.", + "failure_treatment": "A failed regeneration remains FAIL and blocks promotion." + }, + "epochs": [ + { + "epoch_id": "epoch.one", + "parent_epoch_id": null, + "toolchain_id": "analyzer.epoch-one", + "corpus_id": "corpus.epoch-one-v1", + "objective_id": "objective.reduce-incorrect-pass-v1", + "contract_id": "contract.example-v1", + "threshold_policy_id": "threshold.policy-v1", + "development_partition_id": "partition.development", + "final_partition_id": null, + "change_evidence_ids": [], + "regression_fixture_ids": [] + }, + { + "epoch_id": "epoch.two", + "parent_epoch_id": "epoch.one", + "toolchain_id": "analyzer.epoch-two", + "corpus_id": "corpus.epoch-two-v1", + "objective_id": "objective.reduce-incorrect-pass-v1", + "contract_id": "contract.example-v1", + "threshold_policy_id": "threshold.policy-v1", + "development_partition_id": "partition.selection", + "final_partition_id": "partition.final", + "change_evidence_ids": ["evidence.change-classifier-v1"], + "regression_fixture_ids": ["fixture.alias-call-v1", "fixture.indirect-call-v1"] + } + ], + "mncs_binding": { + "result_or_package_id": "result.analyzer-epoch-two", + "mncs_version": "0.3-rc.1", + "candidate_id": "candidate.epoch-two", + "contract_id": "contract.example-v1", + "environment_id": "environment.study-v1", + "status": "UNKNOWN" + }, + "release_controls": { + "release_artifact_id": "analyzer.epoch-two", + "release_environment_id": "environment.study-v1", + "release_authority_id": "authority.release", + "build_procedure_id": "procedure.build-v1", + "package_procedure_id": "procedure.package-v1", + "monitoring": { + "signal_ids": ["signal.incorrect-pass", "signal.crash", "signal.timeout"], + "threshold_policy_id": "threshold.policy-v1", + "status": "PASS" + }, + "rollback": { + "artifact_id": "analyzer.epoch-one", + "procedure_id": "procedure.rollback-v1", + "test_status": "PASS", + "evidence_id": "evidence.rollback-test-v1" + }, + "regeneration_or_replacement": { + "procedure_id": "procedure.regeneration-v1", + "performed_at": "2026-07-28T00:00:00Z", + "status": "PASS", + "evidence_id": "evidence.regeneration-v1", + "replacement_candidate_id": "candidate.epoch-one" + }, + "retirement": { + "trigger_ids": ["trigger.contract-change", "trigger.evaluator-regression"], + "retired": false, + "retired_at": null, + "reason": null, + "replacement_candidate_id": null + } + }, + "extensions": {} +} diff --git a/examples/mncds-d4/README.md b/examples/mncds-d4/README.md new file mode 100644 index 0000000..7d2b501 --- /dev/null +++ b/examples/mncds-d4/README.md @@ -0,0 +1,20 @@ +# MNCDS-D4 reference record + +This example is a compact cumulative `MNCDS-D4` development record. It demonstrates: + +- a readable development charter and immutable baseline identity; +- bounded generator permissions; +- distinct development, selection, and protected-holdout partitions; +- candidate lineage and recorded rejection/selection dispositions; +- reproducible seeded experimentation and repeated measurement; +- an independent final evaluator with separate authority and executable identity; +- a matching MNCS candidate, contract, and environment binding; +- tested rollback, a completed regeneration drill, monitoring thresholds, and retirement triggers. + +Validate it without executing any generator, candidate, evaluator, benchmark, or evidence: + +```bash +mncds validate examples/mncds-d4/development-record.json --require-pass +``` + +The record is illustrative rather than a production assurance claim. Its identities are stable example identifiers, not cryptographic attestations of external facts. diff --git a/examples/mncds-d4/development-record.json b/examples/mncds-d4/development-record.json new file mode 100644 index 0000000..e7be67d --- /dev/null +++ b/examples/mncds-d4/development-record.json @@ -0,0 +1,209 @@ +{ + "schema_version": "0.1", + "mncds_version": "0.1-draft", + "record_id": "mncds-example-d4-record", + "profile": "MNCDS-D4", + "epoch_id": "epoch-2", + "supersedes_record_id": "mncds-example-epoch-1", + "created_at": "2026-07-26T22:30:00Z", + "charter": { + "problem_statement": "Produce a faster contract-equivalent request parser without weakening correctness or safety gates.", + "intended_use": "Demonstrate cumulative MNCDS-D4 controls with a small machine-native optimization search.", + "contract_id": "contract-request-parser-v1", + "baseline_id": "baseline-request-parser-v1", + "environment_id": "environment-linux-x86_64-v1", + "threat_model_id": "threat-model-parser-v1", + "objective": { + "objective_id": "objective-throughput-v1", + "metric": "valid requests processed per second", + "unit": "requests_per_second", + "direction": "maximize", + "minimum_useful_benefit": 0.1 + }, + "selection_policy_id": "selection-policy-v1", + "planned_mncs_level": "MNCS-L4", + "hard_rejection_gates": [ + "behavioral", + "safety", + "resource_bounds" + ] + }, + "roles": [ + { + "role": "contract_authority", + "authority_id": "authority-contract-team", + "executable_id": null + }, + { + "role": "generator_authority", + "authority_id": "authority-generation-team", + "executable_id": "generator-runner-v2" + }, + { + "role": "evaluator_authority", + "authority_id": "authority-evaluation-team", + "executable_id": "development-evaluator-v2" + }, + { + "role": "selection_authority", + "authority_id": "authority-selection-team", + "executable_id": "selection-engine-v1" + }, + { + "role": "release_authority", + "authority_id": "authority-release-team", + "executable_id": null + }, + { + "role": "independent_reviewer", + "authority_id": "authority-independent-review", + "executable_id": "independent-evaluator-v1" + } + ], + "generator": { + "generator_id": "generator-search-v2", + "configuration_id": "generator-config-epoch-2", + "authority_id": "authority-generation-team", + "executable_id": "generator-runner-v2", + "permissions": { + "modify_contract": false, + "modify_baseline": false, + "modify_evaluators": false, + "modify_selection_policy": false, + "modify_thresholds": false, + "access_protected_holdout": false, + "network_access": false + }, + "resource_limits": { + "max_candidates": 100, + "max_wall_seconds": 3600 + } + }, + "partitions": { + "development_id": "partition-development-v2", + "selection_id": "partition-selection-v2", + "holdout_id": "partition-holdout-v2", + "holdout_contaminated": false + }, + "evaluators": [ + { + "evaluator_id": "evaluator-development-v2", + "purpose": "development", + "authority_id": "authority-evaluation-team", + "executable_id": "development-evaluator-v2", + "configuration_id": "development-evaluator-config-v2", + "independent": false, + "regression_corpus_id": "evaluator-corpus-v2" + }, + { + "evaluator_id": "evaluator-independent-v1", + "purpose": "independent", + "authority_id": "authority-independent-review", + "executable_id": "independent-evaluator-v1", + "configuration_id": "independent-evaluator-config-v1", + "independent": true, + "regression_corpus_id": "independent-corpus-v1" + } + ], + "candidates": [ + { + "candidate_id": "candidate-a", + "parent_ids": [], + "generator_id": "generator-search-v2", + "build_status": "PASS", + "disposition": "rejected", + "objective_value": 1040.0, + "evaluator_results": [ + { + "evaluator_id": "evaluator-development-v2", + "gate_id": "behavioral", + "required": true, + "status": "PASS", + "evidence_id": "evidence-a-behavioral" + } + ] + }, + { + "candidate_id": "candidate-b", + "parent_ids": [ + "candidate-a" + ], + "generator_id": "generator-search-v2", + "build_status": "PASS", + "disposition": "selected", + "objective_value": 1180.0, + "evaluator_results": [ + { + "evaluator_id": "evaluator-development-v2", + "gate_id": "behavioral", + "required": true, + "status": "PASS", + "evidence_id": "evidence-b-behavioral" + }, + { + "evaluator_id": "evaluator-development-v2", + "gate_id": "safety", + "required": true, + "status": "PASS", + "evidence_id": "evidence-b-safety" + }, + { + "evaluator_id": "evaluator-development-v2", + "gate_id": "resource_bounds", + "required": true, + "status": "PASS", + "evidence_id": "evidence-b-resources" + }, + { + "evaluator_id": "evaluator-independent-v1", + "gate_id": "holdout", + "required": true, + "status": "PASS", + "evidence_id": "evidence-b-independent-holdout" + } + ] + } + ], + "selection": { + "policy_id": "selection-policy-v1", + "selected_candidate_id": "candidate-b", + "rule_recorded_before_holdout": true, + "unknown_policy": "reject", + "minimum_useful_benefit_met": true, + "rationale": "Candidate B passed all hard gates, exceeded the declared benefit threshold, and passed independent holdout evaluation.", + "human_review": null + }, + "reproducibility": { + "class": "SEEDED", + "seeds_preserved": true, + "protocol": "Rebuild the pinned toolchain, replay the recorded seed set, and repeat each measurement five times.", + "measurement_repetitions": 5 + }, + "mncs_binding": { + "manifest_or_package_id": "mncs-package-request-parser-v1", + "candidate_id": "candidate-b", + "contract_id": "contract-request-parser-v1", + "environment_id": "environment-linux-x86_64-v1" + }, + "release_controls": { + "release_artifact_id": "release-request-parser-v1", + "rollback_artifact_id": "baseline-request-parser-v1", + "rollback_test_status": "PASS", + "monitoring_thresholds": [ + "rollback after any contract regression", + "investigate after three consecutive resource-limit UNKNOWN outcomes" + ], + "regeneration_drill": { + "performed_at": "2026-07-26T22:20:00Z", + "status": "PASS", + "evidence_id": "evidence-regeneration-drill-v1" + }, + "retirement_triggers": [ + "contract changes materially", + "supported environment changes materially", + "critical dependency becomes unsupported", + "artifact cannot be regenerated" + ] + }, + "extensions": {} +} diff --git a/migration/inventory.json b/migration/inventory.json index eb24b16..2a74456 100644 --- a/migration/inventory.json +++ b/migration/inventory.json @@ -102,5 +102,7 @@ "disposition": "SHARED_INTERFACE", "reason": "keep owning semantics with MNCS unless a specific MNCDS validator surface is independently specified; consume through explicit versioned bindings" } - ] + ], + "frozen_source_commit": "f0088c4d46dec84f289d9b4417eec32b0ac028e6", + "migration_date": "2026-08-15" } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a40400e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools>=75", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mncds-validator" +version = "0.1.0rc1" +description = "Offline validator and schemas for the Machine-Native Complexity Development Specification" +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [{name = "MNCS Contributors"}] +keywords = ["mncds", "machine-native-development", "software-standards"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Quality Assurance", +] +dependencies = [ + "jsonschema>=4.23,<5", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3,<10", + "ruff==0.16.0", +] + +[project.scripts] +mncds = "mncds_validator.cli:main" + +[project.urls] +Homepage = "https://github.com/epi13/machine-native-complexity-development-specification" +Repository = "https://github.com/epi13/machine-native-complexity-development-specification" +Issues = "https://github.com/epi13/machine-native-complexity-development-specification/issues" +MNCS = "https://github.com/epi13/machine-native-complexity-standard" +Atlas = "https://github.com/epi13/mncs-atlas" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +"mncds_validator.resources.schemas" = ["*.json"] + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.pytest.ini_options] +addopts = "-q" +testpaths = ["tests"] diff --git a/rfcs/0004-machine-native-development-specification.md b/rfcs/0004-machine-native-development-specification.md new file mode 100644 index 0000000..6989ff7 --- /dev/null +++ b/rfcs/0004-machine-native-development-specification.md @@ -0,0 +1,273 @@ + + +# RFC 0004: Machine-Native Complexity Development Specification + +- Status: Draft +- Authors: Alexander Collamore +- Created: 2026-07-26 +- Review deadline: 2026-08-09 +- Target version: MNCS 0.3 (proposed) +- Conflicts disclosed: Repository owner and proposal author are the same person; independent review is required before acceptance. + +## Summary + +This RFC proposes the **Machine-Native Complexity Development Specification (MNCDS)** as a normative companion to MNCS. + +MNCS defines the evidence and acceptance envelope for machine-native implementations. MNCDS defines the development lifecycle that produces a candidate for that envelope: problem declaration, baseline capture, candidate generation, constrained search, evaluation, selection, independent verification, release, monitoring, regeneration, and retirement. + +The central rule is: + +> A machine-native implementation may be difficult for humans to maintain internally, but the process that creates, selects, validates, reproduces, and replaces it must remain explicit, bounded, inspectable, and reversible. + +MNCDS does not prescribe a model, optimizer, programming language, analyzer, search algorithm, or development platform. It standardizes the control surfaces and records needed to distinguish disciplined machine-native development from unbounded code generation or unverifiable optimization. + +## Motivation + +MNCS 0.2 establishes evidence-derived conformance, reproducible packages, provider interoperability, explicit trust, and scoped certification. Those mechanisms answer whether a submitted implementation is supported by adequate evidence inside a declared contract and environment. + +They do not fully answer how the implementation was produced. + +A development process can satisfy an output test while still being poorly controlled. Examples include: + +- repeatedly tuning against the final acceptance suite until it becomes training data; +- changing the objective after seeing candidate results; +- selecting a candidate without preserving rejected alternatives or the selection rationale; +- accepting an apparent performance gain caused by noise, environment drift, or a weakened baseline; +- using an analyzer whose failures are silently treated as absence of defects; +- allowing a generator to modify the contract, reference implementation, evaluator, or threshold it is supposed to satisfy; +- losing prompts, model identity, seeds, toolchain, datasets, or search history needed to reproduce or replace a candidate; +- deploying a machine-native artifact without rollback, regeneration, monitoring, or retirement controls. + +A standard for machine-native complexity therefore needs two distinct but connected layers: + +1. **MNCS:** what evidence is required to accept a candidate. +2. **MNCDS:** what controls are required while developing and selecting that candidate. + +This separation preserves tool neutrality and prevents historical MNCS bundles from becoming retroactively invalid because their development histories were not recorded. + +## Normative proposal + +The proposed normative text consists of: + +- `spec/MNCDS-v0.1-draft.md` — lifecycle and profile requirements; +- `spec/MNCDS-v0.1-records-and-decisions.md` — initial interoperable record, stochastic reproducibility, evaluator independence, rejected-candidate retention, privacy-extension, and reporting semantics. + +MNCDS defines ten lifecycle stages: + +1. Development charter +2. Baseline and environment lock +3. Evaluation partitioning +4. Candidate-generation envelope +5. Search and experiment ledger +6. Progressive evaluation +7. Candidate selection +8. Independent verification +9. Release and operational controls +10. Regeneration, replacement, and retirement + +### Core requirements + +A conforming MNCDS process MUST: + +- bind development to a readable contract, threat model, resource envelope, and declared useful-benefit objective; +- preserve an immutable baseline before candidate search begins; +- separate development evidence from selection and protected-holdout evidence; +- prevent the candidate generator from silently changing the contract, evaluator, reference behavior, threshold, or acceptance policy; +- record each materially evaluated candidate and its lineage in an append-only experiment ledger; +- report evaluator and analyzer outcomes as PASS, FAIL, or UNKNOWN without converting missing or unsupported analysis into PASS; +- predeclare the candidate-selection rule before final holdout evaluation; +- preserve rejected candidates or auditable aggregates under the rules in the records module; +- require independent verification of the selected candidate against fresh or previously inaccessible evidence for D3 and above; +- produce an MNCS bundle for any MNCS conformance claim; +- define rollback, regeneration, monitoring, and retirement conditions before deployment. + +### Development conformance profiles + +Profiles are cumulative: + +- **MNCDS-D1 — Controlled generation:** charter, baseline, bounded generator authority, candidate identity, lineage, and basic ledger. +- **MNCDS-D2 — Reproducible experimentation:** pinned environment, evaluation partitions, declared reproducibility class, repeated measurement, and evaluator regression corpus. +- **MNCDS-D3 — Independent selection:** predeclared selection policy, protected holdout, independent final evaluator, explicit UNKNOWN treatment, and MNCS binding when applicable. +- **MNCDS-D4 — Operational regeneration:** release binding, rollback triggers, monitoring, regeneration drill, and retirement records. + +These profiles describe development-process assurance and MUST NOT substitute for MNCS conformance levels. A project may state both, for example `MNCDS-D3 / MNCS-L4`, provided each claim is independently supported. + +### Separation of authority + +MNCDS distinguishes: + +- contract authority; +- generator authority; +- evaluator authority; +- selection authority; +- release authority; +- independent reviewer. + +One person or system MAY hold multiple roles in small experiments, but overlap MUST be disclosed. For D3 and above, the final evaluator MUST use authority and executable identities distinct from the generator and MUST be bound to the independent-reviewer role. + +Identity separation demonstrates a declared control boundary. It does not prove honesty, competence, or absence of collusion. + +### Recursive improvement + +MNCDS explicitly permits generated evidence to improve a generator, harness, analyzer, or search strategy, including using alternative implementations to improve a Joern-based harness. + +Recursive improvement MUST preserve epoch boundaries: + +- evidence from epoch `n` MAY inform tools and search policy in epoch `n+1`; +- the updated toolchain MUST receive a new identity and version; +- protected holdout evidence from epoch `n` MUST NOT silently become development evidence for the same acceptance claim; +- prior claims remain bound to their old toolchain and evidence identities; +- a materially changed harness MUST be revalidated against its regression or conformance corpus; +- unresolved disagreement cases MUST remain UNKNOWN rather than being forced into agreement. + +## Experimental schema and validator + +The first implementation uses one aggregate, additive schema: + +- `schemas/mncds-development-record.schema.json` + +The installed validator exposes the same schema through: + +```text +mncs schema mncds-development-record +``` + +The separate process validator is: + +```text +mncds validate DEVELOPMENT_RECORD +``` + +The validator performs offline schema and cross-record semantic checks. It MUST NOT execute or import generators, candidates, analyzers, evaluators, benchmarks, or evidence binaries during ordinary validation. + +The initial validator checks: + +- required role presence and uniqueness; +- forbidden generator permissions; +- partition overlap and holdout contamination; +- candidate identity uniqueness, parent existence, and lineage cycles; +- selected-candidate existence and disposition; +- required FAIL and UNKNOWN handling; +- D2 reproducibility and evaluator-corpus requirements; +- D3 protected holdout, predeclared selection, authority separation, executable separation, reviewer binding, and independent evidence; +- MNCS candidate, contract, and environment binding; +- D4 rollback and regeneration-drill results. + +The aggregate schema is the minimum interoperable unit for 0.1. Future implementations MAY split it into content-addressed records if they expose an equivalent offline-resolvable aggregate view. + +A later RFC may define MNCDS attestation predicates after at least one independent implementation exists. + +## Security, privacy, and vendor-neutrality impact + +MNCDS reduces risks from evaluator tampering, hidden objective drift, benchmark contamination, selective reporting, unreproducible generation, and unsafe deployment. + +It does not eliminate risks from compromised tools, dishonest operators, weak contracts, inadequate threat models, leaked holdouts, colluding evaluators, or undeclared external access. + +Sensitive prompts, proprietary datasets, model weights, and private source code MAY remain undisclosed, but a conformance claim MUST still expose stable identities, role bindings, declared access boundaries, evaluation methods, and sufficient evidence for the claimed profile. Redaction MUST NOT be treated as verification merely because disclosure is restricted. + +Privacy-preserving proofs, confidential-computing attestations, commitments, and transparency logs MAY be used as namespaced experimental extensions. They do not replace required evidence until a future RFC defines predicate, trust, failure, expiration, revocation, and interoperability semantics. + +No model, agent framework, analyzer, compiler, graph system, proof assistant, or orchestration platform is normative. Joern remains one optional structural-analysis provider. + +## Compatibility and migration + +MNCDS is additive. Existing MNCS 0.1 and 0.2 bundles remain valid under their original rules and do not acquire an MNCDS claim retroactively. + +Projects adopting MNCDS MAY wrap an existing development history by identifying the oldest reliably reproducible baseline and marking earlier lineage as UNKNOWN or unavailable. They MUST NOT fabricate missing records or claim protected-holdout separation when none existed. + +MNCDS remains separately versioned throughout the MNCS 0.x series. Any future merger into MNCS requires a major-version RFC and explicit migration rules. + +## Alternatives + +### Put all development rules directly into MNCS + +Rejected for this draft because acceptance evidence and development-process governance are related but separable concerns. Combining them would make the core standard harder to adopt and could invalidate otherwise sound historical bundles. + +### Publish only non-normative guidance + +Rejected because benchmark contamination, objective drift, and unverifiable candidate selection directly affect the credibility of conformance claims. + +### Require complete disclosure of prompts, models, datasets, and source + +Rejected because it would prevent legitimate proprietary or sensitive use. Stable identities, bounded disclosure, independent evidence, and explicit UNKNOWN are more tool-neutral. + +### Require a human to approve every generated change + +Rejected because the purpose of MNCS is to permit machine-owned internal complexity under stronger external controls. Human line-by-line review is neither sufficient nor always feasible. + +### Forbid recursive improvement + +Rejected because evidence-driven improvement of generators and verification harnesses is a central benefit. The standard instead requires versioned epochs, new identities, regression testing, and holdout discipline. + +## Test and evidence plan + +The repository now contains an executable evidence baseline: + +1. **D1 multiple-candidate ledger:** implemented through the D4 reference record reduced to D1 in unit tests. +2. **Reject evaluator or threshold mutation:** implemented in unit tests and deterministic corpus cases. +3. **Reject UNKNOWN promotion:** implemented in unit tests and corpus. +4. **D2 reproducible generation and repeated measurement:** implemented with seeded reproducibility and repeated-measurement checks. +5. **D3 protected holdout and independent evaluator:** implemented in the cumulative reference record and tests. +6. **Recursive harness improvement:** completed by the frozen two-epoch analyzer study + with retained disagreements, fresh developer-withheld final inputs, resource + measurements, and an explicit non-promotion boundary. +7. **D4 rollback, regeneration, monitoring, and retirement:** implemented in the reference record and rejection tests. +8. **Independent validator agreement:** the independent Rust consumer reads the + combined release-candidate corpus directly and agrees with Python on every vector. + +Relevant artifacts include: + +- `examples/mncds-d4/development-record.json`; +- `tests/test_mncds.py`; +- `mncds-conformance-corpus/corpus.json`; +- `scripts/run-mncds-corpus`; +- `docs/mncds-evidence-plan.md`. + +The corpus includes forbidden generator authority, lineage cycles, UNKNOWN promotion, holdout contamination, post-hoc selection, evaluator conflicts, mismatched MNCS binding, untested rollback, and failed regeneration drills. + +Before acceptance, an independent consumer MUST process the corpus without importing the Python validator and publish normalized agreement, disagreement, and unsupported-rule outcomes. + +## Resolved design questions + +The initial 0.1-draft implementation resolves the prior questions as follows: + +- MNCDS remains a permanent companion throughout MNCS 0.x; merger may be reconsidered only at a future major version. +- D1 uses one aggregate record containing charter, roles, generator boundary, partitions, evaluators, candidate ledger, selection, and reproducibility declaration. +- Stochastic generation uses `EXACT`, `SEEDED`, `STATISTICAL`, `DISTRIBUTIONAL`, or `NONE`; D2 and above reject `NONE`. +- Baseline D3 independence requires different generator/evaluator authority and executable identities, immutable evaluator configuration, protected evidence, reviewer binding, and selected-candidate results. +- Every materially evaluated candidate is retained individually; pre-material candidates may be summarized only under a predeclared auditable aggregation rule. +- Privacy proofs and transparency logs remain optional namespaced extensions until an interoperability RFC exists. +- MNCS and MNCDS validation remain separate results and commands; a future summary may display both without collapsing them. + +Detailed semantics are in `spec/MNCDS-v0.1-records-and-decisions.md`. + +## Deferred non-core research questions + +- What optional aggregate commitments best preserve million-candidate search history + while protecting proprietary candidate bodies? +- Which privacy-preserving proof systems are practical for restricted prompts, + datasets, and model configurations? +- What externally evidenced threshold should define a future higher-assurance + organizational-independence profile? + +## Acceptance gate + +RFC 0004 MUST remain Draft until: + +- CI passes for schema, validator, tests, example, corpus, package, and documentation; +- an independent corpus consumer publishes normalized agreement; +- a reproducible two-epoch recursive harness study is complete; +- security and privacy review identifies no unresolved claim-broadening issue; +- the independent approvals required by governance are recorded. + +## Release-candidate decision record + +The repository's 0.1-rc.1 implementation resolves the internally decidable questions +in `spec/MNCS-v0.3-MNCDS-v0.1-decisions.md` and provides a self-contained proposed +specification in `spec/MNCDS-v0.1-rc.1.md`. The aggregate record remains the portable +interoperability unit. Draft records remain valid as draft artifacts and are not +upgraded by changing a version string. + +This implementation work does not change this RFC's Draft status. Implementation +agreement, a recursive study, internal review, and local readiness cannot supply the +required non-conflicted approvals or organizational independence. diff --git a/schemas/README.md b/schemas/README.md index 9ed472a..dff76bc 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -1,7 +1,9 @@ # Schemas -This directory contains versioned machine-readable schemas owned by MNCDS. +This directory is the canonical home of MNCDS development-record schemas. -Schema identifiers and released schema behavior are compatibility surfaces. Migration MUST preserve existing IDs, required fields, validation behavior, and historical versions unless a reviewed normative change explicitly supersedes them. +- `mncds-development-record.schema.json` +- `mncds-development-record-0.1.schema.json` -MNCS-owned schemas remain in the MNCS repository. If an MNCDS record binds to an MNCS artifact, validate the MNCDS side of that binding here and consume the MNCS artifact through its declared version/interface rather than copying the MNCS schema into this directory. +MNCS may keep consumed copies for its local consumer. Those copies are not +authoritative. diff --git a/schemas/mncds-development-record-0.1.schema.json b/schemas/mncds-development-record-0.1.schema.json new file mode 100644 index 0000000..26d7d0e --- /dev/null +++ b/schemas/mncds-development-record-0.1.schema.json @@ -0,0 +1,386 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mncs.dev/schema/mncds/0.1-rc.1/mncds-development-record.schema.json", + "title": "MNCDS 0.1-rc.1 development record", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", "mncds_version", "record_id", "profile", "epoch_id", + "created_at", "supersedes_record_id", "charter", "baseline", "environment_lock", + "roles", "authority_overlaps", "generator", "partitions", "protected_evidence", + "evaluators", "candidates", "candidate_aggregates", "selection", + "reproducibility", "epochs", "mncs_binding", "release_controls", "extensions" + ], + "properties": { + "schema_version": {"const": "0.1-rc.1"}, + "mncds_version": {"const": "0.1-rc.1"}, + "record_id": {"$ref": "#/$defs/id"}, + "profile": {"enum": ["MNCDS-D1", "MNCDS-D2", "MNCDS-D3", "MNCDS-D4"]}, + "epoch_id": {"$ref": "#/$defs/id"}, + "created_at": {"type": "string", "format": "date-time"}, + "supersedes_record_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "charter": {"$ref": "#/$defs/charter"}, + "baseline": {"$ref": "#/$defs/baseline"}, + "environment_lock": {"$ref": "#/$defs/environment"}, + "roles": {"type": "array", "minItems": 6, "items": {"$ref": "#/$defs/role"}}, + "authority_overlaps": {"type": "array", "items": {"$ref": "#/$defs/overlap"}}, + "generator": {"$ref": "#/$defs/generator"}, + "partitions": {"$ref": "#/$defs/partitions"}, + "protected_evidence": {"type": "array", "items": {"$ref": "#/$defs/protectedEvidence"}}, + "evaluators": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/evaluator"}}, + "candidates": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/candidate"}}, + "candidate_aggregates": {"type": "array", "items": {"$ref": "#/$defs/candidateAggregate"}}, + "selection": {"$ref": "#/$defs/selection"}, + "reproducibility": {"$ref": "#/$defs/reproducibility"}, + "epochs": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/epoch"}}, + "mncs_binding": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/mncsBinding"}]}, + "release_controls": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/releaseControls"}]}, + "extensions": {"$ref": "#/$defs/extensions"} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "status": {"enum": ["PASS", "FAIL", "UNKNOWN"]}, + "text": {"type": "string", "minLength": 1}, + "texts": {"type": "array", "items": {"$ref": "#/$defs/text"}}, + "ids": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "charter": { + "type": "object", "additionalProperties": false, + "required": [ + "charter_id", "problem_statement", "intended_use", "exclusions", "contract_id", + "baseline_id", "environment_id", "threat_model_id", "objective", + "selection_policy_id", "planned_mncs_level", "hard_rejection_gates", + "release_owner_id", "rollback_owner_id", "retirement_owner_id" + ], + "properties": { + "charter_id": {"$ref": "#/$defs/id"}, + "problem_statement": {"$ref": "#/$defs/text"}, + "intended_use": {"$ref": "#/$defs/text"}, + "exclusions": {"$ref": "#/$defs/texts"}, + "contract_id": {"$ref": "#/$defs/id"}, + "baseline_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "threat_model_id": {"$ref": "#/$defs/id"}, + "objective": { + "type": "object", "additionalProperties": false, + "required": ["objective_id", "metric", "unit", "direction", "minimum_useful_benefit", "operational_rationale"], + "properties": { + "objective_id": {"$ref": "#/$defs/id"}, + "metric": {"$ref": "#/$defs/text"}, + "unit": {"$ref": "#/$defs/text"}, + "direction": {"enum": ["minimize", "maximize"]}, + "minimum_useful_benefit": {"type": "number"}, + "operational_rationale": {"$ref": "#/$defs/text"} + } + }, + "selection_policy_id": {"$ref": "#/$defs/id"}, + "planned_mncs_level": {"enum": [null, "MNCS-L1", "MNCS-L2", "MNCS-L3", "MNCS-L4", "MNCS-L5"]}, + "hard_rejection_gates": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "release_owner_id": {"$ref": "#/$defs/id"}, + "rollback_owner_id": {"$ref": "#/$defs/id"}, + "retirement_owner_id": {"$ref": "#/$defs/id"} + } + }, + "baseline": { + "type": "object", "additionalProperties": false, + "required": ["baseline_id", "artifact_id", "source_id", "build_id", "dependency_ids", "environment_id", "evaluator_ids", "results", "captured_at", "immutable"], + "properties": { + "baseline_id": {"$ref": "#/$defs/id"}, + "artifact_id": {"$ref": "#/$defs/id"}, + "source_id": {"$ref": "#/$defs/id"}, + "build_id": {"$ref": "#/$defs/id"}, + "dependency_ids": {"$ref": "#/$defs/ids"}, + "environment_id": {"$ref": "#/$defs/id"}, + "evaluator_ids": {"$ref": "#/$defs/ids"}, + "results": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/result"}}, + "captured_at": {"type": "string", "format": "date-time"}, + "immutable": {"const": true} + } + }, + "environment": { + "type": "object", "additionalProperties": false, + "required": ["environment_id", "toolchain_id", "dependency_ids", "hardware_id", "configuration_id", "permitted_variance", "locked"], + "properties": { + "environment_id": {"$ref": "#/$defs/id"}, + "toolchain_id": {"$ref": "#/$defs/id"}, + "dependency_ids": {"$ref": "#/$defs/ids"}, + "hardware_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "permitted_variance": {"$ref": "#/$defs/texts"}, + "locked": {"type": "boolean"} + } + }, + "role": { + "type": "object", "additionalProperties": false, + "required": ["role", "authority_id", "executable_id"], + "properties": { + "role": {"enum": ["contract_authority", "generator_authority", "evaluator_authority", "selection_authority", "release_authority", "independent_reviewer"]}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + }, + "overlap": { + "type": "object", "additionalProperties": false, + "required": ["authority_id", "roles", "scope", "rationale", "risk", "recusal_or_control"], + "properties": { + "authority_id": {"$ref": "#/$defs/id"}, + "roles": {"type": "array", "minItems": 2, "uniqueItems": true, "items": {"$ref": "#/$defs/text"}}, + "scope": {"$ref": "#/$defs/text"}, + "rationale": {"$ref": "#/$defs/text"}, + "risk": {"$ref": "#/$defs/text"}, + "recusal_or_control": {"$ref": "#/$defs/text"} + } + }, + "generator": { + "type": "object", "additionalProperties": false, + "required": ["generator_id", "configuration_id", "authority_id", "executable_id", "permissions", "resource_limits"], + "properties": { + "generator_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "permissions": { + "type": "object", "additionalProperties": false, + "required": ["modify_contract", "modify_baseline", "modify_evaluators", "modify_selection_policy", "modify_thresholds", "access_protected_holdout", "network_access", "filesystem_scope", "process_scope", "tool_ids", "mutation_scope"], + "properties": { + "modify_contract": {"type": "boolean"}, + "modify_baseline": {"type": "boolean"}, + "modify_evaluators": {"type": "boolean"}, + "modify_selection_policy": {"type": "boolean"}, + "modify_thresholds": {"type": "boolean"}, + "access_protected_holdout": {"type": "boolean"}, + "network_access": {"type": "boolean"}, + "filesystem_scope": {"$ref": "#/$defs/texts"}, + "process_scope": {"$ref": "#/$defs/texts"}, + "tool_ids": {"$ref": "#/$defs/ids"}, + "mutation_scope": {"$ref": "#/$defs/texts"} + } + }, + "resource_limits": { + "type": "object", "additionalProperties": false, + "required": ["max_candidates", "max_wall_seconds", "max_memory_bytes", "max_processes"], + "properties": { + "max_candidates": {"type": "integer", "minimum": 1}, + "max_wall_seconds": {"type": "number", "exclusiveMinimum": 0}, + "max_memory_bytes": {"type": "integer", "minimum": 1}, + "max_processes": {"type": "integer", "minimum": 1} + } + } + } + }, + "partitions": { + "type": "object", "additionalProperties": false, + "required": ["development_id", "selection_id", "final_evaluation_id", "holdout_contaminated", "access_policy_ids"], + "properties": { + "development_id": {"$ref": "#/$defs/id"}, + "selection_id": {"$ref": "#/$defs/id"}, + "final_evaluation_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "holdout_contaminated": {"type": "boolean"}, + "access_policy_ids": {"$ref": "#/$defs/ids"} + } + }, + "protectedEvidence": { + "type": "object", "additionalProperties": false, + "required": ["evidence_id", "partition_id", "commitment_id", "custodian_id", "custody_class", "disclosed_at", "generator_access", "reuse_claim_ids", "contaminated", "status"], + "properties": { + "evidence_id": {"$ref": "#/$defs/id"}, + "partition_id": {"$ref": "#/$defs/id"}, + "commitment_id": {"$ref": "#/$defs/id"}, + "custodian_id": {"$ref": "#/$defs/id"}, + "custody_class": {"enum": ["developer_withheld", "independent_operator", "organizationally_independent"]}, + "disclosed_at": {"oneOf": [{"type": "null"}, {"type": "string", "format": "date-time"}]}, + "generator_access": {"type": "boolean"}, + "reuse_claim_ids": {"$ref": "#/$defs/ids"}, + "contaminated": {"type": "boolean"}, + "status": {"$ref": "#/$defs/status"} + } + }, + "evaluator": { + "type": "object", "additionalProperties": false, + "required": ["evaluator_id", "purpose", "authority_id", "executable_id", "configuration_id", "environment_id", "independent", "operator_independence", "organizational_independence", "regression_corpus_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "purpose": {"enum": ["development", "selection", "holdout", "independent"]}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "independent": {"type": "boolean"}, + "operator_independence": {"$ref": "#/$defs/status"}, + "organizational_independence": {"$ref": "#/$defs/status"}, + "regression_corpus_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + }, + "result": { + "type": "object", "additionalProperties": false, + "required": ["evaluator_id", "gate_id", "partition_id", "required", "status", "evidence_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "gate_id": {"$ref": "#/$defs/id"}, + "partition_id": {"$ref": "#/$defs/id"}, + "required": {"type": "boolean"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + }, + "candidate": { + "type": "object", "additionalProperties": false, + "required": ["candidate_id", "parent_ids", "epoch_id", "generator_id", "generation_sequence", "materially_evaluated", "retained", "build_status", "disposition", "objective_value", "evaluator_results"], + "properties": { + "candidate_id": {"$ref": "#/$defs/id"}, + "parent_ids": {"$ref": "#/$defs/ids"}, + "epoch_id": {"$ref": "#/$defs/id"}, + "generator_id": {"$ref": "#/$defs/id"}, + "generation_sequence": {"type": "integer", "minimum": 0}, + "materially_evaluated": {"type": "boolean"}, + "retained": {"type": "boolean"}, + "build_status": {"$ref": "#/$defs/status"}, + "disposition": {"enum": ["rejected", "retained", "promoted", "selected"]}, + "objective_value": {"type": ["number", "null"]}, + "evaluator_results": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/result"}} + } + }, + "candidateAggregate": { + "type": "object", "additionalProperties": false, + "required": ["aggregate_id", "predeclared_rule_id", "count", "reason_class", "generator_id", "configuration_id", "sequence_start", "sequence_end", "digest"], + "properties": { + "aggregate_id": {"$ref": "#/$defs/id"}, + "predeclared_rule_id": {"$ref": "#/$defs/id"}, + "count": {"type": "integer", "minimum": 1}, + "reason_class": {"$ref": "#/$defs/text"}, + "generator_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "sequence_start": {"type": "integer", "minimum": 0}, + "sequence_end": {"type": "integer", "minimum": 0}, + "digest": {"$ref": "#/$defs/hash"} + } + }, + "selection": { + "type": "object", "additionalProperties": false, + "required": ["policy_id", "selection_epoch_id", "selected_candidate_id", "rule_recorded_before_final_evaluation", "unknown_policy", "minimum_useful_benefit_met", "hard_gates_passed", "rationale", "human_review"], + "properties": { + "policy_id": {"$ref": "#/$defs/id"}, + "selection_epoch_id": {"$ref": "#/$defs/id"}, + "selected_candidate_id": {"$ref": "#/$defs/id"}, + "rule_recorded_before_final_evaluation": {"type": "boolean"}, + "unknown_policy": {"enum": ["reject", "human_review"]}, + "minimum_useful_benefit_met": {"type": "boolean"}, + "hard_gates_passed": {"type": "boolean"}, + "rationale": {"$ref": "#/$defs/text"}, + "human_review": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", "additionalProperties": false, + "required": ["reviewer_id", "decision", "rationale"], + "properties": { + "reviewer_id": {"$ref": "#/$defs/id"}, + "decision": {"enum": ["accept_with_unknown", "reject"]}, + "rationale": {"$ref": "#/$defs/text"} + } + } + ] + } + } + }, + "reproducibility": { + "type": "object", "additionalProperties": false, + "required": ["class", "seeds_preserved", "protocol", "measurement_repetitions", "comparison_statistic", "acceptance_bounds", "failure_treatment"], + "properties": { + "class": {"enum": ["EXACT", "SEEDED", "STATISTICAL", "DISTRIBUTIONAL", "NONE"]}, + "seeds_preserved": {"type": "boolean"}, + "protocol": {"$ref": "#/$defs/text"}, + "measurement_repetitions": {"type": "integer", "minimum": 1}, + "comparison_statistic": {"$ref": "#/$defs/text"}, + "acceptance_bounds": {"$ref": "#/$defs/text"}, + "failure_treatment": {"$ref": "#/$defs/text"} + } + }, + "epoch": { + "type": "object", "additionalProperties": false, + "required": ["epoch_id", "parent_epoch_id", "toolchain_id", "corpus_id", "objective_id", "contract_id", "threshold_policy_id", "development_partition_id", "final_partition_id", "change_evidence_ids", "regression_fixture_ids"], + "properties": { + "epoch_id": {"$ref": "#/$defs/id"}, + "parent_epoch_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "toolchain_id": {"$ref": "#/$defs/id"}, + "corpus_id": {"$ref": "#/$defs/id"}, + "objective_id": {"$ref": "#/$defs/id"}, + "contract_id": {"$ref": "#/$defs/id"}, + "threshold_policy_id": {"$ref": "#/$defs/id"}, + "development_partition_id": {"$ref": "#/$defs/id"}, + "final_partition_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "change_evidence_ids": {"$ref": "#/$defs/ids"}, + "regression_fixture_ids": {"$ref": "#/$defs/ids"} + } + }, + "mncsBinding": { + "type": "object", "additionalProperties": false, + "required": ["result_or_package_id", "mncs_version", "candidate_id", "contract_id", "environment_id", "status"], + "properties": { + "result_or_package_id": {"$ref": "#/$defs/id"}, + "mncs_version": {"enum": ["0.1", "0.1.1", "0.2", "0.3-rc.1"]}, + "candidate_id": {"$ref": "#/$defs/id"}, + "contract_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "status": {"$ref": "#/$defs/status"} + } + }, + "releaseControls": { + "type": "object", "additionalProperties": false, + "required": ["release_artifact_id", "release_environment_id", "release_authority_id", "build_procedure_id", "package_procedure_id", "monitoring", "rollback", "regeneration_or_replacement", "retirement"], + "properties": { + "release_artifact_id": {"$ref": "#/$defs/id"}, + "release_environment_id": {"$ref": "#/$defs/id"}, + "release_authority_id": {"$ref": "#/$defs/id"}, + "build_procedure_id": {"$ref": "#/$defs/id"}, + "package_procedure_id": {"$ref": "#/$defs/id"}, + "monitoring": { + "type": "object", "additionalProperties": false, + "required": ["signal_ids", "threshold_policy_id", "status"], + "properties": { + "signal_ids": {"$ref": "#/$defs/ids"}, + "threshold_policy_id": {"$ref": "#/$defs/id"}, + "status": {"$ref": "#/$defs/status"} + } + }, + "rollback": { + "type": "object", "additionalProperties": false, + "required": ["artifact_id", "procedure_id", "test_status", "evidence_id"], + "properties": { + "artifact_id": {"$ref": "#/$defs/id"}, + "procedure_id": {"$ref": "#/$defs/id"}, + "test_status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + }, + "regeneration_or_replacement": { + "type": "object", "additionalProperties": false, + "required": ["procedure_id", "performed_at", "status", "evidence_id", "replacement_candidate_id"], + "properties": { + "procedure_id": {"$ref": "#/$defs/id"}, + "performed_at": {"type": "string", "format": "date-time"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"}, + "replacement_candidate_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + }, + "retirement": { + "type": "object", "additionalProperties": false, + "required": ["trigger_ids", "retired", "retired_at", "reason", "replacement_candidate_id"], + "properties": { + "trigger_ids": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/id"}}, + "retired": {"type": "boolean"}, + "retired_at": {"oneOf": [{"type": "null"}, {"type": "string", "format": "date-time"}]}, + "reason": {"type": ["string", "null"], "minLength": 1}, + "replacement_candidate_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + } + } + }, + "extensions": { + "type": "object", + "propertyNames": {"pattern": "^[a-z][a-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._-]*$"}, + "additionalProperties": true + } + } +} diff --git a/schemas/mncds-development-record.schema.json b/schemas/mncds-development-record.schema.json new file mode 100644 index 0000000..fce00ad --- /dev/null +++ b/schemas/mncds-development-record.schema.json @@ -0,0 +1,344 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mncs.dev/schema/mncds/0.1/mncds-development-record.schema.json", + "title": "MNCDS 0.1 development record", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "mncds_version", + "record_id", + "profile", + "epoch_id", + "created_at", + "charter", + "roles", + "generator", + "partitions", + "evaluators", + "candidates", + "selection", + "reproducibility", + "extensions" + ], + "properties": { + "schema_version": {"const": "0.1"}, + "mncds_version": {"const": "0.1-draft"}, + "record_id": {"$ref": "#/$defs/id"}, + "profile": {"enum": ["MNCDS-D1", "MNCDS-D2", "MNCDS-D3", "MNCDS-D4"]}, + "epoch_id": {"$ref": "#/$defs/id"}, + "supersedes_record_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "created_at": {"type": "string", "format": "date-time"}, + "charter": {"$ref": "#/$defs/charter"}, + "roles": { + "type": "array", + "minItems": 6, + "items": {"$ref": "#/$defs/role"} + }, + "generator": {"$ref": "#/$defs/generator"}, + "partitions": {"$ref": "#/$defs/partitions"}, + "evaluators": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/evaluator"} + }, + "candidates": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/candidate"} + }, + "selection": {"$ref": "#/$defs/selection"}, + "reproducibility": {"$ref": "#/$defs/reproducibility"}, + "mncs_binding": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/mncsBinding"} + ] + }, + "release_controls": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/releaseControls"} + ] + }, + "extensions": {"$ref": "#/$defs/extensions"} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "status": {"enum": ["PASS", "FAIL", "UNKNOWN"]}, + "extensions": { + "type": "object", + "propertyNames": {"pattern": "^[a-z][a-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._-]*$"}, + "additionalProperties": true + }, + "charter": { + "type": "object", + "additionalProperties": false, + "required": [ + "problem_statement", + "intended_use", + "contract_id", + "baseline_id", + "environment_id", + "threat_model_id", + "objective", + "selection_policy_id", + "planned_mncs_level", + "hard_rejection_gates" + ], + "properties": { + "problem_statement": {"type": "string", "minLength": 1}, + "intended_use": {"type": "string", "minLength": 1}, + "contract_id": {"$ref": "#/$defs/id"}, + "baseline_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "threat_model_id": {"$ref": "#/$defs/id"}, + "objective": { + "type": "object", + "additionalProperties": false, + "required": ["objective_id", "metric", "unit", "direction", "minimum_useful_benefit"], + "properties": { + "objective_id": {"$ref": "#/$defs/id"}, + "metric": {"type": "string", "minLength": 1}, + "unit": {"type": "string", "minLength": 1}, + "direction": {"enum": ["minimize", "maximize"]}, + "minimum_useful_benefit": {"type": "number"} + } + }, + "selection_policy_id": {"$ref": "#/$defs/id"}, + "planned_mncs_level": { + "type": ["string", "null"], + "enum": [null, "MNCS-L1", "MNCS-L2", "MNCS-L3", "MNCS-L4", "MNCS-L5"] + }, + "hard_rejection_gates": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + } + } + }, + "role": { + "type": "object", + "additionalProperties": false, + "required": ["role", "authority_id", "executable_id"], + "properties": { + "role": { + "enum": [ + "contract_authority", + "generator_authority", + "evaluator_authority", + "selection_authority", + "release_authority", + "independent_reviewer" + ] + }, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"} + } + }, + "generator": { + "type": "object", + "additionalProperties": false, + "required": ["generator_id", "configuration_id", "authority_id", "executable_id", "permissions", "resource_limits"], + "properties": { + "generator_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "permissions": { + "type": "object", + "additionalProperties": false, + "required": [ + "modify_contract", + "modify_baseline", + "modify_evaluators", + "modify_selection_policy", + "modify_thresholds", + "access_protected_holdout", + "network_access" + ], + "properties": { + "modify_contract": {"type": "boolean"}, + "modify_baseline": {"type": "boolean"}, + "modify_evaluators": {"type": "boolean"}, + "modify_selection_policy": {"type": "boolean"}, + "modify_thresholds": {"type": "boolean"}, + "access_protected_holdout": {"type": "boolean"}, + "network_access": {"type": "boolean"} + } + }, + "resource_limits": { + "type": "object", + "additionalProperties": false, + "required": ["max_candidates", "max_wall_seconds"], + "properties": { + "max_candidates": {"type": "integer", "minimum": 1}, + "max_wall_seconds": {"type": "number", "exclusiveMinimum": 0} + } + } + } + }, + "partitions": { + "type": "object", + "additionalProperties": false, + "required": ["development_id", "selection_id", "holdout_id", "holdout_contaminated"], + "properties": { + "development_id": {"$ref": "#/$defs/id"}, + "selection_id": {"$ref": "#/$defs/id"}, + "holdout_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "holdout_contaminated": {"type": "boolean"} + } + }, + "evaluator": { + "type": "object", + "additionalProperties": false, + "required": ["evaluator_id", "purpose", "authority_id", "executable_id", "configuration_id", "independent", "regression_corpus_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "purpose": {"enum": ["development", "selection", "holdout", "independent"]}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "independent": {"type": "boolean"}, + "regression_corpus_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"} + } + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_id", + "parent_ids", + "generator_id", + "build_status", + "disposition", + "objective_value", + "evaluator_results" + ], + "properties": { + "candidate_id": {"$ref": "#/$defs/id"}, + "parent_ids": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + }, + "generator_id": {"$ref": "#/$defs/id"}, + "build_status": {"$ref": "#/$defs/status"}, + "disposition": {"enum": ["rejected", "retained", "promoted", "selected"]}, + "objective_value": {"type": ["number", "null"]}, + "evaluator_results": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["evaluator_id", "gate_id", "required", "status", "evidence_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "gate_id": {"$ref": "#/$defs/id"}, + "required": {"type": "boolean"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + } + } + } + }, + "selection": { + "type": "object", + "additionalProperties": false, + "required": [ + "policy_id", + "selected_candidate_id", + "rule_recorded_before_holdout", + "unknown_policy", + "minimum_useful_benefit_met", + "rationale", + "human_review" + ], + "properties": { + "policy_id": {"$ref": "#/$defs/id"}, + "selected_candidate_id": {"$ref": "#/$defs/id"}, + "rule_recorded_before_holdout": {"type": "boolean"}, + "unknown_policy": {"enum": ["reject", "human_review"]}, + "minimum_useful_benefit_met": {"type": "boolean"}, + "rationale": {"type": "string", "minLength": 1}, + "human_review": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["reviewer_id", "decision", "rationale"], + "properties": { + "reviewer_id": {"$ref": "#/$defs/id"}, + "decision": {"enum": ["accept_with_unknown", "reject"]}, + "rationale": {"type": "string", "minLength": 1} + } + } + ] + } + } + }, + "reproducibility": { + "type": "object", + "additionalProperties": false, + "required": ["class", "seeds_preserved", "protocol", "measurement_repetitions"], + "properties": { + "class": {"enum": ["EXACT", "SEEDED", "STATISTICAL", "DISTRIBUTIONAL", "NONE"]}, + "seeds_preserved": {"type": "boolean"}, + "protocol": {"type": "string", "minLength": 1}, + "measurement_repetitions": {"type": "integer", "minimum": 1} + } + }, + "mncsBinding": { + "type": "object", + "additionalProperties": false, + "required": ["manifest_or_package_id", "candidate_id", "contract_id", "environment_id"], + "properties": { + "manifest_or_package_id": {"$ref": "#/$defs/id"}, + "candidate_id": {"$ref": "#/$defs/id"}, + "contract_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"} + } + }, + "releaseControls": { + "type": "object", + "additionalProperties": false, + "required": [ + "release_artifact_id", + "rollback_artifact_id", + "rollback_test_status", + "monitoring_thresholds", + "regeneration_drill", + "retirement_triggers" + ], + "properties": { + "release_artifact_id": {"$ref": "#/$defs/id"}, + "rollback_artifact_id": {"$ref": "#/$defs/id"}, + "rollback_test_status": {"$ref": "#/$defs/status"}, + "monitoring_thresholds": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + }, + "regeneration_drill": { + "type": "object", + "additionalProperties": false, + "required": ["performed_at", "status", "evidence_id"], + "properties": { + "performed_at": {"type": "string", "format": "date-time"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + }, + "retirement_triggers": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + } + } + } + } +} diff --git a/spec/MNCDS-v0.1-draft.md b/spec/MNCDS-v0.1-draft.md new file mode 100644 index 0000000..7ddd902 --- /dev/null +++ b/spec/MNCDS-v0.1-draft.md @@ -0,0 +1,340 @@ + + +# Machine-Native Complexity Development Specification 0.1 (Draft) + +MNCDS 0.1 is a proposed, experimental, tool-neutral companion to the Machine-Native Complexity Standard. Normative terms use the meanings defined in `normative-language.md`. + +MNCDS governs the process used to create, evaluate, select, release, regenerate, replace, and retire machine-native implementations. MNCS governs the evidence required to accept such an implementation. An MNCDS claim MUST NOT be represented as an MNCS conformance claim, and an MNCS claim MUST NOT imply compliance with MNCDS unless both are separately established. + +> Machine-owned implementation complexity requires human- and machine-auditable development control. + +## 1. Scope + +MNCDS applies when a development process intentionally permits generated or machine-optimized implementation structure to exceed ordinary human-maintainability limits in exchange for a declared measurable benefit. + +MNCDS standardizes lifecycle records, authority boundaries, evaluation discipline, selection controls, provenance, reproducibility, rollback, and regeneration. It does not prescribe a programming language, model, optimizer, analyzer, benchmark framework, compiler, proof system, or orchestration platform. + +A process MAY use code-generating models, evolutionary search, superoptimization, synthesis, automated repair, graph transformation, reinforcement learning, compiler optimization, program induction, or combinations of these methods. + +## 2. Required identities and records + +Every conforming process MUST assign stable content or canonical identities to: + +- the readable contract; +- the reference behavior or baseline implementation; +- the declared environment; +- the threat and misuse model; +- the objective function and constraints; +- the generator and its effective configuration; +- each materially evaluated candidate; +- each evaluator, analyzer, benchmark, and measurement method; +- the development, validation, and holdout partitions; +- the selection policy; +- the selected candidate; +- the resulting MNCS bundle or package, when an MNCS claim is made. + +The process MUST preserve an append-only development record sufficient to reconstruct which identities and policies were in force at each decision point. Correcting a record MUST create a new record that references the superseded record; it MUST NOT silently rewrite history. + +## 3. Roles and authority boundaries + +A development record MUST identify these logical roles: + +- **Contract authority:** approves the readable contract, limits, and acceptance intent. +- **Generator authority:** operates or configures candidate generation. +- **Evaluator authority:** controls development-time tests, analyzers, and measurements. +- **Selection authority:** applies the candidate-selection rule. +- **Release authority:** authorizes deployment or publication. +- **Independent reviewer:** evaluates the selected candidate using protected evidence for profiles that require independence. + +One person, organization, or system MAY hold multiple roles unless a claimed profile prohibits it. All overlaps MUST be disclosed. + +The generator MUST NOT have undeclared authority to modify the contract, reference behavior, evaluator, acceptance policy, safety limits, holdout evidence, or release criteria. Any authorized change to those items MUST begin a new development epoch and receive a new identity. + +## 4. Development charter + +Before materially evaluating candidates, the process MUST create a development charter containing: + +- problem statement and intended use; +- readable contract and exclusions; +- declared environment and resource limits; +- threat model and prohibited behavior; +- baseline implementation or reference behavior; +- useful-benefit objective; +- mandatory constraints and hard rejection gates; +- evaluation methods and data partitions; +- candidate-selection policy; +- planned MNCDS profile and MNCS level, if any; +- rollback, regeneration, and retirement ownership. + +The useful-benefit objective MUST be measurable. Complexity, novelty, model preference, or code size alone MUST NOT count as a useful benefit unless the charter explains why the metric is operationally valuable. + +The process MUST record any later charter change, its rationale, affected evidence, and whether previous measurements remain valid. A material change MUST begin a new epoch. + +## 5. Baseline and environment lock + +Before candidate search begins, the process MUST preserve an immutable baseline and its evaluation results under the declared environment. + +The baseline record MUST include: + +- source or artifact identity; +- build and dependency identities; +- environment identity; +- evaluator identities; +- functional, safety, resource, and performance results; +- known failures and UNKNOWN outcomes. + +A candidate MUST NOT be credited with improvement against a baseline measured under materially different conditions unless the difference is declared and justified. The comparison method MUST prevent a weaker, stale, or intentionally degraded baseline from creating a false benefit. + +## 6. Evaluation partitioning + +The process MUST distinguish at least: + +- **development evidence**, which MAY guide generation and tuning; +- **selection evidence**, which MAY rank candidates under the predeclared policy; +- **protected holdout evidence**, which MUST remain inaccessible to the generator and ordinary ranking process when required by the claimed profile. + +Partition identities and access rules MUST be recorded before use. Evidence that has influenced generation or selection MUST NOT later be described as protected holdout evidence for the same claim. + +A leaked, inspected, inferred, or repeatedly queried holdout MUST be marked contaminated. The process MUST replace it or downgrade the claim. + +## 7. Candidate-generation envelope + +The development charter MUST declare the generator's permitted inputs, outputs, tools, network access, filesystem access, executable authority, and mutation scope. + +The generator MUST operate within explicit resource and time bounds. Unbounded search MUST NOT be represented as controlled development. + +Each materially evaluated candidate MUST have: + +- a stable identity; +- parent or source lineage; +- generator and configuration identity; +- generation time or sequence position; +- declared transformations; +- build status; +- evaluation status; +- disposition. + +A stochastic generator SHOULD record seeds and sampling parameters. When exact regeneration is not possible, the process MUST state the reproducibility level and preserve enough information to reproduce the distribution, search policy, or candidate family rather than claiming bitwise regeneration. + +## 8. Experiment ledger + +The process MUST maintain an append-only experiment ledger. + +For each materially evaluated candidate, the ledger MUST record: + +- candidate identity and lineage; +- active charter and epoch identity; +- evaluator and environment identities; +- objective values and constraint outcomes; +- PASS, FAIL, and UNKNOWN results; +- measurement uncertainty where applicable; +- rejection, retention, promotion, or selection reason; +- operator or automated decision identity. + +The process MAY summarize candidates rejected before material evaluation, but it MUST define the summarization rule. It MUST NOT omit evaluated candidates merely because they weaken the apparent success rate or reveal undesirable selection pressure. + +Evaluator crashes, unsupported syntax, timeouts, missing data, and inconclusive analysis MUST be recorded as UNKNOWN or operational error. They MUST NOT be converted to PASS by omission or by the absence of a detected defect. + +## 9. Progressive evaluation + +Candidate evaluation SHOULD proceed through increasingly expensive gates, such as: + +1. build and format checks; +2. contract and regression tests; +3. malformed-input and safety tests; +4. structural or semantic analysis; +5. fuzz, property, symbolic, or model-based testing; +6. resource and performance measurement; +7. adversarial and holdout evaluation; +8. independent verification. + +A later gate MUST NOT erase an earlier FAIL. A candidate MAY be repaired and re-enter as a new identity. + +Performance claims MUST use a declared measurement protocol with sufficient repetitions, warmup policy, environment controls, summary statistics, and uncertainty treatment. The selection process MUST NOT choose a candidate from noise and then report only its best measurement. + +## 10. Candidate selection + +The candidate-selection rule MUST be recorded before final holdout evaluation. + +The rule MUST state: + +- hard rejection gates; +- objective metrics; +- metric direction and weighting; +- tie-breaking behavior; +- treatment of UNKNOWN; +- minimum useful-benefit threshold; +- maximum accepted regressions; +- whether human judgment is permitted and how it is recorded. + +Selection MUST apply the declared rule to the recorded candidate set. A post hoc change to the rule MUST begin a new selection epoch and MUST be disclosed. + +The selected candidate MUST satisfy all mandatory constraints. A superior aggregate score MUST NOT compensate for failure of a hard safety, correctness, legal, privacy, or resource limit. + +The process MUST preserve enough rejected-candidate information to audit why the selected candidate won and whether the search exploited evaluator weaknesses. + +## 11. Independent verification + +When the claimed profile requires independence, the selected candidate MUST be evaluated by an independent reviewer using fresh or previously inaccessible evidence. + +The independent evaluator MUST differ from the generator and ordinary ranking process by executable identity, immutable configuration, key identity, or organizational control sufficient to prevent the generator from silently shaping the final result. + +Independent verification MUST include: + +- confirmation of candidate and contract identities; +- confirmation of environment and evaluator identities; +- protected holdout or fresh challenge evidence; +- explicit PASS, FAIL, and UNKNOWN outcomes; +- confirmation that the predeclared selection rule was followed; +- review of material role conflicts and deviations. + +Failure of independent verification MUST block the associated profile claim. Repair creates a new candidate identity and requires reevaluation. + +## 12. MNCS binding + +A project claiming MNCS conformance for a selected candidate MUST produce an MNCS manifest or package whose candidate, contract, environment, evidence, and policy identities agree with the final MNCDS records. + +MNCDS process evidence MAY be included in or referenced by an MNCS evidence graph, but the two conformance results MUST remain distinguishable. + +A truthful combined claim SHOULD use the form: + +`MNCDS-D / MNCS-L` + +A process record MUST NOT state or imply that disciplined development proves implementation correctness. An MNCS validator result MUST NOT state or imply that the candidate was developed under MNCDS controls unless those controls were separately validated. + +## 13. Recursive improvement and development epochs + +Evidence from one development epoch MAY be used to improve a generator, evaluator, analyzer, benchmark harness, or search strategy in a later epoch. + +Recursive improvement MUST satisfy all of these requirements: + +- the prior and updated toolchains receive distinct identities; +- the evidence used for improvement is identified; +- the improvement objective and observed failure modes are recorded; +- protected holdout evidence MUST NOT be reused as development evidence for the same acceptance claim; +- the updated harness or evaluator MUST be revalidated against its own regression or conformance corpus; +- historical claims remain bound to their original toolchain and evidence identities; +- candidate results from incompatible epochs MUST NOT be pooled without a declared normalization method. + +A process MAY use differences between competing implementations, analyzers, or experimental ideas to improve the original harness. Such recursive use SHOULD preserve disagreement cases as regression fixtures. Cases that cannot be resolved MUST remain UNKNOWN rather than being forced into agreement. + +## 14. Release and operational controls + +Before deployment or publication, the process MUST bind the selected candidate to: + +- release artifact identity; +- build and packaging procedure; +- supported environment; +- MNCS package or manifest identity, when applicable; +- known limitations and UNKNOWN outcomes; +- rollback artifact and procedure; +- monitoring signals and thresholds; +- responsible release authority. + +A release MUST NOT broaden the contract or supported environment beyond the evaluated scope. + +Operational monitoring MUST distinguish observed failure, suspected drift, unavailable evidence, and normal operation. Absence of a detected incident MUST NOT be treated as proof of continuing conformance. + +## 15. Regeneration, replacement, and retirement + +The process MUST preserve a regeneration specification containing: + +- generator and toolchain identities; +- required inputs and access boundaries; +- build and evaluation procedures; +- expected stochastic or deterministic reproducibility level; +- selection policy; +- required credentials or trust material by reference; +- fallback and rollback instructions. + +For profile D4, the project MUST perform and record a regeneration or replacement drill. The drill MAY use a non-production environment but MUST exercise the documented control path sufficiently to reveal missing dependencies, inaccessible evidence, or irreproducible steps. + +The process MUST declare retirement triggers, including contract change, environment drift, dependency obsolescence, evidence invalidation, security findings, repeated operational UNKNOWN, or inability to regenerate. + +Retirement MUST preserve the final artifact identity, reason, replacement identity when applicable, and affected claims. Historical records MUST remain immutable. + +## 16. Development conformance profiles + +Profiles are cumulative. + +### MNCDS-D1 — Controlled generation + +D1 requires: + +- development charter; +- immutable baseline; +- bounded generator authority; +- stable candidate identities and lineage; +- experiment ledger; +- explicit PASS, FAIL, and UNKNOWN; +- recorded candidate-selection rationale. + +### MNCDS-D2 — Reproducible experimentation + +D2 adds: + +- pinned development environment; +- identified evidence partitions; +- reproducible or statistically characterized generation; +- declared measurement protocol and uncertainty; +- versioned evaluators and harness regression corpus; +- preserved epoch boundaries. + +### MNCDS-D3 — Independent selection + +D3 adds: + +- predeclared selection rule; +- protected holdout or fresh challenge evidence; +- independent final evaluator; +- audited role separation and conflicts; +- verification that selection followed the declared rule; +- binding to an MNCS candidate package when an MNCS claim is made. + +### MNCDS-D4 — Operational regeneration + +D4 adds: + +- release identity and environment binding; +- monitoring thresholds; +- tested rollback; +- regeneration or replacement drill; +- explicit retirement triggers and records. + +A profile claim MUST identify the MNCDS version, development-record identity, selected-candidate identity, and evaluation time. Claiming D4 means satisfying D1 through D4. + +## 17. Deviations and UNKNOWN + +A SHOULD-level deviation MUST include a recorded rationale and risk assessment. + +Missing required records, inaccessible evidence, unsupported analysis, unresolved identity mismatch, contaminated holdout, or unverified role independence MUST produce FAIL or UNKNOWN according to the applicable rule. They MUST NOT silently satisfy a profile. + +A policy MAY reject UNKNOWN or require human review. It MUST NOT convert UNKNOWN to PASS. + +## 18. Privacy and restricted disclosure + +A project MAY protect proprietary prompts, datasets, source code, model weights, or credentials. Restricted material MUST still receive stable identities and access-boundary descriptions. + +Redaction MUST NOT broaden a claim. A reviewer unable to inspect required evidence MUST report UNKNOWN unless an accepted independent attestation or privacy-preserving proof establishes the requirement. + +## 19. Tool and vendor neutrality + +No model vendor, agent framework, analyzer, compiler, graph database, proof assistant, benchmark service, or orchestration platform is normative. + +Joern, compiler CFG analysis, LLVM passes, abstract interpretation, symbolic execution, fuzzing, model checking, proof assistants, custom analyzers, and independent combinations MAY serve as evaluators or providers. + +Unsupported operations MUST be reported as unsupported or UNKNOWN, never PASS. + +## 20. Claim limitations + +MNCDS conformance demonstrates that the declared development process followed the specified controls within its recorded scope. It does not prove that: + +- the contract is complete; +- the objective is socially desirable; +- the evidence is truthful; +- the implementation is free from defects; +- independent parties are honest; +- deployment outside the declared environment is safe; +- regeneration will remain possible indefinitely. + +MNCDS 0.1 is a draft proposal for experimentation and public review. It is not an accredited standard or blanket assurance claim. diff --git a/spec/MNCDS-v0.1-rc.1.md b/spec/MNCDS-v0.1-rc.1.md new file mode 100644 index 0000000..6220ef2 --- /dev/null +++ b/spec/MNCDS-v0.1-rc.1.md @@ -0,0 +1,173 @@ + + +# Machine-Native Complexity Development Specification 0.1-rc.1 + +Status: release-candidate proposal under Draft RFC 0004. It is not Accepted or Final. +Normative terms use RFC 2119/8174 meanings from `normative-language.md`. + +## 1. Scope and relationship to MNCS + +MNCDS governs development used to create, evaluate, select, release, monitor, +regenerate, replace, and retire a machine-native implementation. MNCS governs +implementation evidence. Results MUST remain separately versioned and MUST NOT collapse. + +MNCDS is tool-neutral. Models, generators, analyzers, compilers, providers, benchmarks, +languages, and case studies are non-normative. + +Results are `PASS`, `FAIL`, and `UNKNOWN`, with `FAIL > UNKNOWN > PASS`. Missing or +inaccessible evidence, unsupported analysis, crashes, and timeouts MUST NOT be `PASS`. + +## 2. Portable aggregate record + +The normative interoperability unit is one `mncds-development-record-0.1` aggregate, +resolvable offline. Optional content-addressed subrecords MUST expose an equivalent +aggregate containing record/epoch/contract/baseline/environment/threat/objective/tool/ +partition/selection/candidate/release/lifecycle identities, authorities and overlaps, +all materially evaluated candidates, auditable aggregates, evaluation and selection +results, reproducibility, MNCS binding, and applicable D4 controls. + +Corrections MUST create a new record and reference the superseded record. Historical +records MUST NOT be rewritten. + +## 3. Charter and contract binding + +Before material evaluation, the process MUST freeze a charter with problem, intended +use, exclusions, readable contract, environment/resource limits, threat model, +baseline, operationally meaningful benefit objective, hard gates, partitions, +selection policy, planned profiles/levels, and lifecycle owners. + +Complexity, novelty, model preference, source size, or a higher score alone MUST NOT be +the useful objective. A material charter, contract, policy, evaluator, threshold, +environment, or baseline change starts a new epoch and gets new identities. + +## 4. Baseline and environment lock + +The baseline MUST predate search and record source/artifact, build, dependency, +environment, evaluator, functional, safety, resource, performance, failure, and UNKNOWN +facts. The environment lock identifies platform, toolchain, dependencies, hardware, +configuration, and permitted variance. A stale, weakened, or materially different +baseline MUST NOT create false benefit. + +## 5. Roles and authority + +Records MUST bind contract, generator, evaluator, selection, release, and +independent-review authorities. Each overlap MUST disclose scope, rationale, risk, and +recusal or compensating control. D3/D4 require final-evaluator authority and executable +identities distinct from the generator and reviewer-role binding. + +The generator MUST NOT modify contract, baseline, evaluators, selection policy, +thresholds, protected evidence, or release criteria. Filesystem, process, network, tool, +mutation, time, and candidate-count permissions MUST be bounded. + +Local code can test implementation and executable identities. Independent operator and +organizational independence need external evidence and remain `UNKNOWN` when absent. + +## 6. Partitions and protected evidence + +Development, selection, and final-evaluation partitions MUST have distinct identities +and access rules. Evidence influencing generation/ranking cannot be protected for the +same claim. + +Protected evidence records commitment, custodian, access boundary, disclosure, reuse, +and contamination. Developer-controlled withholding MUST NOT be called external +custody or organizational independence. Contaminated required holdout is `FAIL`; +unverifiable custody is `UNKNOWN`. + +## 7. Generator, evaluators, and progressive evaluation + +Generator/evaluator authority, executable, configuration, environment, and corpus +identities remain stable within an epoch. Mutation starts a new epoch. D2 requires +evaluator regression corpora. + +Later gates MUST NOT erase earlier required `FAIL` or `UNKNOWN`. Repair creates a new +candidate identity. No evaluator may promote itself to selection or release authority. + +## 8. Candidate ledger and lineage + +Every materially evaluated candidate MUST be retained with identity, parents, epoch, +generator, sequence, build, objective, evaluator results, disposition, and retention. +Identities are unique; parents exist; lineage is acyclic. The selected candidate exists, +is retained, and has disposition `selected`. + +Pre-material candidates MAY use a predeclared aggregate recording count, reason/stage, +generator/configuration, time/sequence range, and digest/query. Material failures MUST +NOT be hidden in an aggregate. + +## 9. Reproducibility + +Exactly one class is declared: `EXACT`, `SEEDED`, `STATISTICAL`, `DISTRIBUTIONAL`, or +`NONE`. D1 MAY use `NONE`; D2-D4 MUST NOT. Exact/seeded claims retain randomness. +Statistical/distributional claims define repetitions, protocol, statistic, bounds, and +failure treatment. Failed regeneration is `FAIL`; unavailable regeneration `UNKNOWN`. + +## 10. Selection + +Before final evidence opens, selection declares hard gates, metrics/directions/weights, +ties, UNKNOWN policy, benefit threshold, maximum regression, and human judgment. +Post-hoc changes start a new selection epoch. + +Required `FAIL` fails selection. Under `reject`, required `UNKNOWN` fails. Under +`human_review`, acceptance preserves overall `UNKNOWN`; review cannot promote it. +The selected candidate meets useful benefit and all hard gates. + +## 11. Independent final evaluation and MNCS binding + +D3/D4 final evaluation requires authority/executable separation, immutable +configuration, reviewer binding, fresh/protected evidence, complete identities, and an +explicit result. Identity separation proves only a declared technical boundary. + +When MNCS is planned, candidate, contract, and environment bindings MUST agree. +Mismatch is `FAIL`. MNCS and MNCDS reports remain separate. + +## 12. Release and lifecycle + +D4 binds release artifact/environment, build/package, limitations, monitoring, +rollback, regeneration/replacement, release authority, and retirement triggers. +Rollback MUST be tested. `FAIL` fails D4; absent, stale, or `UNKNOWN` remains `UNKNOWN`. +Regeneration/replacement MUST be exercised and measured. + +Monitoring distinguishes failure, drift, unavailable evidence, and normal operation. +No incident is not proof. Replacement creates new artifact/candidate/claim/epoch +identities. Retirement records reason, time, affected claims, artifact, and replacement. + +## 13. Recursive improvement + +Epoch `n` evidence MAY improve epoch `n+1` tools. The later epoch MUST freeze prior +tool/corpus identities, retain failures/disagreements, identify feedback, predeclare an +operational objective, create new tool/config identities, use fresh inputs and separate +partitions, retain regressions, avoid silent contract/threshold change, and compare +correctness, UNKNOWN, crash, timeout, resources, determinism, and diagnostics. + +Protected final evidence MUST NOT enter same-claim repair feedback. Unresolved +disagreement remains `UNKNOWN`. Higher score alone is not a useful objective. + +## 14. Cumulative profiles + +- D1: charter, contract, baseline, roles/overlaps, bounded generator, identities, + ledger/lineage, selection rationale, explicit results. +- D2 adds environment lock, partitions, reproducibility above `NONE`, repetitions, + evaluator regression, aggregates, and epochs. +- D3 adds predeclared selection, protected/fresh evidence, evaluator separation, + reviewer binding, conflicts, and MNCS binding when claimed. +- D4 adds release, monitoring, tested rollback, exercised regeneration/replacement, + retirement, and lifecycle identities. + +## 15. Privacy, contamination, reporting, and migration + +Restricted material still gets stable identities/access boundaries. Redaction is not +verification. Required hidden evidence is `UNKNOWN` absent an accepted proof. +Protected-evidence reuse identifies prior epochs/claims; development exposure +contaminates later protected use. + +Reports include version, record/epoch, profile, selected candidate, computed status, +normalized issues, warnings, scope, limitations, unsupported rules, and executable +identity. Validation is offline and executes nothing. + +Independent consumers read the golden corpus directly and distinguish agreement, +disagreement, unsupported, invalid, and implementation errors. Local diversity does not +prove independent operation, custody, organization, governance, or accreditation. + +Draft records remain valid as drafts. Migration is optional, creates a new identity, +and starts at the oldest reliable baseline. Missing history remains `UNKNOWN`. Changing +a version string is not migration. Exact versions dispatch; unknown versions are +`UNSUPPORTED`. diff --git a/spec/MNCDS-v0.1-records-and-decisions.md b/spec/MNCDS-v0.1-records-and-decisions.md new file mode 100644 index 0000000..2f4bf67 --- /dev/null +++ b/spec/MNCDS-v0.1-records-and-decisions.md @@ -0,0 +1,180 @@ + + +# MNCDS 0.1 Development Record and Initial Decisions (Draft) + +This module is part of the proposed MNCDS 0.1 normative text under RFC 0004. Normative +terms use the meanings in `normative-language.md`. + +It defines the first interoperable development-record profile and resolves questions +needed to test the draft. It does not change MNCS 0.2 conformance semantics. + +## 1. Companion relationship + +MNCDS MUST remain separately versioned from MNCS throughout the MNCS 0.x series. A +future merger into MNCS MAY be proposed only through a major-version RFC that defines +migration and preserves the validity of historical MNCS claims made without MNCDS +records. + +An MNCS result and an MNCDS result MUST remain distinguishable in storage, APIs, CLI +output, attestations, and human-readable claims. + +## 2. Aggregate development record + +The minimum interoperable unit for MNCDS 0.1 is one aggregate development record. A D1 +record MUST contain: + +- MNCDS and schema versions, record identity, epoch identity, creation time, and claimed profile; +- problem statement, intended use, contract, baseline, environment, threat-model, objective, and selection-policy identities; +- the six logical role bindings defined by MNCDS; +- generator identity, effective configuration, executable and authority identities, permissions, and resource limits; +- development and selection partition identities; +- evaluator identities and configurations; +- every materially evaluated candidate's identity, parent lineage, generator identity, build status, objective value, evaluator results, and disposition; +- selected-candidate identity, selection rationale, useful-benefit result, and UNKNOWN policy; +- a declared reproducibility class; +- an MNCS binding when the charter plans an MNCS claim; +- release controls when D4 is claimed. + +A project MAY split this information among content-addressed records, but it MUST expose +an offline-resolvable aggregate view with equivalent semantics. + +Correcting a record MUST create a new record identity and reference the superseded +record. Historical records MUST NOT be silently rewritten. + +## 3. Material evaluation and rejected candidates + +A candidate is **materially evaluated** when it reaches any gate whose result can affect +ranking, promotion, rejection, selection, a reported success rate, or a conformance +claim. + +Every materially evaluated candidate MUST be retained individually by identity, +lineage, gate outcomes, objective value where applicable, and disposition. + +Candidates rejected before material evaluation MAY be aggregated only when the charter +predeclares the aggregation boundary. An aggregate MUST preserve: + +- count; +- rejection stage or reason class; +- generator and effective-configuration identities; +- time or sequence range; +- a stable digest, Merkle root, content-addressed index, or reproducible query over the omitted set. + +Search scale MUST NOT justify selective omission of materially evaluated candidates or +failures that reveal selection pressure or evaluator exploitation. + +## 4. Stochastic reproducibility classes + +A record MUST declare exactly one class: + +- **EXACT:** the declared process reproduces byte-identical candidates and records under the bound environment; +- **SEEDED:** the same algorithm, effective configuration, environment, and preserved seeds reproduce the run; +- **STATISTICAL:** repeated runs reproduce predeclared summary statistics within declared uncertainty bounds; +- **DISTRIBUTIONAL:** repeated runs reproduce a declared candidate family or outcome distribution under a declared comparison method; +- **NONE:** no credible exact, seeded, statistical, or distributional claim is made. + +D1 MAY declare `NONE`. D2 and above MUST NOT declare `NONE`. + +`EXACT` and `SEEDED` MUST preserve all randomness inputs needed by the claim. Statistical +and distributional claims MUST declare repetition count, measurement protocol, summary +or distance statistic, bounds, and failure treatment. A stronger class MUST NOT be +inferred from a weaker one. + +## 5. Evaluator independence at D3 + +A D3 final evaluator MUST: + +- use an authority identity different from the generator authority; +- use an executable identity different from the generator executable; +- use an immutable configuration identity; +- be bound to the independent-reviewer role; +- use protected holdout or fresh challenge evidence unavailable to ordinary generation and ranking; +- record PASS, FAIL, or UNKNOWN against the selected candidate; +- preserve the evidence identity used for that result. + +Sharing generator authority or generator executable identity MUST fail D3. Missing or +unverifiable independence evidence MUST produce FAIL or UNKNOWN according to policy; it +MUST NOT silently pass. + +Organizational separation, separate infrastructure, multiple independent evaluators, +and threshold attestations MAY support future higher-assurance profiles. They are not +required by baseline D3. + +Different identities demonstrate a declared control boundary; they do not prove honesty, +competence, or absence of collusion. + +## 6. Selection and UNKNOWN + +The selected candidate MUST exist in the candidate ledger and MUST have disposition +`selected`. The selection-policy identity MUST match the charter. + +A selected candidate with a required FAIL MUST fail the process claim. + +When a selected candidate has required UNKNOWN evidence: + +- policy `reject` MUST fail the process claim; +- policy `human_review` MUST include reviewer identity, explicit decision, and rationale; +- an explicit acceptance with unresolved UNKNOWN MUST preserve overall status UNKNOWN; +- human review MUST NOT convert UNKNOWN to PASS. + +D3 and above MUST record the selection rule before protected-holdout evaluation. A +post-hoc rule change begins a new selection epoch. + +## 7. MNCS binding + +When the charter plans an MNCS claim, the development record MUST bind a resulting MNCS +manifest or package identity. + +The MNCS binding's selected-candidate, contract, and environment identities MUST match +the final MNCDS record. Any mismatch MUST fail the combined claim. + +A validator MUST report MNCS and MNCDS results separately even when the records bind to +one another. + +## 8. D4 operational evidence + +D4 MUST include: + +- release artifact identity; +- rollback artifact identity and a passing rollback test; +- monitoring signals or thresholds; +- a regeneration or replacement drill with time, status, and evidence identity; +- retirement triggers. + +A rollback test with FAIL or UNKNOWN MUST NOT satisfy D4. A regeneration drill with FAIL +or UNKNOWN MUST NOT satisfy D4. + +## 9. Privacy-preserving extensions + +Privacy-preserving proofs, confidential-computing attestations, commitments, and +transparency logs MAY be used as namespaced experimental extensions. + +They MUST NOT replace required evidence unless a future RFC defines their statement, +predicate, trust, expiration, revocation, failure, and interoperability semantics and at +least two implementations agree on a versioned corpus. + +Redaction without an accepted proof MUST remain UNKNOWN whenever the hidden material is +required to establish the claimed profile. + +## 10. CLI and reporting + +The baseline command families are separate: + +```text +mncs validate ... +mncds validate ... +``` + +A future combined summary MAY display both results. It MUST preserve separate statuses, +versions, record identities, issue sets, scopes, and trust decisions. It MUST NOT collapse +the results into one boolean or imply that one result proves the other. + +## 11. Experimental schema + +The initial machine-readable aggregate record is +`schemas/mncds-development-record.schema.json`. + +Ordinary validation MUST be offline and MUST NOT execute or import generators, +candidates, evaluators, analyzers, benchmarks, or evidence binaries. + +The schema and validator are experimental implementations of this draft module. Their +presence in the repository does not bypass RFC review or establish accepted core status. diff --git a/spec/README.md b/spec/README.md index 490c953..3fb5f42 100644 --- a/spec/README.md +++ b/spec/README.md @@ -1,9 +1,18 @@ # Specification -This directory is the home of normative MNCDS specification text. +This directory is the canonical home of normative MNCDS specification text. -During extraction, preserve historical filenames and version identities such as `MNCDS-v0.1-draft.md` and `MNCDS-v0.1-rc.1.md`. Do not silently rewrite a historical release candidate into a new repository-native edition. +Current files: -Future versions SHOULD use stable versioned filenames and release notes that identify accepted RFCs, compatibility, known limitations, and the supported MNCS interoperability envelope. +- [`MNCDS-v0.1-draft.md`](MNCDS-v0.1-draft.md) — historical draft lifecycle and cumulative profiles +- [`MNCDS-v0.1-rc.1.md`](MNCDS-v0.1-rc.1.md) — current independently consumable release candidate +- [`MNCDS-v0.1-records-and-decisions.md`](MNCDS-v0.1-records-and-decisions.md) — aggregate record and decision semantics -Normative changes require the process in `GOVERNANCE.md`. Editorial fixes that do not change meaning should still preserve released artifacts and issue corrected successor material where necessary. +These files were extracted from `epi13/machine-native-complexity-standard` commit +`f0088c4d46dec84f289d9b4417eec32b0ac028e6`. Historical filenames and version +identities are preserved. + +MNCS 0.3 may bind to an MNCDS object. That binding does not move MNCDS +normative authority back into MNCS. + +Normative changes require `GOVERNANCE.md`. diff --git a/src/mncds_validator/__init__.py b/src/mncds_validator/__init__.py new file mode 100644 index 0000000..bacbf83 --- /dev/null +++ b/src/mncds_validator/__init__.py @@ -0,0 +1,9 @@ +"""Reference MNCDS validator. + +This package is the MNCDS-owned consumer for MNCDS 0.1 development records. +It does not define MNCS implementation-evidence semantics. +""" + +from .mncds import validate_development_record, validate_development_value + +__all__ = ["validate_development_record", "validate_development_value"] diff --git a/src/mncds_validator/cli.py b/src/mncds_validator/cli.py new file mode 100644 index 0000000..d23a52e --- /dev/null +++ b/src/mncds_validator/cli.py @@ -0,0 +1,86 @@ +"""Command-line interface for MNCDS validation.""" + +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from .errors import MncdsError +from .mncds import validate_development_record + +MNCDS_VERSION = "0.1-rc.1" +MNCDS_SCHEMA_VERSION = "0.1-rc.1" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="mncds", + description="Offline validator for Machine-Native Complexity Development records", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser("validate", help="validate a development record") + validate.add_argument("record", type=Path) + validate.add_argument( + "--require-pass", + action="store_true", + help="return exit 3 for a valid record whose computed status is not PASS", + ) + validate.add_argument("--json", action="store_true", help="emit machine-readable JSON") + + version = subparsers.add_parser("version", help="print MNCDS validator version") + version.add_argument("--json", action="store_true", help="emit machine-readable JSON") + return parser + + +def run(args: argparse.Namespace) -> int: + if args.command == "validate": + if not args.record.is_file(): + raise FileNotFoundError(args.record) + report = validate_development_record(args.record) + if args.json: + print(json.dumps(report.as_dict(), indent=2, sort_keys=True)) + else: + print(report.category) + for issue in report.issues: + suffix = f" [{issue.path}]" if issue.path else "" + print(f"{issue.code}: {issue.message}{suffix}") + for warning in report.warnings: + suffix = f" [{warning.path}]" if warning.path else "" + print(f"warning {warning.code}: {warning.message}{suffix}") + if not report.supported: + return 4 + if not report.valid: + return 1 + return 3 if args.require_pass and report.computed_status != "PASS" else 0 + + if args.command == "version": + result = { + "package": "mncds-validator", + "mncds_version": MNCDS_VERSION, + "schema_version": MNCDS_SCHEMA_VERSION, + "status": "release_candidate", + "supported_versions": ["0.1-draft", "0.1-rc.1"], + "canonical_repository": ( + "https://github.com/epi13/machine-native-complexity-development-specification" + ), + } + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + print(f"mncds {MNCDS_VERSION} (schema {MNCDS_SCHEMA_VERSION}; release candidate)") + return 0 + + raise AssertionError(f"unhandled command: {args.command}") + + +def main(argv: list[str] | None = None) -> int: + try: + return run(build_parser().parse_args(argv)) + except (MncdsError, FileNotFoundError, PermissionError) as exc: + print(f"mncds: error: {exc}", file=sys.stderr) + return 2 diff --git a/src/mncds_validator/errors.py b/src/mncds_validator/errors.py new file mode 100644 index 0000000..911637a --- /dev/null +++ b/src/mncds_validator/errors.py @@ -0,0 +1,19 @@ +"""Validator exceptions for the MNCDS reference consumer.""" + +# SPDX-License-Identifier: Apache-2.0 + + +class MncdsError(Exception): + """Base class for expected MNCDS validator failures.""" + + +class MncsError(MncdsError): + """Compatibility alias used by the extracted MNCDS validator module.""" + + +class SchemaNotFoundError(MncdsError): + """Raised when a bundled schema name cannot be resolved.""" + + +class ManifestError(MncdsError): + """Raised when a development record cannot be loaded safely.""" diff --git a/src/mncds_validator/mncds.py b/src/mncds_validator/mncds.py new file mode 100644 index 0000000..0aa44de --- /dev/null +++ b/src/mncds_validator/mncds.py @@ -0,0 +1,1087 @@ +"""Offline validation for experimental MNCDS development records.""" + +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Literal, cast + +from .errors import ManifestError +from .schemas import schema_errors +from .validation import load_json_object + +MncdsStatus = Literal["PASS", "FAIL", "UNKNOWN"] + +PROFILE_ORDER = { + "MNCDS-D1": 1, + "MNCDS-D2": 2, + "MNCDS-D3": 3, + "MNCDS-D4": 4, +} +REQUIRED_ROLES = { + "contract_authority", + "generator_authority", + "evaluator_authority", + "selection_authority", + "release_authority", + "independent_reviewer", +} +FORBIDDEN_GENERATOR_PERMISSIONS = { + "modify_contract", + "modify_baseline", + "modify_evaluators", + "modify_selection_policy", + "modify_thresholds", + "access_protected_holdout", +} + + +@dataclass(frozen=True) +class MncdsIssue: + """One deterministic MNCDS validation finding.""" + + code: str + message: str + path: str = "" + + +@dataclass +class MncdsValidationReport: + """Validation and profile result for one MNCDS development record.""" + + target: str + valid: bool = True + supported: bool = True + computed_status: MncdsStatus = "PASS" + profile: str | None = None + record_id: str | None = None + issues: list[MncdsIssue] = field(default_factory=list) + warnings: list[MncdsIssue] = field(default_factory=list) + + @property + def category(self) -> str: + if not self.supported: + return "UNSUPPORTED" + if not self.valid: + return "INVALID" + return self.computed_status + + def add(self, code: str, message: str, path: str = "") -> None: + self.valid = False + self.computed_status = "FAIL" + self.issues.append(MncdsIssue(code, message, path)) + + def warn(self, code: str, message: str, path: str = "") -> None: + self.warnings.append(MncdsIssue(code, message, path)) + + def fail(self, code: str, message: str, path: str = "") -> None: + self.computed_status = "FAIL" + self.issues.append(MncdsIssue(code, message, path)) + + def unknown(self, code: str, message: str, path: str = "") -> None: + if self.valid and self.computed_status == "PASS": + self.computed_status = "UNKNOWN" + self.warnings.append(MncdsIssue(code, message, path)) + + def as_dict(self) -> dict[str, Any]: + result = asdict(self) + result["category"] = self.category + return result + + +def _objects(value: object) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + return [cast(dict[str, Any], item) for item in value if isinstance(item, dict)] + + +def _profile_at_least(profile: object, required: str) -> bool: + return isinstance(profile, str) and PROFILE_ORDER.get(profile, 0) >= PROFILE_ORDER[required] + + +def _check_unique_ids( + values: list[dict[str, Any]], + key: str, + report: MncdsValidationReport, + path: str, +) -> set[str]: + identifiers: set[str] = set() + for index, value in enumerate(values): + identifier = value.get(key) + if not isinstance(identifier, str): + continue + if identifier in identifiers: + report.add( + "duplicate-id", + f"duplicate {key}: {identifier}", + f"{path}/{index}/{key}", + ) + identifiers.add(identifier) + return identifiers + + +def _check_lineage( + candidates: list[dict[str, Any]], + candidate_ids: set[str], + report: MncdsValidationReport, +) -> None: + parents: dict[str, list[str]] = {} + for index, candidate in enumerate(candidates): + candidate_id = candidate.get("candidate_id") + if not isinstance(candidate_id, str): + continue + parent_ids = [item for item in candidate.get("parent_ids", []) if isinstance(item, str)] + parents[candidate_id] = parent_ids + for parent_id in parent_ids: + if parent_id not in candidate_ids: + report.add( + "unknown-parent", + f"candidate parent is not recorded: {parent_id}", + f"$/candidates/{index}/parent_ids", + ) + if parent_id == candidate_id: + report.add( + "lineage-cycle", + "candidate cannot be its own parent", + f"$/candidates/{index}/parent_ids", + ) + + visiting: set[str] = set() + visited: set[str] = set() + + def visit(candidate_id: str) -> None: + if candidate_id in visited: + return + if candidate_id in visiting: + report.add("lineage-cycle", f"candidate lineage cycle includes {candidate_id}") + return + visiting.add(candidate_id) + for parent_id in parents.get(candidate_id, []): + if parent_id in parents: + visit(parent_id) + visiting.remove(candidate_id) + visited.add(candidate_id) + + for candidate_id in sorted(parents): + visit(candidate_id) + + +def _check_roles(value: dict[str, Any], report: MncdsValidationReport) -> dict[str, dict[str, Any]]: + roles = _objects(value.get("roles")) + role_names = [item.get("role") for item in roles if isinstance(item.get("role"), str)] + role_name_set = {cast(str, name) for name in role_names} + for missing in sorted(REQUIRED_ROLES.difference(role_name_set)): + report.add("missing-role", f"required logical role is missing: {missing}", "$/roles") + if len(role_names) != len(role_name_set): + report.add( + "duplicate-role", + "each logical role must appear exactly once", + "$/roles", + ) + return {cast(str, item["role"]): item for item in roles if isinstance(item.get("role"), str)} + + +def _check_generator(value: dict[str, Any], report: MncdsValidationReport) -> None: + generator = value.get("generator") + if not isinstance(generator, dict): + return + permissions = generator.get("permissions") + if not isinstance(permissions, dict): + return + for permission in sorted(FORBIDDEN_GENERATOR_PERMISSIONS): + if permissions.get(permission) is True: + report.add( + "generator-authority-violation", + f"generator has forbidden authority: {permission}", + f"$/generator/permissions/{permission}", + ) + + +def _check_partitions(value: dict[str, Any], report: MncdsValidationReport) -> None: + partitions = value.get("partitions") + if not isinstance(partitions, dict): + return + identifiers = [ + partitions.get("development_id"), + partitions.get("selection_id"), + partitions.get("holdout_id"), + ] + present = [item for item in identifiers if isinstance(item, str)] + if len(present) != len(set(present)): + report.add( + "partition-identity-overlap", + "development, selection, and holdout partitions must have distinct identities", + "$/partitions", + ) + if partitions.get("holdout_contaminated") is True: + report.add( + "holdout-contaminated", + "a contaminated holdout cannot support the claimed profile", + "$/partitions/holdout_contaminated", + ) + + +def _selected_candidate( + value: dict[str, Any], + candidate_ids: set[str], + report: MncdsValidationReport, +) -> dict[str, Any] | None: + selection = value.get("selection") + if not isinstance(selection, dict): + return None + selected_id = selection.get("selected_candidate_id") + if not isinstance(selected_id, str) or selected_id not in candidate_ids: + report.add( + "selected-candidate-missing", + "selected candidate is not present in the candidate ledger", + "$/selection/selected_candidate_id", + ) + return None + for candidate in _objects(value.get("candidates")): + if candidate.get("candidate_id") == selected_id: + if candidate.get("disposition") != "selected": + report.add( + "selection-disposition-mismatch", + "selected candidate must have disposition 'selected'", + "$/candidates", + ) + return candidate + return None + + +def _check_selection( + value: dict[str, Any], + selected: dict[str, Any] | None, + evaluator_ids: set[str], + report: MncdsValidationReport, +) -> None: + selection = value.get("selection") + charter = value.get("charter") + if not isinstance(selection, dict) or not isinstance(charter, dict): + return + if selection.get("policy_id") != charter.get("selection_policy_id"): + report.add( + "selection-policy-mismatch", + "selection record does not bind the charter selection policy", + "$/selection/policy_id", + ) + if selection.get("minimum_useful_benefit_met") is not True: + report.add( + "benefit-threshold-not-met", + "selected candidate did not meet the predeclared useful-benefit threshold", + "$/selection/minimum_useful_benefit_met", + ) + if selected is None: + return + + required_unknown = False + for index, result in enumerate(_objects(selected.get("evaluator_results"))): + evaluator_id = result.get("evaluator_id") + if isinstance(evaluator_id, str) and evaluator_id not in evaluator_ids: + report.add( + "unknown-evaluator", + f"candidate references unrecorded evaluator: {evaluator_id}", + f"$/candidates/evaluator_results/{index}", + ) + if result.get("required") is True and result.get("status") == "FAIL": + report.add( + "selected-required-fail", + "selected candidate has a required FAIL", + "$/selection/selected_candidate_id", + ) + if result.get("required") is True and result.get("status") == "UNKNOWN": + required_unknown = True + + if not required_unknown: + return + unknown_policy = selection.get("unknown_policy") + if unknown_policy == "reject": + report.add( + "unknown-promoted", + "selected candidate has required UNKNOWN evidence under a reject policy", + "$/selection/unknown_policy", + ) + return + review = selection.get("human_review") + if not isinstance(review, dict) or review.get("decision") != "accept_with_unknown": + report.add( + "unknown-review-missing", + "human-review policy requires an explicit accept-with-UNKNOWN decision", + "$/selection/human_review", + ) + return + report.unknown( + "selected-with-unknown", + ( + "record is structurally valid, but the selected candidate retains " + "required UNKNOWN evidence" + ), + "$/selection/selected_candidate_id", + ) + + +def _check_d2(value: dict[str, Any], report: MncdsValidationReport) -> None: + reproducibility = value.get("reproducibility") + if not isinstance(reproducibility, dict): + return + reproduction_class = reproducibility.get("class") + if reproduction_class == "NONE": + report.add( + "d2-reproducibility-missing", + "D2 and above require reproducible or statistically characterized generation", + "$/reproducibility/class", + ) + if ( + reproduction_class in {"EXACT", "SEEDED"} + and reproducibility.get("seeds_preserved") is not True + ): + report.add( + "seed-record-missing", + f"{reproduction_class} reproducibility requires preserved seeds", + "$/reproducibility/seeds_preserved", + ) + repetitions = reproducibility.get("measurement_repetitions") + if not isinstance(repetitions, int) or repetitions < 2: + report.add( + "measurement-repetition-insufficient", + "D2 and above require repeated measurements", + "$/reproducibility/measurement_repetitions", + ) + for index, evaluator in enumerate(_objects(value.get("evaluators"))): + if evaluator.get("regression_corpus_id") is None: + report.add( + "evaluator-corpus-missing", + "D2 and above require a versioned evaluator regression corpus", + f"$/evaluators/{index}/regression_corpus_id", + ) + + +def _check_d3( + value: dict[str, Any], + roles: dict[str, dict[str, Any]], + selected: dict[str, Any] | None, + report: MncdsValidationReport, +) -> None: + partitions = value.get("partitions") + selection = value.get("selection") + generator = value.get("generator") + if isinstance(partitions, dict) and not isinstance(partitions.get("holdout_id"), str): + report.add( + "holdout-missing", + "D3 and above require a protected holdout", + "$/partitions", + ) + if isinstance(selection, dict) and selection.get("rule_recorded_before_holdout") is not True: + report.add( + "selection-rule-post-hoc", + "D3 and above require the selection rule before holdout evaluation", + "$/selection/rule_recorded_before_holdout", + ) + + independent = [ + item + for item in _objects(value.get("evaluators")) + if item.get("independent") is True and item.get("purpose") in {"holdout", "independent"} + ] + if not independent: + report.add( + "independent-evaluator-missing", + "D3 and above require an independent final evaluator", + "$/evaluators", + ) + return + generator_authority = generator.get("authority_id") if isinstance(generator, dict) else None + generator_executable = generator.get("executable_id") if isinstance(generator, dict) else None + reviewer = roles.get("independent_reviewer", {}) + for evaluator in independent: + if evaluator.get("authority_id") == generator_authority: + report.add( + "independence-authority-conflict", + "independent evaluator shares generator authority", + "$/evaluators", + ) + if evaluator.get("executable_id") == generator_executable: + report.add( + "independence-executable-conflict", + "independent evaluator shares generator executable identity", + "$/evaluators", + ) + if reviewer and evaluator.get("authority_id") != reviewer.get("authority_id"): + report.add( + "independent-role-mismatch", + "independent evaluator authority does not match the declared reviewer role", + "$/evaluators", + ) + if selected is not None: + used = { + item.get("evaluator_id") + for item in _objects(selected.get("evaluator_results")) + if item.get("status") in {"PASS", "FAIL", "UNKNOWN"} + } + if not any(item.get("evaluator_id") in used for item in independent): + report.add( + "independent-evidence-missing", + "selected candidate has no result from the independent evaluator", + "$/candidates", + ) + + +def _check_d4(value: dict[str, Any], report: MncdsValidationReport) -> None: + controls = value.get("release_controls") + if not isinstance(controls, dict): + report.add( + "release-controls-missing", + "D4 requires release controls", + "$/release_controls", + ) + return + if controls.get("rollback_test_status") != "PASS": + report.add( + "rollback-not-tested", + "D4 requires a passing rollback test", + "$/release_controls/rollback_test_status", + ) + drill = controls.get("regeneration_drill") + if not isinstance(drill, dict) or drill.get("status") != "PASS": + report.add( + "regeneration-drill-failed", + "D4 requires a passing regeneration or replacement drill", + "$/release_controls/regeneration_drill", + ) + + +def _check_mncs_binding(value: dict[str, Any], report: MncdsValidationReport) -> None: + charter = value.get("charter") + binding = value.get("mncs_binding") + selection = value.get("selection") + if not isinstance(charter, dict) or charter.get("planned_mncs_level") is None: + return + if not isinstance(binding, dict): + report.add( + "mncs-binding-missing", + "a planned MNCS claim requires an explicit final binding", + "$/mncs_binding", + ) + return + expected = { + "candidate_id": ( + selection.get("selected_candidate_id") if isinstance(selection, dict) else None + ), + "contract_id": charter.get("contract_id"), + "environment_id": charter.get("environment_id"), + } + for key, expected_value in expected.items(): + if binding.get(key) != expected_value: + report.add( + "mncs-binding-mismatch", + f"MNCS binding {key} does not match the development record", + f"$/mncs_binding/{key}", + ) + + +def _validate_draft_value( + value: dict[str, Any], + *, + target: str = "$", +) -> MncdsValidationReport: + """Validate one decoded MNCDS development record.""" + + report = MncdsValidationReport(target=target) + report.profile = value.get("profile") if isinstance(value.get("profile"), str) else None + report.record_id = value.get("record_id") if isinstance(value.get("record_id"), str) else None + + for error in schema_errors(value, "mncds-development-record"): + report.add("schema", error) + if not report.valid: + return report + + roles = _check_roles(value, report) + _check_generator(value, report) + _check_partitions(value, report) + + evaluators = _objects(value.get("evaluators")) + evaluator_ids = _check_unique_ids(evaluators, "evaluator_id", report, "$/evaluators") + candidates = _objects(value.get("candidates")) + candidate_ids = _check_unique_ids(candidates, "candidate_id", report, "$/candidates") + _check_lineage(candidates, candidate_ids, report) + selected = _selected_candidate(value, candidate_ids, report) + _check_selection(value, selected, evaluator_ids, report) + _check_mncs_binding(value, report) + + profile = value.get("profile") + if _profile_at_least(profile, "MNCDS-D2"): + _check_d2(value, report) + if _profile_at_least(profile, "MNCDS-D3"): + _check_d3(value, roles, selected, report) + if _profile_at_least(profile, "MNCDS-D4"): + _check_d4(value, report) + + return report + + +def _check_rc_authority_overlaps( + value: dict[str, Any], + report: MncdsValidationReport, +) -> dict[str, dict[str, Any]]: + roles = _check_roles(value, report) + grouped: dict[str, set[str]] = {} + for role_name, role in roles.items(): + authority = role.get("authority_id") + if isinstance(authority, str): + grouped.setdefault(authority, set()).add(role_name) + disclosures = { + item.get("authority_id"): item + for item in _objects(value.get("authority_overlaps")) + if isinstance(item.get("authority_id"), str) + } + for authority, role_names in grouped.items(): + if len(role_names) > 1 and authority not in disclosures: + report.fail( + "authority-overlap-undisclosed", + f"authority {authority} holds multiple roles without disclosure", + "$/authority_overlaps", + ) + return roles + + +def _check_rc_generator(value: dict[str, Any], report: MncdsValidationReport) -> None: + generator = value.get("generator") + permissions = generator.get("permissions") if isinstance(generator, dict) else None + if not isinstance(permissions, dict): + return + for permission in sorted(FORBIDDEN_GENERATOR_PERMISSIONS): + if permissions.get(permission) is True: + report.fail( + "generator-authority-violation", + f"generator has forbidden authority: {permission}", + f"$/generator/permissions/{permission}", + ) + + +def _check_rc_partitions(value: dict[str, Any], report: MncdsValidationReport) -> set[str]: + partitions = value.get("partitions") + if not isinstance(partitions, dict): + return set() + identities = { + item + for item in ( + partitions.get("development_id"), + partitions.get("selection_id"), + partitions.get("final_evaluation_id"), + ) + if isinstance(item, str) + } + present = [ + item + for item in ( + partitions.get("development_id"), + partitions.get("selection_id"), + partitions.get("final_evaluation_id"), + ) + if isinstance(item, str) + ] + if len(present) != len(identities): + report.fail( + "partition-identity-overlap", + "development, selection, and final partitions must be distinct", + "$/partitions", + ) + if partitions.get("holdout_contaminated") is True: + report.fail( + "holdout-contaminated", + "a contaminated final partition cannot support the profile", + "$/partitions/holdout_contaminated", + ) + return identities + + +def _check_rc_epochs( + value: dict[str, Any], + report: MncdsValidationReport, +) -> set[str]: + epochs = _objects(value.get("epochs")) + epoch_ids = _check_unique_ids(epochs, "epoch_id", report, "$/epochs") + parents: dict[str, str | None] = {} + for index, epoch in enumerate(epochs): + epoch_id = epoch.get("epoch_id") + parent_id = epoch.get("parent_epoch_id") + if not isinstance(epoch_id, str): + continue + parents[epoch_id] = parent_id if isinstance(parent_id, str) else None + if isinstance(parent_id, str) and parent_id not in epoch_ids: + report.fail( + "unknown-epoch-parent", + f"epoch parent is not recorded: {parent_id}", + f"$/epochs/{index}/parent_epoch_id", + ) + if parent_id == epoch_id: + report.fail( + "epoch-cycle", + "an epoch cannot be its own parent", + f"$/epochs/{index}/parent_epoch_id", + ) + for epoch_id in parents: + seen: set[str] = set() + current: str | None = epoch_id + while current is not None: + if current in seen: + report.fail("epoch-cycle", f"recursive epoch cycle includes {current}") + break + seen.add(current) + current = parents.get(current) + current_epoch = value.get("epoch_id") + if current_epoch not in epoch_ids: + report.fail( + "current-epoch-missing", + "record epoch_id is not present in epochs", + "$/epoch_id", + ) + return epoch_ids + + +def _check_rc_protected_evidence( + value: dict[str, Any], + partition_ids: set[str], + report: MncdsValidationReport, +) -> None: + evidence = _objects(value.get("protected_evidence")) + _check_unique_ids(evidence, "evidence_id", report, "$/protected_evidence") + for index, item in enumerate(evidence): + if item.get("partition_id") not in partition_ids: + report.fail( + "protected-partition-missing", + "protected evidence references an unknown partition", + f"$/protected_evidence/{index}/partition_id", + ) + if item.get("generator_access") is True or item.get("contaminated") is True: + report.fail( + "protected-evidence-contaminated", + "generator access or contamination invalidates protected use", + f"$/protected_evidence/{index}", + ) + + +def _check_rc_candidates( + value: dict[str, Any], + evaluator_ids: set[str], + epoch_ids: set[str], + partition_ids: set[str], + report: MncdsValidationReport, +) -> tuple[set[str], dict[str, Any] | None]: + candidates = _objects(value.get("candidates")) + candidate_ids = _check_unique_ids(candidates, "candidate_id", report, "$/candidates") + _check_lineage(candidates, candidate_ids, report) + generator = value.get("generator") + generator_id = generator.get("generator_id") if isinstance(generator, dict) else None + for index, candidate in enumerate(candidates): + if candidate.get("epoch_id") not in epoch_ids: + report.fail( + "candidate-epoch-missing", + "candidate references an unknown epoch", + f"$/candidates/{index}/epoch_id", + ) + if candidate.get("generator_id") != generator_id: + report.fail( + "candidate-generator-mismatch", + "candidate generator does not match the bound generator", + f"$/candidates/{index}/generator_id", + ) + if candidate.get("materially_evaluated") is True and candidate.get("retained") is not True: + report.fail( + "material-candidate-not-retained", + "every materially evaluated candidate must be retained", + f"$/candidates/{index}/retained", + ) + for result_index, result in enumerate(_objects(candidate.get("evaluator_results"))): + if result.get("evaluator_id") not in evaluator_ids: + report.fail( + "unknown-evaluator", + "candidate result references an unknown evaluator", + f"$/candidates/{index}/evaluator_results/{result_index}/evaluator_id", + ) + if result.get("partition_id") not in partition_ids: + report.fail( + "unknown-partition", + "candidate result references an unknown partition", + f"$/candidates/{index}/evaluator_results/{result_index}/partition_id", + ) + + selection = value.get("selection") + selected_id = selection.get("selected_candidate_id") if isinstance(selection, dict) else None + selected = next( + (candidate for candidate in candidates if candidate.get("candidate_id") == selected_id), + None, + ) + if selected is None: + report.fail( + "selected-candidate-missing", + "selected candidate is not present in the ledger", + "$/selection/selected_candidate_id", + ) + elif selected.get("disposition") != "selected" or selected.get("retained") is not True: + report.fail( + "selection-disposition-mismatch", + "selected candidate must be retained with disposition selected", + "$/candidates", + ) + return candidate_ids, selected + + +def _check_rc_selection( + value: dict[str, Any], + selected: dict[str, Any] | None, + report: MncdsValidationReport, +) -> None: + selection = value.get("selection") + charter = value.get("charter") + if not isinstance(selection, dict) or not isinstance(charter, dict): + return + if selection.get("policy_id") != charter.get("selection_policy_id"): + report.fail( + "selection-policy-mismatch", + "selection does not bind the charter policy", + "$/selection/policy_id", + ) + if selection.get("minimum_useful_benefit_met") is not True: + report.fail( + "benefit-threshold-not-met", + "selected candidate does not meet useful benefit", + "$/selection/minimum_useful_benefit_met", + ) + if selection.get("hard_gates_passed") is not True: + report.fail( + "hard-gate-failed", + "selected candidate does not satisfy all hard gates", + "$/selection/hard_gates_passed", + ) + if selected is None: + return + required = [ + result + for result in _objects(selected.get("evaluator_results")) + if result.get("required") is True + ] + statuses = {result.get("status") for result in required} + if "FAIL" in statuses: + report.fail( + "selected-required-fail", + "selected candidate has a required FAIL", + "$/selection/selected_candidate_id", + ) + if "UNKNOWN" not in statuses: + return + if selection.get("unknown_policy") == "reject": + report.fail( + "unknown-promoted", + "required UNKNOWN cannot pass a reject policy", + "$/selection/unknown_policy", + ) + return + review = selection.get("human_review") + if not isinstance(review, dict) or review.get("decision") != "accept_with_unknown": + report.fail( + "unknown-review-missing", + "human review requires explicit accept-with-UNKNOWN", + "$/selection/human_review", + ) + return + report.unknown( + "selected-with-unknown", + "selected candidate retains required UNKNOWN evidence", + "$/selection/selected_candidate_id", + ) + + +def _check_rc_d2(value: dict[str, Any], report: MncdsValidationReport) -> None: + environment = value.get("environment_lock") + if isinstance(environment, dict) and environment.get("locked") is not True: + report.fail( + "environment-not-locked", + "D2 and above require a locked environment", + "$/environment_lock/locked", + ) + reproducibility = value.get("reproducibility") + if isinstance(reproducibility, dict): + reproduction_class = reproducibility.get("class") + if reproduction_class == "NONE": + report.fail( + "d2-reproducibility-missing", + "D2 and above require a reproducible or characterized process", + "$/reproducibility/class", + ) + if ( + reproduction_class in {"EXACT", "SEEDED"} + and reproducibility.get("seeds_preserved") is not True + ): + report.fail( + "seed-record-missing", + f"{reproduction_class} reproducibility requires preserved seeds", + "$/reproducibility/seeds_preserved", + ) + repetitions = reproducibility.get("measurement_repetitions") + if not isinstance(repetitions, int) or repetitions < 2: + report.fail( + "measurement-repetition-insufficient", + "D2 and above require repeated measurements", + "$/reproducibility/measurement_repetitions", + ) + for index, evaluator in enumerate(_objects(value.get("evaluators"))): + if evaluator.get("regression_corpus_id") is None: + report.fail( + "evaluator-corpus-missing", + "D2 and above require a versioned evaluator regression corpus", + f"$/evaluators/{index}/regression_corpus_id", + ) + for index, aggregate in enumerate(_objects(value.get("candidate_aggregates"))): + start = aggregate.get("sequence_start") + end = aggregate.get("sequence_end") + if isinstance(start, int) and isinstance(end, int) and end < start: + report.fail( + "candidate-aggregate-range", + "candidate aggregate sequence range is reversed", + f"$/candidate_aggregates/{index}", + ) + + +def _check_rc_d3( + value: dict[str, Any], + roles: dict[str, dict[str, Any]], + selected: dict[str, Any] | None, + report: MncdsValidationReport, +) -> None: + partitions = value.get("partitions") + selection = value.get("selection") + generator = value.get("generator") + final_id = partitions.get("final_evaluation_id") if isinstance(partitions, dict) else None + if not isinstance(final_id, str): + report.fail( + "holdout-missing", + "D3 and above require a final-evaluation partition", + "$/partitions/final_evaluation_id", + ) + if ( + isinstance(selection, dict) + and selection.get("rule_recorded_before_final_evaluation") is not True + ): + report.fail( + "selection-rule-post-hoc", + "selection rule must predate final evaluation", + "$/selection/rule_recorded_before_final_evaluation", + ) + independent = [ + evaluator + for evaluator in _objects(value.get("evaluators")) + if evaluator.get("independent") is True + and evaluator.get("purpose") in {"holdout", "independent"} + ] + if not independent: + report.fail( + "independent-evaluator-missing", + "D3 and above require a separated final evaluator", + "$/evaluators", + ) + return + generator_authority = generator.get("authority_id") if isinstance(generator, dict) else None + generator_executable = generator.get("executable_id") if isinstance(generator, dict) else None + reviewer = roles.get("independent_reviewer", {}) + for evaluator in independent: + if evaluator.get("authority_id") == generator_authority: + report.fail( + "independence-authority-conflict", + "final evaluator shares generator authority", + "$/evaluators", + ) + if evaluator.get("executable_id") == generator_executable: + report.fail( + "independence-executable-conflict", + "final evaluator shares generator executable", + "$/evaluators", + ) + if reviewer and evaluator.get("authority_id") != reviewer.get("authority_id"): + report.fail( + "independent-role-mismatch", + "final evaluator is not bound to independent-review authority", + "$/evaluators", + ) + if selected is not None: + independent_ids = {evaluator.get("evaluator_id") for evaluator in independent} + used = [ + result + for result in _objects(selected.get("evaluator_results")) + if result.get("evaluator_id") in independent_ids + and result.get("partition_id") == final_id + ] + if not used: + report.fail( + "independent-evidence-missing", + "selected candidate lacks final independent evidence", + "$/candidates", + ) + protected = [ + item + for item in _objects(value.get("protected_evidence")) + if item.get("partition_id") == final_id + ] + if not protected: + report.unknown( + "protected-evidence-missing", + "final evidence custody is unavailable", + "$/protected_evidence", + ) + else: + statuses = [str(item.get("status", "UNKNOWN")) for item in protected] + if "FAIL" in statuses: + report.fail( + "protected-evidence-failed", + "protected evidence failed its custody or evaluation rule", + "$/protected_evidence", + ) + elif "UNKNOWN" in statuses: + report.unknown( + "protected-evidence-unknown", + "protected evidence remains UNKNOWN", + "$/protected_evidence", + ) + + +def _check_rc_binding(value: dict[str, Any], report: MncdsValidationReport) -> None: + charter = value.get("charter") + binding = value.get("mncs_binding") + selection = value.get("selection") + if not isinstance(charter, dict) or charter.get("planned_mncs_level") is None: + return + if not isinstance(binding, dict): + report.fail( + "mncs-binding-missing", + "a planned MNCS claim requires a binding", + "$/mncs_binding", + ) + return + expected = { + "candidate_id": selection.get("selected_candidate_id") + if isinstance(selection, dict) + else None, + "contract_id": charter.get("contract_id"), + "environment_id": charter.get("environment_id"), + } + for key, expected_value in expected.items(): + if binding.get(key) != expected_value: + report.fail( + "mncs-binding-mismatch", + f"MNCS binding {key} does not match the development record", + f"$/mncs_binding/{key}", + ) + + +def _check_rc_d4( + value: dict[str, Any], + candidate_ids: set[str], + report: MncdsValidationReport, +) -> None: + controls = value.get("release_controls") + if not isinstance(controls, dict): + report.unknown( + "release-controls-missing", + "D4 release controls are unavailable", + "$/release_controls", + ) + return + monitoring = controls.get("monitoring") + rollback = controls.get("rollback") + regeneration = controls.get("regeneration_or_replacement") + retirement = controls.get("retirement") + for code, record, field_name in ( + ("monitoring-not-established", monitoring, "status"), + ("rollback-not-tested", rollback, "test_status"), + ("regeneration-drill-failed", regeneration, "status"), + ): + status = record.get(field_name) if isinstance(record, dict) else "UNKNOWN" + if status == "FAIL": + report.fail(code, f"D4 control {code} failed", "$/release_controls") + elif status != "PASS": + report.unknown(code, f"D4 control {code} is UNKNOWN", "$/release_controls") + if isinstance(regeneration, dict): + replacement = regeneration.get("replacement_candidate_id") + if replacement is not None and replacement not in candidate_ids: + report.fail( + "replacement-candidate-missing", + "replacement candidate is not in the ledger", + "$/release_controls/regeneration_or_replacement/replacement_candidate_id", + ) + if isinstance(retirement, dict) and retirement.get("retired") is True: + report.fail( + "selected-candidate-retired", + "a retired candidate cannot support a current D4 release", + "$/release_controls/retirement", + ) + + +def _validate_rc_value( + value: dict[str, Any], + *, + target: str, +) -> MncdsValidationReport: + report = MncdsValidationReport(target=target) + report.profile = value.get("profile") if isinstance(value.get("profile"), str) else None + report.record_id = value.get("record_id") if isinstance(value.get("record_id"), str) else None + for error in schema_errors(value, "mncds-development-record-0.1"): + report.add("schema", error) + if not report.valid: + return report + + roles = _check_rc_authority_overlaps(value, report) + _check_rc_generator(value, report) + partition_ids = _check_rc_partitions(value, report) + epoch_ids = _check_rc_epochs(value, report) + _check_rc_protected_evidence(value, partition_ids, report) + evaluators = _objects(value.get("evaluators")) + evaluator_ids = _check_unique_ids(evaluators, "evaluator_id", report, "$/evaluators") + candidate_ids, selected = _check_rc_candidates( + value, evaluator_ids, epoch_ids, partition_ids, report + ) + _check_rc_selection(value, selected, report) + _check_rc_binding(value, report) + + profile = value.get("profile") + if _profile_at_least(profile, "MNCDS-D2"): + _check_rc_d2(value, report) + if _profile_at_least(profile, "MNCDS-D3"): + _check_rc_d3(value, roles, selected, report) + if _profile_at_least(profile, "MNCDS-D4"): + _check_rc_d4(value, candidate_ids, report) + return report + + +def validate_development_value( + value: dict[str, Any], + *, + target: str = "$", +) -> MncdsValidationReport: + """Dispatch draft and release-candidate MNCDS records by exact version.""" + + version = value.get("mncds_version") + if version == "0.1-draft": + return _validate_draft_value(value, target=target) + if version == "0.1-rc.1": + return _validate_rc_value(value, target=target) + report = MncdsValidationReport(target=target, valid=False, supported=False) + report.add( + "unsupported-version", + f"unsupported MNCDS version: {version!r}", + "$/mncds_version", + ) + return report + + +def validate_development_record(path: Path) -> MncdsValidationReport: + """Load and validate an MNCDS development record without executing evidence.""" + + try: + value = load_json_object(path) + except ManifestError as exc: + report = MncdsValidationReport(target=str(path)) + report.add("invalid-json", str(exc), str(path)) + return report + return validate_development_value(value, target=str(path)) diff --git a/src/mncds_validator/resources/__init__.py b/src/mncds_validator/resources/__init__.py new file mode 100644 index 0000000..16756fc --- /dev/null +++ b/src/mncds_validator/resources/__init__.py @@ -0,0 +1 @@ +"""Packaged MNCDS resources.""" diff --git a/src/mncds_validator/resources/schemas/.gitkeep b/src/mncds_validator/resources/schemas/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/mncds_validator/resources/schemas/mncds-development-record-0.1.schema.json b/src/mncds_validator/resources/schemas/mncds-development-record-0.1.schema.json new file mode 100644 index 0000000..26d7d0e --- /dev/null +++ b/src/mncds_validator/resources/schemas/mncds-development-record-0.1.schema.json @@ -0,0 +1,386 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mncs.dev/schema/mncds/0.1-rc.1/mncds-development-record.schema.json", + "title": "MNCDS 0.1-rc.1 development record", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", "mncds_version", "record_id", "profile", "epoch_id", + "created_at", "supersedes_record_id", "charter", "baseline", "environment_lock", + "roles", "authority_overlaps", "generator", "partitions", "protected_evidence", + "evaluators", "candidates", "candidate_aggregates", "selection", + "reproducibility", "epochs", "mncs_binding", "release_controls", "extensions" + ], + "properties": { + "schema_version": {"const": "0.1-rc.1"}, + "mncds_version": {"const": "0.1-rc.1"}, + "record_id": {"$ref": "#/$defs/id"}, + "profile": {"enum": ["MNCDS-D1", "MNCDS-D2", "MNCDS-D3", "MNCDS-D4"]}, + "epoch_id": {"$ref": "#/$defs/id"}, + "created_at": {"type": "string", "format": "date-time"}, + "supersedes_record_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "charter": {"$ref": "#/$defs/charter"}, + "baseline": {"$ref": "#/$defs/baseline"}, + "environment_lock": {"$ref": "#/$defs/environment"}, + "roles": {"type": "array", "minItems": 6, "items": {"$ref": "#/$defs/role"}}, + "authority_overlaps": {"type": "array", "items": {"$ref": "#/$defs/overlap"}}, + "generator": {"$ref": "#/$defs/generator"}, + "partitions": {"$ref": "#/$defs/partitions"}, + "protected_evidence": {"type": "array", "items": {"$ref": "#/$defs/protectedEvidence"}}, + "evaluators": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/evaluator"}}, + "candidates": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/candidate"}}, + "candidate_aggregates": {"type": "array", "items": {"$ref": "#/$defs/candidateAggregate"}}, + "selection": {"$ref": "#/$defs/selection"}, + "reproducibility": {"$ref": "#/$defs/reproducibility"}, + "epochs": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/epoch"}}, + "mncs_binding": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/mncsBinding"}]}, + "release_controls": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/releaseControls"}]}, + "extensions": {"$ref": "#/$defs/extensions"} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "hash": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "status": {"enum": ["PASS", "FAIL", "UNKNOWN"]}, + "text": {"type": "string", "minLength": 1}, + "texts": {"type": "array", "items": {"$ref": "#/$defs/text"}}, + "ids": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "charter": { + "type": "object", "additionalProperties": false, + "required": [ + "charter_id", "problem_statement", "intended_use", "exclusions", "contract_id", + "baseline_id", "environment_id", "threat_model_id", "objective", + "selection_policy_id", "planned_mncs_level", "hard_rejection_gates", + "release_owner_id", "rollback_owner_id", "retirement_owner_id" + ], + "properties": { + "charter_id": {"$ref": "#/$defs/id"}, + "problem_statement": {"$ref": "#/$defs/text"}, + "intended_use": {"$ref": "#/$defs/text"}, + "exclusions": {"$ref": "#/$defs/texts"}, + "contract_id": {"$ref": "#/$defs/id"}, + "baseline_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "threat_model_id": {"$ref": "#/$defs/id"}, + "objective": { + "type": "object", "additionalProperties": false, + "required": ["objective_id", "metric", "unit", "direction", "minimum_useful_benefit", "operational_rationale"], + "properties": { + "objective_id": {"$ref": "#/$defs/id"}, + "metric": {"$ref": "#/$defs/text"}, + "unit": {"$ref": "#/$defs/text"}, + "direction": {"enum": ["minimize", "maximize"]}, + "minimum_useful_benefit": {"type": "number"}, + "operational_rationale": {"$ref": "#/$defs/text"} + } + }, + "selection_policy_id": {"$ref": "#/$defs/id"}, + "planned_mncs_level": {"enum": [null, "MNCS-L1", "MNCS-L2", "MNCS-L3", "MNCS-L4", "MNCS-L5"]}, + "hard_rejection_gates": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "release_owner_id": {"$ref": "#/$defs/id"}, + "rollback_owner_id": {"$ref": "#/$defs/id"}, + "retirement_owner_id": {"$ref": "#/$defs/id"} + } + }, + "baseline": { + "type": "object", "additionalProperties": false, + "required": ["baseline_id", "artifact_id", "source_id", "build_id", "dependency_ids", "environment_id", "evaluator_ids", "results", "captured_at", "immutable"], + "properties": { + "baseline_id": {"$ref": "#/$defs/id"}, + "artifact_id": {"$ref": "#/$defs/id"}, + "source_id": {"$ref": "#/$defs/id"}, + "build_id": {"$ref": "#/$defs/id"}, + "dependency_ids": {"$ref": "#/$defs/ids"}, + "environment_id": {"$ref": "#/$defs/id"}, + "evaluator_ids": {"$ref": "#/$defs/ids"}, + "results": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/result"}}, + "captured_at": {"type": "string", "format": "date-time"}, + "immutable": {"const": true} + } + }, + "environment": { + "type": "object", "additionalProperties": false, + "required": ["environment_id", "toolchain_id", "dependency_ids", "hardware_id", "configuration_id", "permitted_variance", "locked"], + "properties": { + "environment_id": {"$ref": "#/$defs/id"}, + "toolchain_id": {"$ref": "#/$defs/id"}, + "dependency_ids": {"$ref": "#/$defs/ids"}, + "hardware_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "permitted_variance": {"$ref": "#/$defs/texts"}, + "locked": {"type": "boolean"} + } + }, + "role": { + "type": "object", "additionalProperties": false, + "required": ["role", "authority_id", "executable_id"], + "properties": { + "role": {"enum": ["contract_authority", "generator_authority", "evaluator_authority", "selection_authority", "release_authority", "independent_reviewer"]}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + }, + "overlap": { + "type": "object", "additionalProperties": false, + "required": ["authority_id", "roles", "scope", "rationale", "risk", "recusal_or_control"], + "properties": { + "authority_id": {"$ref": "#/$defs/id"}, + "roles": {"type": "array", "minItems": 2, "uniqueItems": true, "items": {"$ref": "#/$defs/text"}}, + "scope": {"$ref": "#/$defs/text"}, + "rationale": {"$ref": "#/$defs/text"}, + "risk": {"$ref": "#/$defs/text"}, + "recusal_or_control": {"$ref": "#/$defs/text"} + } + }, + "generator": { + "type": "object", "additionalProperties": false, + "required": ["generator_id", "configuration_id", "authority_id", "executable_id", "permissions", "resource_limits"], + "properties": { + "generator_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "permissions": { + "type": "object", "additionalProperties": false, + "required": ["modify_contract", "modify_baseline", "modify_evaluators", "modify_selection_policy", "modify_thresholds", "access_protected_holdout", "network_access", "filesystem_scope", "process_scope", "tool_ids", "mutation_scope"], + "properties": { + "modify_contract": {"type": "boolean"}, + "modify_baseline": {"type": "boolean"}, + "modify_evaluators": {"type": "boolean"}, + "modify_selection_policy": {"type": "boolean"}, + "modify_thresholds": {"type": "boolean"}, + "access_protected_holdout": {"type": "boolean"}, + "network_access": {"type": "boolean"}, + "filesystem_scope": {"$ref": "#/$defs/texts"}, + "process_scope": {"$ref": "#/$defs/texts"}, + "tool_ids": {"$ref": "#/$defs/ids"}, + "mutation_scope": {"$ref": "#/$defs/texts"} + } + }, + "resource_limits": { + "type": "object", "additionalProperties": false, + "required": ["max_candidates", "max_wall_seconds", "max_memory_bytes", "max_processes"], + "properties": { + "max_candidates": {"type": "integer", "minimum": 1}, + "max_wall_seconds": {"type": "number", "exclusiveMinimum": 0}, + "max_memory_bytes": {"type": "integer", "minimum": 1}, + "max_processes": {"type": "integer", "minimum": 1} + } + } + } + }, + "partitions": { + "type": "object", "additionalProperties": false, + "required": ["development_id", "selection_id", "final_evaluation_id", "holdout_contaminated", "access_policy_ids"], + "properties": { + "development_id": {"$ref": "#/$defs/id"}, + "selection_id": {"$ref": "#/$defs/id"}, + "final_evaluation_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "holdout_contaminated": {"type": "boolean"}, + "access_policy_ids": {"$ref": "#/$defs/ids"} + } + }, + "protectedEvidence": { + "type": "object", "additionalProperties": false, + "required": ["evidence_id", "partition_id", "commitment_id", "custodian_id", "custody_class", "disclosed_at", "generator_access", "reuse_claim_ids", "contaminated", "status"], + "properties": { + "evidence_id": {"$ref": "#/$defs/id"}, + "partition_id": {"$ref": "#/$defs/id"}, + "commitment_id": {"$ref": "#/$defs/id"}, + "custodian_id": {"$ref": "#/$defs/id"}, + "custody_class": {"enum": ["developer_withheld", "independent_operator", "organizationally_independent"]}, + "disclosed_at": {"oneOf": [{"type": "null"}, {"type": "string", "format": "date-time"}]}, + "generator_access": {"type": "boolean"}, + "reuse_claim_ids": {"$ref": "#/$defs/ids"}, + "contaminated": {"type": "boolean"}, + "status": {"$ref": "#/$defs/status"} + } + }, + "evaluator": { + "type": "object", "additionalProperties": false, + "required": ["evaluator_id", "purpose", "authority_id", "executable_id", "configuration_id", "environment_id", "independent", "operator_independence", "organizational_independence", "regression_corpus_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "purpose": {"enum": ["development", "selection", "holdout", "independent"]}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "independent": {"type": "boolean"}, + "operator_independence": {"$ref": "#/$defs/status"}, + "organizational_independence": {"$ref": "#/$defs/status"}, + "regression_corpus_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + }, + "result": { + "type": "object", "additionalProperties": false, + "required": ["evaluator_id", "gate_id", "partition_id", "required", "status", "evidence_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "gate_id": {"$ref": "#/$defs/id"}, + "partition_id": {"$ref": "#/$defs/id"}, + "required": {"type": "boolean"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + }, + "candidate": { + "type": "object", "additionalProperties": false, + "required": ["candidate_id", "parent_ids", "epoch_id", "generator_id", "generation_sequence", "materially_evaluated", "retained", "build_status", "disposition", "objective_value", "evaluator_results"], + "properties": { + "candidate_id": {"$ref": "#/$defs/id"}, + "parent_ids": {"$ref": "#/$defs/ids"}, + "epoch_id": {"$ref": "#/$defs/id"}, + "generator_id": {"$ref": "#/$defs/id"}, + "generation_sequence": {"type": "integer", "minimum": 0}, + "materially_evaluated": {"type": "boolean"}, + "retained": {"type": "boolean"}, + "build_status": {"$ref": "#/$defs/status"}, + "disposition": {"enum": ["rejected", "retained", "promoted", "selected"]}, + "objective_value": {"type": ["number", "null"]}, + "evaluator_results": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/result"}} + } + }, + "candidateAggregate": { + "type": "object", "additionalProperties": false, + "required": ["aggregate_id", "predeclared_rule_id", "count", "reason_class", "generator_id", "configuration_id", "sequence_start", "sequence_end", "digest"], + "properties": { + "aggregate_id": {"$ref": "#/$defs/id"}, + "predeclared_rule_id": {"$ref": "#/$defs/id"}, + "count": {"type": "integer", "minimum": 1}, + "reason_class": {"$ref": "#/$defs/text"}, + "generator_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "sequence_start": {"type": "integer", "minimum": 0}, + "sequence_end": {"type": "integer", "minimum": 0}, + "digest": {"$ref": "#/$defs/hash"} + } + }, + "selection": { + "type": "object", "additionalProperties": false, + "required": ["policy_id", "selection_epoch_id", "selected_candidate_id", "rule_recorded_before_final_evaluation", "unknown_policy", "minimum_useful_benefit_met", "hard_gates_passed", "rationale", "human_review"], + "properties": { + "policy_id": {"$ref": "#/$defs/id"}, + "selection_epoch_id": {"$ref": "#/$defs/id"}, + "selected_candidate_id": {"$ref": "#/$defs/id"}, + "rule_recorded_before_final_evaluation": {"type": "boolean"}, + "unknown_policy": {"enum": ["reject", "human_review"]}, + "minimum_useful_benefit_met": {"type": "boolean"}, + "hard_gates_passed": {"type": "boolean"}, + "rationale": {"$ref": "#/$defs/text"}, + "human_review": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", "additionalProperties": false, + "required": ["reviewer_id", "decision", "rationale"], + "properties": { + "reviewer_id": {"$ref": "#/$defs/id"}, + "decision": {"enum": ["accept_with_unknown", "reject"]}, + "rationale": {"$ref": "#/$defs/text"} + } + } + ] + } + } + }, + "reproducibility": { + "type": "object", "additionalProperties": false, + "required": ["class", "seeds_preserved", "protocol", "measurement_repetitions", "comparison_statistic", "acceptance_bounds", "failure_treatment"], + "properties": { + "class": {"enum": ["EXACT", "SEEDED", "STATISTICAL", "DISTRIBUTIONAL", "NONE"]}, + "seeds_preserved": {"type": "boolean"}, + "protocol": {"$ref": "#/$defs/text"}, + "measurement_repetitions": {"type": "integer", "minimum": 1}, + "comparison_statistic": {"$ref": "#/$defs/text"}, + "acceptance_bounds": {"$ref": "#/$defs/text"}, + "failure_treatment": {"$ref": "#/$defs/text"} + } + }, + "epoch": { + "type": "object", "additionalProperties": false, + "required": ["epoch_id", "parent_epoch_id", "toolchain_id", "corpus_id", "objective_id", "contract_id", "threshold_policy_id", "development_partition_id", "final_partition_id", "change_evidence_ids", "regression_fixture_ids"], + "properties": { + "epoch_id": {"$ref": "#/$defs/id"}, + "parent_epoch_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "toolchain_id": {"$ref": "#/$defs/id"}, + "corpus_id": {"$ref": "#/$defs/id"}, + "objective_id": {"$ref": "#/$defs/id"}, + "contract_id": {"$ref": "#/$defs/id"}, + "threshold_policy_id": {"$ref": "#/$defs/id"}, + "development_partition_id": {"$ref": "#/$defs/id"}, + "final_partition_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]}, + "change_evidence_ids": {"$ref": "#/$defs/ids"}, + "regression_fixture_ids": {"$ref": "#/$defs/ids"} + } + }, + "mncsBinding": { + "type": "object", "additionalProperties": false, + "required": ["result_or_package_id", "mncs_version", "candidate_id", "contract_id", "environment_id", "status"], + "properties": { + "result_or_package_id": {"$ref": "#/$defs/id"}, + "mncs_version": {"enum": ["0.1", "0.1.1", "0.2", "0.3-rc.1"]}, + "candidate_id": {"$ref": "#/$defs/id"}, + "contract_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "status": {"$ref": "#/$defs/status"} + } + }, + "releaseControls": { + "type": "object", "additionalProperties": false, + "required": ["release_artifact_id", "release_environment_id", "release_authority_id", "build_procedure_id", "package_procedure_id", "monitoring", "rollback", "regeneration_or_replacement", "retirement"], + "properties": { + "release_artifact_id": {"$ref": "#/$defs/id"}, + "release_environment_id": {"$ref": "#/$defs/id"}, + "release_authority_id": {"$ref": "#/$defs/id"}, + "build_procedure_id": {"$ref": "#/$defs/id"}, + "package_procedure_id": {"$ref": "#/$defs/id"}, + "monitoring": { + "type": "object", "additionalProperties": false, + "required": ["signal_ids", "threshold_policy_id", "status"], + "properties": { + "signal_ids": {"$ref": "#/$defs/ids"}, + "threshold_policy_id": {"$ref": "#/$defs/id"}, + "status": {"$ref": "#/$defs/status"} + } + }, + "rollback": { + "type": "object", "additionalProperties": false, + "required": ["artifact_id", "procedure_id", "test_status", "evidence_id"], + "properties": { + "artifact_id": {"$ref": "#/$defs/id"}, + "procedure_id": {"$ref": "#/$defs/id"}, + "test_status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + }, + "regeneration_or_replacement": { + "type": "object", "additionalProperties": false, + "required": ["procedure_id", "performed_at", "status", "evidence_id", "replacement_candidate_id"], + "properties": { + "procedure_id": {"$ref": "#/$defs/id"}, + "performed_at": {"type": "string", "format": "date-time"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"}, + "replacement_candidate_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + }, + "retirement": { + "type": "object", "additionalProperties": false, + "required": ["trigger_ids", "retired", "retired_at", "reason", "replacement_candidate_id"], + "properties": { + "trigger_ids": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/id"}}, + "retired": {"type": "boolean"}, + "retired_at": {"oneOf": [{"type": "null"}, {"type": "string", "format": "date-time"}]}, + "reason": {"type": ["string", "null"], "minLength": 1}, + "replacement_candidate_id": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/id"}]} + } + } + } + }, + "extensions": { + "type": "object", + "propertyNames": {"pattern": "^[a-z][a-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._-]*$"}, + "additionalProperties": true + } + } +} diff --git a/src/mncds_validator/resources/schemas/mncds-development-record.schema.json b/src/mncds_validator/resources/schemas/mncds-development-record.schema.json new file mode 100644 index 0000000..fce00ad --- /dev/null +++ b/src/mncds_validator/resources/schemas/mncds-development-record.schema.json @@ -0,0 +1,344 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mncs.dev/schema/mncds/0.1/mncds-development-record.schema.json", + "title": "MNCDS 0.1 development record", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "mncds_version", + "record_id", + "profile", + "epoch_id", + "created_at", + "charter", + "roles", + "generator", + "partitions", + "evaluators", + "candidates", + "selection", + "reproducibility", + "extensions" + ], + "properties": { + "schema_version": {"const": "0.1"}, + "mncds_version": {"const": "0.1-draft"}, + "record_id": {"$ref": "#/$defs/id"}, + "profile": {"enum": ["MNCDS-D1", "MNCDS-D2", "MNCDS-D3", "MNCDS-D4"]}, + "epoch_id": {"$ref": "#/$defs/id"}, + "supersedes_record_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "created_at": {"type": "string", "format": "date-time"}, + "charter": {"$ref": "#/$defs/charter"}, + "roles": { + "type": "array", + "minItems": 6, + "items": {"$ref": "#/$defs/role"} + }, + "generator": {"$ref": "#/$defs/generator"}, + "partitions": {"$ref": "#/$defs/partitions"}, + "evaluators": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/evaluator"} + }, + "candidates": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/candidate"} + }, + "selection": {"$ref": "#/$defs/selection"}, + "reproducibility": {"$ref": "#/$defs/reproducibility"}, + "mncs_binding": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/mncsBinding"} + ] + }, + "release_controls": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/releaseControls"} + ] + }, + "extensions": {"$ref": "#/$defs/extensions"} + }, + "$defs": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "status": {"enum": ["PASS", "FAIL", "UNKNOWN"]}, + "extensions": { + "type": "object", + "propertyNames": {"pattern": "^[a-z][a-z0-9.-]*:[A-Za-z0-9][A-Za-z0-9._-]*$"}, + "additionalProperties": true + }, + "charter": { + "type": "object", + "additionalProperties": false, + "required": [ + "problem_statement", + "intended_use", + "contract_id", + "baseline_id", + "environment_id", + "threat_model_id", + "objective", + "selection_policy_id", + "planned_mncs_level", + "hard_rejection_gates" + ], + "properties": { + "problem_statement": {"type": "string", "minLength": 1}, + "intended_use": {"type": "string", "minLength": 1}, + "contract_id": {"$ref": "#/$defs/id"}, + "baseline_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"}, + "threat_model_id": {"$ref": "#/$defs/id"}, + "objective": { + "type": "object", + "additionalProperties": false, + "required": ["objective_id", "metric", "unit", "direction", "minimum_useful_benefit"], + "properties": { + "objective_id": {"$ref": "#/$defs/id"}, + "metric": {"type": "string", "minLength": 1}, + "unit": {"type": "string", "minLength": 1}, + "direction": {"enum": ["minimize", "maximize"]}, + "minimum_useful_benefit": {"type": "number"} + } + }, + "selection_policy_id": {"$ref": "#/$defs/id"}, + "planned_mncs_level": { + "type": ["string", "null"], + "enum": [null, "MNCS-L1", "MNCS-L2", "MNCS-L3", "MNCS-L4", "MNCS-L5"] + }, + "hard_rejection_gates": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + } + } + }, + "role": { + "type": "object", + "additionalProperties": false, + "required": ["role", "authority_id", "executable_id"], + "properties": { + "role": { + "enum": [ + "contract_authority", + "generator_authority", + "evaluator_authority", + "selection_authority", + "release_authority", + "independent_reviewer" + ] + }, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"} + } + }, + "generator": { + "type": "object", + "additionalProperties": false, + "required": ["generator_id", "configuration_id", "authority_id", "executable_id", "permissions", "resource_limits"], + "properties": { + "generator_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "permissions": { + "type": "object", + "additionalProperties": false, + "required": [ + "modify_contract", + "modify_baseline", + "modify_evaluators", + "modify_selection_policy", + "modify_thresholds", + "access_protected_holdout", + "network_access" + ], + "properties": { + "modify_contract": {"type": "boolean"}, + "modify_baseline": {"type": "boolean"}, + "modify_evaluators": {"type": "boolean"}, + "modify_selection_policy": {"type": "boolean"}, + "modify_thresholds": {"type": "boolean"}, + "access_protected_holdout": {"type": "boolean"}, + "network_access": {"type": "boolean"} + } + }, + "resource_limits": { + "type": "object", + "additionalProperties": false, + "required": ["max_candidates", "max_wall_seconds"], + "properties": { + "max_candidates": {"type": "integer", "minimum": 1}, + "max_wall_seconds": {"type": "number", "exclusiveMinimum": 0} + } + } + } + }, + "partitions": { + "type": "object", + "additionalProperties": false, + "required": ["development_id", "selection_id", "holdout_id", "holdout_contaminated"], + "properties": { + "development_id": {"$ref": "#/$defs/id"}, + "selection_id": {"$ref": "#/$defs/id"}, + "holdout_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "holdout_contaminated": {"type": "boolean"} + } + }, + "evaluator": { + "type": "object", + "additionalProperties": false, + "required": ["evaluator_id", "purpose", "authority_id", "executable_id", "configuration_id", "independent", "regression_corpus_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "purpose": {"enum": ["development", "selection", "holdout", "independent"]}, + "authority_id": {"$ref": "#/$defs/id"}, + "executable_id": {"$ref": "#/$defs/id"}, + "configuration_id": {"$ref": "#/$defs/id"}, + "independent": {"type": "boolean"}, + "regression_corpus_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"} + } + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_id", + "parent_ids", + "generator_id", + "build_status", + "disposition", + "objective_value", + "evaluator_results" + ], + "properties": { + "candidate_id": {"$ref": "#/$defs/id"}, + "parent_ids": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + }, + "generator_id": {"$ref": "#/$defs/id"}, + "build_status": {"$ref": "#/$defs/status"}, + "disposition": {"enum": ["rejected", "retained", "promoted", "selected"]}, + "objective_value": {"type": ["number", "null"]}, + "evaluator_results": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["evaluator_id", "gate_id", "required", "status", "evidence_id"], + "properties": { + "evaluator_id": {"$ref": "#/$defs/id"}, + "gate_id": {"$ref": "#/$defs/id"}, + "required": {"type": "boolean"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + } + } + } + }, + "selection": { + "type": "object", + "additionalProperties": false, + "required": [ + "policy_id", + "selected_candidate_id", + "rule_recorded_before_holdout", + "unknown_policy", + "minimum_useful_benefit_met", + "rationale", + "human_review" + ], + "properties": { + "policy_id": {"$ref": "#/$defs/id"}, + "selected_candidate_id": {"$ref": "#/$defs/id"}, + "rule_recorded_before_holdout": {"type": "boolean"}, + "unknown_policy": {"enum": ["reject", "human_review"]}, + "minimum_useful_benefit_met": {"type": "boolean"}, + "rationale": {"type": "string", "minLength": 1}, + "human_review": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["reviewer_id", "decision", "rationale"], + "properties": { + "reviewer_id": {"$ref": "#/$defs/id"}, + "decision": {"enum": ["accept_with_unknown", "reject"]}, + "rationale": {"type": "string", "minLength": 1} + } + } + ] + } + } + }, + "reproducibility": { + "type": "object", + "additionalProperties": false, + "required": ["class", "seeds_preserved", "protocol", "measurement_repetitions"], + "properties": { + "class": {"enum": ["EXACT", "SEEDED", "STATISTICAL", "DISTRIBUTIONAL", "NONE"]}, + "seeds_preserved": {"type": "boolean"}, + "protocol": {"type": "string", "minLength": 1}, + "measurement_repetitions": {"type": "integer", "minimum": 1} + } + }, + "mncsBinding": { + "type": "object", + "additionalProperties": false, + "required": ["manifest_or_package_id", "candidate_id", "contract_id", "environment_id"], + "properties": { + "manifest_or_package_id": {"$ref": "#/$defs/id"}, + "candidate_id": {"$ref": "#/$defs/id"}, + "contract_id": {"$ref": "#/$defs/id"}, + "environment_id": {"$ref": "#/$defs/id"} + } + }, + "releaseControls": { + "type": "object", + "additionalProperties": false, + "required": [ + "release_artifact_id", + "rollback_artifact_id", + "rollback_test_status", + "monitoring_thresholds", + "regeneration_drill", + "retirement_triggers" + ], + "properties": { + "release_artifact_id": {"$ref": "#/$defs/id"}, + "rollback_artifact_id": {"$ref": "#/$defs/id"}, + "rollback_test_status": {"$ref": "#/$defs/status"}, + "monitoring_thresholds": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + }, + "regeneration_drill": { + "type": "object", + "additionalProperties": false, + "required": ["performed_at", "status", "evidence_id"], + "properties": { + "performed_at": {"type": "string", "format": "date-time"}, + "status": {"$ref": "#/$defs/status"}, + "evidence_id": {"$ref": "#/$defs/id"} + } + }, + "retirement_triggers": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1} + } + } + } + } +} diff --git a/src/mncds_validator/schemas.py b/src/mncds_validator/schemas.py new file mode 100644 index 0000000..14c3ea5 --- /dev/null +++ b/src/mncds_validator/schemas.py @@ -0,0 +1,62 @@ +"""Packaged MNCDS JSON Schema discovery and validation.""" + +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import math +from importlib.resources import files +from typing import Any, cast + +from jsonschema import Draft202012Validator, FormatChecker +from jsonschema.exceptions import SchemaError + +from .errors import SchemaNotFoundError + +SCHEMA_NAMES = { + "mncds-development-record": "mncds-development-record.schema.json", + "mncds-development-record-0.1": "mncds-development-record-0.1.schema.json", +} + + +def load_schema(name: str) -> dict[str, Any]: + filename = SCHEMA_NAMES.get(name, name) + if filename not in SCHEMA_NAMES.values(): + raise SchemaNotFoundError(f"unknown schema: {name}") + candidate = files("mncds_validator.resources.schemas").joinpath(filename) + if not candidate.is_file(): + raise SchemaNotFoundError(f"schema is not installed: {filename}") + value: Any = json.loads(candidate.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise SchemaError(f"schema {name} is not an object") + Draft202012Validator.check_schema(value) + return cast(dict[str, Any], value) + + +def _nonfinite_paths(value: Any, path: str = "$") -> list[str]: + if isinstance(value, float) and not math.isfinite(value): + return [f"{path}: nonfinite numbers are forbidden"] + if isinstance(value, dict): + return [ + finding + for key, child in value.items() + for finding in _nonfinite_paths(child, f"{path}/{key}") + ] + if isinstance(value, list): + return [ + finding + for index, child in enumerate(value) + for finding in _nonfinite_paths(child, f"{path}/{index}") + ] + return [] + + +def schema_errors(instance: Any, name: str) -> list[str]: + validator = Draft202012Validator(load_schema(name), format_checker=FormatChecker()) + errors = sorted(validator.iter_errors(instance), key=lambda item: list(item.absolute_path)) + rendered = _nonfinite_paths(instance) + for error in errors: + location = "/".join(str(part) for part in error.absolute_path) or "$" + rendered.append(f"{location}: {error.message}") + return sorted(rendered) diff --git a/src/mncds_validator/validation.py b/src/mncds_validator/validation.py new file mode 100644 index 0000000..a96b8c7 --- /dev/null +++ b/src/mncds_validator/validation.py @@ -0,0 +1,46 @@ +"""Safe JSON object loading for MNCDS development records.""" + +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .errors import ManifestError + +JSON_MAX_BYTES = 4 * 1024 * 1024 + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate object key: {key}") + result[key] = value + return result + + +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-standard JSON constant is forbidden: {value}") + + +def load_json_object(path: Path, *, expected_sha256: str | None = None) -> dict[str, Any]: + del expected_sha256 + try: + content = path.read_bytes() + if len(content) > JSON_MAX_BYTES: + raise ManifestError(f"JSON object exceeds {JSON_MAX_BYTES} bytes: {path}") + value: Any = json.loads( + content.decode("utf-8"), + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_json_constant, + ) + except ManifestError: + raise + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc: + raise ManifestError(f"cannot read JSON object {path}: {exc}") from exc + if not isinstance(value, dict): + raise ManifestError(f"expected a JSON object: {path}") + return value diff --git a/tests/test_mncds.py b/tests/test_mncds.py new file mode 100644 index 0000000..b6bf0af --- /dev/null +++ b/tests/test_mncds.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 + +import copy +import json +from pathlib import Path +from typing import Any + +from mncds_validator.mncds import validate_development_value +from mncds_validator.schemas import schema_errors + +ROOT = Path(__file__).resolve().parents[1] + + +def _d4_record() -> dict[str, Any]: + path = ROOT / "examples/mncds-d4/development-record.json" + value = json.loads(path.read_text(encoding="utf-8")) + assert isinstance(value, dict) + return value + + +def _d1_record() -> dict[str, Any]: + value = copy.deepcopy(_d4_record()) + value["profile"] = "MNCDS-D1" + value["charter"]["planned_mncs_level"] = None + value["partitions"]["holdout_id"] = None + value["evaluators"] = [value["evaluators"][0]] + value["candidates"][1]["evaluator_results"] = value["candidates"][1]["evaluator_results"][:3] + value["reproducibility"] = { + "class": "NONE", + "seeds_preserved": False, + "protocol": "Candidate identities and generator configuration are preserved.", + "measurement_repetitions": 1, + } + value["mncs_binding"] = None + value["release_controls"] = None + return value + + +def _d2_record() -> dict[str, Any]: + value = _d1_record() + value["profile"] = "MNCDS-D2" + value["reproducibility"] = { + "class": "SEEDED", + "seeds_preserved": True, + "protocol": "Replay the pinned environment and recorded seed set.", + "measurement_repetitions": 3, + } + return value + + +def _d3_record() -> dict[str, Any]: + value = copy.deepcopy(_d4_record()) + value["profile"] = "MNCDS-D3" + value["release_controls"] = None + return value + + +def test_development_record_schema_is_packaged() -> None: + assert schema_errors(_d4_record(), "mncds-development-record") == [] + + +def test_cumulative_profiles_pass() -> None: + for record in (_d1_record(), _d2_record(), _d3_record(), _d4_record()): + report = validate_development_value(record) + assert report.valid, report.as_dict() + assert report.computed_status == "PASS" + + +def test_generator_cannot_modify_evaluator_or_threshold() -> None: + record = _d4_record() + record["generator"]["permissions"]["modify_evaluators"] = True + record["generator"]["permissions"]["modify_thresholds"] = True + report = validate_development_value(record) + assert not report.valid + assert {issue.code for issue in report.issues} >= {"generator-authority-violation"} + + +def test_candidate_lineage_cycle_is_rejected() -> None: + record = _d4_record() + record["candidates"][0]["parent_ids"] = ["candidate-b"] + report = validate_development_value(record) + assert not report.valid + assert "lineage-cycle" in {issue.code for issue in report.issues} + + +def test_required_unknown_cannot_be_promoted_under_reject_policy() -> None: + record = _d4_record() + record["candidates"][1]["evaluator_results"][0]["status"] = "UNKNOWN" + report = validate_development_value(record) + assert not report.valid + assert "unknown-promoted" in {issue.code for issue in report.issues} + + +def test_explicit_human_review_preserves_unknown_status() -> None: + record = _d1_record() + record["candidates"][1]["evaluator_results"][0]["status"] = "UNKNOWN" + record["selection"]["unknown_policy"] = "human_review" + record["selection"]["human_review"] = { + "reviewer_id": "authority-contract-team", + "decision": "accept_with_unknown", + "rationale": "Proceed only as an experimental record; no PASS claim is made.", + } + report = validate_development_value(record) + assert report.valid + assert report.computed_status == "UNKNOWN" + + +def test_d3_requires_independent_authority_and_executable() -> None: + record = _d3_record() + record["evaluators"][1]["authority_id"] = "authority-generation-team" + record["evaluators"][1]["executable_id"] = "generator-runner-v2" + report = validate_development_value(record) + assert not report.valid + codes = {issue.code for issue in report.issues} + assert "independence-authority-conflict" in codes + assert "independence-executable-conflict" in codes + + +def test_mncs_binding_must_match_selected_candidate_and_charter() -> None: + record = _d3_record() + record["mncs_binding"]["candidate_id"] = "candidate-a" + record["mncs_binding"]["environment_id"] = "other-environment" + report = validate_development_value(record) + assert not report.valid + assert "mncs-binding-mismatch" in {issue.code for issue in report.issues} + + +def test_d4_requires_passing_rollback_and_regeneration_drill() -> None: + record = _d4_record() + record["release_controls"]["rollback_test_status"] = "UNKNOWN" + record["release_controls"]["regeneration_drill"]["status"] = "FAIL" + report = validate_development_value(record) + assert not report.valid + codes = {issue.code for issue in report.issues} + assert "rollback-not-tested" in codes + assert "regeneration-drill-failed" in codes From e577d641ac5c8b055e338c74faface2fb247f149 Mon Sep 17 00:00:00 2001 From: epi13 Date: Fri, 14 Aug 2026 17:47:40 -0800 Subject: [PATCH 2/3] Complete the MNCDS bootstrap check after the specification import. The repository is now the canonical MNCDS home, so CI no longer requires the old bootstrap warning. Add a local normative-language note so 0.1-rc.1 does not depend on an MNCS checkout. --- scripts/check_repository.py | 2 +- spec/MNCDS-v0.1-rc.1.md | 2 +- spec/normative-language.md | 27 +++++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 spec/normative-language.md diff --git a/scripts/check_repository.py b/scripts/check_repository.py index 312c3a9..640b28f 100644 --- a/scripts/check_repository.py +++ b/scripts/check_repository.py @@ -100,7 +100,7 @@ def main() -> int: required_phrases = [ "independently versioned, governed, and released", "Machine-Native Complexity Standard (MNCS)", - "Repository bootstrap in progress", + "canonical home of MNCDS", ] for phrase in required_phrases: if phrase not in text: diff --git a/spec/MNCDS-v0.1-rc.1.md b/spec/MNCDS-v0.1-rc.1.md index 6220ef2..cc91fbc 100644 --- a/spec/MNCDS-v0.1-rc.1.md +++ b/spec/MNCDS-v0.1-rc.1.md @@ -3,7 +3,7 @@ # Machine-Native Complexity Development Specification 0.1-rc.1 Status: release-candidate proposal under Draft RFC 0004. It is not Accepted or Final. -Normative terms use RFC 2119/8174 meanings from `normative-language.md`. +Normative terms use RFC 2119/8174 meanings from [`normative-language.md`](normative-language.md). ## 1. Scope and relationship to MNCS diff --git a/spec/normative-language.md b/spec/normative-language.md new file mode 100644 index 0000000..50ffa7f --- /dev/null +++ b/spec/normative-language.md @@ -0,0 +1,27 @@ +# Normative language + + + +The terms in this document have these MNCDS 0.1 meanings when capitalized: + +- **MUST**, **REQUIRED**, and **SHALL** express an unconditional requirement. +- **MUST NOT** and **SHALL NOT** express an unconditional prohibition. +- **SHOULD** and **RECOMMENDED** express a strong recommendation. A deviation needs a + recorded reason and risk assessment. +- **SHOULD NOT** expresses a strongly discouraged action. A deviation needs the same + record. +- **MAY** and **OPTIONAL** identify a permitted choice. +- **PASS** means the declared method produced sufficient evidence for the requirement + within its stated assumptions and bounds. +- **FAIL** means evidence demonstrates a violation or a required gate did not pass. +- **UNKNOWN** means the available method could not establish either PASS or FAIL, + including tool error, unsupported syntax, exceeded bounds, ambiguity, or missing + evidence. + +UNKNOWN MUST NOT be converted to PASS by omission, truthiness, aggregation, provider +reputation, or absence of a counterexample. An acceptance policy MAY reject UNKNOWN +or route it to explicit human review; it MUST NOT silently accept it. + +Normative text uses these words only in uppercase. Lowercase words are descriptive. From 3187d1e78cbaf907b416c470399b95fc4454149c Mon Sep 17 00:00:00 2001 From: epi13 Date: Fri, 14 Aug 2026 17:47:59 -0800 Subject: [PATCH 3/3] List the local MNCDS normative-language note in the spec index. --- spec/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/README.md b/spec/README.md index 3fb5f42..e8e46b0 100644 --- a/spec/README.md +++ b/spec/README.md @@ -4,6 +4,7 @@ This directory is the canonical home of normative MNCDS specification text. Current files: +- [`normative-language.md`](normative-language.md) — local MUST/SHOULD/PASS/FAIL/UNKNOWN vocabulary - [`MNCDS-v0.1-draft.md`](MNCDS-v0.1-draft.md) — historical draft lifecycle and cumulative profiles - [`MNCDS-v0.1-rc.1.md`](MNCDS-v0.1-rc.1.md) — current independently consumable release candidate - [`MNCDS-v0.1-records-and-decisions.md`](MNCDS-v0.1-records-and-decisions.md) — aggregate record and decision semantics