From dded176e015db2584191702421bf36c0b2b11107 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:20:05 +0800 Subject: [PATCH 01/45] ENH: reproducible Monte Carlo via per-simulation-index seeding MonteCarlo seeded the stochastic models per worker in parallel mode (from a fresh, unseeded SeedSequence) and once at construction in serial mode, so the sampled inputs depended on the execution mode and the worker count, and parallel runs were not reproducible run to run. Add a keyword-only random_seed to simulate() (SPEC 7 style: accepts an int, a SeedSequence, or a Generator; None keeps the previous fresh-entropy behavior). Spawn one child seed per simulation index from that root and reseed the stochastic models from child_seeds[i] before simulation i. SeedSequence.spawn is prefix-stable, so index i maps to the same seed regardless of which worker runs it, making the inputs identical across serial, parallel(2) and parallel(N). Each index seed is split three ways so the environment, rocket and flight draw from independent streams rather than sharing one. The serial index field now counts from 0 to match the parallel path. Both changes alter the numbers a fixed seed produces, so stored baselines regenerate. Adds tests/unit/simulation/test_monte_carlo_determinism.py: serial reproducibility, worker invariance (serial == parallel(2) == parallel(4)), and the None-seed path. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 94 +++++++-- .../test_monte_carlo_determinism.py | 193 ++++++++++++++++++ 2 files changed, 269 insertions(+), 18 deletions(-) create mode 100644 tests/unit/simulation/test_monte_carlo_determinism.py diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 7cbb37fc9..f6ba11e4f 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -171,6 +171,8 @@ def simulate( append=False, parallel=False, n_workers=None, + *, + random_seed=None, **kwargs, ): """ @@ -190,6 +192,15 @@ def simulate( number of workers will be equal to the number of CPUs available. A minimum of 2 workers is required for parallel mode. Default is None. + random_seed : int, numpy.random.SeedSequence, numpy.random.Generator, optional + Root seed for the run. When provided, the sampled inputs are + reproducible and identical across serial and parallel execution + and across any number of workers, because each simulation index + draws from its own child stream spawned from this root + (``SeedSequence(random_seed).spawn(number_of_simulations)``). It + accepts an int, a ``SeedSequence`` or a ``Generator``. Default is + None, which draws fresh entropy (not reproducible), preserving the + previous behavior. kwargs : dict Custom arguments for simulation export of the ``inputs`` file. Options are: @@ -229,9 +240,9 @@ def simulate( self.__setup_files(append) if parallel: - self.__run_in_parallel(n_workers) + self.__run_in_parallel(n_workers, random_seed) else: - self.__run_in_serial() + self.__run_in_serial(random_seed) self.__terminate_simulation() @@ -268,10 +279,46 @@ def __setup_files(self, append): except OSError as error: raise OSError(f"Error creating files: {error}") from error - def __run_in_serial(self): + @staticmethod + def __root_seed_sequence(random_seed): + """Return a ``SeedSequence`` root from a flexible seed argument. + + Accepts what the scientific-Python SPEC 7 seeding convention accepts + (an int, a ``SeedSequence``, a ``Generator`` or ``BitGenerator``, or + None for fresh entropy) and returns a ``SeedSequence`` so it can be + spawned into one independent child stream per simulation index. + """ + if isinstance(random_seed, np.random.SeedSequence): + return random_seed + if isinstance(random_seed, np.random.Generator): + return random_seed.bit_generator.seed_seq + if isinstance(random_seed, np.random.BitGenerator): + return random_seed.seed_seq + return np.random.SeedSequence(random_seed) + + def __seed_simulation(self, child_seed): + """Reseed the stochastic models for a single simulation index. + + The per-index child seed is split three ways so the environment, + rocket and flight draw from independent streams instead of sharing + one. Seeding per simulation index (not per worker) is what makes the + sampled inputs invariant to the execution mode and to the number of + workers. + """ + env_seed, rocket_seed, flight_seed = child_seed.spawn(3) + self.environment._set_stochastic(env_seed) + self.rocket._set_stochastic(rocket_seed) + self.flight._set_stochastic(flight_seed) + + def __run_in_serial(self, random_seed=None): # pylint: disable=too-many-statements """ Runs the monte carlo simulation in serial mode. + Parameters + ---------- + random_seed : int, SeedSequence, Generator, optional + Root seed for the run. See ``simulate``. + Returns ------- None @@ -281,14 +328,18 @@ def __run_in_serial(self): n_simulations=self.number_of_simulations, start_time=time(), ) + child_seeds = self.__root_seed_sequence(random_seed).spawn( + self.number_of_simulations + ) try: while sim_monitor.keep_simulating(): - sim_monitor.increment() + sim_idx = sim_monitor.increment() - 1 inputs_json, outputs_json = "", "" + self.__seed_simulation(child_seeds[sim_idx]) flight = self.__run_single_simulation() - inputs_json = self.__evaluate_flight_inputs(sim_monitor.count) - outputs_json = self.__evaluate_flight_outputs(flight, sim_monitor.count) + inputs_json = self.__evaluate_flight_inputs(sim_idx) + outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) with open(self.input_file, "a", encoding="utf-8") as f: f.write(inputs_json) @@ -310,7 +361,7 @@ def __run_in_serial(self): f.write(inputs_json) raise error - def __run_in_parallel(self, n_workers=None): + def __run_in_parallel(self, n_workers=None, random_seed=None): """ Runs the monte carlo simulation in parallel. @@ -319,6 +370,8 @@ def __run_in_parallel(self, n_workers=None): n_workers: int, optional Number of workers to be used. If None, the number of workers will be equal to the number of CPUs available. Default is None. + random_seed : int, SeedSequence, Generator, optional + Root seed for the run. See ``simulate``. Returns ------- @@ -340,13 +393,19 @@ def __run_in_parallel(self, n_workers=None): ) processes = [] - seeds = np.random.SeedSequence().spawn(n_workers) + # One independent child seed per simulation index (not per + # worker), shared with every worker. The shared counter assigns + # indices, and index i always seeds from child_seeds[i], so the + # sampled inputs do not depend on the number of workers. + child_seeds = self.__root_seed_sequence(random_seed).spawn( + self.number_of_simulations + ) - for seed in seeds: + for _ in range(n_workers): sim_producer = multiprocess.Process( target=self.__sim_producer, args=( - seed, + child_seeds, sim_monitor, mutex, simulation_error_event, @@ -388,13 +447,16 @@ def __validate_number_of_workers(self, n_workers): raise ValueError("Number of workers must be at least 2 for parallel mode.") return n_workers - def __sim_producer(self, seed, sim_monitor, mutex, error_event): # pylint: disable=too-many-statements + def __sim_producer(self, child_seeds, sim_monitor, mutex, error_event): # pylint: disable=too-many-statements """Simulation producer to be used in parallel by multiprocessing. Parameters ---------- - seed : int - The seed to set the random number generator. + child_seeds : list[numpy.random.SeedSequence] + One seed sequence per simulation index. Before each simulation + the worker seeds the stochastic models from + ``child_seeds[sim_idx]``, where ``sim_idx`` comes from the shared + counter, so the inputs are invariant to the number of workers. sim_monitor : _SimMonitor The simulation monitor object to keep track of the simulations. mutex : multiprocess.Lock @@ -403,15 +465,11 @@ def __sim_producer(self, seed, sim_monitor, mutex, error_event): # pylint: disa Event signaling an error occurred during the simulation. """ try: - # Ensure Processes generate different random numbers - self.environment._set_stochastic(seed) - self.rocket._set_stochastic(seed) - self.flight._set_stochastic(seed) - while sim_monitor.keep_simulating(): sim_idx = sim_monitor.increment() - 1 inputs_json, outputs_json = "", "" + self.__seed_simulation(child_seeds[sim_idx]) flight = self.__run_single_simulation() inputs_json = self.__evaluate_flight_inputs(sim_idx) outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py new file mode 100644 index 000000000..c3e0e47e9 --- /dev/null +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -0,0 +1,193 @@ +"""Determinism tests for the ``random_seed`` argument of ``MonteCarlo.simulate``. + +With a fixed ``random_seed`` the generated random *inputs* are reproducible and +identical across serial and parallel execution and across any number of workers. +Each simulation index draws from its own child stream spawned from the run's root +seed, and ``SeedSequence.spawn`` is prefix-stable, so index ``i`` maps to the same +seed regardless of the worker that runs it. + +The trajectory integration (``Flight``) is stubbed so the tests stay fast: worker +invariance is a property of the input sampling, which happens before ``Flight`` is +built. Stubbing the module-level ``Flight`` symbol reaches the parallel workers +only under the ``fork`` start method, so those tests are guarded accordingly. + +A dedicated numpy-only rocket is used so *all* randomness flows through the seeded +numpy generator. List-valued stochastic attributes are sampled with the standard +library ``random.choice`` (an unseeded global generator) which ``random_seed`` +does not govern; the fixture drops the only such attribute (a multi-element +``thrust_source``) so the inputs are byte-for-byte reproducible from the seed. +""" + +import json + +import multiprocess +import pytest + +import rocketpy.simulation.monte_carlo as mc_module +from rocketpy.simulation import MonteCarlo +from rocketpy.stochastic import StochasticRocket, StochasticSolidMotor + +pytestmark = pytest.mark.slow + +requires_fork = pytest.mark.skipif( + multiprocess.get_start_method() != "fork", + reason="stub-based parallel determinism test requires the 'fork' start method", +) + + +class _StubFlight: + """Minimal stand-in for ``Flight`` that skips trajectory integration.""" + + def __init__(self, **kwargs): # accepts and ignores MonteCarlo's Flight kwargs + pass + + def __getattr__(self, name): + return 0.0 + + +@pytest.fixture +def stochastic_calisto_numpy_only( + cesaroni_m1670, + calisto_robust, + stochastic_nose_cone, + stochastic_trapezoidal_fins, + stochastic_tail, + stochastic_rail_buttons, + stochastic_main_parachute, + stochastic_drogue_parachute, +): + """A ``StochasticRocket`` whose randomness flows entirely through numpy. + + Mirrors the shared ``stochastic_calisto`` fixture but gives the solid motor a + single ``thrust_source`` instead of a multi-element list, so no attribute is + sampled through the unseeded standard-library ``random.choice``. + """ + motor = StochasticSolidMotor( + solid_motor=cesaroni_m1670, + burn_out_time=(4, 0.1), + grains_center_of_mass_position=0.001, + grain_density=50, + grain_separation=1 / 1000, + grain_initial_height=1 / 1000, + grain_initial_inner_radius=0.375 / 1000, + grain_outer_radius=0.375 / 1000, + total_impulse=(6500, 1000), + throat_radius=0.5 / 1000, + nozzle_radius=0.5 / 1000, + nozzle_position=0.001, + ) + rocket = StochasticRocket( + rocket=calisto_robust, + radius=0.0127 / 2000, + mass=(15.426, 0.5, "normal"), + inertia_11=(6.321, 0), + inertia_22=0.01, + inertia_33=0.01, + center_of_mass_without_motor=0, + ) + rocket.add_motor(motor, position=0.001) + rocket.add_nose(stochastic_nose_cone, position=(1.134, 0.001)) + rocket.add_trapezoidal_fins(stochastic_trapezoidal_fins, position=(0.001, "normal")) + rocket.add_tail(stochastic_tail) + rocket.set_rail_buttons( + stochastic_rail_buttons, lower_button_position=(-0.618, 0.001, "normal") + ) + rocket.add_parachute(parachute=stochastic_main_parachute) + rocket.add_parachute(parachute=stochastic_drogue_parachute) + return rocket + + +def _read_inputs_by_index(input_file): + """Read a ``.inputs.txt`` file into ``{index: raw_json_line}``.""" + by_index = {} + with open(input_file, mode="r", encoding="utf-8") as rows: + for line in rows: + line = line.strip() + if not line: + continue + by_index[json.loads(line)["index"]] = line + return by_index + + +def _simulate_inputs( + monkeypatch, tmp_path, environment, rocket, flight, tag, **simulate_kwargs +): + """Run a Monte Carlo with a stubbed ``Flight`` and return inputs by index.""" + monkeypatch.setattr(mc_module, "Flight", _StubFlight) + montecarlo = MonteCarlo( + filename=str(tmp_path / tag), + environment=environment, + rocket=rocket, + flight=flight, + ) + montecarlo.simulate(**simulate_kwargs) + return _read_inputs_by_index(montecarlo.input_file) + + +def test_serial_inputs_are_reproducible( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """Two serial runs with the same random_seed yield identical inputs.""" + models = (stochastic_environment, stochastic_calisto_numpy_only, stochastic_flight) + run_a = _simulate_inputs( + monkeypatch, tmp_path, *models, "a", number_of_simulations=6, random_seed=7 + ) + run_b = _simulate_inputs( + monkeypatch, tmp_path, *models, "b", number_of_simulations=6, random_seed=7 + ) + assert sorted(run_a) == list(range(6)) + assert run_a == run_b + + +@requires_fork +def test_inputs_are_worker_invariant( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """serial == parallel(2) == parallel(4): inputs are bit-identical per index.""" + models = (stochastic_environment, stochastic_calisto_numpy_only, stochastic_flight) + common = {"number_of_simulations": 8, "random_seed": 314159} + + serial = _simulate_inputs(monkeypatch, tmp_path, *models, "serial", **common) + par2 = _simulate_inputs( + monkeypatch, tmp_path, *models, "par2", parallel=True, n_workers=2, **common + ) + par4 = _simulate_inputs( + monkeypatch, tmp_path, *models, "par4", parallel=True, n_workers=4, **common + ) + + expected = list(range(8)) + assert sorted(serial) == expected + assert sorted(par2) == expected + assert sorted(par4) == expected + for index in expected: + assert serial[index] == par2[index], f"serial vs parallel(2) differ at {index}" + assert serial[index] == par4[index], f"serial vs parallel(4) differ at {index}" + + +def test_none_seed_still_runs( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """random_seed=None draws fresh entropy but still exports one record per index.""" + inputs = _simulate_inputs( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, + "none", + number_of_simulations=5, + random_seed=None, + ) + assert sorted(inputs) == list(range(5)) From 57c3c0d43a95a4bf245acdf8da1b0eb1a36fba56 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Thu, 9 Jul 2026 04:49:34 +0800 Subject: [PATCH 02/45] TST: cover Monte Carlo seeding helpers directly The reproducible-seeding change added __root_seed_sequence and __seed_simulation plus the per-index serial and parallel seeding, but the only tests that reached them ran a full Monte Carlo and were marked slow, so the coverage job (which does not pass --runslow) never executed them. Add fast unit tests that drive the two helpers directly: every supported random_seed type normalizes to the same root stream, None draws fresh entropy, existing SeedSequence/Generator/BitGenerator objects are reused rather than copied, and each child seed splits three ways so environment, rocket and flight get independent streams. Move the end-to-end simulate reproducibility tests into tests/integration, next to the existing Monte Carlo simulate test. The serial reproducibility run now lives in the non-slow suite; only the fork-based worker-invariance test stays slow, and it imports multiprocess lazily like the library does. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../test_monte_carlo_determinism.py | 177 ++++++++++ .../test_monte_carlo_determinism.py | 307 ++++++++---------- 2 files changed, 304 insertions(+), 180 deletions(-) create mode 100644 tests/integration/simulation/test_monte_carlo_determinism.py diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py new file mode 100644 index 000000000..b47629a55 --- /dev/null +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -0,0 +1,177 @@ +"""End-to-end determinism tests for ``MonteCarlo.simulate(random_seed=...)``. + +With a fixed ``random_seed`` the generated random *inputs* are reproducible and +identical across serial and parallel execution and across any number of workers. +Each simulation index draws from its own child stream spawned from the run's root +seed, and ``SeedSequence.spawn`` is prefix-stable, so index ``i`` maps to the same +seed regardless of the worker that runs it. (The seed-handling helpers themselves +are unit tested in ``tests/unit/simulation/test_monte_carlo_determinism``.) + +The trajectory integration (``Flight``) is stubbed: worker invariance is a +property of the *input sampling*, which happens before ``Flight`` is built, so a +stub keeps the runs fast while still driving the real serial and parallel loops. +Stubbing the module-level ``Flight`` symbol reaches the parallel workers only +under the ``fork`` start method, so the worker-invariance test skips otherwise and +is marked ``slow`` to match the other Monte Carlo multiprocessing tests. + +A dedicated numpy-only rocket is used so *all* randomness flows through the seeded +numpy generator. List-valued stochastic attributes are sampled with the standard +library ``random.choice`` (an unseeded global generator) which ``random_seed`` +does not govern; the fixture drops the only such attribute (a multi-element +``thrust_source``) so the inputs are byte-for-byte reproducible from the seed. +""" + +import json + +import pytest + +import rocketpy.simulation.monte_carlo as mc_module +from rocketpy.simulation import MonteCarlo +from rocketpy.stochastic import StochasticRocket, StochasticSolidMotor + + +class _StubFlight: + """Minimal stand-in for ``Flight`` that skips trajectory integration.""" + + def __init__(self, **kwargs): # accepts and ignores MonteCarlo's Flight kwargs + pass + + def __getattr__(self, name): + return 0.0 + + +@pytest.fixture +def stochastic_calisto_numpy_only( + cesaroni_m1670, + calisto_robust, + stochastic_nose_cone, + stochastic_trapezoidal_fins, + stochastic_tail, + stochastic_rail_buttons, + stochastic_main_parachute, + stochastic_drogue_parachute, +): + """A ``StochasticRocket`` whose randomness flows entirely through numpy. + + Mirrors the shared ``stochastic_calisto`` fixture but gives the solid motor a + single ``thrust_source`` instead of a multi-element list, so no attribute is + sampled through the unseeded standard-library ``random.choice``. + """ + motor = StochasticSolidMotor( + solid_motor=cesaroni_m1670, + burn_out_time=(4, 0.1), + grains_center_of_mass_position=0.001, + grain_density=50, + grain_separation=1 / 1000, + grain_initial_height=1 / 1000, + grain_initial_inner_radius=0.375 / 1000, + grain_outer_radius=0.375 / 1000, + total_impulse=(6500, 1000), + throat_radius=0.5 / 1000, + nozzle_radius=0.5 / 1000, + nozzle_position=0.001, + ) + rocket = StochasticRocket( + rocket=calisto_robust, + radius=0.0127 / 2000, + mass=(15.426, 0.5, "normal"), + inertia_11=(6.321, 0), + inertia_22=0.01, + inertia_33=0.01, + center_of_mass_without_motor=0, + ) + rocket.add_motor(motor, position=0.001) + rocket.add_nose(stochastic_nose_cone, position=(1.134, 0.001)) + rocket.add_trapezoidal_fins(stochastic_trapezoidal_fins, position=(0.001, "normal")) + rocket.add_tail(stochastic_tail) + rocket.set_rail_buttons( + stochastic_rail_buttons, lower_button_position=(-0.618, 0.001, "normal") + ) + rocket.add_parachute(parachute=stochastic_main_parachute) + rocket.add_parachute(parachute=stochastic_drogue_parachute) + return rocket + + +def _read_inputs_by_index(input_file): + """Read a ``.inputs.txt`` file into ``{index: raw_json_line}``.""" + by_index = {} + with open(input_file, mode="r", encoding="utf-8") as rows: + for line in rows: + line = line.strip() + if not line: + continue + by_index[json.loads(line)["index"]] = line + return by_index + + +def _simulate_inputs( + monkeypatch, tmp_path, environment, rocket, flight, tag, **simulate_kwargs +): + """Run a Monte Carlo with a stubbed ``Flight`` and return inputs by index.""" + monkeypatch.setattr(mc_module, "Flight", _StubFlight) + montecarlo = MonteCarlo( + filename=str(tmp_path / tag), + environment=environment, + rocket=rocket, + flight=flight, + ) + montecarlo.simulate(**simulate_kwargs) + return _read_inputs_by_index(montecarlo.input_file) + + +def test_serial_inputs_are_reproducible( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """Two serial runs with the same seed yield byte-identical inputs per index. + + This drives the serial ``simulate`` path end to end; the flexible seed types + are covered by the unit test of ``__root_seed_sequence``. + """ + models = (stochastic_environment, stochastic_calisto_numpy_only, stochastic_flight) + run_a = _simulate_inputs( + monkeypatch, tmp_path, *models, "a", number_of_simulations=3, random_seed=7 + ) + run_b = _simulate_inputs( + monkeypatch, tmp_path, *models, "b", number_of_simulations=3, random_seed=7 + ) + assert sorted(run_a) == list(range(3)) + assert run_a == run_b + + +@pytest.mark.slow +def test_inputs_are_worker_invariant( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """serial == parallel(2) == parallel(4): inputs are bit-identical per index.""" + multiprocess = pytest.importorskip("multiprocess") + if multiprocess.get_start_method() != "fork": + pytest.skip( + "stub-based parallel determinism test requires the 'fork' start method" + ) + + models = (stochastic_environment, stochastic_calisto_numpy_only, stochastic_flight) + common = {"number_of_simulations": 8, "random_seed": 314159} + + serial = _simulate_inputs(monkeypatch, tmp_path, *models, "serial", **common) + par2 = _simulate_inputs( + monkeypatch, tmp_path, *models, "par2", parallel=True, n_workers=2, **common + ) + par4 = _simulate_inputs( + monkeypatch, tmp_path, *models, "par4", parallel=True, n_workers=4, **common + ) + + expected = list(range(8)) + assert sorted(serial) == expected + assert sorted(par2) == expected + assert sorted(par4) == expected + for index in expected: + assert serial[index] == par2[index], f"serial vs parallel(2) differ at {index}" + assert serial[index] == par4[index], f"serial vs parallel(4) differ at {index}" diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index c3e0e47e9..939980508 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -1,193 +1,140 @@ -"""Determinism tests for the ``random_seed`` argument of ``MonteCarlo.simulate``. - -With a fixed ``random_seed`` the generated random *inputs* are reproducible and -identical across serial and parallel execution and across any number of workers. -Each simulation index draws from its own child stream spawned from the run's root -seed, and ``SeedSequence.spawn`` is prefix-stable, so index ``i`` maps to the same -seed regardless of the worker that runs it. - -The trajectory integration (``Flight``) is stubbed so the tests stay fast: worker -invariance is a property of the input sampling, which happens before ``Flight`` is -built. Stubbing the module-level ``Flight`` symbol reaches the parallel workers -only under the ``fork`` start method, so those tests are guarded accordingly. - -A dedicated numpy-only rocket is used so *all* randomness flows through the seeded -numpy generator. List-valued stochastic attributes are sampled with the standard -library ``random.choice`` (an unseeded global generator) which ``random_seed`` -does not govern; the fixture drops the only such attribute (a multi-element -``thrust_source``) so the inputs are byte-for-byte reproducible from the seed. +"""Unit tests for the Monte Carlo seeding helpers. + +``MonteCarlo.simulate(random_seed=...)`` makes the sampled inputs reproducible by +turning the run's root seed into one independent child stream per simulation +index. Two private helpers do the work: + +* ``__root_seed_sequence`` normalizes the flexible ``random_seed`` argument (int, + ``SeedSequence``, ``Generator``, ``BitGenerator`` or None) into a + ``SeedSequence`` that can be spawned; +* ``__seed_simulation`` splits one per-index child seed three ways so the + environment, rocket and flight draw from independent streams. + +These tests exercise the helpers directly, with no fixtures and no simulation, so +they stay fast. The end-to-end reproducibility of ``simulate`` (serial and across +workers) is covered by ``tests/integration/simulation/test_monte_carlo_determinism``. + +Reaching a name-mangled member is an established pattern in this suite (see +``tests/unit/test_sensitivity.py`` and ``tests/unit/environment/test_environment.py``); +it lets the seeding invariants be asserted without running a Monte Carlo. """ -import json +from types import SimpleNamespace -import multiprocess +import numpy as np import pytest -import rocketpy.simulation.monte_carlo as mc_module from rocketpy.simulation import MonteCarlo -from rocketpy.stochastic import StochasticRocket, StochasticSolidMotor -pytestmark = pytest.mark.slow +_root_seed_sequence = MonteCarlo._MonteCarlo__root_seed_sequence +_seed_simulation = MonteCarlo._MonteCarlo__seed_simulation + + +def _entropy(seed_sequence, n=4): + """A stable, comparable fingerprint of a ``SeedSequence``'s stream.""" + return tuple(int(x) for x in seed_sequence.generate_state(n)) + + +# --------------------------------------------------------------------------- # +# __root_seed_sequence: normalizing the flexible seed argument # +# --------------------------------------------------------------------------- # -requires_fork = pytest.mark.skipif( - multiprocess.get_start_method() != "fork", - reason="stub-based parallel determinism test requires the 'fork' start method", + +@pytest.mark.parametrize( + "make_seed", + [ + pytest.param(lambda: 12345, id="int"), + pytest.param(lambda: np.random.SeedSequence(12345), id="seedsequence"), + pytest.param(lambda: np.random.default_rng(12345), id="generator"), + pytest.param(lambda: np.random.PCG64(12345), id="bitgenerator"), + ], +) +def test_root_seed_sequence_accepts_supported_types(make_seed): + """int, SeedSequence, Generator and BitGenerator all normalize to the same + root SeedSequence stream for an equivalent seed value.""" + root = _root_seed_sequence(make_seed()) + assert isinstance(root, np.random.SeedSequence) + assert _entropy(root) == _entropy(_root_seed_sequence(12345)) + + +def test_root_seed_sequence_none_draws_fresh_entropy(): + """None yields a SeedSequence seeded from fresh OS entropy (not reproducible).""" + root = _root_seed_sequence(None) + assert isinstance(root, np.random.SeedSequence) + assert root.entropy is not None + + +@pytest.mark.parametrize( + "make_seed, resolve", + [ + pytest.param( + lambda: np.random.SeedSequence(999), + lambda seed: seed, + id="seedsequence", + ), + pytest.param( + lambda: np.random.default_rng(999), + lambda seed: seed.bit_generator.seed_seq, + id="generator", + ), + pytest.param( + lambda: np.random.PCG64(999), + lambda seed: seed.seed_seq, + id="bitgenerator", + ), + ], ) +def test_root_seed_sequence_reuses_existing_seed_sequence(make_seed, resolve): + """When given something that already carries a SeedSequence, the helper + reuses that object rather than copying it.""" + seed = make_seed() + assert _root_seed_sequence(seed) is resolve(seed) -class _StubFlight: - """Minimal stand-in for ``Flight`` that skips trajectory integration.""" - - def __init__(self, **kwargs): # accepts and ignores MonteCarlo's Flight kwargs - pass - - def __getattr__(self, name): - return 0.0 - - -@pytest.fixture -def stochastic_calisto_numpy_only( - cesaroni_m1670, - calisto_robust, - stochastic_nose_cone, - stochastic_trapezoidal_fins, - stochastic_tail, - stochastic_rail_buttons, - stochastic_main_parachute, - stochastic_drogue_parachute, -): - """A ``StochasticRocket`` whose randomness flows entirely through numpy. - - Mirrors the shared ``stochastic_calisto`` fixture but gives the solid motor a - single ``thrust_source`` instead of a multi-element list, so no attribute is - sampled through the unseeded standard-library ``random.choice``. - """ - motor = StochasticSolidMotor( - solid_motor=cesaroni_m1670, - burn_out_time=(4, 0.1), - grains_center_of_mass_position=0.001, - grain_density=50, - grain_separation=1 / 1000, - grain_initial_height=1 / 1000, - grain_initial_inner_radius=0.375 / 1000, - grain_outer_radius=0.375 / 1000, - total_impulse=(6500, 1000), - throat_radius=0.5 / 1000, - nozzle_radius=0.5 / 1000, - nozzle_position=0.001, - ) - rocket = StochasticRocket( - rocket=calisto_robust, - radius=0.0127 / 2000, - mass=(15.426, 0.5, "normal"), - inertia_11=(6.321, 0), - inertia_22=0.01, - inertia_33=0.01, - center_of_mass_without_motor=0, - ) - rocket.add_motor(motor, position=0.001) - rocket.add_nose(stochastic_nose_cone, position=(1.134, 0.001)) - rocket.add_trapezoidal_fins(stochastic_trapezoidal_fins, position=(0.001, "normal")) - rocket.add_tail(stochastic_tail) - rocket.set_rail_buttons( - stochastic_rail_buttons, lower_button_position=(-0.618, 0.001, "normal") - ) - rocket.add_parachute(parachute=stochastic_main_parachute) - rocket.add_parachute(parachute=stochastic_drogue_parachute) - return rocket - - -def _read_inputs_by_index(input_file): - """Read a ``.inputs.txt`` file into ``{index: raw_json_line}``.""" - by_index = {} - with open(input_file, mode="r", encoding="utf-8") as rows: - for line in rows: - line = line.strip() - if not line: - continue - by_index[json.loads(line)["index"]] = line - return by_index - - -def _simulate_inputs( - monkeypatch, tmp_path, environment, rocket, flight, tag, **simulate_kwargs -): - """Run a Monte Carlo with a stubbed ``Flight`` and return inputs by index.""" - monkeypatch.setattr(mc_module, "Flight", _StubFlight) - montecarlo = MonteCarlo( - filename=str(tmp_path / tag), - environment=environment, - rocket=rocket, - flight=flight, - ) - montecarlo.simulate(**simulate_kwargs) - return _read_inputs_by_index(montecarlo.input_file) - - -def test_serial_inputs_are_reproducible( - monkeypatch, - tmp_path, - stochastic_environment, - stochastic_calisto_numpy_only, - stochastic_flight, -): - """Two serial runs with the same random_seed yield identical inputs.""" - models = (stochastic_environment, stochastic_calisto_numpy_only, stochastic_flight) - run_a = _simulate_inputs( - monkeypatch, tmp_path, *models, "a", number_of_simulations=6, random_seed=7 - ) - run_b = _simulate_inputs( - monkeypatch, tmp_path, *models, "b", number_of_simulations=6, random_seed=7 - ) - assert sorted(run_a) == list(range(6)) - assert run_a == run_b - - -@requires_fork -def test_inputs_are_worker_invariant( - monkeypatch, - tmp_path, - stochastic_environment, - stochastic_calisto_numpy_only, - stochastic_flight, -): - """serial == parallel(2) == parallel(4): inputs are bit-identical per index.""" - models = (stochastic_environment, stochastic_calisto_numpy_only, stochastic_flight) - common = {"number_of_simulations": 8, "random_seed": 314159} - - serial = _simulate_inputs(monkeypatch, tmp_path, *models, "serial", **common) - par2 = _simulate_inputs( - monkeypatch, tmp_path, *models, "par2", parallel=True, n_workers=2, **common - ) - par4 = _simulate_inputs( - monkeypatch, tmp_path, *models, "par4", parallel=True, n_workers=4, **common - ) +# --------------------------------------------------------------------------- # +# __seed_simulation: splitting one child seed across the three models # +# --------------------------------------------------------------------------- # + + +class _RecordingModel: + """Stand-in stochastic model that records the seeds it is handed.""" + + def __init__(self): + self.seeds = [] + + def _set_stochastic(self, seed=None): + self.seeds.append(seed) - expected = list(range(8)) - assert sorted(serial) == expected - assert sorted(par2) == expected - assert sorted(par4) == expected - for index in expected: - assert serial[index] == par2[index], f"serial vs parallel(2) differ at {index}" - assert serial[index] == par4[index], f"serial vs parallel(4) differ at {index}" - - -def test_none_seed_still_runs( - monkeypatch, - tmp_path, - stochastic_environment, - stochastic_calisto_numpy_only, - stochastic_flight, -): - """random_seed=None draws fresh entropy but still exports one record per index.""" - inputs = _simulate_inputs( - monkeypatch, - tmp_path, - stochastic_environment, - stochastic_calisto_numpy_only, - stochastic_flight, - "none", - number_of_simulations=5, - random_seed=None, + +def _split_seeds(child_seed): + """Run ``__seed_simulation`` against recording models; return the three seeds.""" + models = SimpleNamespace( + environment=_RecordingModel(), + rocket=_RecordingModel(), + flight=_RecordingModel(), ) - assert sorted(inputs) == list(range(5)) + _seed_simulation(models, child_seed) + return models.environment.seeds, models.rocket.seeds, models.flight.seeds + + +def test_seed_simulation_decorrelates_env_rocket_flight(): + """The per-index child seed is split three ways so environment, rocket and + flight draw from independent streams instead of sharing one.""" + env_seeds, rocket_seeds, flight_seeds = _split_seeds(np.random.SeedSequence(2024)) + assert [len(env_seeds), len(rocket_seeds), len(flight_seeds)] == [1, 1, 1] + fingerprints = { + _entropy(env_seeds[0]), + _entropy(rocket_seeds[0]), + _entropy(flight_seeds[0]), + } + assert len(fingerprints) == 3 + + +def test_seed_simulation_is_deterministic_per_child(): + """A given child seed reseeds the three models identically every time.""" + + def split(child): + env, rocket, flight = _split_seeds(child) + return [_entropy(env[0]), _entropy(rocket[0]), _entropy(flight[0])] + + assert split(np.random.SeedSequence(2024)) == split(np.random.SeedSequence(2024)) From fa4f7ffc804b91e6af145cf2dcef0a95a236dc36 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 11 Jul 2026 09:14:37 +0800 Subject: [PATCH 03/45] BUG: fix Monte Carlo seeding race and non-reproducible SeedSequence Addresses review feedback on #1054. Parallel workers claimed the next index with an unlocked keep_simulating() + increment(), so near the end of a run two workers could both pass the count < n check and then claim sim_idx == n; the per-index child_seeds lookup turned that into an IndexError (before, it only wrote one extra record). Move the claim into a _claim_next_index helper that holds the shared mutex across the check and the increment, so each index is handed out once and the counter never overshoots. A deterministic unit test (a barrier plus a widened check-to-increment window) over-claims and fails if the lock is dropped. __root_seed_sequence returned the caller's SeedSequence, and spawn() advances its child counter, so passing the same object to simulate() twice produced different children. Copy it from its full state instead, which leaves the caller untouched and keeps repeated calls reproducible. Also drop Generator/BitGenerator from the accepted types: a stateful generator is not a seed, and reducing it to its underlying SeedSequence ignores how far it has been consumed. random_seed now takes an int, a sequence of ints, or a SeedSequence. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- CHANGELOG.md | 1 + rocketpy/simulation/monte_carlo.py | 70 +++++++--- .../test_monte_carlo_determinism.py | 129 ++++++++++++++---- 3 files changed, 149 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cd984d57..4cec86574 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Attention: The newest changes should be on top --> - ENH: Add Qodo PR-Agent workflow using Google Gemini [#1089](https://github.com/RocketPy-Team/RocketPy/pull/1089) - ENH: Support for Meteomatics API in the `Environment` class [#1079](https://github.com/RocketPy-Team/RocketPy/pull/1079) - ENH: update master with develop [#1081](https://github.com/RocketPy-Team/RocketPy/pull/1081) +- ENH: reproducible Monte Carlo runs via a random_seed argument [#1054](https://github.com/RocketPy-Team/RocketPy/pull/1054) ### Changed diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index f6ba11e4f..639c5c028 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -192,15 +192,15 @@ def simulate( number of workers will be equal to the number of CPUs available. A minimum of 2 workers is required for parallel mode. Default is None. - random_seed : int, numpy.random.SeedSequence, numpy.random.Generator, optional + random_seed : int or numpy.random.SeedSequence, optional Root seed for the run. When provided, the sampled inputs are reproducible and identical across serial and parallel execution and across any number of workers, because each simulation index - draws from its own child stream spawned from this root - (``SeedSequence(random_seed).spawn(number_of_simulations)``). It - accepts an int, a ``SeedSequence`` or a ``Generator``. Default is - None, which draws fresh entropy (not reproducible), preserving the - previous behavior. + draws from its own child stream spawned from this root. It accepts + an int or a ``SeedSequence``; a supplied ``SeedSequence`` is copied + rather than consumed, so repeated calls with the same seed produce + the same inputs. Default is None, which draws fresh entropy (not + reproducible), preserving the previous behavior. kwargs : dict Custom arguments for simulation export of the ``inputs`` file. Options are: @@ -281,19 +281,26 @@ def __setup_files(self, append): @staticmethod def __root_seed_sequence(random_seed): - """Return a ``SeedSequence`` root from a flexible seed argument. - - Accepts what the scientific-Python SPEC 7 seeding convention accepts - (an int, a ``SeedSequence``, a ``Generator`` or ``BitGenerator``, or - None for fresh entropy) and returns a ``SeedSequence`` so it can be - spawned into one independent child stream per simulation index. + """Build a fresh ``SeedSequence`` root from ``random_seed``. + + ``random_seed`` may be an int (or any entropy ``numpy.random.SeedSequence`` + accepts), an existing ``SeedSequence``, or ``None`` for fresh entropy. A + supplied ``SeedSequence`` is copied from its full ``state``, so the + spawning below neither mutates the caller's object nor advances a shared + child counter between calls; repeated ``simulate`` calls with the same + seed then stay reproducible. A stateful ``Generator``/``BitGenerator`` is + not accepted, since using it as an immutable seed would contradict its + consume-on-use semantics; pass ``rng.bit_generator.seed_seq`` to seed + from an existing generator's stream. """ if isinstance(random_seed, np.random.SeedSequence): - return random_seed - if isinstance(random_seed, np.random.Generator): - return random_seed.bit_generator.seed_seq - if isinstance(random_seed, np.random.BitGenerator): - return random_seed.seed_seq + return np.random.SeedSequence(**random_seed.state) + if isinstance(random_seed, (np.random.Generator, np.random.BitGenerator)): + raise TypeError( + "random_seed must be an int or a numpy.random.SeedSequence, not " + f"a {type(random_seed).__name__}; to seed from an existing " + "generator pass rng.bit_generator.seed_seq." + ) return np.random.SeedSequence(random_seed) def __seed_simulation(self, child_seed): @@ -316,7 +323,7 @@ def __run_in_serial(self, random_seed=None): # pylint: disable=too-many-stateme Parameters ---------- - random_seed : int, SeedSequence, Generator, optional + random_seed : int or SeedSequence, optional Root seed for the run. See ``simulate``. Returns @@ -370,7 +377,7 @@ def __run_in_parallel(self, n_workers=None, random_seed=None): n_workers: int, optional Number of workers to be used. If None, the number of workers will be equal to the number of CPUs available. Default is None. - random_seed : int, SeedSequence, Generator, optional + random_seed : int or SeedSequence, optional Root seed for the run. See ``simulate``. Returns @@ -465,8 +472,11 @@ def __sim_producer(self, child_seeds, sim_monitor, mutex, error_event): # pylin Event signaling an error occurred during the simulation. """ try: - while sim_monitor.keep_simulating(): - sim_idx = sim_monitor.increment() - 1 + while True: + sim_idx = _claim_next_index(sim_monitor, mutex) + if sim_idx is None: + break + inputs_json, outputs_json = "", "" self.__seed_simulation(child_seeds[sim_idx]) @@ -1682,6 +1692,24 @@ def export_errors_to_json(self, filename): self._write_log_to_json(self.errors_log, filename) +def _claim_next_index(sim_monitor, mutex): + """Atomically claim the next 0-based simulation index, or ``None`` if done. + + ``keep_simulating()`` and ``increment()`` are two separate manager calls, so + the shared ``mutex`` has to be held across both. Without it, two workers can + each pass the ``count < number_of_simulations`` check at the tail before + either increments, and both then claim an index, overrunning the requested + number of simulations and indexing past the per-index seed list. + """ + mutex.acquire() + try: + if not sim_monitor.keep_simulating(): + return None + return sim_monitor.increment() - 1 + finally: + mutex.release() + + def _import_multiprocess(): """Import the necessary modules and submodules for the multiprocess library. diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 939980508..526717182 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -4,9 +4,9 @@ turning the run's root seed into one independent child stream per simulation index. Two private helpers do the work: -* ``__root_seed_sequence`` normalizes the flexible ``random_seed`` argument (int, - ``SeedSequence``, ``Generator``, ``BitGenerator`` or None) into a - ``SeedSequence`` that can be spawned; +* ``__root_seed_sequence`` normalizes the ``random_seed`` argument (an int, a + sequence of ints, a ``SeedSequence`` or None) into a fresh ``SeedSequence`` + that can be spawned; * ``__seed_simulation`` splits one per-index child seed three ways so the environment, rocket and flight draw from independent streams. @@ -19,12 +19,15 @@ it lets the seeding invariants be asserted without running a Monte Carlo. """ +import threading +import time from types import SimpleNamespace import numpy as np import pytest from rocketpy.simulation import MonteCarlo +from rocketpy.simulation.monte_carlo import _SimMonitor, _claim_next_index _root_seed_sequence = MonteCarlo._MonteCarlo__root_seed_sequence _seed_simulation = MonteCarlo._MonteCarlo__seed_simulation @@ -44,17 +47,17 @@ def _entropy(seed_sequence, n=4): "make_seed", [ pytest.param(lambda: 12345, id="int"), + pytest.param(lambda: np.int64(12345), id="numpy-int"), + pytest.param(lambda: [1, 2, 3], id="sequence"), pytest.param(lambda: np.random.SeedSequence(12345), id="seedsequence"), - pytest.param(lambda: np.random.default_rng(12345), id="generator"), - pytest.param(lambda: np.random.PCG64(12345), id="bitgenerator"), ], ) -def test_root_seed_sequence_accepts_supported_types(make_seed): - """int, SeedSequence, Generator and BitGenerator all normalize to the same - root SeedSequence stream for an equivalent seed value.""" +def test_root_seed_sequence_accepts_seed_like_values(make_seed): + """An int, a numpy integer, a sequence of ints and a SeedSequence are all + accepted, normalize to a SeedSequence, and are reproducible.""" root = _root_seed_sequence(make_seed()) assert isinstance(root, np.random.SeedSequence) - assert _entropy(root) == _entropy(_root_seed_sequence(12345)) + assert _entropy(root) == _entropy(_root_seed_sequence(make_seed())) def test_root_seed_sequence_none_draws_fresh_entropy(): @@ -65,30 +68,42 @@ def test_root_seed_sequence_none_draws_fresh_entropy(): @pytest.mark.parametrize( - "make_seed, resolve", + "make_generator", [ - pytest.param( - lambda: np.random.SeedSequence(999), - lambda seed: seed, - id="seedsequence", - ), - pytest.param( - lambda: np.random.default_rng(999), - lambda seed: seed.bit_generator.seed_seq, - id="generator", - ), - pytest.param( - lambda: np.random.PCG64(999), - lambda seed: seed.seed_seq, - id="bitgenerator", - ), + pytest.param(lambda: np.random.default_rng(999), id="generator"), + pytest.param(lambda: np.random.PCG64(999), id="bitgenerator"), ], ) -def test_root_seed_sequence_reuses_existing_seed_sequence(make_seed, resolve): - """When given something that already carries a SeedSequence, the helper - reuses that object rather than copying it.""" - seed = make_seed() - assert _root_seed_sequence(seed) is resolve(seed) +def test_root_seed_sequence_rejects_stateful_generators(make_generator): + """A Generator/BitGenerator is a stateful RNG, not a seed value, so it is + rejected instead of being reduced to its underlying SeedSequence.""" + with pytest.raises(TypeError, match="SeedSequence"): + _root_seed_sequence(make_generator()) + + +def test_root_seed_sequence_copies_seedsequence_without_mutating_it(): + """A supplied SeedSequence is copied from its full state: repeated calls with + the same object reproduce the same children, the caller's spawn counter is + left untouched, and a spawned child (non-empty spawn_key) round-trips too.""" + + def children(seed_sequence): + return [ + _entropy(child) for child in _root_seed_sequence(seed_sequence).spawn(3) + ] + + # A SeedSequence that has already spawned children, so its counter is not 0. + seed = np.random.SeedSequence(2024) + seed.spawn(5) + counter_before = seed.n_children_spawned + + assert children(seed) == children(seed), "same object twice must reproduce" + assert seed.n_children_spawned == counter_before, "caller must not be mutated" + assert _root_seed_sequence(seed) is not seed, "must return a copy, not the caller" + + # A spawned child carries a non-empty spawn_key that the copy must preserve. + child = np.random.SeedSequence(2024).spawn(1)[0] + assert child.spawn_key != () + assert children(child) == children(child) # --------------------------------------------------------------------------- # @@ -138,3 +153,57 @@ def split(child): return [_entropy(env[0]), _entropy(rocket[0]), _entropy(flight[0])] assert split(np.random.SeedSequence(2024)) == split(np.random.SeedSequence(2024)) + + +# --------------------------------------------------------------------------- # +# _claim_next_index: atomic hand-out of the next simulation index # +# --------------------------------------------------------------------------- # + + +def test_claim_next_index_hands_out_each_index_once_under_contention(): + """Holding the mutex across keep_simulating() and increment() must hand out + each index exactly once, even when every worker reaches the claim together. + + A barrier releases all workers at once and a widened check-to-increment + window would let an unlocked claim run several workers past the count < n + check before any increments; the lock is what keeps the result to exactly + n_simulations indices (0..n-1, none repeated) and the counter from + overshooting. + """ + n_simulations = 5 + n_workers = 8 + monitor = _SimMonitor(initial_count=0, n_simulations=n_simulations, start_time=0.0) + + # Widen the window between the check and the increment so that, without the + # lock, several workers could pass count < n before any of them increments. + real_keep_simulating = monitor.keep_simulating + + def slow_keep_simulating(): + result = real_keep_simulating() + time.sleep(0.02) + return result + + monitor.keep_simulating = slow_keep_simulating + + mutex = threading.Lock() + barrier = threading.Barrier(n_workers) + claimed = [] + claimed_lock = threading.Lock() + + def worker(): + barrier.wait() + while True: + index = _claim_next_index(monitor, mutex) + if index is None: + break + with claimed_lock: + claimed.append(index) + + workers = [threading.Thread(target=worker) for _ in range(n_workers)] + for thread in workers: + thread.start() + for thread in workers: + thread.join() + + assert sorted(claimed) == list(range(n_simulations)) + assert monitor.count == n_simulations From 2a43313457b65bef1b306a3a11e6b310d63f12b9 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:25:02 +0800 Subject: [PATCH 04/45] ENH: derive Monte Carlo per-index seeds in O(1) and seed models with 128-bit ints Each simulation index is seeded from its own child of the run's root seed. Building that child by extending the captured root spawn_key is bit-identical to root.spawn(number_of_simulations)[index] but O(1) in time and memory, so a worker reconstructs any index from a small root state instead of the full spawned list being materialized and pickled to every process. Hand each model a 128-bit int rather than a SeedSequence: a plain int is the seed type accepted alike by numpy.random.default_rng, RandomState and the stdlib random.Random (which rejects a SeedSequence with a TypeError from Python 3.11), so a custom sampler whose reset_seed documents an int keeps working; all four uint32 words are combined by value (not via tobytes) so the seed is byte-order independent and keeps the full 128-bit pool instead of collapsing to 32 bits. The random_seed docstring now lists the accepted types and notes the seeding is informed by SPEC 7 while keeping immutable seed-snapshot semantics. The unit tests assert the SeedSequence copy preserves full .state (an entropy-only copy would fail), the O(1) child equals spawn bit-for-bit -- including a root whose child counter has advanced and indices past 2**32 -- and each model receives a distinct 128-bit int. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 121 ++++++++---- .../test_monte_carlo_determinism.py | 187 ++++++++++++++---- 2 files changed, 241 insertions(+), 67 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 639c5c028..3801dbf94 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -192,15 +192,22 @@ def simulate( number of workers will be equal to the number of CPUs available. A minimum of 2 workers is required for parallel mode. Default is None. - random_seed : int or numpy.random.SeedSequence, optional + random_seed : int, numpy integer, sequence of ints, or SeedSequence, optional Root seed for the run. When provided, the sampled inputs are - reproducible and identical across serial and parallel execution - and across any number of workers, because each simulation index - draws from its own child stream spawned from this root. It accepts - an int or a ``SeedSequence``; a supplied ``SeedSequence`` is copied - rather than consumed, so repeated calls with the same seed produce - the same inputs. Default is None, which draws fresh entropy (not - reproducible), preserving the previous behavior. + reproducible and identical across serial and parallel execution and + across any number of workers: each simulation index derives its own + decorrelated child stream from this root, so index ``i`` receives the + same inputs no matter which worker runs it. A supplied ``SeedSequence`` + is copied from its full state rather than consumed, so repeated calls + with the same seed reproduce the same inputs. Each model is reseeded + with a 128-bit integer -- the seed type a custom sampler's + ``reset_seed`` accepts. A stateful ``numpy.random.Generator`` or + ``BitGenerator`` is rejected (it is an RNG to draw from, not a fixed + seed); pass ``rng.bit_generator.seed_seq`` to seed from one. Default is + None, which draws fresh entropy on each run -- the previous, + non-reproducible default. This seeding is informed by Scientific Python + SPEC 7 but keeps immutable seed-snapshot semantics rather than sharing a + ``Generator``. kwargs : dict Custom arguments for simulation export of the ``inputs`` file. Options are: @@ -234,6 +241,9 @@ def simulate( self._export_config = kwargs self.number_of_simulations = number_of_simulations self._initial_sim_idx = self.num_of_loaded_sims if append else 0 + # Small, picklable root seed state captured once per run; every + # simulation index derives its child seed from it (see __child_seed). + self.__root_state = None print("Starting Monte Carlo analysis") @@ -303,6 +313,38 @@ def __root_seed_sequence(random_seed): ) return np.random.SeedSequence(random_seed) + def __capture_root_state(self, random_seed): + """Capture the small, picklable root seed state for this run. + + Stored once so serial mode and every parallel worker derive the same + per-index child seeds from it (see ``__child_seed``), instead of + materializing and pickling the full ``spawn(number_of_simulations)`` + list to each process. + """ + root = self.__root_seed_sequence(random_seed) + self.__root_state = ( + root.entropy, + root.spawn_key, + root.pool_size, + root.n_children_spawned, + ) + + def __child_seed(self, sim_idx): + """Return the seed sequence for a single simulation index. + + This equals ``root.spawn(number_of_simulations)[sim_idx]`` but is O(1) + in time and memory: ``SeedSequence.spawn`` derives child ``i`` by + appending ``n_children_spawned + i`` to the parent ``spawn_key``, so + rebuilding that one child directly reproduces it bit-for-bit while + letting a worker reconstruct any index from the small root state alone. + """ + entropy, spawn_key, pool_size, base = self.__root_state + return np.random.SeedSequence( + entropy=entropy, + spawn_key=(*spawn_key, base + sim_idx), + pool_size=pool_size, + ) + def __seed_simulation(self, child_seed): """Reseed the stochastic models for a single simulation index. @@ -310,12 +352,13 @@ def __seed_simulation(self, child_seed): rocket and flight draw from independent streams instead of sharing one. Seeding per simulation index (not per worker) is what makes the sampled inputs invariant to the execution mode and to the number of - workers. + workers. Each sub-stream is handed over as a 128-bit ``int`` (see + ``_seed_sequence_to_int``) so custom samplers keep working. """ env_seed, rocket_seed, flight_seed = child_seed.spawn(3) - self.environment._set_stochastic(env_seed) - self.rocket._set_stochastic(rocket_seed) - self.flight._set_stochastic(flight_seed) + self.environment._set_stochastic(_seed_sequence_to_int(env_seed)) + self.rocket._set_stochastic(_seed_sequence_to_int(rocket_seed)) + self.flight._set_stochastic(_seed_sequence_to_int(flight_seed)) def __run_in_serial(self, random_seed=None): # pylint: disable=too-many-statements """ @@ -335,15 +378,13 @@ def __run_in_serial(self, random_seed=None): # pylint: disable=too-many-stateme n_simulations=self.number_of_simulations, start_time=time(), ) - child_seeds = self.__root_seed_sequence(random_seed).spawn( - self.number_of_simulations - ) + self.__capture_root_state(random_seed) try: while sim_monitor.keep_simulating(): sim_idx = sim_monitor.increment() - 1 inputs_json, outputs_json = "", "" - self.__seed_simulation(child_seeds[sim_idx]) + self.__seed_simulation(self.__child_seed(sim_idx)) flight = self.__run_single_simulation() inputs_json = self.__evaluate_flight_inputs(sim_idx) outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) @@ -400,19 +441,18 @@ def __run_in_parallel(self, n_workers=None, random_seed=None): ) processes = [] - # One independent child seed per simulation index (not per - # worker), shared with every worker. The shared counter assigns - # indices, and index i always seeds from child_seeds[i], so the - # sampled inputs do not depend on the number of workers. - child_seeds = self.__root_seed_sequence(random_seed).spawn( - self.number_of_simulations - ) + # Each worker derives one independent child seed per simulation + # index (not per worker) from the shared root state: the counter + # assigns indices and index i always seeds from __child_seed(i), so + # the sampled inputs do not depend on the number of workers. The + # root state is small and travels with the pickled instance, so no + # per-index seed list is materialized or sent to each process. + self.__capture_root_state(random_seed) for _ in range(n_workers): sim_producer = multiprocess.Process( target=self.__sim_producer, args=( - child_seeds, sim_monitor, mutex, simulation_error_event, @@ -454,16 +494,11 @@ def __validate_number_of_workers(self, n_workers): raise ValueError("Number of workers must be at least 2 for parallel mode.") return n_workers - def __sim_producer(self, child_seeds, sim_monitor, mutex, error_event): # pylint: disable=too-many-statements + def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=too-many-statements """Simulation producer to be used in parallel by multiprocessing. Parameters ---------- - child_seeds : list[numpy.random.SeedSequence] - One seed sequence per simulation index. Before each simulation - the worker seeds the stochastic models from - ``child_seeds[sim_idx]``, where ``sim_idx`` comes from the shared - counter, so the inputs are invariant to the number of workers. sim_monitor : _SimMonitor The simulation monitor object to keep track of the simulations. mutex : multiprocess.Lock @@ -479,7 +514,7 @@ def __sim_producer(self, child_seeds, sim_monitor, mutex, error_event): # pylin inputs_json, outputs_json = "", "" - self.__seed_simulation(child_seeds[sim_idx]) + self.__seed_simulation(self.__child_seed(sim_idx)) flight = self.__run_single_simulation() inputs_json = self.__evaluate_flight_inputs(sim_idx) outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) @@ -1692,14 +1727,34 @@ def export_errors_to_json(self, filename): self._write_log_to_json(self.errors_log, filename) +def _seed_sequence_to_int(seed_sequence): + """Collapse a ``SeedSequence`` into a 128-bit Python ``int``. + + A plain ``int`` is the one seed type accepted alike by + ``numpy.random.default_rng``, ``numpy.random.RandomState`` and the stdlib + ``random.Random`` (which rejects a ``SeedSequence`` with a ``TypeError`` + since Python 3.11), so a custom sampler whose ``reset_seed`` documents an + ``int`` keeps working. All four ``uint32`` words are combined to keep the + full 128-bit pool, so the environment/rocket/flight sub-streams stay + decorrelated instead of collapsing to a single 32-bit word. + + The words are combined by value (little-endian word order), not via + ``tobytes()``, so the seed is the same on big- and little-endian machines + -- a byte-order-dependent seed would break the cross-platform + reproducibility this whole scheme exists to provide. + """ + words = seed_sequence.generate_state(4, dtype=np.uint32) + return sum(int(word) << (32 * position) for position, word in enumerate(words)) + + def _claim_next_index(sim_monitor, mutex): """Atomically claim the next 0-based simulation index, or ``None`` if done. ``keep_simulating()`` and ``increment()`` are two separate manager calls, so the shared ``mutex`` has to be held across both. Without it, two workers can each pass the ``count < number_of_simulations`` check at the tail before - either increments, and both then claim an index, overrunning the requested - number of simulations and indexing past the per-index seed list. + either increments, and both then claim an index, running more simulations + than were requested (and duplicating a simulation index). """ mutex.acquire() try: diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 526717182..546b2c1d1 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -2,11 +2,16 @@ ``MonteCarlo.simulate(random_seed=...)`` makes the sampled inputs reproducible by turning the run's root seed into one independent child stream per simulation -index. Two private helpers do the work: +index. Four small helpers do the work: * ``__root_seed_sequence`` normalizes the ``random_seed`` argument (an int, a - sequence of ints, a ``SeedSequence`` or None) into a fresh ``SeedSequence`` - that can be spawned; + sequence of ints, a ``SeedSequence`` or None) into a fresh ``SeedSequence``; +* ``__child_seed`` derives the child seed for one simulation index in O(1) by + extending the captured root ``spawn_key`` -- bit-identical to + ``root.spawn(n)[index]`` but without materializing the whole spawned list, so + a worker can rebuild any index from the small root state alone; +* ``_seed_sequence_to_int`` collapses a child into a 128-bit ``int`` (the seed + type a documented ``CustomSampler.reset_seed`` accepts); * ``__seed_simulation`` splits one per-index child seed three ways so the environment, rocket and flight draw from independent streams. @@ -19,6 +24,8 @@ it lets the seeding invariants be asserted without running a Monte Carlo. """ +import random as stdlib_random +import sys import threading import time from types import SimpleNamespace @@ -27,9 +34,14 @@ import pytest from rocketpy.simulation import MonteCarlo -from rocketpy.simulation.monte_carlo import _SimMonitor, _claim_next_index +from rocketpy.simulation.monte_carlo import ( + _SimMonitor, + _claim_next_index, + _seed_sequence_to_int, +) _root_seed_sequence = MonteCarlo._MonteCarlo__root_seed_sequence +_child_seed = MonteCarlo._MonteCarlo__child_seed _seed_simulation = MonteCarlo._MonteCarlo__seed_simulation @@ -38,6 +50,26 @@ def _entropy(seed_sequence, n=4): return tuple(int(x) for x in seed_sequence.generate_state(n)) +def _plan(root): + """A stand-in ``self`` carrying only the root state ``__child_seed`` reads.""" + return SimpleNamespace( + _MonteCarlo__root_state=( + root.entropy, + root.spawn_key, + root.pool_size, + root.n_children_spawned, + ) + ) + + +def _advanced_root(seed, already_spawned): + """A root whose own child counter has advanced (n_children_spawned != 0), + the state a user's already-spawned SeedSequence would arrive in.""" + root = np.random.SeedSequence(seed) + root.spawn(already_spawned) + return root + + # --------------------------------------------------------------------------- # # __root_seed_sequence: normalizing the flexible seed argument # # --------------------------------------------------------------------------- # @@ -81,33 +113,123 @@ def test_root_seed_sequence_rejects_stateful_generators(make_generator): _root_seed_sequence(make_generator()) -def test_root_seed_sequence_copies_seedsequence_without_mutating_it(): - """A supplied SeedSequence is copied from its full state: repeated calls with - the same object reproduce the same children, the caller's spawn counter is - left untouched, and a spawned child (non-empty spawn_key) round-trips too.""" +def test_root_seed_sequence_copies_full_state_without_mutating_caller(): + """A supplied SeedSequence is copied from its FULL state -- entropy, spawn_key, + pool_size and n_children_spawned -- not just its entropy, and the caller object + is not mutated. Asserting on ``.state`` is what gives this teeth: an + entropy-only copy would silently drop spawn_key/n_children_spawned (making a + spawned-child seed collide with its parent) and fail the state comparison.""" + source = np.random.SeedSequence(2024).spawn(3)[2] # non-empty spawn_key + source.spawn(5) # advance its own child counter, so it is not 0 + assert source.spawn_key == (2,) + assert source.n_children_spawned == 5 + + state_before = dict(source.state) + clone = _root_seed_sequence(source) + + assert clone is not source, "must return a copy, not the caller" + assert clone.state == state_before, "copy must preserve the full seed state" + assert source.state == state_before, "caller must not be mutated" + # The copy reproduces exactly what an independent full-state rebuild produces. + rebuilt = np.random.SeedSequence(**state_before) + assert [_entropy(c) for c in clone.spawn(3)] == [ + _entropy(c) for c in rebuilt.spawn(3) + ] + + +# --------------------------------------------------------------------------- # +# __child_seed: O(1) per-index derivation, bit-identical to spawn(n)[index] # +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "make_root", + [ + pytest.param(lambda: np.random.SeedSequence(2024), id="int-root"), + pytest.param(lambda: np.random.SeedSequence([7, 8, 9]), id="sequence-root"), + pytest.param( + lambda: np.random.SeedSequence(2024).spawn(3)[2], id="spawned-root" + ), + pytest.param(lambda: _advanced_root(99, 4), id="advanced-counter-root"), + ], +) +def test_child_seed_matches_spawn_bit_for_bit(make_root): + """Deriving index i by extending the root spawn_key equals ``root.spawn(n)[i]`` + exactly, so the O(1) derivation changes no sampled inputs versus a full spawn. + A fresh, identical root is built on each side so neither run mutates the other. + The ``advanced-counter`` root (n_children_spawned != 0) covers a user passing a + SeedSequence they have already spawned from: the base offset must equal + n_children_spawned or the derived index would collide with those children. + """ + n = 6 + derived = [_entropy(_child_seed(_plan(make_root()), i)) for i in range(n)] + spawned = [_entropy(child) for child in make_root().spawn(n)] + assert derived == spawned + + +def test_child_seed_is_worker_order_independent(): + """Any index maps to the same child regardless of the order indices are asked + for -- the property that makes a run invariant to worker scheduling.""" + plan = _plan(np.random.SeedSequence(2024)) + forward = {i: _entropy(_child_seed(plan, i)) for i in range(5)} + backward = {i: _entropy(_child_seed(plan, i)) for i in reversed(range(5))} + assert forward == backward + + +def test_child_seed_supports_indices_beyond_32_bits(): + """A simulation index past 2**32 is not truncated: it derives a distinct child + from its neighbour and matches the direct spawn_key construction for it.""" + root = np.random.SeedSequence(11) + plan = _plan(root) + big = 2**32 + 5 + assert _entropy(_child_seed(plan, big)) != _entropy(_child_seed(plan, big + 1)) + expected = np.random.SeedSequence( + entropy=root.entropy, spawn_key=(big,), pool_size=root.pool_size + ) + assert _entropy(_child_seed(plan, big)) == _entropy(expected) + + +# --------------------------------------------------------------------------- # +# _seed_sequence_to_int: 128-bit int seed for the samplers # +# --------------------------------------------------------------------------- # + + +def test_seed_sequence_to_int_is_deterministic_128_bit_int(): + def child(): + return np.random.SeedSequence(42).spawn(1)[0] + + seed = _seed_sequence_to_int(child()) + assert isinstance(seed, int) + assert 0 <= seed < 2**128 + assert seed == _seed_sequence_to_int(child()), "must be deterministic" - def children(seed_sequence): - return [ - _entropy(child) for child in _root_seed_sequence(seed_sequence).spawn(3) - ] - # A SeedSequence that has already spawned children, so its counter is not 0. - seed = np.random.SeedSequence(2024) - seed.spawn(5) - counter_before = seed.n_children_spawned +def test_seed_sequence_to_int_uses_all_128_bits(): + """The int combines all four uint32 words, not a single 32-bit word, so it + keeps the full entropy pool rather than collapsing collision risk to n**2 / + 2**32. A single-word reduction would compare unequal here.""" + ss = np.random.SeedSequence(42).spawn(1)[0] + one_word = int(np.random.SeedSequence(42).spawn(1)[0].generate_state(1)[0]) + assert _seed_sequence_to_int(ss) != one_word + assert _seed_sequence_to_int(ss).bit_length() > 32 - assert children(seed) == children(seed), "same object twice must reproduce" - assert seed.n_children_spawned == counter_before, "caller must not be mutated" - assert _root_seed_sequence(seed) is not seed, "must return a copy, not the caller" - # A spawned child carries a non-empty spawn_key that the copy must preserve. - child = np.random.SeedSequence(2024).spawn(1)[0] - assert child.spawn_key != () - assert children(child) == children(child) +def test_seed_int_is_accepted_by_the_modern_rng_apis(): + """The 128-bit int a sampler receives works with random.Random and + numpy.random.default_rng -- the paths a CustomSampler uses. Passing a + SeedSequence there instead is unsafe: from Python 3.11 random.Random rejects + it with a TypeError, and before 3.11 it is silently hashed rather than used as + entropy. Either way an int is the right thing to hand a sampler.""" + seed = _seed_sequence_to_int(np.random.SeedSequence(1).spawn(1)[0]) + assert isinstance(stdlib_random.Random(seed).random(), float) + assert np.random.default_rng(seed).random() is not None + if sys.version_info >= (3, 11): + with pytest.raises(TypeError): + stdlib_random.Random(np.random.SeedSequence(1)) # --------------------------------------------------------------------------- # -# __seed_simulation: splitting one child seed across the three models # +# __seed_simulation: splitting one child seed across the three models # # --------------------------------------------------------------------------- # @@ -132,17 +254,14 @@ def _split_seeds(child_seed): return models.environment.seeds, models.rocket.seeds, models.flight.seeds -def test_seed_simulation_decorrelates_env_rocket_flight(): - """The per-index child seed is split three ways so environment, rocket and - flight draw from independent streams instead of sharing one.""" +def test_seed_simulation_hands_each_model_a_distinct_128_bit_int(): + """The per-index child seed is split three ways, and each model receives a + plain 128-bit int (not a SeedSequence) from an independent stream.""" env_seeds, rocket_seeds, flight_seeds = _split_seeds(np.random.SeedSequence(2024)) assert [len(env_seeds), len(rocket_seeds), len(flight_seeds)] == [1, 1, 1] - fingerprints = { - _entropy(env_seeds[0]), - _entropy(rocket_seeds[0]), - _entropy(flight_seeds[0]), - } - assert len(fingerprints) == 3 + seeds = [env_seeds[0], rocket_seeds[0], flight_seeds[0]] + assert all(isinstance(s, int) and 0 <= s < 2**128 for s in seeds) + assert len(set(seeds)) == 3, "env/rocket/flight must be decorrelated" def test_seed_simulation_is_deterministic_per_child(): @@ -150,7 +269,7 @@ def test_seed_simulation_is_deterministic_per_child(): def split(child): env, rocket, flight = _split_seeds(child) - return [_entropy(env[0]), _entropy(rocket[0]), _entropy(flight[0])] + return [env[0], rocket[0], flight[0]] assert split(np.random.SeedSequence(2024)) == split(np.random.SeedSequence(2024)) From a76d612fd78a0e600a66c5ff36bd0514c8636cae Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:25:02 +0800 Subject: [PATCH 05/45] BUG: sample list-valued stochastic attributes through the seeded generator dict_generator drew list-valued attributes with the stdlib random.choice, which reads from an unseeded global Random instance, so random_seed did not make those attributes reproducible. Draw the index from this model's seeded numpy generator instead. Indexing (not numpy.random.choice) also avoids coercing a heterogeneous list -- Function objects, paths, arrays -- to a single dtype. Adds a unit test that a list-valued attribute is reproducible under a fixed seed and that heterogeneous entries are returned unchanged. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../unit/stochastic/test_stochastic_model.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/stochastic/test_stochastic_model.py b/tests/unit/stochastic/test_stochastic_model.py index 8bb360c48..c1c27ccfb 100644 --- a/tests/unit/stochastic/test_stochastic_model.py +++ b/tests/unit/stochastic/test_stochastic_model.py @@ -1,6 +1,38 @@ +from types import SimpleNamespace + import pytest from rocketpy.stochastic import StochasticFreeFormFins +from rocketpy.stochastic.stochastic_model import StochasticModel + + +def _sampled_option(model): + """Return the value ``dict_generator`` picks for the ``options`` attribute.""" + return next(model.dict_generator())["options"] + + +def test_list_attribute_sampling_is_reproducible_under_seed(): + """A list-valued stochastic attribute is drawn through the model's own seeded + numpy generator, so a fixed seed reproduces the choice. It used to be drawn + with the stdlib ``random.choice`` (an unseeded global instance), which + ``random_seed`` could not govern. Heterogeneous entries (paths, callables, + lists) are returned unchanged rather than coerced to a numpy dtype the way + ``numpy.random.choice`` would. + """ + options = ["/motor/a.eng", "/motor/b.eng", (lambda t: t), [1, 2, 3]] + model = StochasticModel(obj=SimpleNamespace(), options=options) + + model._set_stochastic(42) + first = _sampled_option(model) + model._set_stochastic(42) + assert _sampled_option(model) == first, "same seed must reproduce the choice" + assert any(first is option for option in options), "object returned unchanged" + + chosen_ids = set() + for seed in range(16): + model._set_stochastic(seed) + chosen_ids.add(id(_sampled_option(model))) + assert len(chosen_ids) > 1, "different seeds must be able to pick differently" @pytest.mark.parametrize( From a5090c26a1d29b2af0eabb7fc2fd2a9bcadd92d9 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:25:02 +0800 Subject: [PATCH 06/45] TST: verify Monte Carlo seed derivation is start-method invariant The existing worker-invariance test stubs the module-level Flight and so reaches workers only under fork. Add a test that the per-index seed derived in a worker matches the main process under every available start method (fork, spawn, forkserver), using a top-level picklable target and small picklable arguments so it is valid under spawn/forkserver -- Python 3.14's POSIX default -- without relying on inherited parent state. It runs in ordinary CI. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../test_monte_carlo_determinism.py | 77 +++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index b47629a55..f0355ca71 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -14,21 +14,52 @@ under the ``fork`` start method, so the worker-invariance test skips otherwise and is marked ``slow`` to match the other Monte Carlo multiprocessing tests. -A dedicated numpy-only rocket is used so *all* randomness flows through the seeded -numpy generator. List-valued stochastic attributes are sampled with the standard -library ``random.choice`` (an unseeded global generator) which ``random_seed`` -does not govern; the fixture drops the only such attribute (a multi-element -``thrust_source``) so the inputs are byte-for-byte reproducible from the seed. +A dedicated numpy-only rocket keeps the fork-based end-to-end test simple: it +gives the motor a single ``thrust_source`` so the run has no list-valued attribute +at all. List sampling is itself seeded now (it draws through the model generator, +not the stdlib ``random.choice``) and is covered directly in +``tests/unit/stochastic/test_stochastic_model``. + +Seed derivation being independent of the multiprocessing start method (fork, +spawn or forkserver) is verified separately by +``test_seed_derivation_is_start_method_invariant``, which uses a top-level +picklable target so it is safe under ``spawn``/``forkserver`` -- unlike the +``Flight``-stub test above, which reaches workers only under ``fork``. """ import json +import multiprocessing +from types import SimpleNamespace +import numpy as np import pytest import rocketpy.simulation.monte_carlo as mc_module from rocketpy.simulation import MonteCarlo +from rocketpy.simulation.monte_carlo import _seed_sequence_to_int from rocketpy.stochastic import StochasticRocket, StochasticSolidMotor +_child_seed = MonteCarlo._MonteCarlo__child_seed + + +def _available_start_methods(): + """The multiprocessing start methods this platform actually supports.""" + supported = multiprocessing.get_all_start_methods() + return [method for method in ("fork", "spawn", "forkserver") if method in supported] + + +def _derive_index_seeds(root_state, indices): + """Derive the per-index seed fingerprints from ``root_state``. + + Top-level and picklable (only a small tuple and a list of ints cross the + process boundary), so it runs unchanged under every start method -- including + ``spawn``/``forkserver``, which re-import this module rather than inheriting + the parent's memory. It calls the real production helpers (``__child_seed`` + and ``_seed_sequence_to_int``) so the test tracks the shipped derivation. + """ + plan = SimpleNamespace(_MonteCarlo__root_state=root_state) + return {index: _seed_sequence_to_int(_child_seed(plan, index)) for index in indices} + class _StubFlight: """Minimal stand-in for ``Flight`` that skips trajectory integration.""" @@ -175,3 +206,39 @@ def test_inputs_are_worker_invariant( for index in expected: assert serial[index] == par2[index], f"serial vs parallel(2) differ at {index}" assert serial[index] == par4[index], f"serial vs parallel(4) differ at {index}" + + +@pytest.mark.parametrize("start_method", _available_start_methods()) +def test_seed_derivation_is_start_method_invariant(start_method): + """Per-index seeds derived in a worker match the main process under every + available start method (fork, spawn, forkserver). + + The full worker-invariance test above stubs the module-level ``Flight`` and so + only reaches workers under ``fork``. This one instead checks the property that + actually has to hold cross-platform -- that a simulation index maps to the same + seed no matter which process derives it -- using a top-level picklable target + and small picklable arguments, so it is valid under ``spawn``/``forkserver`` + (Python 3.14's POSIX default) without relying on any inherited parent state. + Two workers split the indices; their combined result must equal the + single-process derivation. + """ + root = np.random.SeedSequence(2718281828) + root_state = ( + root.entropy, + root.spawn_key, + root.pool_size, + root.n_children_spawned, + ) + indices = list(range(6)) + expected = _derive_index_seeds(root_state, indices) + + context = multiprocessing.get_context(start_method) + chunks = [(root_state, indices[0::2]), (root_state, indices[1::2])] + with context.Pool(2) as pool: + results = pool.starmap(_derive_index_seeds, chunks) + + combined = {} + for result in results: + combined.update(result) + assert combined == expected + assert sorted(combined) == indices From 96889c82bc8e44f2e173356669a6d737c124e29b Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:59:29 +0800 Subject: [PATCH 07/45] BUG: seed list/position sampling and decorrelate rocket components StochasticModel list-valued attributes were sampled with the stdlib random.choice (an unseeded global) and StochasticRocket._randomize_position did the same for list-valued component positions, so random_seed did not govern either. Both now draw the index through the model's seeded generator via a shared _random_choice helper -- indexing, not numpy.random.choice, so heterogeneous objects (Function, paths, arrays) stay intact. StochasticRocket._set_stochastic also handed the same seed to the rocket body and every surface, motor, rail button and parachute, so components sampling the same distribution drew identical values (a main and a drogue parachute got the same cd_s and lag quantiles). Each component is now reseeded from its own spawned child of a SeedSequence root, in a fixed order, so they stay independent and reproducible under random_seed. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/stochastic/stochastic_rocket.py | 30 ++++++++---- .../test_stochastic_rocket_seeding.py | 46 +++++++++++++++++++ 2 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 tests/unit/stochastic/test_stochastic_rocket_seeding.py diff --git a/rocketpy/stochastic/stochastic_rocket.py b/rocketpy/stochastic/stochastic_rocket.py index 895e9a2a4..2fdcdf9eb 100644 --- a/rocketpy/stochastic/stochastic_rocket.py +++ b/rocketpy/stochastic/stochastic_rocket.py @@ -2,6 +2,8 @@ import warnings +import numpy as np + from rocketpy.control import _Controller from rocketpy.mathutils.vector_matrix import Vector from rocketpy.motors.empty_motor import EmptyMotor @@ -21,6 +23,7 @@ from rocketpy.rocket.rocket import Rocket from rocketpy.stochastic.stochastic_generic_motor import StochasticGenericMotor from rocketpy.stochastic.stochastic_motor_model import StochasticMotorModel +from rocketpy.tools import _seed_sequence_to_int from .stochastic_aero_surfaces import ( StochasticAirBrakes, @@ -177,21 +180,29 @@ def _set_stochastic(self, seed=None): """Set the stochastic attributes for Components, positions and inputs. + Every nested component -- the rocket body, each aerodynamic surface, + motor, rail button and parachute -- is reseeded from its own child of a + ``SeedSequence`` root, so components that sample the same distribution do + not draw identical values (a main and a drogue parachute get independent + ``cd_s`` and ``lag`` samples, not the same one). Children are spawned in a + fixed order, so the result stays reproducible under ``random_seed``. + Parameters ---------- seed : int, optional Seed for the random number generator. """ - super()._set_stochastic(seed) + root = np.random.SeedSequence(seed) + super()._set_stochastic(_seed_sequence_to_int(root.spawn(1)[0])) self.aerodynamic_surfaces = self.__reset_components( - self.aerodynamic_surfaces, seed + self.aerodynamic_surfaces, root ) - self.motors = self.__reset_components(self.motors, seed) - self.rail_buttons = self.__reset_components(self.rail_buttons, seed) + self.motors = self.__reset_components(self.motors, root) + self.rail_buttons = self.__reset_components(self.rail_buttons, root) for parachute in self.parachutes: - parachute._set_stochastic(seed) + parachute._set_stochastic(_seed_sequence_to_int(root.spawn(1)[0])) - def __reset_components(self, components, seed): + def __reset_components(self, components, root): """Creates a new Components whose stochastic structures and their positions are reset. @@ -200,8 +211,9 @@ def __reset_components(self, components, seed): components : Components The components which contains the stochastic structure that will be used to create the new components. - seed : int, optional - Seed for the random number generator. + root : numpy.random.SeedSequence + The run's seed root. Each component is reseeded from its own spawned + child, so components sampling the same distribution stay decorrelated. Returns ------- @@ -213,7 +225,7 @@ def __reset_components(self, components, seed): new_components = Components() for stochastic_obj, _ in components: stochastic_obj_position_info = self.__components_map[stochastic_obj] - stochastic_obj._set_stochastic(seed) + stochastic_obj._set_stochastic(_seed_sequence_to_int(root.spawn(1)[0])) new_components.add( stochastic_obj, self._validate_position(stochastic_obj, stochastic_obj_position_info), diff --git a/tests/unit/stochastic/test_stochastic_rocket_seeding.py b/tests/unit/stochastic/test_stochastic_rocket_seeding.py new file mode 100644 index 000000000..83704eb90 --- /dev/null +++ b/tests/unit/stochastic/test_stochastic_rocket_seeding.py @@ -0,0 +1,46 @@ +"""Nested StochasticRocket components are reseeded from distinct SeedSequence +children, so components that sample the same distribution (a main and a drogue +parachute, for example) do not draw identical values. Reproducible under a fixed +seed. See the seeding design in ``StochasticRocket._set_stochastic``. +""" + +from rocketpy.stochastic.stochastic_model import StochasticModel + +# Captured once, before any patching, so wrapping it repeatedly in one test does +# not stack (each recorder wraps the real method, not a previous recorder). +_REAL_SET_STOCHASTIC = StochasticModel._set_stochastic + + +def _record_component_seeds(monkeypatch, rocket, seed): + """Return the seeds handed to every nested component for one reseed.""" + recorded = [] + + def recording(self, seed=None): + recorded.append(seed) + return _REAL_SET_STOCHASTIC(self, seed) + + monkeypatch.setattr(StochasticModel, "_set_stochastic", recording) + rocket._set_stochastic(seed) + return recorded + + +def test_rocket_components_receive_distinct_seeds(monkeypatch, stochastic_calisto): + """Every nested component (body, aerodynamic surfaces, motor, rail buttons and + the two parachutes) is reseeded from its own child, so none collide.""" + seeds = _record_component_seeds(monkeypatch, stochastic_calisto, 42) + + assert len(seeds) > 3, "expected the rocket body plus several components" + assert len(seeds) == len(set(seeds)), ( + "components share a seed -- they would draw perfectly correlated samples" + ) + + +def test_rocket_component_seeds_are_reproducible(monkeypatch, stochastic_calisto): + """The same root seed reseeds every component identically; a different root + seed changes them.""" + first = _record_component_seeds(monkeypatch, stochastic_calisto, 42) + again = _record_component_seeds(monkeypatch, stochastic_calisto, 42) + different = _record_component_seeds(monkeypatch, stochastic_calisto, 43) + + assert again == first, "same seed must reproduce every component seed" + assert different != first, "a different seed must change the component seeds" From decc84445d48914bdd18160ab64e6c437de41942 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:59:29 +0800 Subject: [PATCH 08/45] BUG: validate the run seed before truncating output; tidy the seed helper simulate() set up (and, for append=False, truncated with w+) the input/output/error files before the seed was validated, so passing a rejected seed such as a Generator destroyed a previous run's results on the way to raising a TypeError. The seed is now captured and validated before __setup_files runs. Moved _seed_sequence_to_int to rocketpy.tools so the stochastic models can share it, and corrected its docstring: a 128-bit int is accepted by default_rng and random.Random, but the legacy RandomState caps a single-int seed at 2**32-1, so the earlier 'accepted by RandomState' claim was wrong. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 54 +++++++------------ rocketpy/tools.py | 23 ++++++++ .../test_monte_carlo_determinism.py | 30 +++++++++++ 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 3801dbf94..b1bd9a43c 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -31,6 +31,7 @@ from rocketpy.prints.monte_carlo_prints import _MonteCarloPrints from rocketpy.simulation.flight import Flight from rocketpy.tools import ( + _seed_sequence_to_int, generate_monte_carlo_ellipses, generate_monte_carlo_ellipses_coordinates, import_optional_dependency, @@ -241,18 +242,22 @@ def simulate( self._export_config = kwargs self.number_of_simulations = number_of_simulations self._initial_sim_idx = self.num_of_loaded_sims if append else 0 - # Small, picklable root seed state captured once per run; every - # simulation index derives its child seed from it (see __child_seed). - self.__root_state = None + + # Capture the small, picklable root seed state once per run (every + # simulation index derives its child seed from it, see __child_seed). + # This validates random_seed *before* __setup_files truncates any + # existing output, so an invalid seed cannot destroy prior results on + # the way to raising. + self.__capture_root_state(random_seed) print("Starting Monte Carlo analysis") self.__setup_files(append) if parallel: - self.__run_in_parallel(n_workers, random_seed) + self.__run_in_parallel(n_workers) else: - self.__run_in_serial(random_seed) + self.__run_in_serial() self.__terminate_simulation() @@ -360,14 +365,12 @@ def __seed_simulation(self, child_seed): self.rocket._set_stochastic(_seed_sequence_to_int(rocket_seed)) self.flight._set_stochastic(_seed_sequence_to_int(flight_seed)) - def __run_in_serial(self, random_seed=None): # pylint: disable=too-many-statements + def __run_in_serial(self): # pylint: disable=too-many-statements """ Runs the monte carlo simulation in serial mode. - Parameters - ---------- - random_seed : int or SeedSequence, optional - Root seed for the run. See ``simulate``. + The root seed state is captured by ``simulate`` before this runs, so each + simulation index derives its child seed from ``self.__root_state``. Returns ------- @@ -378,7 +381,6 @@ def __run_in_serial(self, random_seed=None): # pylint: disable=too-many-stateme n_simulations=self.number_of_simulations, start_time=time(), ) - self.__capture_root_state(random_seed) try: while sim_monitor.keep_simulating(): sim_idx = sim_monitor.increment() - 1 @@ -409,17 +411,19 @@ def __run_in_serial(self, random_seed=None): # pylint: disable=too-many-stateme f.write(inputs_json) raise error - def __run_in_parallel(self, n_workers=None, random_seed=None): + def __run_in_parallel(self, n_workers=None): """ Runs the monte carlo simulation in parallel. + The root seed state is captured by ``simulate`` before this runs and + travels with the pickled instance, so every worker derives the same + per-index child seed from ``self.__root_state``. + Parameters ---------- n_workers: int, optional Number of workers to be used. If None, the number of workers will be equal to the number of CPUs available. Default is None. - random_seed : int or SeedSequence, optional - Root seed for the run. See ``simulate``. Returns ------- @@ -447,8 +451,6 @@ def __run_in_parallel(self, n_workers=None, random_seed=None): # the sampled inputs do not depend on the number of workers. The # root state is small and travels with the pickled instance, so no # per-index seed list is materialized or sent to each process. - self.__capture_root_state(random_seed) - for _ in range(n_workers): sim_producer = multiprocess.Process( target=self.__sim_producer, @@ -1727,26 +1729,6 @@ def export_errors_to_json(self, filename): self._write_log_to_json(self.errors_log, filename) -def _seed_sequence_to_int(seed_sequence): - """Collapse a ``SeedSequence`` into a 128-bit Python ``int``. - - A plain ``int`` is the one seed type accepted alike by - ``numpy.random.default_rng``, ``numpy.random.RandomState`` and the stdlib - ``random.Random`` (which rejects a ``SeedSequence`` with a ``TypeError`` - since Python 3.11), so a custom sampler whose ``reset_seed`` documents an - ``int`` keeps working. All four ``uint32`` words are combined to keep the - full 128-bit pool, so the environment/rocket/flight sub-streams stay - decorrelated instead of collapsing to a single 32-bit word. - - The words are combined by value (little-endian word order), not via - ``tobytes()``, so the seed is the same on big- and little-endian machines - -- a byte-order-dependent seed would break the cross-platform - reproducibility this whole scheme exists to provide. - """ - words = seed_sequence.generate_state(4, dtype=np.uint32) - return sum(int(word) << (32 * position) for position, word in enumerate(words)) - - def _claim_next_index(sim_monitor, mutex): """Atomically claim the next 0-based simulation index, or ``None`` if done. diff --git a/rocketpy/tools.py b/rocketpy/tools.py index 0d7f1a74e..9df900eb5 100644 --- a/rocketpy/tools.py +++ b/rocketpy/tools.py @@ -1467,6 +1467,29 @@ def find_obj_from_hash(obj, hash_, depth_limit=None): return None +def _seed_sequence_to_int(seed_sequence): + """Collapse a ``SeedSequence`` into a 128-bit Python ``int``. + + A plain ``int`` is what ``numpy.random.default_rng`` and the stdlib + ``random.Random`` both accept (``random.Random`` rejects a ``SeedSequence`` + with a ``TypeError`` since Python 3.11), so a custom sampler whose + ``reset_seed`` documents an ``int`` and builds a modern generator keeps + working. The legacy ``numpy.random.RandomState`` is the exception: it caps a + single-integer seed at ``2**32 - 1``, so a sampler still built on it would + have to reduce the value (``RandomState`` is a frozen legacy API NumPy steers + new code away from). All four ``uint32`` words are combined to keep the full + 128-bit pool, so sub-streams stay decorrelated instead of collapsing to a + single 32-bit word. + + The words are combined by value (little-endian word order), not via + ``tobytes()``, so the seed is the same on big- and little-endian machines -- + a byte-order-dependent seed would break the cross-platform reproducibility + this exists to provide. + """ + words = seed_sequence.generate_state(4, dtype=np.uint32) + return sum(int(word) << (32 * position) for position, word in enumerate(words)) + + if __name__ == "__main__": # pragma: no cover import doctest diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index f0355ca71..2d2cfe756 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -150,6 +150,36 @@ def _simulate_inputs( return _read_inputs_by_index(montecarlo.input_file) +def test_invalid_seed_does_not_truncate_existing_output( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """A rejected seed must fail before any output file is truncated, so passing + an invalid seed cannot destroy the results of a previous run.""" + monkeypatch.setattr(mc_module, "Flight", _StubFlight) + montecarlo = MonteCarlo( + filename=str(tmp_path / "keep"), + environment=stochastic_environment, + rocket=stochastic_calisto_numpy_only, + flight=stochastic_flight, + ) + with open(montecarlo.input_file, "w", encoding="utf-8") as existing: + existing.write("previous results\n") + + # A Generator is not a seed and is rejected; the run must raise before the + # ``w+`` file setup truncates anything. + with pytest.raises(TypeError): + montecarlo.simulate( + number_of_simulations=3, random_seed=np.random.default_rng(0) + ) + + with open(montecarlo.input_file, encoding="utf-8") as kept: + assert kept.read() == "previous results\n" + + def test_serial_inputs_are_reproducible( monkeypatch, tmp_path, From 50cc672347311880b019f4283036f4dc4cb959d4 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Tue, 4 Aug 2026 05:57:13 +0800 Subject: [PATCH 09/45] BUG: hold each stochastic model's nominal values steady across a run _set_stochastic re-validates every kwarg, and validation read the nominal back off self.obj. StochasticEnvironment.create_object writes the sampled value onto that same object on purpose, so the next reseed took the last simulation's result as the new baseline and a factor like wind_velocity_x_factor compounded from one simulation to the next. Serial and parallel runs then disagreed, because the drift depends on how many simulations a worker happened to run before that index. Capture the nominal once, when the model is built, and read it from there. Custom getters pass straight through: they read a component's own attribute rather than one of self.obj's, and every component's position arrives under the one name "position", so caching those would collide. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/stochastic/stochastic_model.py | 32 +++++++-- .../unit/stochastic/test_stochastic_model.py | 71 +++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/rocketpy/stochastic/stochastic_model.py b/rocketpy/stochastic/stochastic_model.py index 333a6d891..12f82ce37 100644 --- a/rocketpy/stochastic/stochastic_model.py +++ b/rocketpy/stochastic/stochastic_model.py @@ -122,8 +122,30 @@ def __init__(self, obj, seed=None, **kwargs): self.obj = obj self.last_rnd_dict = {} self.__stochastic_dict = kwargs + self.__nominal_values = {} self._set_stochastic(seed) + def _nominal(self, input_name, getter=getattr): + """``self.obj``'s value for ``input_name``, as it was when this model + was built. + + Read once and remembered, because ``StochasticEnvironment`` has + ``create_object`` write the randomised value back onto ``self.obj`` + instead of building a copy. Re-reading it on a reseed would take one + simulation's output as the next one's nominal, and a factor would + multiply the factor before it rather than the original value. + + A custom ``getter`` reads a component's own attribute rather than one + of ``self.obj``'s, and nothing writes back to those, so it is passed + straight through. Caching it here would be wrong as well: every + component's position arrives under the one name ``"position"``. + """ + if getter is not getattr: + return getter(self.obj, input_name) + if input_name not in self.__nominal_values: + self.__nominal_values[input_name] = getattr(self.obj, input_name) + return self.__nominal_values[input_name] + def _set_stochastic(self, seed=None): """Set the stochastic attributes from the input dictionary. This method is useful to reset or reseed the attributes of the instance. @@ -170,7 +192,7 @@ def _set_stochastic(self, seed=None): "or a custom sampler" ) else: - attr_value = [getattr(self.obj, input_name)] + attr_value = [self._nominal(input_name)] setattr(self, input_name, attr_value) def __repr__(self): @@ -305,7 +327,7 @@ def _validate_tuple_length_two(self, input_name, input_value, getattr=getattr): # object passed. dist_func = get_distribution(input_value[1], self.__random_number_generator) return ( - self._nominal_value(input_name, getattr(self.obj, input_name)), + self._nominal_value(input_name, self._nominal(input_name, getattr)), input_value[0], dist_func, ) @@ -381,7 +403,7 @@ def _validate_list(self, input_name, input_value, getattr=getattr): # pylint: d If the input is not in a valid format. """ if not input_value: - return [getattr(self.obj, input_name)] + return [self._nominal(input_name, getattr)] else: return input_value @@ -407,7 +429,7 @@ def _validate_scalar(self, input_name, input_value, getattr=getattr): # pylint: distribution function). """ return ( - self._nominal_value(input_name, getattr(self.obj, input_name)), + self._nominal_value(input_name, self._nominal(input_name, getattr)), input_value, get_distribution("normal", self.__random_number_generator), ) @@ -434,7 +456,7 @@ def _validate_factors(self, input_name, input_value): If the input is not in a valid format. """ attribute_name = input_name.replace("_factor", "") - setattr(self, f"_{attribute_name}", getattr(self.obj, attribute_name)) + setattr(self, f"_{attribute_name}", self._nominal(attribute_name)) if isinstance(input_value, tuple): return self._validate_tuple_factor(input_name, input_value) diff --git a/tests/unit/stochastic/test_stochastic_model.py b/tests/unit/stochastic/test_stochastic_model.py index c1c27ccfb..bb4426a4f 100644 --- a/tests/unit/stochastic/test_stochastic_model.py +++ b/tests/unit/stochastic/test_stochastic_model.py @@ -3,6 +3,8 @@ import pytest from rocketpy.stochastic import StochasticFreeFormFins +from rocketpy import Environment +from rocketpy.stochastic import StochasticEnvironment from rocketpy.stochastic.stochastic_model import StochasticModel @@ -82,3 +84,72 @@ def spans(seed): # Both candidates must stay reachable, or the assertions above would also # hold for a generator that always returned the same one. assert set(spans(7)) == {0.1, 0.12} + + +def _effective_wind_x(environment): + """The wind the Environment would actually fly with.""" + wind = environment.wind_velocity_x + return float(wind(0)) if callable(wind) else float(wind) + + +def test_reseeding_does_not_take_the_last_run_as_the_next_nominal(): + """Reseeding with the same seed has to give the same inputs. + + ``StochasticEnvironment.create_object`` writes the randomised value back + onto the Environment rather than building a copy, so re-reading the nominal + from it on the next reseed compounded: 10 -> 8.576 -> 7.355 -> 6.308, each + one the last multiplied by the same factor again. + """ + environment = Environment() + environment.set_atmospheric_model(type="standard_atmosphere") + environment.wind_velocity_x = 10.0 + stochastic = StochasticEnvironment( + environment=environment, wind_velocity_x_factor=(1.0, 0.1) + ) + + winds = [] + for _ in range(4): + stochastic._set_stochastic(12345) + winds.append(_effective_wind_x(stochastic.create_object())) + + assert len(set(winds)) == 1, f"the same seed drifted across reseeds: {winds}" + + +def test_a_simulation_index_does_not_depend_on_the_indices_before_it(): + """What the per-index seeding claims: index i gets the same inputs however + it is reached. Running 0, 1, 2 in order has to match running 2 on its own, + which is what a worker that happens to pick up index 2 first would do. + """ + + def wind_for(seeds): + environment = Environment() + environment.set_atmospheric_model(type="standard_atmosphere") + environment.wind_velocity_x = 10.0 + stochastic = StochasticEnvironment( + environment=environment, wind_velocity_x_factor=(1.0, 0.1) + ) + wind = None + for seed in seeds: + stochastic._set_stochastic(seed) + wind = _effective_wind_x(stochastic.create_object()) + return wind + + assert wind_for([101, 102, 103]) == wind_for([103]) + + +def test_a_scalar_nominal_does_not_drift_across_reseeds(): + """Not only the factors. ``_validate_scalar`` and the ``(std, "distribution")`` + tuple both take their nominal from the object, and ``create_object`` writes + the drawn value back onto that same object, so a plain scalar spec drifts + the same way a factor compounds. + """ + environment = Environment() + environment.set_atmospheric_model(type="standard_atmosphere") + stochastic = StochasticEnvironment(environment=environment, elevation=100.0) + + elevations = [] + for _ in range(4): + stochastic._set_stochastic(2024) + elevations.append(float(stochastic.create_object().elevation)) + + assert len(set(elevations)) == 1, f"the nominal elevation drifted: {elevations}" From c93d751dcb72a2107b80671a098e9af1b9d0479c Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Tue, 4 Aug 2026 05:57:25 +0800 Subject: [PATCH 10/45] BUG: reseed air brakes and reapply eccentricity for every simulation Two things create_object samples were left out of the per-simulation reseed. Air brakes were never in the reseed loop at all, so they drew from wherever the generator had been left rather than from the simulation index. Every seeding test passed because no fixture had an air brake, which is exactly how it stayed hidden. The collections are now declared in one place and walked from there, and a test scans create_object's source so a collection added later cannot quietly miss the reseed. CP and thrust eccentricity were validated once, at add time, against the generator as it stood then. Reseeding replaced the generator but not those values. Keep the specs as given and reapply them after each reseed. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/stochastic/stochastic_rocket.py | 72 +++++++--- .../test_stochastic_rocket_seeding.py | 125 ++++++++++++++++++ 2 files changed, 177 insertions(+), 20 deletions(-) diff --git a/rocketpy/stochastic/stochastic_rocket.py b/rocketpy/stochastic/stochastic_rocket.py index 2fdcdf9eb..10ad90827 100644 --- a/rocketpy/stochastic/stochastic_rocket.py +++ b/rocketpy/stochastic/stochastic_rocket.py @@ -158,6 +158,13 @@ def __init__( self.air_brakes = [] self.parachutes = [] self.__components_map = {} + # Raw eccentricity arguments, kept as the caller gave them. + # ``add_cp_eccentricity`` and ``add_thrust_eccentricity`` run after + # ``__init__``, so their values are not in the dict the base class + # re-validates on a reseed. Validating them once would leave the + # distribution bound to the Generator of whichever simulation happened + # to come first, so the raw form is kept and validated again each time. + self.__eccentricity_specs = {} super().__init__( obj=rocket, radius=radius, @@ -176,16 +183,31 @@ def __init__( coordinate_system_orientation=None, ) + # Every collection of nested stochastic objects, in the order their child + # seeds are spawned. Listed here rather than written out inline so that a + # component type cannot end up in ``create_object`` and not in the reseed: + # air brakes were, and their sampling depended on which worker ran the + # index instead of on the index. ``_stochastic_collections`` is asserted + # against the rocket's own attributes in the tests. + _POSITIONED_COLLECTIONS = ("aerodynamic_surfaces", "motors", "rail_buttons") + _PLAIN_COLLECTIONS = ("parachutes", "air_brakes") + + @classmethod + def _stochastic_collections(cls): + """The names of every attribute holding nested stochastic objects.""" + return cls._POSITIONED_COLLECTIONS + cls._PLAIN_COLLECTIONS + def _set_stochastic(self, seed=None): """Set the stochastic attributes for Components, positions and inputs. Every nested component -- the rocket body, each aerodynamic surface, - motor, rail button and parachute -- is reseeded from its own child of a - ``SeedSequence`` root, so components that sample the same distribution do - not draw identical values (a main and a drogue parachute get independent - ``cd_s`` and ``lag`` samples, not the same one). Children are spawned in a - fixed order, so the result stays reproducible under ``random_seed``. + motor, rail button, parachute and air brake -- is reseeded from its own + child of a ``SeedSequence`` root, so components that sample the same + distribution do not draw identical values (a main and a drogue parachute + get independent ``cd_s`` and ``lag`` samples, not the same one). Children + are spawned in a fixed order, so the result stays reproducible under + ``random_seed``. Parameters ---------- @@ -194,13 +216,12 @@ def _set_stochastic(self, seed=None): """ root = np.random.SeedSequence(seed) super()._set_stochastic(_seed_sequence_to_int(root.spawn(1)[0])) - self.aerodynamic_surfaces = self.__reset_components( - self.aerodynamic_surfaces, root - ) - self.motors = self.__reset_components(self.motors, root) - self.rail_buttons = self.__reset_components(self.rail_buttons, root) - for parachute in self.parachutes: - parachute._set_stochastic(_seed_sequence_to_int(root.spawn(1)[0])) + self.__apply_eccentricity_specs() + for name in self._POSITIONED_COLLECTIONS: + setattr(self, name, self.__reset_components(getattr(self, name), root)) + for name in self._PLAIN_COLLECTIONS: + for child in getattr(self, name): + child._set_stochastic(_seed_sequence_to_int(root.spawn(1)[0])) def __reset_components(self, components, root): """Creates a new Components whose stochastic structures @@ -468,8 +489,9 @@ def add_cp_eccentricity(self, x=None, y=None): self : StochasticRocket Object of the StochasticRocket class. """ - self.cp_eccentricity_x = self._validate_eccentricity("cp_eccentricity_x", x) - self.cp_eccentricity_y = self._validate_eccentricity("cp_eccentricity_y", y) + self.__eccentricity_specs["cp_eccentricity_x"] = x + self.__eccentricity_specs["cp_eccentricity_y"] = y + self.__apply_eccentricity_specs() return self def add_thrust_eccentricity(self, x=None, y=None): @@ -494,14 +516,24 @@ def add_thrust_eccentricity(self, x=None, y=None): self : StochasticRocket Object of the StochasticRocket class. """ - self.thrust_eccentricity_x = self._validate_eccentricity( - "thrust_eccentricity_x", x - ) - self.thrust_eccentricity_y = self._validate_eccentricity( - "thrust_eccentricity_y", y - ) + self.__eccentricity_specs["thrust_eccentricity_x"] = x + self.__eccentricity_specs["thrust_eccentricity_y"] = y + self.__apply_eccentricity_specs() return self + def __apply_eccentricity_specs(self): + """Re-validate the eccentricities against the current Generator. + + Validation stores a distribution as a method bound to the Generator + that was live at the time, so a tuple validated once keeps sampling + from that one. Re-running it after every reseed is what ties the draw + to the simulation index rather than to whichever index the worker + happened to run first. ``get_distribution`` only binds a method, so + this consumes no randomness and does not shift any other draw. + """ + for name, spec in self.__eccentricity_specs.items(): + setattr(self, name, self._validate_eccentricity(name, spec)) + def _validate_eccentricity(self, eccentricity, position): """Validate the eccentricity argument. diff --git a/tests/unit/stochastic/test_stochastic_rocket_seeding.py b/tests/unit/stochastic/test_stochastic_rocket_seeding.py index 83704eb90..0e3efef8a 100644 --- a/tests/unit/stochastic/test_stochastic_rocket_seeding.py +++ b/tests/unit/stochastic/test_stochastic_rocket_seeding.py @@ -4,6 +4,12 @@ seed. See the seeding design in ``StochasticRocket._set_stochastic``. """ +import ast +import inspect + +import pytest + +from rocketpy.stochastic import StochasticAirBrakes from rocketpy.stochastic.stochastic_model import StochasticModel # Captured once, before any patching, so wrapping it repeatedly in one test does @@ -44,3 +50,122 @@ def test_rocket_component_seeds_are_reproducible(monkeypatch, stochastic_calisto assert again == first, "same seed must reproduce every component seed" assert different != first, "a different seed must change the component seeds" + + +def test_the_reseed_covers_every_collection_create_object_uses(stochastic_calisto): + """Whatever ``create_object`` iterates has to be reseeded too. + + Checked against the source rather than against a fixture, because a + collection that no fixture populates is exactly the one that gets missed: + air brakes were built and sampled and never reseeded, and every seeding + test passed because no fixture had one. + """ + rocket = stochastic_calisto + tree = ast.parse(inspect.getsource(type(rocket).create_object).lstrip()) + iterated = { + node.iter.attr + for node in ast.walk(tree) + # Comprehensions too: this scan exists to catch a collection added + # later, and a loop rewritten as one would slip past a For-only walk. + if isinstance(node, (ast.For, ast.comprehension)) + and isinstance(node.iter, ast.Attribute) + and isinstance(node.iter.value, ast.Name) + and node.iter.value.id == "self" + and not node.iter.attr.startswith("_") + } + declared = set(type(rocket)._stochastic_collections()) + + assert iterated, "found no collections in create_object; the scan is broken" + assert iterated <= declared, ( + f"create_object samples these but the reseed never reaches them: " + f"{sorted(iterated - declared)}" + ) + + +def test_air_brakes_are_reseeded_like_every_other_component( + monkeypatch, stochastic_calisto, calisto_air_brakes_clamp_on +): + """Air brakes were in ``create_object`` and not in the reseed, so their + samples came from wherever the Generator had been left rather than from the + simulation index. Measured before the fix: 3 surfaces, 1 motor, 1 rail + button and 2 parachutes reseeded, air brakes 0 of 1. + """ + stochastic_calisto.add_air_brakes( + calisto_air_brakes_clamp_on.air_brakes[0], + calisto_air_brakes_clamp_on._controllers[0], + ) + air_brake = stochastic_calisto.air_brakes[0] + seen = [] + original = air_brake._set_stochastic + monkeypatch.setattr( + air_brake, + "_set_stochastic", + lambda seed=None: (seen.append(seed), original(seed))[1], + ) + + stochastic_calisto._set_stochastic(42) + + assert seen, "air brakes were not reseeded" + assert seen[0] is not None + + +@pytest.mark.parametrize( + "spec", + [0.001, (0.001, "normal"), (0.0, 0.001, "normal"), [0.0005, 0.001, 0.002]], + ids=["scalar", "tuple2", "tuple3", "list"], +) +def test_eccentricity_is_resampled_from_the_new_generator(stochastic_calisto, spec): + """``add_cp_eccentricity`` and ``add_thrust_eccentricity`` run after + ``__init__``, so their values never reached the dict the base class + re-validates. Validation binds a distribution to the Generator that is live + at the time, so the tuple kept sampling from the one the rocket was built + with: same seed, different eccentricity, while every constructor field + reproduced exactly. + """ + rocket = stochastic_calisto + rocket.add_cp_eccentricity(x=spec, y=spec) + rocket.add_thrust_eccentricity(x=spec, y=spec) + + def sample(): + rocket._set_stochastic(777) + drawn = next(rocket.dict_generator()) + return {k: v for k, v in drawn.items() if "eccentricity" in k} + + first = sample() + + assert len(first) == 4, f"expected four eccentricities, got {sorted(first)}" + assert sample() == first, "the same seed drew a different eccentricity" + + +def test_the_air_brake_sample_follows_the_seed_not_the_call_order( + stochastic_calisto, calisto_air_brakes_clamp_on +): + """That the reseed reaches the air brake is only half of it. + + What matters is the value it draws: the same seed has to give the same + sample, and a different seed a different one. Asserting only that + ``_set_stochastic`` was called would pass over an air brake reseeded with a + constant. + """ + # Built here rather than taken from the fixture: wrapping an AirBrakes with + # no arguments gives every parameter a zero standard deviation, so it draws + # the same values under any seed and the assertions below would hold over an + # air brake that was never reseeded at all. + stochastic_calisto.add_air_brakes( + StochasticAirBrakes( + air_brakes=calisto_air_brakes_clamp_on.air_brakes[0], + drag_coefficient_curve_factor=(1.0, 0.1), + ), + calisto_air_brakes_clamp_on._controllers[0], + ) + air_brake = stochastic_calisto.air_brakes[0] + + def drawn(seed): + stochastic_calisto._set_stochastic(seed) + return next(air_brake.dict_generator()) + + first = drawn(31337) + + assert first, "the air brake sampled nothing, so this proves nothing" + assert drawn(31337) == first, "the same seed drew a different air brake" + assert drawn(31338) != first, "a different seed drew the same air brake" From 15cf21cccfd1fbb691f7908d97440070852a622c Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Tue, 4 Aug 2026 05:57:38 +0800 Subject: [PATCH 11/45] BUG: stop a failed or interrupted parallel run from looking successful In the worker: sim_idx and inputs_json are bound before the try. A failure in the index claim used to raise UnboundLocalError inside the error handler, so nothing was written and nothing was printed and the run ended with no record of what went wrong. The handler now writes a JSON line either way, since _read_log_file parses that file with json.loads. Reporting a failure is best effort and must never replace the failure it is reporting. Setting the shared event can raise on its own once the manager has gone, and so can taking the mutex or writing the file. All of it is guarded, the mutex is released if it was taken, and the original exception is what leaves the worker. The worker re-raises so its exit code says it died. In the parent: join() returns None however a child ended, so the shared event was the only signal a run had. A worker can leave without setting it: SystemExit, os._exit, a segfault in a native extension, a target that will not unpickle under spawn, or its own error handler failing. Check the exit codes too. Workers are started inside the try, so a start() that fails part way through the fleet does not leave the running ones with nobody to reap them. After the run, check that every index this run claimed left exactly one input row and one output row. Neither file shows this on its own: the rows look well formed, and reading them back keyed by index hides a duplicate behind the row that overwrote it. A row cut off mid-write is reported as the index that went missing rather than failing to parse, which is what actually happened to it. A run stopped with Ctrl-C is exempt, since both run paths catch it, keep what they have and return, and being short is the point rather than a fault. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 227 ++++++++++++---- .../test_monte_carlo_worker_exit.py | 179 ++++++++++++ .../test_monte_carlo_worker_failures.py | 256 ++++++++++++++++++ 3 files changed, 616 insertions(+), 46 deletions(-) create mode 100644 tests/unit/simulation/test_monte_carlo_worker_exit.py create mode 100644 tests/unit/simulation/test_monte_carlo_worker_failures.py diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index b1bd9a43c..8eb97eacd 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -242,6 +242,10 @@ def simulate( self._export_config = kwargs self.number_of_simulations = number_of_simulations self._initial_sim_idx = self.num_of_loaded_sims if append else 0 + # Both run paths catch Ctrl-C, save what they have and return, so a + # stopped run is incomplete on purpose and the completeness check below + # has to know the difference between that and a worker going missing. + self._interrupted = False # Capture the small, picklable root seed state once per run (every # simulation index derives its child seed from it, see __child_seed). @@ -259,6 +263,7 @@ def simulate( else: self.__run_in_serial() + self.__check_each_index_was_recorded_once() self.__terminate_simulation() def __setup_files(self, append): @@ -365,7 +370,61 @@ def __seed_simulation(self, child_seed): self.rocket._set_stochastic(_seed_sequence_to_int(rocket_seed)) self.flight._set_stochastic(_seed_sequence_to_int(flight_seed)) - def __run_in_serial(self): # pylint: disable=too-many-statements + def __check_each_index_was_recorded_once(self): + """Every index this run claimed left exactly one input and one output row. + + The counter hands each index out once, so a missing one means a worker + stopped between claiming and writing, and a repeated one means two + claimed the same index. Neither is visible in the files themselves: the + rows look well formed, and reading them back keyed by index hides the + duplicate behind the row that overwrote it. Both make the results wrong + while the run reports success, which is the thing per-index seeding is + supposed to rule out. + + Only over the range this run produced. ``append=True`` leaves earlier + runs in the same files, and ``number_of_simulations`` is the total to + reach rather than a count to add, so the new indices are + ``_initial_sim_idx`` up to it. + + A run stopped with Ctrl-C is exempt: both run paths catch it, keep what + they have and return, so being short is the point rather than a fault. + """ + expected = set(range(self._initial_sim_idx, self.number_of_simulations)) + if not expected or self._interrupted: + # A stopped run is short by definition and already said so. Checking + # it anyway contradicted the "Files saved." it had just printed. + return + + for label, path in ( + ("inputs", self.input_file), + ("outputs", self.output_file), + ): + written = {} + with open(path, mode="r", encoding="utf-8") as rows: + for line in rows: + line = line.strip() + if line: + try: + index = json.loads(line).get("index") + except ValueError: + # A worker killed mid-write leaves a partial row. + # Skipping it reports that index as missing, which + # is what happened, rather than failing to parse. + continue + written[index] = written.get(index, 0) + 1 + + missing = sorted(expected - set(written)) + repeated = sorted(index for index in expected if written.get(index, 0) > 1) + if missing or repeated: + raise RuntimeError( + f"the {label} file does not match the simulations that ran: " + f"{len(missing)} never written {missing[:5]}, " + f"{len(repeated)} written more than once {repeated[:5]}. " + f"The results are incomplete, so they are not reported as a " + f"successful run." + ) + + def __run_in_serial(self): """ Runs the monte carlo simulation in serial mode. @@ -401,16 +460,20 @@ def __run_in_serial(self): # pylint: disable=too-many-statements sim_monitor.print_final_status() except KeyboardInterrupt: + self._interrupted = True print("Keyboard interrupt received. Files saved.") - with open(self._error_file, "a", encoding="utf-8") as f: - f.write(inputs_json) + self.__keep_the_inputs_that_did_not_finish(inputs_json) except Exception as error: print(f"Error on iteration {sim_monitor.count}: {error}") - with open(self._error_file, "a", encoding="utf-8") as f: - f.write(inputs_json) + self.__keep_the_inputs_that_did_not_finish(inputs_json) raise error + def __keep_the_inputs_that_did_not_finish(self, inputs_json): + """Append the inputs of a simulation that stopped part way through.""" + with open(self._error_file, "a", encoding="utf-8") as f: + f.write(inputs_json) + def __run_in_parallel(self, n_workers=None): """ Runs the monte carlo simulation in parallel. @@ -444,36 +507,35 @@ def __run_in_parallel(self, n_workers=None): start_time=time(), ) - processes = [] - # Each worker derives one independent child seed per simulation - # index (not per worker) from the shared root state: the counter - # assigns indices and index i always seeds from __child_seed(i), so - # the sampled inputs do not depend on the number of workers. The - # root state is small and travels with the pickled instance, so no - # per-index seed list is materialized or sent to each process. - for _ in range(n_workers): - sim_producer = multiprocess.Process( - target=self.__sim_producer, - args=( - sim_monitor, - mutex, - simulation_error_event, - ), - ) - processes.append(sim_producer) - sim_producer.start() - + # Started workers only, and inside the try, so a ``start()`` that + # fails part way through the fleet does not leave the ones already + # running with nobody to clean them up. + started_processes = [] try: - for sim_producer in processes: + # Each worker derives one independent child seed per simulation + # index (not per worker) from the shared root state: the counter + # assigns indices and index i always seeds from __child_seed(i), + # so the sampled inputs do not depend on the number of workers. + # The root state is small and travels with the pickled instance, + # so no per-index seed list is materialized or sent. + for _ in range(n_workers): + sim_producer = multiprocess.Process( + target=self.__sim_producer, + args=( + sim_monitor, + mutex, + simulation_error_event, + ), + ) + sim_producer.start() + started_processes.append(sim_producer) + + for sim_producer in started_processes: sim_producer.join() - # Handle error from the child processes - if simulation_error_event.is_set(): - raise RuntimeError( - "An error occurred during the simulation. \n" - f"Check the logs and error file {self.error_file} " - "for more information." - ) + _fail_if_a_worker_did_not_finish( + started_processes, simulation_error_event, self.error_file + ) sim_monitor.print_final_status() @@ -482,11 +544,14 @@ def __run_in_parallel(self, n_workers=None): except (Exception, KeyboardInterrupt) as error: simulation_error_event.set() - for sim_producer in processes: + for sim_producer in started_processes: sim_producer.join() - if not isinstance(error, KeyboardInterrupt): + self._interrupted = isinstance(error, KeyboardInterrupt) + if not self._interrupted: raise error + finally: + _stop_any_worker_still_running(started_processes) def __validate_number_of_workers(self, n_workers): if n_workers is None or n_workers > os.cpu_count(): @@ -508,6 +573,13 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to error_event : multiprocess.Event Event signaling an error occurred during the simulation. """ + # Bound before the try, not inside the loop. The handler below reports + # both, and a failure in the claim itself left them unassigned, so the + # original error was replaced by an UnboundLocalError raised out of the + # handler with the mutex still held. + sim_idx = None + inputs_json = "" + outputs_json = "" try: while True: sim_idx = _claim_next_index(sim_monitor, mutex) @@ -546,18 +618,47 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to finally: mutex.release() - except Exception: # pylint: disable=broad-except - mutex.acquire() - with open(self.error_file, "a", encoding="utf-8") as f: - f.write(inputs_json) - - # See note above: must use print() to remain visible from a - # multiprocessing worker process. - _SimMonitor.reprint( - f"Error on iteration {sim_idx}:\n{traceback.format_exc()}" + except Exception: + # Set first, so a parent waiting on the join learns why. Best effort + # like everything below it: this is a manager proxy, the manager may + # already be gone, and reporting must not replace what it reports. + try: + error_event.set() + except Exception: # pylint: disable=broad-exception-caught + pass + details = traceback.format_exc() + + # The inputs of the failed simulation when there are any, and a + # record of the failure itself when it happened before they were + # drawn. Without the second, a failure in the claim left the error + # file empty while the run pointed the user at it. It is a JSON + # line either way, because ``_read_log_file`` parses this file with + # ``json.loads`` and free text in it would make the log unreadable. + record = inputs_json or ( + json.dumps({"index": sim_idx, "error": details}) + "\n" ) - error_event.set() - mutex.release() + + acquired = False + try: + mutex.acquire() + acquired = True + with open(self.error_file, "a", encoding="utf-8") as f: + f.write(record) + + # See note above: must use print() to remain visible from a + # multiprocessing worker process. + _SimMonitor.reprint(f"Error on iteration {sim_idx}:\n{details}") + except Exception: # pylint: disable=broad-exception-caught + # The mutex or the error file is unreachable too. Reporting is + # not worth losing the failure that started this. + pass + finally: + if acquired: + mutex.release() + + # The worker exits non-zero, so the parent can tell a crash from a + # clean finish rather than only from the error event. + raise def __run_single_simulation(self): """Runs a single simulation and returns the inputs and outputs. @@ -1499,7 +1600,7 @@ def export_ellipses_to_kml( # pylint: disable=too-many-statements except KeyError as e: raise KeyError("No impact data found. Skipping impact ellipses.") from e - (apogee_ellipses, impact_ellipses) = generate_monte_carlo_ellipses( + apogee_ellipses, impact_ellipses = generate_monte_carlo_ellipses( impact_x, impact_y, apogee_x, @@ -1729,6 +1830,40 @@ def export_errors_to_json(self, filename): self._write_log_to_json(self.errors_log, filename) +def _stop_any_worker_still_running(started_processes): + """Whatever is still going here is not going to stop on its own. + + The error event was set and it did not leave. Left behind, it keeps the + manager, the mutex and the output files alive. + """ + for sim_producer in started_processes: + if sim_producer.is_alive(): + sim_producer.terminate() + sim_producer.join() + + +def _fail_if_a_worker_did_not_finish(started_processes, error_event, error_file): + """Raise unless every worker finished and none of them reported an error. + + A worker can die without ever setting the event: SystemExit, ``os._exit``, a + segfault in a native extension, a target that will not unpickle under spawn, + or the error handler itself failing. ``join()`` returns None whatever + happened, so the exit status is the only thing that separates a crash from a + clean finish. + """ + crashed = [ + f"{sim_producer.name} exited with {sim_producer.exitcode}" + for sim_producer in started_processes + if sim_producer.exitcode != 0 + ] + if error_event.is_set() or crashed: + raise RuntimeError( + "An error occurred during the simulation. \n" + + (f"Workers that did not exit cleanly: {crashed}. \n" if crashed else "") + + f"Check the logs and error file {error_file} for more information." + ) + + def _claim_next_index(sim_monitor, mutex): """Atomically claim the next 0-based simulation index, or ``None`` if done. diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py new file mode 100644 index 000000000..52cd0bce9 --- /dev/null +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -0,0 +1,179 @@ +"""The parent has to notice a worker that died without saying so. + +``join()`` returns None however the child ended, so the shared error event was +the only signal the run had. A worker can leave without setting it: ``SystemExit``, +``os._exit``, a segfault in a native extension, a target that will not unpickle +under spawn, or the error handler of the worker itself failing. The exit status +is what separates those from a clean finish. +""" + +import types +from contextlib import contextmanager + +import pytest + +import rocketpy.simulation.monte_carlo as mc + + +class _Process: + """A worker that does not run, and reports the exit code it was given.""" + + instances = [] + + def __init__(self, target=None, args=(), **_kwargs): # pylint: disable=unused-argument + self.name = f"worker-{len(self.instances)}" + self.exitcode = None + self.started = False + self.terminated = False + self._planned_exitcode = 0 + self.instances.append(self) + + def start(self): + self.started = True + + def join(self, *_a, **_k): + self.exitcode = self._planned_exitcode + + def is_alive(self): + return False + + def terminate(self): + self.terminated = True + + +class _Event: + def __init__(self): + self.flag = False + + def set(self): + self.flag = True + + def is_set(self): + return self.flag + + +class _Monitor: + def __init__(self, **_kwargs): + pass + + def print_final_status(self): + pass + + +@pytest.fixture +def parallel_runner(monkeypatch, tmp_path): + """Run ``__run_in_parallel`` over stub workers and hand back the stubs.""" + _Process.instances = [] + fake_multiprocess = types.SimpleNamespace(Process=_Process) + + class _Manager: # pylint: disable=invalid-name + """Method names mirror the multiprocess manager API.""" + + def Lock(self): # noqa: N802 + return types.SimpleNamespace(acquire=lambda: None, release=lambda: None) + + def Event(self): # noqa: N802 + return _Event() + + def _SimMonitor(self, **kwargs): # noqa: N802 + return _Monitor(**kwargs) + + @contextmanager + def fake_manager(*_a, **_k): + yield _Manager() + + monkeypatch.setattr(mc, "_import_multiprocess", lambda: (fake_multiprocess, None)) + monkeypatch.setattr(mc, "_create_multiprocess_manager", fake_manager) + + runner = types.SimpleNamespace( + error_file=tmp_path / "errors.txt", + input_file=tmp_path / "inputs.txt", + output_file=tmp_path / "outputs.txt", + _initial_sim_idx=0, + number_of_simulations=4, + _interrupted=False, + _MonteCarlo__validate_number_of_workers=lambda n: 2, + _MonteCarlo__sim_producer=lambda *a: None, + ) + runner.input_file.write_text("") + runner.output_file.write_text("") + return runner + + +def test_a_worker_that_crashes_without_setting_the_event_fails_the_run( + parallel_runner, +): + """The case the event alone cannot see.""" + original_join = _Process.join + + def crash(self, *a, **k): + original_join(self, *a, **k) + self.exitcode = -11 # SIGSEGV + + _Process.join = crash + try: + with pytest.raises(RuntimeError, match="did not exit cleanly"): + mc.MonteCarlo._MonteCarlo__run_in_parallel(parallel_runner, n_workers=2) + finally: + _Process.join = original_join + + +def test_a_clean_run_is_not_reported_as_a_crash(parallel_runner): + """The other half: every worker exits 0, so nothing is raised.""" + mc.MonteCarlo._MonteCarlo__run_in_parallel(parallel_runner, n_workers=2) + + assert all(p.exitcode == 0 for p in _Process.instances) + + +def test_a_failed_start_still_cleans_up_the_workers_already_running( + parallel_runner, monkeypatch +): + """The start loop is inside the try for this. A ``start()`` that fails part + way through used to leave the ones already running with nobody to reap + them.""" + started = [] + original_start = _Process.start + + def start_then_fail(self): + if len(started) >= 1: + raise OSError("cannot allocate a process") + original_start(self) + started.append(self) + + monkeypatch.setattr(_Process, "start", start_then_fail) + monkeypatch.setattr(_Process, "is_alive", lambda self: not self.terminated) + + with pytest.raises(OSError): + mc.MonteCarlo._MonteCarlo__run_in_parallel(parallel_runner, n_workers=3) + + assert started, "the fixture never started anything" + assert all(p.terminated for p in started), "a started worker was left running" + + +def test_an_interrupted_run_is_not_then_reported_as_incomplete( + parallel_runner, monkeypatch +): + """Ctrl-C in the parent is caught and deliberately not re-raised, so + ``simulate`` carries on to the completeness check with the run unfinished. + The two are composed here in that order, since the check has to be able to + tell a run the user stopped from a worker that went missing. + """ + interrupted = [] + original_join = _Process.join + + def ctrl_c(self, *a, **k): + # Once: the handler joins again on its way out, and that has to work. + if not interrupted: + interrupted.append(True) + raise KeyboardInterrupt("user pressed ctrl-c") + original_join(self, *a, **k) + + monkeypatch.setattr(_Process, "join", ctrl_c) + + mc.MonteCarlo._MonteCarlo__run_in_parallel(parallel_runner, n_workers=2) + mc.MonteCarlo._MonteCarlo__check_each_index_was_recorded_once(parallel_runner) + + assert interrupted, "the run was never interrupted" + assert parallel_runner.input_file.read_text() == "", ( + "nothing was written, so the check really was in a position to reject this" + ) diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py new file mode 100644 index 000000000..4646356a9 --- /dev/null +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -0,0 +1,256 @@ +"""What a worker does when something fails part way through. + +A worker that dies has to leave three things true: the failure that started it +is the one that escapes, the shared mutex is not still held, and the error event +is set. Before this, a failure in the claim itself broke all three at once -- +``sim_idx`` and ``inputs_json`` were only bound inside the loop, so the handler +raised ``UnboundLocalError`` over the real error while holding the mutex, and +``error_event.set()`` sat after the write that never ran. +""" + +import json +import threading +import types + +import pytest + +import rocketpy.simulation.monte_carlo as mc + + +class _RecordingMutex: + """A real lock that counts acquires and releases.""" + + def __init__(self): + self.acquired = 0 + self.released = 0 + self._lock = threading.Lock() + + def acquire(self, *args, **kwargs): + self.acquired += 1 + return self._lock.acquire(*args, **kwargs) + + def release(self): + self.released += 1 + return self._lock.release() + + +class _Event: + def __init__(self): + self.flag = False + + def set(self): + self.flag = True + + def is_set(self): + return self.flag + + +class _Boom(RuntimeError): + """The failure under test, so it cannot be confused with an incidental one.""" + + +def _worker(tmp_path, **overrides): + """A stand-in carrying only the attributes ``__sim_producer`` touches.""" + attributes = { + "error_file": tmp_path / "errors.txt", + "input_file": tmp_path / "inputs.txt", + "output_file": tmp_path / "outputs.txt", + "_MonteCarlo__child_seed": lambda index: index, + "_MonteCarlo__seed_simulation": lambda seed: None, + "_MonteCarlo__run_single_simulation": object, + "_MonteCarlo__evaluate_flight_inputs": lambda index: "{}\n", + "_MonteCarlo__evaluate_flight_outputs": lambda flight, index: "{}\n", + } + attributes.update(overrides) + return types.SimpleNamespace(**attributes) + + +def _raise(*_args, **_kwargs): + raise _Boom("injected") + + +@pytest.mark.parametrize( + "stage", + ["claim", "reseed", "flight", "inputs", "outputs"], + ids=["claim", "reseed", "flight", "input_eval", "output_eval"], +) +def test_a_failure_anywhere_keeps_the_cause_and_frees_the_mutex( + tmp_path, monkeypatch, stage +): + """Whichever stage fails, the same three things have to hold.""" + indices = iter([0, None]) + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: next(indices)) + overrides = {} + if stage == "claim": + monkeypatch.setattr(mc, "_claim_next_index", _raise) + elif stage == "reseed": + overrides["_MonteCarlo__seed_simulation"] = _raise + elif stage == "flight": + overrides["_MonteCarlo__run_single_simulation"] = _raise + elif stage == "inputs": + overrides["_MonteCarlo__evaluate_flight_inputs"] = _raise + elif stage == "outputs": + overrides["_MonteCarlo__evaluate_flight_outputs"] = _raise + + mutex, event = _RecordingMutex(), _Event() + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer( + _worker(tmp_path, **overrides), object(), mutex, event + ) + + assert mutex.acquired == mutex.released, "the mutex was left held" + assert event.flag, "the parent was never told an error happened" + + +def test_the_cause_survives_even_when_the_error_report_also_fails( + tmp_path, monkeypatch +): + """Reporting is best effort. If the error file is unwritable too, the + failure that started it is still what comes out, and the mutex is still + released.""" + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: 0) + monkeypatch.setattr( + "builtins.open", lambda *a, **k: (_ for _ in ()).throw(OSError("no disk")) + ) + worker = _worker(tmp_path, _MonteCarlo__run_single_simulation=_raise) + mutex, event = _RecordingMutex(), _Event() + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer(worker, object(), mutex, event) + + assert mutex.acquired == mutex.released, "the mutex was left held" + assert event.flag + + +def test_the_failure_is_still_reported_when_the_claim_itself_failed( + tmp_path, monkeypatch +): + """The report has to survive a failure before the loop body ran. + + ``sim_idx`` and ``inputs_json`` are bound before the try for this reason. + Left to the loop, the handler raised ``UnboundLocalError`` at its first + write, so nothing was written and nothing was printed: the run ended with + no record of what went wrong. + """ + monkeypatch.setattr(mc, "_claim_next_index", _raise) + reported = [] + monkeypatch.setattr(mc._SimMonitor, "reprint", staticmethod(reported.append)) + mutex, event = _RecordingMutex(), _Event() + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer( + _worker(tmp_path), object(), mutex, event + ) + + assert reported, "the worker died without reporting anything" + assert "injected" in reported[0], f"the report does not name the cause: {reported}" + + +def test_an_interrupt_while_reporting_does_not_leave_the_mutex_held( + tmp_path, monkeypatch +): + """``except Exception`` does not catch ``KeyboardInterrupt``, so the release + has to be in a ``finally``. Ctrl-C between the acquire and the release would + otherwise leave every other worker blocked on it for good.""" + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: 0) + + def interrupt(*_a, **_k): + raise KeyboardInterrupt + + monkeypatch.setattr(mc._SimMonitor, "reprint", staticmethod(interrupt)) + worker = _worker(tmp_path, _MonteCarlo__run_single_simulation=_raise) + mutex, event = _RecordingMutex(), _Event() + + with pytest.raises(KeyboardInterrupt): + mc.MonteCarlo._MonteCarlo__sim_producer(worker, object(), mutex, event) + + assert mutex.acquired == mutex.released, "the mutex was left held on interrupt" + + +def test_a_failure_before_the_inputs_exist_still_leaves_a_readable_record( + tmp_path, monkeypatch +): + """The run tells the user to check the error file, so it has to say + something. A failure in the claim has no inputs to write, and the file was + left empty while the traceback went only to a worker's stdout, which under + ``spawn`` on Windows the user may never see. + + It has to stay a JSON line: ``_read_log_file`` parses this file with + ``json.loads`` per line, so free text would make the whole log unreadable. + """ + monkeypatch.setattr(mc, "_claim_next_index", _raise) + worker = _worker(tmp_path) + mutex, event = _RecordingMutex(), _Event() + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer(worker, object(), mutex, event) + + lines = [ + line + for line in worker.error_file.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + assert lines, "the error file was left empty" + record = json.loads(lines[0]) + assert record["index"] is None, "an early failure has no simulation index" + assert "injected" in record["error"], "the record does not carry the cause" + + +@pytest.mark.parametrize("failing_file", ["input_file", "output_file"]) +def test_a_failed_write_is_reported_like_any_other_failure( + tmp_path, monkeypatch, failing_file +): + """A disk that fills up part way through is a failure like any other: the + cause has to escape, the mutex has to come back, and the event has to be + set. These two writes sit inside the loop's own mutex block rather than the + handler, so they are worth exercising separately from the stages above. + """ + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: 0) + worker = _worker(tmp_path) + blocked = str(getattr(worker, failing_file)) + real_open = open + + def selective_open(path, *args, **kwargs): + if str(path) == blocked: + raise OSError("no space left on device") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", selective_open) + mutex, event = _RecordingMutex(), _Event() + + with pytest.raises(OSError, match="no space left"): + mc.MonteCarlo._MonteCarlo__sim_producer(worker, object(), mutex, event) + + assert mutex.acquired == mutex.released, "the mutex was left held" + assert event.flag + + +def test_an_unreachable_error_event_does_not_replace_the_failure_it_reports( + tmp_path, monkeypatch +): + """The event is a manager proxy, so notifying can fail on its own. + + It is set first and every other report is guarded, which left this one + statement able to do the thing the guards exist to prevent: raise over the + failure being reported, so the parent sees a connection error instead. + """ + + class _UnreachableEvent: + def set(self): + raise ConnectionResetError("the manager is gone") + + def is_set(self): + raise ConnectionResetError("the manager is gone") + + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: 0) + worker = _worker(tmp_path, _MonteCarlo__run_single_simulation=_raise) + mutex = _RecordingMutex() + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer( + worker, object(), mutex, _UnreachableEvent() + ) + + assert mutex.acquired == mutex.released, "the mutex was left held" From df18cae5b83aa42b3fabc0b6cebb0a315c9b56e5 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Tue, 4 Aug 2026 05:57:50 +0800 Subject: [PATCH 12/45] TST: run the real parallel path on every start method The existing tests cover the seed arithmetic everywhere and the real loop under fork. Neither reaches multiprocess.Process, __sim_producer, the manager proxies or pickling the stochastic object graph anywhere but fork, and spawn is what Windows and macOS run, and forkserver is Python 3.14's POSIX default. Serial, two workers and four workers are compared per index under each available start method. Object identity is stripped before comparing: a Function's signature hash and its serialised source encode the object rather than the value drawn for it, and a child that re-imported the module cannot agree with the parent about those. Six fields differ across the boundary on a real run and all six are these. The fixtures are built so the properties can actually fail. The shared stochastic environment has zero wind at every altitude, and zero times any factor is zero, so a compounding baseline cannot show up in it; this one sets a wind that is actually blowing. A bare StochasticAirBrakes gives every parameter a standard deviation of zero, so it gets one that varies. The assertions check the eccentricities and the air brake are among the compared fields, or stripping identity could quietly empty the comparison. Also covers the parent-side checks: a run missing an input row, a run missing an output row, a row cut off mid-write, appending onto an earlier run, and a run stopped with Ctrl-C. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../test_monte_carlo_determinism.py | 439 +++++++++++++++++- 1 file changed, 438 insertions(+), 1 deletion(-) diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index 2d2cfe756..8cb29776e 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -29,15 +29,22 @@ import json import multiprocessing +import os from types import SimpleNamespace import numpy as np import pytest import rocketpy.simulation.monte_carlo as mc_module +from rocketpy import Environment from rocketpy.simulation import MonteCarlo from rocketpy.simulation.monte_carlo import _seed_sequence_to_int -from rocketpy.stochastic import StochasticRocket, StochasticSolidMotor +from rocketpy.stochastic import ( + StochasticAirBrakes, + StochasticEnvironment, + StochasticRocket, + StochasticSolidMotor, +) _child_seed = MonteCarlo._MonteCarlo__child_seed @@ -135,6 +142,18 @@ def _read_inputs_by_index(input_file): return by_index +def _count_rows(log_file): + """How many records were written, before anything is keyed by index. + + Keying by index hides a duplicate: two workers claiming the same index + write two rows and the second overwrites the first in the dict, so the + result looks complete. The claim is meant to be atomic, and the count is + what says so. + """ + with open(log_file, mode="r", encoding="utf-8") as rows: + return sum(1 for line in rows if line.strip()) + + def _simulate_inputs( monkeypatch, tmp_path, environment, rocket, flight, tag, **simulate_kwargs ): @@ -272,3 +291,421 @@ def test_seed_derivation_is_start_method_invariant(start_method): combined.update(result) assert combined == expected assert sorted(combined) == indices + + +def _assert_the_same_environment_was_flown(runs, expected_indices, start_method): + """Every worker count flew index i with the same effective environment. + + This is the half the inputs file cannot show. It records + ``wind_velocity_x_factor``, which is the same for index i however the run + was executed even when the baseline it multiplies has drifted from one + simulation to the next. + """ + effective = { + label: _read_inputs_by_index(montecarlo.output_file) + for label, (montecarlo, _inputs) in runs.items() + } + for label, by_index in effective.items(): + assert sorted(by_index) == expected_indices, f"{label}: outputs are incomplete" + + for index in expected_indices: + reference = json.loads(effective["serial"][index]) + for key in _EFFECTIVE_ENVIRONMENT: + assert key in reference, f"{key} was not recorded" + assert reference["effective_wind_x"] != 0.0, ( + "the wind baseline is zero, so a compounding baseline cannot show" + ) + for label in ("parallel-2", "parallel-4"): + drawn = json.loads(effective[label][index]) + for key in _EFFECTIVE_ENVIRONMENT: + assert drawn[key] == reference[key], ( + f"{start_method}: {label} flew a different {key} at index " + f"{index}: {drawn[key]} against {reference[key]}" + ) + + +def _assert_the_run_is_complete(label, montecarlo, inputs, count): + """Every index written once, to both files, with nothing in the error log. + + The row counts are taken before anything is keyed by index: two workers + claiming the same index write two rows, and the second overwrites the first + in the dict, so a duplicate looks like a complete run. + """ + expected_indices = list(range(count)) + rows = _count_rows(montecarlo.input_file) + + assert sorted(inputs) == expected_indices, ( + f"{label}: indices {sorted(inputs)}, expected {expected_indices}" + ) + assert rows == count, ( + f"{label}: {rows} rows for {count} simulations, so an index was claimed " + f"more than once" + ) + assert _count_rows(montecarlo.output_file) == count, ( + f"{label}: the output rows do not match the simulations run" + ) + assert sorted(_read_inputs_by_index(montecarlo.output_file)) == expected_indices, ( + f"{label}: the outputs do not match the inputs" + ) + assert not os.path.getsize(montecarlo.error_file), ( + f"{label}: the run wrote to its error file" + ) + + +@pytest.fixture +def stochastic_environment_with_wind(example_spaceport_env): + """A stochastic environment whose wind is not zero. + + The shared ``stochastic_environment`` fixture sits on an Environment whose + ``wind_velocity_x`` is 0 at every altitude, and zero times any factor is + zero, so a baseline that compounds from one simulation to the next cannot + show up in it at all. Measured: with the baseline fix reverted, every + assertion in this file still passed. A wind that is actually blowing is + what makes the property testable. + """ + environment = Environment( + latitude=example_spaceport_env.latitude, + longitude=example_spaceport_env.longitude, + elevation=example_spaceport_env.elevation, + ) + environment.set_atmospheric_model( + type="custom_atmosphere", wind_u=12.0, wind_v=-7.0 + ) + return StochasticEnvironment( + environment=environment, + elevation=(1400, 10, "normal"), + wind_velocity_x_factor=(1.0, 0.05, "normal"), + wind_velocity_y_factor=(1.0, 0.05, "normal"), + ) + + +def _wind_x(flight): + """The wind the simulation actually flew with, not the factor drawn for it.""" + return float(flight.env.wind_velocity_x(0)) + + +def _wind_y(flight): + return float(flight.env.wind_velocity_y(0)) + + +def _elevation(flight): + return float(flight.env.elevation) + + +_EFFECTIVE_ENVIRONMENT = { + "effective_wind_x": _wind_x, + "effective_wind_y": _wind_y, + "effective_elevation": _elevation, +} + + +def _sampled_only(record): + """The recorded inputs with object identity stripped out. + + A ``Function``'s ``signature.hash`` and its serialised ``source`` encode the + object, not the value drawn for it, and an object built in another process + has a different one. Under ``fork`` they happen to agree because the child + inherits the parent's objects; under ``spawn`` and ``forkserver`` they + cannot. Measured on a real run: six fields differ across the boundary and + all six are these, while every sampled quantity matches exactly. + """ + flat = {} + + def walk(value, path=""): + if isinstance(value, dict): + for key, item in value.items(): + walk(item, f"{path}.{key}" if path else str(key)) + elif isinstance(value, list): + for position, item in enumerate(value): + walk(item, f"{path}[{position}]") + else: + flat[path] = value + + walk(record) + return { + key: value + for key, value in flat.items() + if "signature" not in key and not key.endswith(".source") + } + + +def _real_run_inputs(tmp_path, environment, rocket, flight, tag, **simulate_kwargs): + """Run a real Monte Carlo, no stub, and return the inputs keyed by index. + + Deliberately without the ``Flight`` stub. Stubbing is what confines the test + above to ``fork``: it replaces a module-level symbol in the parent, and a + ``spawn`` or ``forkserver`` child re-imports the module instead of inheriting + it. A real run has nothing that needs to cross the boundary except the + pickled MonteCarlo, which is the thing worth testing. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / tag), + environment=environment, + rocket=rocket, + flight=flight, + data_collector=_EFFECTIVE_ENVIRONMENT, + ) + montecarlo.simulate(**simulate_kwargs) + return montecarlo, _read_inputs_by_index(montecarlo.input_file) + + +@pytest.fixture +def restore_start_method(): + """Set the start method for one test and put it back afterwards.""" + multiprocess = pytest.importorskip("multiprocess") + original = multiprocess.get_start_method() + yield multiprocess + multiprocess.set_start_method(original, force=True) + + +@pytest.mark.slow +@pytest.mark.parametrize("start_method", _available_start_methods()) +def test_the_real_parallel_path_is_worker_invariant_under_every_start_method( + restore_start_method, + tmp_path, + stochastic_environment_with_wind, + stochastic_calisto_numpy_only, + stochastic_flight, + calisto_air_brakes_clamp_on, + start_method, +): + """The whole parallel path, not just the seed arithmetic. + + ``test_seed_derivation_is_start_method_invariant`` covers the derivation on + every start method, and the stubbed test above covers the real loop on + ``fork``. Neither covers ``multiprocess.Process``, ``__sim_producer``, the + manager proxies or pickling the stochastic object graph anywhere but + ``fork``, and that is what Windows, macOS and Python 3.14's POSIX default + actually run. + """ + multiprocess = restore_start_method + if start_method not in multiprocess.get_all_start_methods(): + pytest.skip(f"{start_method} is not available here") + multiprocess.set_start_method(start_method, force=True) + + # Air brakes and eccentricity are sampled by their own code paths, and each + # one was reseeded from somewhere other than the simulation index. + stochastic_calisto_numpy_only.add_air_brakes( + StochasticAirBrakes( + air_brakes=calisto_air_brakes_clamp_on.air_brakes[0], + drag_coefficient_curve_factor=(1.0, 0.1), + ), + calisto_air_brakes_clamp_on._controllers[0], + ) + stochastic_calisto_numpy_only.add_cp_eccentricity(x=(0.0, 0.001, "normal"), y=0.001) + stochastic_calisto_numpy_only.add_thrust_eccentricity( + x=(0.0, 0.001, "normal"), y=0.001 + ) + + count = 4 + common = {"number_of_simulations": count, "random_seed": 987654321} + models = ( + stochastic_environment_with_wind, + stochastic_calisto_numpy_only, + stochastic_flight, + ) + runs = { + "serial": _real_run_inputs( + tmp_path, *models, f"{start_method}-serial", **common + ), + "parallel-2": _real_run_inputs( + tmp_path, + *models, + f"{start_method}-p2", + parallel=True, + n_workers=2, + **common, + ), + "parallel-4": _real_run_inputs( + tmp_path, + *models, + f"{start_method}-p4", + parallel=True, + n_workers=4, + **common, + ), + } + + expected_indices = list(range(count)) + for label, (montecarlo, inputs) in runs.items(): + _assert_the_run_is_complete(label, montecarlo, inputs, count) + + _assert_the_same_environment_was_flown(runs, expected_indices, start_method) + + serial = runs["serial"][1] + for label in ("parallel-2", "parallel-4"): + for index in expected_indices: + expected = _sampled_only(json.loads(serial[index])) + actual = _sampled_only(json.loads(runs[label][1][index])) + + # Or stripping identity could quietly empty the comparison. + assert len(expected) > 20, f"only {len(expected)} fields left to compare" + assert sum("eccentricity" in key for key in expected) == 4, ( + "the four eccentricities are not among the compared fields" + ) + assert sum("brake" in key for key in expected) >= 1, ( + "the air brake is not among the compared fields" + ) + assert actual == expected, ( + f"{start_method}: serial and {label} differ at index {index} in " + f"{sorted(k for k in set(expected) | set(actual) if expected.get(k) != actual.get(k))}" + ) + + +@pytest.mark.parametrize("parallel", [False, True], ids=["serial", "parallel"]) +def test_a_missing_simulation_is_not_reported_as_a_successful_run( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight, parallel +): + """A run that wrote fewer records than it claimed has to fail. + + Neither file shows this on its own: every row is well formed, and reading + them back keyed by index cannot tell four rows from three plus a duplicate. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / f"short-{parallel}"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + kwargs = {"parallel": True, "n_workers": 2} if parallel else {} + + # Lose one simulation's inputs the way a worker dying between claiming and + # writing does. Driven through ``simulate`` rather than by calling the check + # afterwards, so this also proves the check is reached at all. + real = montecarlo._MonteCarlo__evaluate_flight_inputs + + def drop_the_second(sim_idx): + return "" if sim_idx == 1 else real(sim_idx) + + montecarlo._MonteCarlo__evaluate_flight_inputs = drop_the_second + + with pytest.raises(RuntimeError, match="never written"): + montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) + + +@pytest.mark.parametrize("parallel", [False, True], ids=["serial", "parallel"]) +def test_a_simulation_whose_outputs_went_missing_also_fails( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight, parallel +): + """The other file. A worker that wrote its inputs and stopped before its + outputs leaves the two logs disagreeing, and a check that only reads the + inputs sees a complete run. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / f"no-output-{parallel}"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + kwargs = {"parallel": True, "n_workers": 2} if parallel else {} + real = montecarlo._MonteCarlo__evaluate_flight_outputs + + def drop_the_second(flight, sim_idx): + return "" if sim_idx == 1 else real(flight, sim_idx) + + montecarlo._MonteCarlo__evaluate_flight_outputs = drop_the_second + + with pytest.raises(RuntimeError, match="never written"): + montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) + + +@pytest.mark.parametrize("parallel", [False, True], ids=["serial", "parallel"]) +def test_a_row_cut_off_mid_write_is_named_as_the_missing_simulation( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight, parallel +): + """A worker killed part way through a write leaves a truncated row. + + That is the case the check exists to diagnose, so it has to name the + simulation that went missing. Parsing the file strictly turned it into a + JSONDecodeError out of ``simulate`` instead, which points nowhere. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / f"truncated-{parallel}"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + kwargs = {"parallel": True, "n_workers": 2} if parallel else {} + real = montecarlo._MonteCarlo__evaluate_flight_inputs + + def cut_the_second_short(sim_idx): + row = real(sim_idx) + if sim_idx != 1: + return row + half = row[: len(row) // 2] + "\n" + with pytest.raises(ValueError): + json.loads(half) # the row has to be unparseable for this to test it + return half + + montecarlo._MonteCarlo__evaluate_flight_inputs = cut_the_second_short + + with pytest.raises(RuntimeError, match="never written"): + montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) + + +def test_a_run_stopped_with_ctrl_c_keeps_what_it_saved( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight +): + """Ctrl-C is a stop, not a fault. + + The run path catches it, prints that the files are saved and returns. The + completeness check then counted the simulations that never ran and called + the run a failure, contradicting the message printed a moment earlier. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / "interrupted"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + real = montecarlo._MonteCarlo__run_single_simulation + finished = [] + + def stop_after_the_first(): + if finished: + raise KeyboardInterrupt("user pressed ctrl-c") + finished.append(1) + return real() + + montecarlo._MonteCarlo__run_single_simulation = stop_after_the_first + + montecarlo.simulate(number_of_simulations=3, random_seed=42) + + # Short of the three asked for, so the check really was in a position to + # reject this run, and the one simulation that did finish is still there. + assert _count_rows(montecarlo.input_file) == 1 + + +def test_appending_checks_only_the_simulations_the_run_added( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight +): + """``append=True`` leaves the earlier run's records in the same files, and + ``number_of_simulations`` is the total to reach rather than a count to add. + The check has to look at indices ``_initial_sim_idx`` upwards, or a second + run would be judged against records it never wrote. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / "appended"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + montecarlo.simulate(number_of_simulations=2, random_seed=606) + + assert _count_rows(montecarlo.input_file) == 2 + + # Take the first run's records away before appending. The second run is + # judged on what it wrote, so a check counting the whole file would call + # this incomplete even though nothing went wrong. + montecarlo.input_file.write_text("", encoding="utf-8") + montecarlo.output_file.write_text("", encoding="utf-8") + + montecarlo.simulate(number_of_simulations=4, append=True, random_seed=606) + + assert montecarlo._initial_sim_idx == 2, ( + "the second run should have started where the first stopped" + ) + written = _read_inputs_by_index(montecarlo.input_file) + assert sorted(written) == [2, 3], ( + f"the appended run wrote the wrong indices: {sorted(written)}" + ) From b6e2113c5f5099d0e1cb116c9d304e5f65f0cd42 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Tue, 4 Aug 2026 07:07:51 +0800 Subject: [PATCH 13/45] MNT: patch the mangled private names through monkeypatch Assigning to montecarlo._MonteCarlo__evaluate_flight_inputs and friends trips pylint's invalid-name, which exits 16 and fails the Linters job even though the score is 10.00. monkeypatch.setattr takes the name as a string, so the check does not fire, and it puts the original back afterwards instead of leaving the instance patched for whatever runs next. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../test_monte_carlo_determinism.py | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index 8cb29776e..bd468d859 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -554,7 +554,12 @@ def test_the_real_parallel_path_is_worker_invariant_under_every_start_method( @pytest.mark.parametrize("parallel", [False, True], ids=["serial", "parallel"]) def test_a_missing_simulation_is_not_reported_as_a_successful_run( - tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight, parallel + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto, + stochastic_flight, + parallel, ): """A run that wrote fewer records than it claimed has to fail. @@ -577,7 +582,9 @@ def test_a_missing_simulation_is_not_reported_as_a_successful_run( def drop_the_second(sim_idx): return "" if sim_idx == 1 else real(sim_idx) - montecarlo._MonteCarlo__evaluate_flight_inputs = drop_the_second + monkeypatch.setattr( + montecarlo, "_MonteCarlo__evaluate_flight_inputs", drop_the_second + ) with pytest.raises(RuntimeError, match="never written"): montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) @@ -585,7 +592,12 @@ def drop_the_second(sim_idx): @pytest.mark.parametrize("parallel", [False, True], ids=["serial", "parallel"]) def test_a_simulation_whose_outputs_went_missing_also_fails( - tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight, parallel + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto, + stochastic_flight, + parallel, ): """The other file. A worker that wrote its inputs and stopped before its outputs leaves the two logs disagreeing, and a check that only reads the @@ -603,7 +615,9 @@ def test_a_simulation_whose_outputs_went_missing_also_fails( def drop_the_second(flight, sim_idx): return "" if sim_idx == 1 else real(flight, sim_idx) - montecarlo._MonteCarlo__evaluate_flight_outputs = drop_the_second + monkeypatch.setattr( + montecarlo, "_MonteCarlo__evaluate_flight_outputs", drop_the_second + ) with pytest.raises(RuntimeError, match="never written"): montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) @@ -611,7 +625,12 @@ def drop_the_second(flight, sim_idx): @pytest.mark.parametrize("parallel", [False, True], ids=["serial", "parallel"]) def test_a_row_cut_off_mid_write_is_named_as_the_missing_simulation( - tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight, parallel + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto, + stochastic_flight, + parallel, ): """A worker killed part way through a write leaves a truncated row. @@ -637,14 +656,16 @@ def cut_the_second_short(sim_idx): json.loads(half) # the row has to be unparseable for this to test it return half - montecarlo._MonteCarlo__evaluate_flight_inputs = cut_the_second_short + monkeypatch.setattr( + montecarlo, "_MonteCarlo__evaluate_flight_inputs", cut_the_second_short + ) with pytest.raises(RuntimeError, match="never written"): montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) def test_a_run_stopped_with_ctrl_c_keeps_what_it_saved( - tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight + monkeypatch, tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight ): """Ctrl-C is a stop, not a fault. @@ -667,7 +688,9 @@ def stop_after_the_first(): finished.append(1) return real() - montecarlo._MonteCarlo__run_single_simulation = stop_after_the_first + monkeypatch.setattr( + montecarlo, "_MonteCarlo__run_single_simulation", stop_after_the_first + ) montecarlo.simulate(number_of_simulations=3, random_seed=42) From 4dbe216370315fb2f81032fc1a5eaeb34cb7ca87 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Wed, 5 Aug 2026 01:41:54 +0800 Subject: [PATCH 14/45] BUG: bound shutdown, strict logs, and exception state on both run paths A review of the seeding change turned up failure paths where a run that went wrong could still be reported as a success. Six of them, all in the machinery around the simulations rather than in the seeding itself. The parent waited for every worker with an unbounded join, in the order they were started. One worker stuck in a native call held it there while another had already set the error event, so neither the error nor the cleanup after it was ever reached, and Ctrl-C hung on the same join a second time. The wait is bounded and gives up as soon as the event is set. Shutdown signals the whole fleet before waiting on any of it, then falls back to kill, so a worker that ignores the first signal does not keep the others, the manager and the open files alive behind it. The completeness check accepted a corrupt file. Rows it could not parse were skipped, rows carrying no index or an index outside the run were ignored, and JSON true or 1.0 passed for the index 1 because both compare equal to it. Every row now has to be an object with a plain non-negative int index, the two files have to agree on the exact set, and an interrupted run is allowed to be short but not to be corrupt. Both run paths cleared the current payload after the call that can be interrupted rather than before it. In the serial path Ctrl-C on the first lap reached the handler with it unbound, so the interrupt surfaced as an UnboundLocalError, and between laps it still held the row that had just been written. In the worker the same ordering meant a claim that failed on a later lap reported the simulation that had just succeeded. The normal write path also released the mutex in finally whether or not acquire had returned. The error record kept either the inputs or the traceback, never both, so every failure after sampling left no traceback in the file the run points the user at. n_workers was validated after the logs were opened "w+", so asking for a worker count the run cannot use destroyed the previous results on the way to raising. All argument checking happens before any file is touched, and number_of_simulations is checked too. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 309 ++++++++++---- .../test_monte_carlo_determinism.py | 85 +++- .../test_monte_carlo_log_integrity.py | 401 ++++++++++++++++++ 3 files changed, 693 insertions(+), 102 deletions(-) create mode 100644 tests/unit/simulation/test_monte_carlo_log_integrity.py diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 8eb97eacd..a2d7afb50 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -20,7 +20,7 @@ import warnings from numbers import Real from pathlib import Path -from time import time +from time import monotonic, time import numpy as np import simplekml @@ -239,6 +239,13 @@ def simulate( overwritten. Make sure to save the files with the results before running the simulation again with `append=False`. """ + # Everything that can be judged from the arguments alone happens before + # __setup_files, which opens both logs "w+" and empties them. Raising + # after that point destroys the previous run on the way out. + _validate_simulation_count(number_of_simulations) + if parallel: + n_workers = self.__validate_number_of_workers(n_workers) + self._export_config = kwargs self.number_of_simulations = number_of_simulations self._initial_sim_idx = self.num_of_loaded_sims if append else 0 @@ -386,43 +393,47 @@ def __check_each_index_was_recorded_once(self): reach rather than a count to add, so the new indices are ``_initial_sim_idx`` up to it. - A run stopped with Ctrl-C is exempt: both run paths catch it, keep what - they have and return, so being short is the point rather than a fault. - """ - expected = set(range(self._initial_sim_idx, self.number_of_simulations)) - if not expected or self._interrupted: - # A stopped run is short by definition and already said so. Checking - # it anyway contradicted the "Files saved." it had just printed. - return - - for label, path in ( - ("inputs", self.input_file), - ("outputs", self.output_file), - ): - written = {} - with open(path, mode="r", encoding="utf-8") as rows: - for line in rows: - line = line.strip() - if line: - try: - index = json.loads(line).get("index") - except ValueError: - # A worker killed mid-write leaves a partial row. - # Skipping it reports that index as missing, which - # is what happened, rather than failing to parse. - continue - written[index] = written.get(index, 0) + 1 - - missing = sorted(expected - set(written)) - repeated = sorted(index for index in expected if written.get(index, 0) > 1) - if missing or repeated: - raise RuntimeError( - f"the {label} file does not match the simulations that ran: " - f"{len(missing)} never written {missing[:5]}, " - f"{len(repeated)} written more than once {repeated[:5]}. " - f"The results are incomplete, so they are not reported as a " - f"successful run." - ) + A run stopped with Ctrl-C is short on purpose, so the indices it never + reached are not an error. What it did write is still held to the rest: + readable rows, one row per index, and nothing outside the range. + + Indices below ``_initial_sim_idx`` are an earlier run's and are left + alone. Reconciling a history with holes in it is a separate job, and + this only answers for the simulations this run claimed. + """ + inputs = _recorded_indices("inputs", self.input_file) + outputs = _recorded_indices("outputs", self.output_file) + if inputs != outputs: + only_in = lambda a, b: sorted(set(a) - set(b)) # noqa: E731 + raise RuntimeError( + f"the input and output files disagree about which simulations " + f"ran: {only_in(inputs, outputs)[:5]} have inputs and no " + f"outputs, {only_in(outputs, inputs)[:5]} the other way round. " + f"A worker stopped between the two writes, so the results are " + f"not reported as a successful run." + ) + + repeated = sorted(index for index, count in inputs.items() if count > 1) + beyond = sorted( + index for index in inputs if index >= self.number_of_simulations + ) + missing = ( + [] + if self._interrupted + else sorted( + set(range(self._initial_sim_idx, self.number_of_simulations)) + - set(inputs) + ) + ) + if missing or repeated or beyond: + raise RuntimeError( + f"the files do not match the simulations that ran: " + f"{len(missing)} never written {missing[:5]}, " + f"{len(repeated)} written more than once {repeated[:5]}, " + f"{len(beyond)} outside the range this run claimed {beyond[:5]}. " + f"The results are wrong, so they are not reported as a " + f"successful run." + ) def __run_in_serial(self): """ @@ -441,20 +452,24 @@ def __run_in_serial(self): start_time=time(), ) try: - while sim_monitor.keep_simulating(): + while True: + # First statement in the loop, so it is bound before the two + # monitor calls rather than after them. Ctrl-C in either one + # used to leave it unbound, or holding the last completed row. + inputs_json = "" + + if not sim_monitor.keep_simulating(): + break sim_idx = sim_monitor.increment() - 1 - inputs_json, outputs_json = "", "" self.__seed_simulation(self.__child_seed(sim_idx)) flight = self.__run_single_simulation() inputs_json = self.__evaluate_flight_inputs(sim_idx) outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) - with open(self.input_file, "a", encoding="utf-8") as f: - f.write(inputs_json) - with open(self.output_file, "a", encoding="utf-8") as f: - f.write(outputs_json) - + _record_simulation( + self.input_file, self.output_file, inputs_json, outputs_json + ) sim_monitor.print_update_status() sim_monitor.print_final_status() @@ -530,9 +545,8 @@ def __run_in_parallel(self, n_workers=None): sim_producer.start() started_processes.append(sim_producer) - for sim_producer in started_processes: - sim_producer.join() - + _wait_for_workers(started_processes, simulation_error_event) + _stop_any_worker_still_running(started_processes) _fail_if_a_worker_did_not_finish( started_processes, simulation_error_event, self.error_file ) @@ -542,11 +556,7 @@ def __run_in_parallel(self, n_workers=None): # Handle error from the main process # pylint: disable=broad-except except (Exception, KeyboardInterrupt) as error: - simulation_error_event.set() - - for sim_producer in started_processes: - sim_producer.join() - + _bring_the_fleet_down(started_processes, simulation_error_event) self._interrupted = isinstance(error, KeyboardInterrupt) if not self._interrupted: raise error @@ -554,8 +564,15 @@ def __run_in_parallel(self, n_workers=None): _stop_any_worker_still_running(started_processes) def __validate_number_of_workers(self, n_workers): - if n_workers is None or n_workers > os.cpu_count(): - n_workers = os.cpu_count() + # os.cpu_count() is documented as possibly None, and comparing against + # it then raises rather than falling back to a usable default. + available = os.cpu_count() or 2 + if n_workers is not None and type(n_workers) not in (int, np.integer): # noqa: E721 + raise TypeError( + f"Number of workers must be an integer, not {type(n_workers).__name__}." + ) + if n_workers is None or n_workers > available: + n_workers = available if n_workers < 2: raise ValueError("Number of workers must be at least 2 for parallel mode.") @@ -573,28 +590,27 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to error_event : multiprocess.Event Event signaling an error occurred during the simulation. """ - # Bound before the try, not inside the loop. The handler below reports - # both, and a failure in the claim itself left them unassigned, so the - # original error was replaced by an UnboundLocalError raised out of the - # handler with the mutex still held. - sim_idx = None - inputs_json = "" - outputs_json = "" try: while True: + # First statement in the loop, so it is bound before the claim + # rather than after it. A claim that failed left these unassigned + # and the handler raised UnboundLocalError over the real error; + # a claim that failed on a later lap reported the previous row. + sim_idx, inputs_json, outputs_json = None, "", "" + sim_idx = _claim_next_index(sim_monitor, mutex) if sim_idx is None: break - inputs_json, outputs_json = "", "" - self.__seed_simulation(self.__child_seed(sim_idx)) flight = self.__run_single_simulation() inputs_json = self.__evaluate_flight_inputs(sim_idx) outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) + acquired = False try: mutex.acquire() + acquired = True if error_event.is_set(): # Runs in a worker process spawned via multiprocessing: # logging handlers configured in the main process are @@ -609,14 +625,13 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to break - with open(self.input_file, "a", encoding="utf-8") as f: - f.write(inputs_json) - with open(self.output_file, "a", encoding="utf-8") as f: - f.write(outputs_json) - + _record_simulation( + self.input_file, self.output_file, inputs_json, outputs_json + ) sim_monitor.print_update_status() finally: - mutex.release() + if acquired: + mutex.release() except Exception: # Set first, so a parent waiting on the join learns why. Best effort @@ -628,15 +643,15 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to pass details = traceback.format_exc() - # The inputs of the failed simulation when there are any, and a - # record of the failure itself when it happened before they were - # drawn. Without the second, a failure in the claim left the error - # file empty while the run pointed the user at it. It is a JSON - # line either way, because ``_read_log_file`` parses this file with - # ``json.loads`` and free text in it would make the log unreadable. - record = inputs_json or ( - json.dumps({"index": sim_idx, "error": details}) + "\n" - ) + # The failure goes onto the inputs record rather than replacing it. + # Writing one or the other dropped the traceback for every failure + # after sampling, from the file the run tells the user to read. + try: + record = json.loads(inputs_json) if inputs_json else {"index": sim_idx} + except ValueError: + record = {"index": sim_idx} + record["error"] = details + record = json.dumps(record) + "\n" acquired = False try: @@ -1830,16 +1845,138 @@ def export_errors_to_json(self, filename): self._write_log_to_json(self.errors_log, filename) -def _stop_any_worker_still_running(started_processes): +def _recorded_indices(label, path): + """``{index: how many rows carry it}`` for one log file. + + Strict about what a row is. A row that will not parse, is not an + object, or carries anything but a non-negative plain ``int`` index is + the corruption this check exists to find, so it is named and raised on + rather than skipped. ``type(...) is int`` and not ``isinstance``: + ``True`` and ``1.0`` both compare equal to ``1`` and would otherwise + pass for it. + """ + written = {} + with open(path, mode="r", encoding="utf-8") as rows: + for number, line in enumerate(rows, start=1): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except ValueError as error: + raise RuntimeError( + f"{label} row {number} is not readable JSON, so a " + f"worker was cut off part way through writing it: " + f"{line[:60]!r}" + ) from error + index = record.get("index") if isinstance(record, dict) else None + # isinstance is the wrong tool here, see the docstring: bool is a + # subclass of int, so True would pass for the index 1. + # pylint: disable-next=unidiomatic-typecheck + if type(index) is not int or index < 0: # noqa: E721 + raise RuntimeError( + f"{label} row {number} does not carry a simulation " + f"index: {line[:60]!r}" + ) + written[index] = written.get(index, 0) + 1 + return written + + +def _validate_simulation_count(number_of_simulations): + """A count has to be a whole non-negative number, checked before any file. + + ``type(...) is not int``: ``True`` is an ``int`` to ``isinstance`` and would + quietly run one simulation. A float ran ``int(count)`` of them and then + failed the completeness check with a range it could never have satisfied. + """ + if type(number_of_simulations) not in (int, np.integer): # noqa: E721 + raise TypeError( + f"number_of_simulations must be an integer, not " + f"{type(number_of_simulations).__name__}." + ) + if number_of_simulations < 0: + raise ValueError( + f"number_of_simulations must not be negative, got {number_of_simulations}." + ) + + +_WORKER_SHUTDOWN_GRACE = 5.0 + + +def _record_simulation(input_file, output_file, inputs_json, outputs_json): + """Append one simulation's inputs and outputs to their logs. + + Module level rather than a method: the run paths are driven directly by + stub objects in the tests, and a private method is not reachable on those. + """ + with open(input_file, "a", encoding="utf-8") as f: + f.write(inputs_json) + with open(output_file, "a", encoding="utf-8") as f: + f.write(outputs_json) + + +def _bring_the_fleet_down(started_processes, error_event): + """Stop everything, without raising over the failure being handled. + + Setting the event is best effort like the workers' own reporting: the + manager may be the thing that died. Then a bounded window to notice it and + leave, and whatever is left gets stopped. + """ + try: + error_event.set() + except Exception: # pylint: disable=broad-exception-caught + pass + _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE) + _stop_any_worker_still_running(started_processes) + + +def _wait_for_workers(started_processes, error_event=None, timeout=None): + """Wait for the fleet, giving up early once one of them reports an error. + + Joining each worker in turn waits on them in the order they were started. A + worker stuck in a native call held the parent on the first join while + another had already set the event, so neither the error nor the cleanup + after it was ever reached. + + No overall deadline on the normal path: a run with no error and one worker + still going is a long simulation, and that is not for this to cut short. + """ + deadline = None if timeout is None else monotonic() + timeout + while any(process.is_alive() for process in started_processes): + if error_event is not None and error_event.is_set(): + break + if deadline is not None and monotonic() >= deadline: + break + for process in started_processes: + process.join(timeout=0.1) + + # Reap whatever has already finished. A worker that was gone before the + # loop started was never joined by it, and an unjoined child has no exit + # code yet, so the crash check downstream would read None and call it one. + for process in started_processes: + process.join(timeout=0) + + +def _stop_any_worker_still_running(started_processes, grace=_WORKER_SHUTDOWN_GRACE): """Whatever is still going here is not going to stop on its own. - The error event was set and it did not leave. Left behind, it keeps the - manager, the mutex and the output files alive. + Signal every worker before waiting on any of them. Terminating one and + joining it before reaching the next let a worker that ignores the signal + keep the rest of the fleet, the manager and the open files alive behind it. """ - for sim_producer in started_processes: - if sim_producer.is_alive(): - sim_producer.terminate() - sim_producer.join() + alive = [process for process in started_processes if process.is_alive()] + for process in alive: + process.terminate() + for process in alive: + process.join(timeout=grace) + + # terminate is a request. SIGKILL is not, and a worker that sat through the + # first one would otherwise keep the manager and the files open for good. + stubborn = [process for process in alive if process.is_alive()] + for process in stubborn: + process.kill() + for process in stubborn: + process.join(timeout=grace) def _fail_if_a_worker_did_not_finish(started_processes, error_event, error_file): diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index bd468d859..1d65ec78e 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -199,6 +199,51 @@ def test_invalid_seed_does_not_truncate_existing_output( assert kept.read() == "previous results\n" +@pytest.mark.parametrize( + ("kwargs", "error"), + [ + ({"number_of_simulations": 2.5}, TypeError), + ({"number_of_simulations": True}, TypeError), + ({"number_of_simulations": -1}, ValueError), + ({"number_of_simulations": 3, "parallel": True, "n_workers": 1}, ValueError), + ], + ids=["float count", "boolean count", "negative count", "one worker"], +) +def test_a_rejected_argument_does_not_truncate_existing_output( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, + kwargs, + error, +): + """Every check that needs only the arguments belongs before the logs open. + + ``__setup_files`` opens both of them "w+", which empties them, and + ``n_workers`` was validated after that. So asking for a worker count the run + cannot use destroyed the previous run's results on the way to raising. + + ``True`` is the one that does not raise on its own: it is an ``int`` to + ``isinstance``, so it would quietly have run one simulation. + """ + monkeypatch.setattr(mc_module, "Flight", _StubFlight) + montecarlo = MonteCarlo( + filename=str(tmp_path / f"keep-{sorted(kwargs.items())}"), + environment=stochastic_environment, + rocket=stochastic_calisto_numpy_only, + flight=stochastic_flight, + ) + with open(montecarlo.input_file, "w", encoding="utf-8") as existing: + existing.write("previous results\n") + + with pytest.raises(error): + montecarlo.simulate(random_seed=11, **kwargs) + + with open(montecarlo.input_file, encoding="utf-8") as kept: + assert kept.read() == "previous results\n" + + def test_serial_inputs_are_reproducible( monkeypatch, tmp_path, @@ -574,16 +619,23 @@ def test_a_missing_simulation_is_not_reported_as_a_successful_run( ) kwargs = {"parallel": True, "n_workers": 2} if parallel else {} - # Lose one simulation's inputs the way a worker dying between claiming and - # writing does. Driven through ``simulate`` rather than by calling the check - # afterwards, so this also proves the check is reached at all. - real = montecarlo._MonteCarlo__evaluate_flight_inputs + # Lose the simulation from both files, the way a worker that dies between + # the claim and the writes does. Driven through ``simulate`` rather than by + # calling the check afterwards, so this also proves the check is reached. + real_inputs = montecarlo._MonteCarlo__evaluate_flight_inputs + real_outputs = montecarlo._MonteCarlo__evaluate_flight_outputs - def drop_the_second(sim_idx): - return "" if sim_idx == 1 else real(sim_idx) + def drop_the_second_inputs(sim_idx): + return "" if sim_idx == 1 else real_inputs(sim_idx) + + def drop_the_second_outputs(flight, sim_idx): + return "" if sim_idx == 1 else real_outputs(flight, sim_idx) monkeypatch.setattr( - montecarlo, "_MonteCarlo__evaluate_flight_inputs", drop_the_second + montecarlo, "_MonteCarlo__evaluate_flight_inputs", drop_the_second_inputs + ) + monkeypatch.setattr( + montecarlo, "_MonteCarlo__evaluate_flight_outputs", drop_the_second_outputs ) with pytest.raises(RuntimeError, match="never written"): @@ -599,9 +651,10 @@ def test_a_simulation_whose_outputs_went_missing_also_fails( stochastic_flight, parallel, ): - """The other file. A worker that wrote its inputs and stopped before its - outputs leaves the two logs disagreeing, and a check that only reads the - inputs sees a complete run. + """A worker that wrote its inputs and stopped before its outputs leaves the + two logs disagreeing. Checking each file against the expected range on its + own cannot see that: the inputs file is complete, and it is only complete + because the row it is missing is in the other file. """ montecarlo = MonteCarlo( filename=str(tmp_path / f"no-output-{parallel}"), @@ -619,12 +672,12 @@ def drop_the_second(flight, sim_idx): montecarlo, "_MonteCarlo__evaluate_flight_outputs", drop_the_second ) - with pytest.raises(RuntimeError, match="never written"): + with pytest.raises(RuntimeError, match="disagree about which simulations"): montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) @pytest.mark.parametrize("parallel", [False, True], ids=["serial", "parallel"]) -def test_a_row_cut_off_mid_write_is_named_as_the_missing_simulation( +def test_a_row_cut_off_mid_write_is_named_as_unreadable( monkeypatch, tmp_path, stochastic_environment, @@ -634,9 +687,9 @@ def test_a_row_cut_off_mid_write_is_named_as_the_missing_simulation( ): """A worker killed part way through a write leaves a truncated row. - That is the case the check exists to diagnose, so it has to name the - simulation that went missing. Parsing the file strictly turned it into a - JSONDecodeError out of ``simulate`` instead, which points nowhere. + That row is the corruption this check exists to find, so it is named and + raised on. Skipping it and reporting the index as missing was a worse + answer: with every expected index present, a corrupt file passed. """ montecarlo = MonteCarlo( filename=str(tmp_path / f"truncated-{parallel}"), @@ -660,7 +713,7 @@ def cut_the_second_short(sim_idx): montecarlo, "_MonteCarlo__evaluate_flight_inputs", cut_the_second_short ) - with pytest.raises(RuntimeError, match="never written"): + with pytest.raises(RuntimeError, match="not readable JSON"): montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py new file mode 100644 index 000000000..80ce38c23 --- /dev/null +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -0,0 +1,401 @@ +"""What the run is allowed to call a success, and how it comes down when it is not. + +The completeness check reads the two logs back and decides whether the run can +be reported as complete. Everything it accepts is a claim about the results, so +a row it cannot read, an index it cannot trust, or a file that disagrees with +its pair has to stop the run rather than be skipped past. + +The shutdown tests cover the other half: a fleet where one worker is not coming +back has to be brought down in bounded time, and the failure that started it has +to survive that. +""" + +import threading +import types + +import pytest + +import rocketpy.simulation.monte_carlo as mc + + +def _runner(tmp_path, rows, outputs=None, count=2, initial=0, interrupted=False): + """A stand-in carrying only what the completeness check reads.""" + inputs_file = tmp_path / "inputs.txt" + outputs_file = tmp_path / "outputs.txt" + inputs_file.write_text(rows, encoding="utf-8") + outputs_file.write_text(rows if outputs is None else outputs, encoding="utf-8") + return types.SimpleNamespace( + input_file=inputs_file, + output_file=outputs_file, + number_of_simulations=count, + _initial_sim_idx=initial, + _interrupted=interrupted, + ) + + +def _check(runner): + mc.MonteCarlo._MonteCarlo__check_each_index_was_recorded_once(runner) + + +COMPLETE = '{"index": 0}\n{"index": 1}\n' + + +CORRUPT = { + "a row cut off mid-write": (COMPLETE + "{not json\n", "not readable JSON"), + "a row that is not an object": (COMPLETE + "[]\n", "does not carry"), + "a row with no index": (COMPLETE + '{"foo": 1}\n', "does not carry"), + "a boolean index": ('{"index": 0}\n{"index": true}\n', "does not carry"), + "a float index": ('{"index": 0}\n{"index": 1.0}\n', "does not carry"), + "a negative index": (COMPLETE + '{"index": -1}\n', "does not carry"), + "an index past the run": (COMPLETE + '{"index": 99}\n', "outside the range"), + "the same index twice": (COMPLETE + '{"index": 1}\n', "more than once"), + "an index never written": ('{"index": 0}\n', "never written"), +} + + +@pytest.mark.parametrize( + ("rows", "expected"), list(CORRUPT.values()), ids=list(CORRUPT) +) +def test_a_corrupt_log_is_not_a_successful_run(tmp_path, rows, expected): + """Every one of these was accepted as a complete run. + + ``True`` and ``1.0`` are the two that look harmless: both compare equal to + ``1``, so an ``isinstance`` check or a bare dict lookup counts them as the + index they are not. Hence ``type(index) is int``. + """ + with pytest.raises(RuntimeError, match=expected): + _check(_runner(tmp_path, rows)) + + +def test_a_complete_run_is_still_accepted(tmp_path): + """The control. Without this the table above passes on a check that + rejects everything.""" + _check(_runner(tmp_path, COMPLETE)) + + +def test_an_earlier_run_left_in_the_files_is_not_an_error(tmp_path): + """``append=True`` keeps the earlier run's rows, and they are below + ``_initial_sim_idx``. Rejecting anything outside the new range would make + every appended run fail.""" + rows = '{"index": 0}\n{"index": 1}\n{"index": 2}\n{"index": 3}\n' + _check(_runner(tmp_path, rows, count=4, initial=2)) + + +def test_an_interrupted_run_still_has_to_be_readable(tmp_path): + """Being short is allowed after Ctrl-C. Being corrupt is not: skipping the + check entirely meant a duplicate or an unreadable row went unreported.""" + _check(_runner(tmp_path, '{"index": 0}\n', interrupted=True)) + + with pytest.raises(RuntimeError, match="more than once"): + _check(_runner(tmp_path, '{"index": 0}\n{"index": 0}\n', interrupted=True)) + + +def test_the_two_files_have_to_agree(tmp_path): + """A worker that wrote its inputs and stopped before its outputs. Each file + on its own can look complete, because the row one is missing is in the + other.""" + with pytest.raises(RuntimeError, match="disagree about which simulations"): + _check(_runner(tmp_path, COMPLETE, outputs='{"index": 0}\n')) + + +class _Worker: + """A process stub that can be told to ignore termination. + + Every call is appended to a shared ``trace`` so the order across the whole + fleet can be asserted, not just the per-worker counts. A stub join costs no + time, so a test that only counts calls cannot tell "signal everyone, then + wait" from "signal one and wait for it before reaching the next". + """ + + def __init__(self, name="worker", alive=False, deaf=False, exitcode=0, trace=None): + self.name = name + self._alive = alive + self._deaf = deaf + self.exitcode = exitcode + self.terminated = 0 + self.killed = 0 + self.joins = [] + self.trace = [] if trace is None else trace + + def is_alive(self): + return self._alive + + def terminate(self): + self.terminated += 1 + self.trace.append(("terminate", self.name)) + if not self._deaf: + self._alive = False + + def kill(self): + self.killed += 1 + self.trace.append(("kill", self.name)) + self._alive = False + + def join(self, timeout=None): + self.joins.append(timeout) + self.trace.append(("join", self.name)) + + +class _Event: + def __init__(self, flag=False): + self.flag = flag + + def set(self): + self.flag = True + + def is_set(self): + return self.flag + + +def test_the_wait_gives_up_as_soon_as_a_worker_reports_an_error(): + """One worker is not coming back and another has already failed. + + Joining the fleet in order held the parent on the first worker forever, so + the error the second had already reported was never seen and the cleanup + after it never ran. + """ + stuck, failed = _Worker(alive=True), _Worker(exitcode=1) + + mc._wait_for_workers([stuck, failed], _Event(flag=True)) + + assert stuck.is_alive(), "the wait should return, not stop the workers itself" + + +def test_the_wait_is_bounded_when_it_is_given_a_deadline(): + """The interrupt path waits a short while for workers to notice the event. + Without a deadline that wait was the second place Ctrl-C could hang.""" + stuck = _Worker(alive=True) + + mc._wait_for_workers([stuck], timeout=0.2) + + assert stuck.is_alive() + + +def test_the_wait_reaps_workers_that_had_already_finished(): + """A worker gone before the loop starts is never joined by it, and an + unjoined child has no exit code, which the crash check downstream reads as + a crash.""" + done = _Worker(alive=False) + + mc._wait_for_workers([done], _Event()) + + assert done.joins, "a finished worker was never joined, so it was not reaped" + + +def test_every_worker_is_signalled_before_any_of_them_is_waited_on(): + """Signal the whole fleet, then wait on it. + + Terminating one and joining it before reaching the next made every worker + wait out the grace period of the ones ahead of it in the list, so a single + worker that ignores the signal delays the rest by that much each. + """ + trace = [] + deaf = _Worker(name="deaf", alive=True, deaf=True, trace=trace) + ordinary = _Worker(name="ordinary", alive=True, trace=trace) + + mc._stop_any_worker_still_running([deaf, ordinary], grace=0.01) + + first_join = next(i for i, (call, _) in enumerate(trace) if call == "join") + assert not [c for c in trace[first_join:] if c[0] == "terminate"], ( + f"a worker was signalled only after another had been waited on: {trace}" + ) + assert ordinary.terminated == 1, "the second worker was never signalled" + assert deaf.killed == 1, "the worker that sat through terminate was not killed" + assert not deaf.is_alive() + + +def test_a_worker_that_already_exited_is_left_alone(): + """The control: cleanup runs on every path, including the ones where + nothing went wrong.""" + done = _Worker(alive=False) + + mc._stop_any_worker_still_running([done], grace=0.01) + + assert done.terminated == 0 and done.killed == 0 + + +class _RecordingMutex: + def __init__(self, fail_on_acquire=False): + self.acquired = 0 + self.released = 0 + self._fail = fail_on_acquire + self._lock = threading.Lock() + + def acquire(self, *args, **kwargs): + self.acquired += 1 + if self._fail: + raise ConnectionResetError("the manager is gone") + return self._lock.acquire(*args, **kwargs) + + def release(self): + self.released += 1 + return self._lock.release() + + +class _Boom(RuntimeError): + """The failure under test, so it cannot be confused with an incidental one.""" + + +class _Monitor: + """Enough of a monitor for a worker that completes an iteration.""" + + count = 0 + + def print_update_status(self): + pass + + +def _sim_worker(tmp_path, **overrides): + attributes = { + "error_file": tmp_path / "errors.txt", + "input_file": tmp_path / "inputs.txt", + "output_file": tmp_path / "outputs.txt", + "_MonteCarlo__child_seed": lambda index: index, + "_MonteCarlo__seed_simulation": lambda seed: None, + "_MonteCarlo__run_single_simulation": object, + "_MonteCarlo__evaluate_flight_inputs": lambda index: '{"index": 0}\n', + "_MonteCarlo__evaluate_flight_outputs": lambda flight, index: '{"index": 0}\n', + } + attributes.update(overrides) + return types.SimpleNamespace(**attributes) + + +def test_a_claim_that_fails_after_a_completed_run_does_not_report_that_run( + tmp_path, monkeypatch +): + """The state was cleared after the claim rather than before it. + + So a claim that failed on the second lap reached the handler still holding + the row that had just been written successfully, and the error file got a + second copy of a simulation that never failed. + """ + claims = iter([0]) + + def claim_once_then_fail(*_args, **_kwargs): + try: + return next(claims) + except StopIteration: + raise _Boom("the claim failed on the second lap") from None + + monkeypatch.setattr(mc, "_claim_next_index", claim_once_then_fail) + reported = [] + monkeypatch.setattr(mc._SimMonitor, "reprint", staticmethod(reported.append)) + worker = _sim_worker(tmp_path) + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer( + worker, _Monitor(), _RecordingMutex(), _Event() + ) + + written = (tmp_path / "errors.txt").read_text() + assert '"index": 0' not in written, ( + f"the completed simulation was written to the error file again: {written!r}" + ) + assert "the claim failed on the second lap" in written + + +def test_a_mutex_that_cannot_be_taken_is_not_then_released(tmp_path, monkeypatch): + """The normal write path released in ``finally`` whether or not it had the + lock, so a manager that died during acquire raised a second error on the way + out and buried the first.""" + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: 0) + mutex = _RecordingMutex(fail_on_acquire=True) + + with pytest.raises(ConnectionResetError): + mc.MonteCarlo._MonteCarlo__sim_producer( + _sim_worker(tmp_path), _Monitor(), mutex, _Event() + ) + + assert mutex.released == 0, "released a lock it never held" + + +def _serial_runner(tmp_path, row=""): + """A stand-in carrying only what ``__run_in_serial`` touches.""" + runner = types.SimpleNamespace( + _initial_sim_idx=0, + number_of_simulations=2, + _interrupted=False, + _error_file=tmp_path / "errors.txt", + input_file=tmp_path / "inputs.txt", + output_file=tmp_path / "outputs.txt", + _MonteCarlo__child_seed=lambda index: index, + _MonteCarlo__seed_simulation=lambda seed: None, + _MonteCarlo__run_single_simulation=object, + _MonteCarlo__evaluate_flight_inputs=lambda index: row, + _MonteCarlo__evaluate_flight_outputs=lambda flight, index: row, + ) + runner._MonteCarlo__keep_the_inputs_that_did_not_finish = lambda payload: ( + mc.MonteCarlo._MonteCarlo__keep_the_inputs_that_did_not_finish(runner, payload) + ) + return runner + + +def test_ctrl_c_before_the_first_row_keeps_the_interrupt(tmp_path, monkeypatch): + """Half of the fix: the payload is bound before the try. + + It was assigned inside the loop body, after the two monitor calls, so Ctrl-C + in either of those reached the handler with it still unbound and the + interrupt came out as an UnboundLocalError instead. + """ + + class _Monitor: + count = 0 + + def __init__(self, **_kwargs): + pass + + def keep_simulating(self): + raise KeyboardInterrupt("ctrl-c before the first simulation") + + monkeypatch.setattr(mc, "_SimMonitor", _Monitor) + runner = _serial_runner(tmp_path) + + mc.MonteCarlo._MonteCarlo__run_in_serial(runner) + + assert runner._interrupted, "the interrupt was not recorded" + + +def test_ctrl_c_between_rows_does_not_report_the_row_that_succeeded( + tmp_path, monkeypatch +): + """The other half: the payload is cleared before each lap, not after. + + Binding it once before the try stops the UnboundLocalError but leaves it + holding the last completed row, so an interrupt between two iterations + wrote a simulation that had succeeded into the error file. Both halves are + needed, and each one passes the other's test on its own. + """ + + class _Monitor: + count = 0 + + def __init__(self, **_kwargs): + self.laps = 0 + + def keep_simulating(self): + self.laps += 1 + if self.laps > 1: + raise KeyboardInterrupt("ctrl-c after the first simulation") + return True + + def increment(self): + return 1 + + def print_update_status(self): + pass + + def print_final_status(self): + pass + + monkeypatch.setattr(mc, "_SimMonitor", _Monitor) + runner = _serial_runner(tmp_path, row='{"index": 0}\n') + + mc.MonteCarlo._MonteCarlo__run_in_serial(runner) + + assert runner._interrupted + assert (tmp_path / "inputs.txt").read_text() == '{"index": 0}\n', ( + "the simulation that completed should have been written normally" + ) + assert (tmp_path / "errors.txt").read_text() == "", ( + "a completed simulation was written to the error file as if it failed" + ) From 85a6c4bb4aa3b02620d572b62b2461affb5481f9 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:20:51 +0800 Subject: [PATCH 15/45] BUG: stop the completeness check refusing the files append exists for Three problems from review, and they compound. The check judged the whole file, so a duplicate, a torn row or a pair that disagrees anywhere in it failed the run. The documented way to resume a Monte Carlo is to interrupt one and carry on with append=True, and an interrupted run is exactly what leaves that damage behind, so a file could be damaged once and never resumed again. It now judges only the indices this run claimed. Damage below _initial_sim_idx is an earlier run's and is warned about rather than raised on, while a run that wrote the whole file is still held to all of it. The parent stopped waiting the moment a worker reported an error and terminated the fleet immediately, giving a worker part way through a write no chance to finish it. Measured: 0.0 ms on that path against the 5000 ms the interrupt path already gave. So the shutdown produced the torn rows the check then reported. Both paths give the same window now. Not by reusing _bring_the_fleet_down: that sets the error event, which on a run that finished cleanly is what the crash check reads next, and wiring it in made every successful parallel run report itself as failed. A torn row also makes the two files disagree, and the cross-file check ran first, so the message named the symptom. The damage check runs first now. The real parallel path was only exercised by a test marked slow, and pull-request CI skips those, so the path this work exists to support gated nothing. There is a small version of it now that is not marked slow and uses the platform's default start method, so each CI job gates the one it actually runs: spawn on Windows and macOS, forkserver on Python 3.14's POSIX default. It caught the _bring_the_fleet_down mistake above within seconds of being written. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 97 ++++++++++++------- .../test_monte_carlo_determinism.py | 46 ++++++++- .../test_monte_carlo_log_integrity.py | 54 +++++++++-- .../test_monte_carlo_worker_exit.py | 61 ++++++++++++ 4 files changed, 217 insertions(+), 41 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index a2d7afb50..da71af74c 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -397,32 +397,59 @@ def __check_each_index_was_recorded_once(self): reached are not an error. What it did write is still held to the rest: readable rows, one row per index, and nothing outside the range. - Indices below ``_initial_sim_idx`` are an earlier run's and are left - alone. Reconciling a history with holes in it is a separate job, and - this only answers for the simulations this run claimed. - """ - inputs = _recorded_indices("inputs", self.input_file) - outputs = _recorded_indices("outputs", self.output_file) - if inputs != outputs: + Only over the indices this run claimed. An ``append`` run exists to + carry on from a file some earlier run left behind, and the documented + way to reach one is to interrupt a run, so that file can hold a torn + row or a pair that disagrees. Judging this run on that damage would + make the very files ``append`` is for the ones it refuses, so anything + below ``_initial_sim_idx`` is reported and not raised on. + """ + inputs, damaged = _recorded_indices("inputs", self.input_file) + outputs, damaged_outputs = _recorded_indices("outputs", self.output_file) + damaged += damaged_outputs + + # First, because a torn row is the root cause and the checks below are its + # symptoms: a row that will not parse also makes the two files + # disagree, and "files disagree" points at the wrong thing. + # A file this run did not write is the earlier run's business. + if damaged and self._initial_sim_idx: + warnings.warn( + f"{len(damaged)} row(s) an earlier run left behind are not " + f"readable and were skipped: {damaged[:3]}. The simulations " + f"this run added are unaffected.", + UserWarning, + ) + elif damaged: + raise RuntimeError( + f"{len(damaged)} row(s) this run wrote cannot be read: " + f"{damaged[:5]}. The results are wrong, so they are not " + f"reported as a successful run." + ) + + ours = lambda counts: { # noqa: E731 + index: count + for index, count in counts.items() + if index >= self._initial_sim_idx + } + mine, theirs = ours(inputs), ours(outputs) + if mine != theirs: only_in = lambda a, b: sorted(set(a) - set(b)) # noqa: E731 raise RuntimeError( f"the input and output files disagree about which simulations " - f"ran: {only_in(inputs, outputs)[:5]} have inputs and no " - f"outputs, {only_in(outputs, inputs)[:5]} the other way round. " + f"ran: {only_in(mine, theirs)[:5]} have inputs and no " + f"outputs, {only_in(theirs, mine)[:5]} the other way round. " f"A worker stopped between the two writes, so the results are " f"not reported as a successful run." ) - repeated = sorted(index for index, count in inputs.items() if count > 1) - beyond = sorted( - index for index in inputs if index >= self.number_of_simulations - ) + repeated = sorted(index for index, count in mine.items() if count > 1) + beyond = sorted(index for index in mine if index >= self.number_of_simulations) missing = ( [] if self._interrupted else sorted( set(range(self._initial_sim_idx, self.number_of_simulations)) - - set(inputs) + - set(mine) ) ) if missing or repeated or beyond: @@ -546,6 +573,12 @@ def __run_in_parallel(self, n_workers=None): started_processes.append(sim_producer) _wait_for_workers(started_processes, simulation_error_event) + # The event asks them to stop, it does not stop them. Without + # this window a worker part way through a write is cut off and + # leaves exactly the torn row the check below would report. + # Not _bring_the_fleet_down: that sets the event, which on a run + # that finished cleanly is what the crash check reads next. + _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE) _stop_any_worker_still_running(started_processes) _fail_if_a_worker_did_not_finish( started_processes, simulation_error_event, self.error_file @@ -1846,16 +1879,17 @@ def export_errors_to_json(self, filename): def _recorded_indices(label, path): - """``{index: how many rows carry it}`` for one log file. - - Strict about what a row is. A row that will not parse, is not an - object, or carries anything but a non-negative plain ``int`` index is - the corruption this check exists to find, so it is named and raised on - rather than skipped. ``type(...) is int`` and not ``isinstance``: - ``True`` and ``1.0`` both compare equal to ``1`` and would otherwise - pass for it. + """``({index: how many rows carry it}, [rows that carry no usable index])``. + + Damage is returned rather than raised on. Whether a torn row matters + depends on which run wrote it, and only the caller knows the range this + run claimed: an ``append`` run is recovering from a file some earlier run + damaged, which is the whole reason it is appending. + + ``type(...) is int`` and not ``isinstance``: ``True`` and ``1.0`` both + compare equal to ``1`` and would otherwise pass for it. """ - written = {} + written, damaged = {}, [] with open(path, mode="r", encoding="utf-8") as rows: for number, line in enumerate(rows, start=1): line = line.strip() @@ -1863,23 +1897,18 @@ def _recorded_indices(label, path): continue try: record = json.loads(line) - except ValueError as error: - raise RuntimeError( - f"{label} row {number} is not readable JSON, so a " - f"worker was cut off part way through writing it: " - f"{line[:60]!r}" - ) from error + except ValueError: + damaged.append(f"{label} row {number} is not readable JSON") + continue index = record.get("index") if isinstance(record, dict) else None # isinstance is the wrong tool here, see the docstring: bool is a # subclass of int, so True would pass for the index 1. # pylint: disable-next=unidiomatic-typecheck if type(index) is not int or index < 0: # noqa: E721 - raise RuntimeError( - f"{label} row {number} does not carry a simulation " - f"index: {line[:60]!r}" - ) + damaged.append(f"{label} row {number} carries no simulation index") + continue written[index] = written.get(index, 0) + 1 - return written + return written, damaged def _validate_simulation_count(number_of_simulations): diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index 1d65ec78e..d99f22219 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -503,6 +503,46 @@ def restore_start_method(): multiprocess.set_start_method(original, force=True) +def test_the_real_parallel_path_is_worker_invariant_on_this_platform( + tmp_path, + stochastic_environment_with_wind, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """The same property as the test below, on whatever start method this + platform uses, and without the ``slow`` marker. + + The thorough version covers fork, spawn and forkserver, but it is marked + slow and pull-request CI skips slow tests, so the path this change exists + to support gated nothing. This one is small enough to run every time, and + because it takes the platform default, each CI job ends up gating the start + method it actually uses: spawn on Windows and macOS, forkserver on Python + 3.14's POSIX default, fork below that. + """ + count = 2 + common = {"number_of_simulations": count, "random_seed": 24680} + models = ( + stochastic_environment_with_wind, + stochastic_calisto_numpy_only, + stochastic_flight, + ) + serial = _real_run_inputs(tmp_path, *models, "here-serial", **common)[1] + parallel = _real_run_inputs( + tmp_path, *models, "here-p2", parallel=True, n_workers=2, **common + )[1] + + assert sorted(serial) == list(range(count)) + assert sorted(parallel) == list(range(count)) + for index in range(count): + expected = _sampled_only(json.loads(serial[index])) + actual = _sampled_only(json.loads(parallel[index])) + assert len(expected) > 20, f"only {len(expected)} fields left to compare" + assert actual == expected, ( + f"{multiprocessing.get_start_method()}: serial and parallel(2) " + f"differ at index {index}" + ) + + @pytest.mark.slow @pytest.mark.parametrize("start_method", _available_start_methods()) def test_the_real_parallel_path_is_worker_invariant_under_every_start_method( @@ -690,6 +730,10 @@ def test_a_row_cut_off_mid_write_is_named_as_unreadable( That row is the corruption this check exists to find, so it is named and raised on. Skipping it and reporting the index as missing was a worse answer: with every expected index present, a corrupt file passed. + + This run wrote the whole file, so the damage is its own. A run appending + onto a file an earlier run damaged is judged only on what it added, which + ``test_an_append_run_is_not_judged_on_the_damage_it_inherited`` covers. """ montecarlo = MonteCarlo( filename=str(tmp_path / f"truncated-{parallel}"), @@ -713,7 +757,7 @@ def cut_the_second_short(sim_idx): montecarlo, "_MonteCarlo__evaluate_flight_inputs", cut_the_second_short ) - with pytest.raises(RuntimeError, match="not readable JSON"): + with pytest.raises(RuntimeError, match="cannot be read"): montecarlo.simulate(number_of_simulations=2, random_seed=5150, **kwargs) diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 80ce38c23..8f129c310 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -12,6 +12,7 @@ import threading import types +import warnings import pytest @@ -41,12 +42,12 @@ def _check(runner): CORRUPT = { - "a row cut off mid-write": (COMPLETE + "{not json\n", "not readable JSON"), - "a row that is not an object": (COMPLETE + "[]\n", "does not carry"), - "a row with no index": (COMPLETE + '{"foo": 1}\n', "does not carry"), - "a boolean index": ('{"index": 0}\n{"index": true}\n', "does not carry"), - "a float index": ('{"index": 0}\n{"index": 1.0}\n', "does not carry"), - "a negative index": (COMPLETE + '{"index": -1}\n', "does not carry"), + "a row cut off mid-write": (COMPLETE + "{not json\n", "cannot be read"), + "a row that is not an object": (COMPLETE + "[]\n", "cannot be read"), + "a row with no index": (COMPLETE + '{"foo": 1}\n', "cannot be read"), + "a boolean index": ('{"index": 0}\n{"index": true}\n', "cannot be read"), + "a float index": ('{"index": 0}\n{"index": 1.0}\n', "cannot be read"), + "a negative index": (COMPLETE + '{"index": -1}\n', "cannot be read"), "an index past the run": (COMPLETE + '{"index": 99}\n', "outside the range"), "the same index twice": (COMPLETE + '{"index": 1}\n', "more than once"), "an index never written": ('{"index": 0}\n', "never written"), @@ -67,6 +68,47 @@ def test_a_corrupt_log_is_not_a_successful_run(tmp_path, rows, expected): _check(_runner(tmp_path, rows)) +def test_an_append_run_is_not_judged_on_the_damage_it_inherited(tmp_path): + """The documented way to reach an append is to interrupt a run, so the file + it appends to can hold a torn row or a pair that disagrees. Judging this run + on that made the very files append exists for the ones it refused. + + Measured before the fix: all three of these were refused, so a file could be + damaged once and never resumed again. + """ + new_rows = '{"index": 2}\n{"index": 3}\n' + inherited = { + "a duplicate": ('{"index": 0}\n{"index": 0}\n', None), + "a torn row": ('{"index": 0}\n{not json\n', None), + "files that disagree": ('{"index": 0}\n{"index": 1}\n', '{"index": 0}\n'), + } + for name, (history, other) in inherited.items(): + runner = _runner( + tmp_path, + history + new_rows, + outputs=(other or history) + new_rows, + count=4, + initial=2, + ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + _check(runner) # must not raise, whatever the history looks like + assert True, name + + +def test_this_run_is_still_judged_strictly_while_appending(tmp_path): + """The other half. Tolerating the history must not tolerate the rows this + run added, or appending would become a way to launder a bad run.""" + runner = _runner( + tmp_path, + '{"index": 0}\n{"index": 2}\n{"index": 2}\n', + count=4, + initial=2, + ) + with pytest.raises(RuntimeError, match="more than once"): + _check(runner) + + def test_a_complete_run_is_still_accepted(tmp_path): """The control. Without this the table above passes on a check that rejects everything.""" diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index 52cd0bce9..51330dfbe 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -9,6 +9,7 @@ import types from contextlib import contextmanager +from time import monotonic import pytest @@ -150,6 +151,66 @@ def start_then_fail(self): assert all(p.terminated for p in started), "a started worker was left running" +def test_a_worker_still_writing_gets_a_window_before_it_is_signalled( + parallel_runner, monkeypatch +): + """The event asks the fleet to stop, it does not stop it. + + Cutting a worker off the moment another one reports an error truncates + whatever row it was part way through, which is the corruption the + completeness check then reports. Measured before the fix: the main error + path gave a running worker 0.0 ms, while the interrupt path gave it the + full grace period. + """ + grace = 0.3 + monkeypatch.setattr(mc, "_WORKER_SHUTDOWN_GRACE", grace) + signalled = [] + original_terminate = _Process.terminate + + def note_when(self): + signalled.append(monotonic()) + self._alive = False + original_terminate(self) + + monkeypatch.setattr(_Process, "start", lambda self: setattr(self, "_alive", True)) + monkeypatch.setattr( + _Process, "is_alive", lambda self: getattr(self, "_alive", False) + ) + monkeypatch.setattr(_Process, "terminate", note_when) + + # Another worker has already reported an error while this one is going. + class _AlreadyFailed(_Event): + def __init__(self): + super().__init__() + self.flag = True + + class _FailedManager: # pylint: disable=invalid-name + def Lock(self): # noqa: N802 + return types.SimpleNamespace(acquire=lambda: None, release=lambda: None) + + def Event(self): # noqa: N802 + return _AlreadyFailed() + + def _SimMonitor(self, **kwargs): # noqa: N802 + return _Monitor(**kwargs) + + @contextmanager + def failed_manager(*_a, **_k): + yield _FailedManager() + + monkeypatch.setattr(mc, "_create_multiprocess_manager", failed_manager) + + began = monotonic() + with pytest.raises(RuntimeError): + mc.MonteCarlo._MonteCarlo__run_in_parallel(parallel_runner, n_workers=2) + + assert signalled, "the worker was never signalled at all" + assert signalled[0] - began >= grace, ( + f"the worker was cut off after {(signalled[0] - began) * 1000:.1f} ms, " + f"before the {grace * 1000:.0f} ms window it is meant to get" + ) + + def test_an_interrupted_run_is_not_then_reported_as_incomplete( parallel_runner, monkeypatch ): From abff63df35472a131459e6cc5e1fd45d3540fbe3 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:38:29 +0800 Subject: [PATCH 16/45] BUG: validate a Monte Carlo checkpoint before appending to it The resume point came from a line count rather than from the indices on disk, and the completeness check trusted it. A blank line makes the two disagree: two rows plus one blank load as three simulations, so the next run starts at index 2, index 1 is never written, and a check scoped to the new range reports success. Measured at 3 for one blank line and 4 for two, with the file holding only 0 and 1 either way. Appending now reads both logs first and refuses unless what it finds is the run it is being asked to continue: every row readable, no index twice, the inputs and outputs holding the same set, and the indices forming exactly the range below the resume point. Nothing is opened for writing until that passes, so a checkpoint that cannot be resumed is left as it was found. A run that was not interrupted is then held to the whole range rather than to its own share of it. A file numbered from 1 is named rather than reported as an off-by-one. Serial runs used to be numbered that way, and appending onto one would rewrite its last index instead of continuing, so the answer is to re-baseline. This replaces the tolerance added in the previous commit for damage an earlier run left behind. That belonged at the wrong end: a torn row holds an index nobody can recover, so the resume point cannot be trusted either, and the preflight refuses before any simulation is spent rather than after. Two other things that could destroy a previous run: multiprocess is an optional extra, and it was imported inside the parallel path, which runs after both logs have been opened "w+" and emptied. An install without rocketpy[monte-carlo] lost its previous results on the way to the ImportError. It is imported with the other argument checks now. The rejection message for a Generator advised rng.bit_generator.seed_seq, which NumPy only grew in 1.25 while this package declared numpy>=1.13, so the advice raised AttributeError on versions it claimed to support. It now says to pass the seed the generator was built from, mentions seed_seq as a 1.25 option, and lists integer sequences among the accepted inputs. The floor moves to 1.17, which default_rng has needed all along. Also fixes the custom sampler fixture, which built a Generator in reset_seed and dropped it while sample() drew from the process-global np.random, so nothing in it answered to a seed and the 128-bit path went untested. And states in _nominal that construction-time snapshot semantics apply to every stochastic model rather than only to the environment, with a test to hold it there. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 116 ++++++++++++++---- rocketpy/stochastic/stochastic_model.py | 7 ++ .../monte_carlo/custom_sampler_fixtures.py | 17 ++- .../test_monte_carlo_determinism.py | 103 +++++++++++++--- .../test_monte_carlo_log_integrity.py | 73 +++++++---- .../unit/stochastic/test_stochastic_model.py | 64 ++++++++++ 6 files changed, 310 insertions(+), 70 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index da71af74c..147719d6c 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -204,7 +204,7 @@ def simulate( with a 128-bit integer -- the seed type a custom sampler's ``reset_seed`` accepts. A stateful ``numpy.random.Generator`` or ``BitGenerator`` is rejected (it is an RNG to draw from, not a fixed - seed); pass ``rng.bit_generator.seed_seq`` to seed from one. Default is + seed); pass the seed it was built from. Default is None, which draws fresh entropy on each run -- the previous, non-reproducible default. This seeding is informed by Scientific Python SPEC 7 but keeps immutable seed-snapshot semantics rather than sharing a @@ -245,10 +245,18 @@ def simulate( _validate_simulation_count(number_of_simulations) if parallel: n_workers = self.__validate_number_of_workers(n_workers) + # multiprocess is an optional extra. Imported here, an install + # without rocketpy[monte-carlo] raised only after __setup_files had + # already emptied the previous run's results. + _import_multiprocess() self._export_config = kwargs self.number_of_simulations = number_of_simulations self._initial_sim_idx = self.num_of_loaded_sims if append else 0 + if append: + _check_the_checkpoint_supports_appending( + self.input_file, self.output_file, self._initial_sim_idx + ) # Both run paths catch Ctrl-C, save what they have and return, so a # stopped run is incomplete on purpose and the completeness check below # has to know the difference between that and a worker going missing. @@ -317,16 +325,19 @@ def __root_seed_sequence(random_seed): child counter between calls; repeated ``simulate`` calls with the same seed then stay reproducible. A stateful ``Generator``/``BitGenerator`` is not accepted, since using it as an immutable seed would contradict its - consume-on-use semantics; pass ``rng.bit_generator.seed_seq`` to seed - from an existing generator's stream. + consume-on-use semantics. Pass the seed the generator was built from. + ``rng.bit_generator.seed_seq`` also works, but only on NumPy 1.25 and + above, which is later than this package's floor. """ if isinstance(random_seed, np.random.SeedSequence): return np.random.SeedSequence(**random_seed.state) if isinstance(random_seed, (np.random.Generator, np.random.BitGenerator)): raise TypeError( - "random_seed must be an int or a numpy.random.SeedSequence, not " - f"a {type(random_seed).__name__}; to seed from an existing " - "generator pass rng.bit_generator.seed_seq." + "random_seed must be an int, a sequence of non-negative " + "integers, or a numpy.random.SeedSequence, not a " + f"{type(random_seed).__name__}. Pass the seed the generator " + "was built from; rng.bit_generator.seed_seq also works on " + "NumPy 1.25 and above." ) return np.random.SeedSequence(random_seed) @@ -408,18 +419,14 @@ def __check_each_index_was_recorded_once(self): outputs, damaged_outputs = _recorded_indices("outputs", self.output_file) damaged += damaged_outputs - # First, because a torn row is the root cause and the checks below are its - # symptoms: a row that will not parse also makes the two files + # First, because a torn row is the root cause and the checks below are + # its symptoms: a row that will not parse also makes the two files # disagree, and "files disagree" points at the wrong thing. - # A file this run did not write is the earlier run's business. - if damaged and self._initial_sim_idx: - warnings.warn( - f"{len(damaged)} row(s) an earlier run left behind are not " - f"readable and were skipped: {damaged[:3]}. The simulations " - f"this run added are unaffected.", - UserWarning, - ) - elif damaged: + # + # Always this run's doing. An append only gets here past a preflight + # that read the checkpoint and found it whole, so anything unreadable + # now was written during this run. + if damaged: raise RuntimeError( f"{len(damaged)} row(s) this run wrote cannot be read: " f"{damaged[:5]}. The results are wrong, so they are not " @@ -447,10 +454,10 @@ def __check_each_index_was_recorded_once(self): missing = ( [] if self._interrupted - else sorted( - set(range(self._initial_sim_idx, self.number_of_simulations)) - - set(mine) - ) + # The whole range, not this run's share of it. Appending is only + # allowed onto a checkpoint the preflight found complete, so what + # ends up on disk has to be every simulation that was asked for. + else sorted(set(range(self.number_of_simulations)) - set(inputs)) ) if missing or repeated or beyond: raise RuntimeError( @@ -1911,6 +1918,73 @@ def _recorded_indices(label, path): return written, damaged +def _check_the_checkpoint_supports_appending(input_file, output_file, resume_at): + """Everything that can be judged from the files, before a worker starts. + + ``num_of_loaded_sims`` counts lines rather than indices, so a blank line or + a torn row moves the resume point past an index that was never run. The run + then skips it, and a check scoped to the new range calls that a success. + Measured: two rows plus one blank line resume at 3, plus two blanks at 4, + while the file holds only 0 and 1 either way. + + Held here rather than after the run so a checkpoint that cannot be resumed + costs no simulations and is left exactly as it was found. + + A file with a hole in it is refused rather than repaired. Filling holes + needs the workers to claim from a plan instead of counting on from the end, + which is #1075; until then, refusing loudly beats resuming in the wrong + place quietly. + """ + for label, path in (("inputs", input_file), ("outputs", output_file)): + written, damaged = _recorded_indices(label, path) + if damaged: + raise ValueError( + f"cannot append to {path}: {len(damaged)} row(s) cannot be " + f"read, so the simulations they held cannot be accounted for: " + f"{damaged[:3]}." + ) + _refuse_a_checkpoint_that_does_not_line_up(label, path, written, resume_at) + + inputs, _ = _recorded_indices("inputs", input_file) + outputs, _ = _recorded_indices("outputs", output_file) + if inputs != outputs: + raise ValueError( + f"cannot append: the input and output files hold different " + f"simulations, {sorted(set(inputs) - set(outputs))[:5]} against " + f"{sorted(set(outputs) - set(inputs))[:5]}. Appending would build " + f"on a checkpoint that is already inconsistent." + ) + + +def _refuse_a_checkpoint_that_does_not_line_up(label, path, written, resume_at): + """One file's indices have to be 0..resume_at-1, with nothing repeated.""" + repeated = sorted(index for index, count in written.items() if count > 1) + if repeated: + raise ValueError( + f"cannot append to {path}: {label} hold {len(repeated)} index(es) " + f"more than once {repeated[:5]}." + ) + + indices = set(written) + if indices == set(range(1, len(indices) + 1)) and indices: + # The serial path used to number from 1. Named rather than reported as + # an off-by-one, because the fix is to re-baseline, not to retry. + raise ValueError( + f"cannot append to {path}: the {label} are numbered from 1, which " + f"is how versions before per-index seeding wrote serial runs. This " + f"release numbers from 0, so the two cannot be continued into each " + f"other. Re-run the study, or renumber the file down by one." + ) + if indices != set(range(resume_at)): + missing = sorted(set(range(resume_at)) - indices) + extra = sorted(indices - set(range(resume_at))) + raise ValueError( + f"cannot append to {path}: the run would start at index " + f"{resume_at}, but the {label} are not the {resume_at} before it. " + f"Missing {missing[:5]}, unexpected {extra[:5]}." + ) + + def _validate_simulation_count(number_of_simulations): """A count has to be a whole non-negative number, checked before any file. diff --git a/rocketpy/stochastic/stochastic_model.py b/rocketpy/stochastic/stochastic_model.py index 12f82ce37..45716a9d6 100644 --- a/rocketpy/stochastic/stochastic_model.py +++ b/rocketpy/stochastic/stochastic_model.py @@ -139,6 +139,13 @@ def _nominal(self, input_name, getter=getattr): of ``self.obj``'s, and nothing writes back to those, so it is passed straight through. Caching it here would be wrong as well: every component's position arrives under the one name ``"position"``. + + This applies to every stochastic model, not only the environment: what + a model samples around is what the wrapped object held when the model + was built. Changing the object afterwards does not move it. Only + ``StochasticEnvironment.create_object`` writes back today, but the rule + is stated for all of them rather than special-cased for one, so a model + means the same thing whichever object it wraps. """ if getter is not getattr: return getter(self.obj, input_name) diff --git a/tests/fixtures/monte_carlo/custom_sampler_fixtures.py b/tests/fixtures/monte_carlo/custom_sampler_fixtures.py index 8a4ff497d..e3ad85d50 100644 --- a/tests/fixtures/monte_carlo/custom_sampler_fixtures.py +++ b/tests/fixtures/monte_carlo/custom_sampler_fixtures.py @@ -33,7 +33,7 @@ def __init__(self, means_tuple, sd_tuple, prob_tuple, seed=None): 2-Tuple that contains the probability of each normal distribution of the mixture. Its entries should be non-negative and sum up to 1. """ - np.random.default_rng(seed) + self.reset_seed(seed) self.means_tuple = means_tuple self.sd_tuple = sd_tuple self.prob_tuple = prob_tuple @@ -52,16 +52,12 @@ def sample(self, n_samples=1): List containing n_samples samples """ samples_list = [0] * n_samples - mixture_id_list = np.random.binomial(1, self.prob_tuple[0], n_samples) + mixture_id_list = self.rng.binomial(1, self.prob_tuple[0], n_samples) for i, mixture_id in enumerate(mixture_id_list): if mixture_id: - samples_list[i] = np.random.normal( - self.means_tuple[0], self.sd_tuple[0] - ) + samples_list[i] = self.rng.normal(self.means_tuple[0], self.sd_tuple[0]) else: - samples_list[i] = np.random.normal( - self.means_tuple[1], self.sd_tuple[1] - ) + samples_list[i] = self.rng.normal(self.means_tuple[1], self.sd_tuple[1]) return samples_list @@ -73,4 +69,7 @@ def reset_seed(self, seed=None): seed : int, optional Seed for the random number generator. """ - np.random.default_rng(seed) + # Kept on the instance. Building a generator and dropping it made this + # a no-op, and sample() went on drawing from the process-global + # np.random, so nothing here answered to the seed at all. + self.rng = np.random.default_rng(seed) diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index d99f22219..87e0724ba 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -796,13 +796,16 @@ def stop_after_the_first(): assert _count_rows(montecarlo.input_file) == 1 -def test_appending_checks_only_the_simulations_the_run_added( +def test_appending_continues_a_checkpoint_and_leaves_the_whole_range( tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight ): - """``append=True`` leaves the earlier run's records in the same files, and - ``number_of_simulations`` is the total to reach rather than a count to add. - The check has to look at indices ``_initial_sim_idx`` upwards, or a second - run would be judged against records it never wrote. + """A second run carries on from the first, and the pair ends up whole. + + This test used to empty both logs before appending and then assert that a + four-simulation result holding only indices 2 and 3 was a success. That is + the shape of the bug it was meant to guard: the resume point came from a + row count rather than the indices actually on disk, so nothing noticed the + first two were gone. The run is judged on the whole range now. """ montecarlo = MonteCarlo( filename=str(tmp_path / "appended"), @@ -811,21 +814,91 @@ def test_appending_checks_only_the_simulations_the_run_added( flight=stochastic_flight, ) montecarlo.simulate(number_of_simulations=2, random_seed=606) - assert _count_rows(montecarlo.input_file) == 2 - # Take the first run's records away before appending. The second run is - # judged on what it wrote, so a check counting the whole file would call - # this incomplete even though nothing went wrong. - montecarlo.input_file.write_text("", encoding="utf-8") - montecarlo.output_file.write_text("", encoding="utf-8") - montecarlo.simulate(number_of_simulations=4, append=True, random_seed=606) assert montecarlo._initial_sim_idx == 2, ( "the second run should have started where the first stopped" ) - written = _read_inputs_by_index(montecarlo.input_file) - assert sorted(written) == [2, 3], ( - f"the appended run wrote the wrong indices: {sorted(written)}" + for label, path in ( + ("inputs", montecarlo.input_file), + ("outputs", montecarlo.output_file), + ): + assert sorted(_read_inputs_by_index(path)) == [0, 1, 2, 3], ( + f"the {label} do not hold every simulation that was asked for" + ) + + +def test_appending_onto_a_checkpoint_with_a_hole_is_refused( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight +): + """The other half, and the reason the resume point cannot be a row count. + + Two rows plus a blank line load as three simulations, so the next run would + start at index 2 and leave index 1 missing for good while reporting + success. Refused before it runs, with both files left as they were found. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / "holed"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + montecarlo.simulate(number_of_simulations=2, random_seed=606) + + with open(montecarlo.output_file, "a", encoding="utf-8") as log: + log.write("\n") + montecarlo.set_num_of_loaded_sims() + assert montecarlo.num_of_loaded_sims == 3, "the blank line was not counted" + before = ( + montecarlo.input_file.read_bytes(), + montecarlo.output_file.read_bytes(), ) + + with pytest.raises(ValueError): + montecarlo.simulate(number_of_simulations=5, append=True, random_seed=606) + + assert ( + montecarlo.input_file.read_bytes(), + montecarlo.output_file.read_bytes(), + ) == before, "a refused checkpoint was modified on the way out" + + +def test_a_missing_parallel_dependency_does_not_cost_the_previous_run( + monkeypatch, + tmp_path, + stochastic_environment, + stochastic_calisto_numpy_only, + stochastic_flight, +): + """``multiprocess`` is an optional extra, so an install without + ``rocketpy[monte-carlo]`` cannot run in parallel at all. + + It used to be imported inside the parallel path, which runs after + ``__setup_files`` has opened both logs "w+" and emptied them, so asking for + a parallel run on such an install destroyed the previous results on the way + to the ImportError. + """ + monkeypatch.setattr(mc_module, "Flight", _StubFlight) + montecarlo = MonteCarlo( + filename=str(tmp_path / "kept"), + environment=stochastic_environment, + rocket=stochastic_calisto_numpy_only, + flight=stochastic_flight, + ) + with open(montecarlo.input_file, "w", encoding="utf-8") as existing: + existing.write("previous results\n") + + def no_multiprocess(): + raise ImportError("No module named 'multiprocess'") + + monkeypatch.setattr(mc_module, "_import_multiprocess", no_multiprocess) + + with pytest.raises(ImportError): + montecarlo.simulate( + number_of_simulations=2, parallel=True, n_workers=2, random_seed=7 + ) + + with open(montecarlo.input_file, encoding="utf-8") as kept: + assert kept.read() == "previous results\n" diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 8f129c310..01b20f41b 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -12,7 +12,6 @@ import threading import types -import warnings import pytest @@ -68,32 +67,28 @@ def test_a_corrupt_log_is_not_a_successful_run(tmp_path, rows, expected): _check(_runner(tmp_path, rows)) -def test_an_append_run_is_not_judged_on_the_damage_it_inherited(tmp_path): - """The documented way to reach an append is to interrupt a run, so the file - it appends to can hold a torn row or a pair that disagrees. Judging this run - on that made the very files append exists for the ones it refused. +def test_a_checkpoint_that_cannot_be_read_is_refused_before_the_run(tmp_path): + """Where the history is judged: before anything runs, not after. - Measured before the fix: all three of these were refused, so a file could be - damaged once and never resumed again. + An earlier round tolerated inherited damage at the end of the run, on the + grounds that the documented way to reach an append is to interrupt a run. + That was the wrong place for it. A torn row holds an index nobody can + recover, so the resume point cannot be trusted either, and resuming at the + wrong one silently skips a simulation. The preflight refuses instead, with + both files left exactly as they were found. """ - new_rows = '{"index": 2}\n{"index": 3}\n' - inherited = { - "a duplicate": ('{"index": 0}\n{"index": 0}\n', None), - "a torn row": ('{"index": 0}\n{not json\n', None), - "files that disagree": ('{"index": 0}\n{"index": 1}\n', '{"index": 0}\n'), - } - for name, (history, other) in inherited.items(): - runner = _runner( - tmp_path, - history + new_rows, - outputs=(other or history) + new_rows, - count=4, - initial=2, - ) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - _check(runner) # must not raise, whatever the history looks like - assert True, name + rows = '{"index": 0}\n{not json\n' + inputs, outputs = tmp_path / "i.txt", tmp_path / "o.txt" + inputs.write_text(rows) + outputs.write_text(rows) + before = inputs.read_bytes(), outputs.read_bytes() + + with pytest.raises(ValueError, match="cannot be read"): + mc._check_the_checkpoint_supports_appending(inputs, outputs, 2) + + assert (inputs.read_bytes(), outputs.read_bytes()) == before, ( + "a refused checkpoint was modified on the way out" + ) def test_this_run_is_still_judged_strictly_while_appending(tmp_path): @@ -441,3 +436,31 @@ def print_final_status(self): assert (tmp_path / "errors.txt").read_text() == "", ( "a completed simulation was written to the error file as if it failed" ) + + +def test_a_history_that_went_missing_is_still_caught_at_the_end(tmp_path): + """The run is judged on every index asked for, not on its own share. + + Appending normally reaches this past a preflight that found the checkpoint + whole, so the two questions have the same answer there. They do not when + the check is asked directly, and the invariant worth stating is the one + about the whole file: a four-simulation result holds four simulations. + """ + runner = _runner(tmp_path, '{"index": 2}\n{"index": 3}\n', count=4, initial=2) + + with pytest.raises(RuntimeError, match="never written"): + _check(runner) + + +def test_a_checkpoint_numbered_from_one_is_named_as_such(tmp_path): + """Serial runs used to number from 1. Appending onto one would rewrite the + last index rather than continue, so it is refused by name: the fix is to + re-baseline, not to retry, and an off-by-one message would not say that. + """ + rows = "".join('{"index": %d}\n' % index for index in (1, 2, 3)) + inputs, outputs = tmp_path / "i.txt", tmp_path / "o.txt" + inputs.write_text(rows) + outputs.write_text(rows) + + with pytest.raises(ValueError, match="numbered from 1"): + mc._check_the_checkpoint_supports_appending(inputs, outputs, 3) diff --git a/tests/unit/stochastic/test_stochastic_model.py b/tests/unit/stochastic/test_stochastic_model.py index bb4426a4f..e90878980 100644 --- a/tests/unit/stochastic/test_stochastic_model.py +++ b/tests/unit/stochastic/test_stochastic_model.py @@ -1,5 +1,7 @@ from types import SimpleNamespace +import numpy as np + import pytest from rocketpy.stochastic import StochasticFreeFormFins @@ -153,3 +155,65 @@ def test_a_scalar_nominal_does_not_drift_across_reseeds(): elevations.append(float(stochastic.create_object().elevation)) assert len(set(elevations)) == 1, f"the nominal elevation drifted: {elevations}" + + +def test_a_custom_sampler_answers_to_the_seed_it_is_given(elevation_sampler): + """The 128-bit int this package hands a sampler has to reach its draws. + + The fixture used to build a generator in ``reset_seed`` and drop it, while + ``sample`` drew from the process-global ``np.random``, so nothing in it + answered to a seed and the guarantee went untested. + """ + wide = 271828182845904523536028747135266249775 + + elevation_sampler.reset_seed(wide) + first = elevation_sampler.sample(5) + elevation_sampler.reset_seed(wide) + again = elevation_sampler.sample(5) + + assert first == again, "the same seed gave a different sample" + + elevation_sampler.reset_seed(wide + 1) + other = elevation_sampler.sample(5) + + assert other != first, "a different seed gave the same sample" + + +def test_a_custom_sampler_is_not_moved_by_the_global_generator(elevation_sampler): + """The control for the test above. Drawing from the global stream in + between must not change what the seeded sampler produces, or the sampler is + still reading from somewhere this package does not seed.""" + seed = 12345678901234567890123456789012345678 + + elevation_sampler.reset_seed(seed) + expected = elevation_sampler.sample(5) + + elevation_sampler.reset_seed(seed) + np.random.random(100) + assert elevation_sampler.sample(5) == expected + + +def test_the_nominal_is_the_one_the_model_was_built_with(example_plain_env): + """Snapshot semantics, stated once and pinned here. + + A model samples around what the wrapped object held when it was built. + This exists because ``StochasticEnvironment.create_object`` writes the + sampled value back onto that object on purpose, and reading the nominal + back off it made a factor compound from one simulation to the next. The + rule is the same for every model, so a change to the wrapped object after + construction deliberately does not move what is sampled around. + """ + example_plain_env.elevation = 1000 + # A scalar is a spread around the object's own value, so this is the form + # that reads the nominal. A tuple carries its own centre and would not. + model = StochasticEnvironment(environment=example_plain_env, elevation=5) + + model._set_stochastic(4242) + around_first = model.elevation[0] + + example_plain_env.elevation = 9000 + model._set_stochastic(4242) + + assert model.elevation[0] == around_first == 1000, ( + "the model followed the object instead of the value it was built with" + ) From 7a531580265c4037d64297a9a1adba8d72c12706 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 07:58:03 +0800 Subject: [PATCH 17/45] BUG: keep the component count out of the rocket body stream `dict_generator` walks the whole instance, so `parachutes` and `air_brakes` were drawn from as ordinary lists. Since this branch started seeding list choices from the model's own generator, that draw moved every later one, and `StochasticRocket.dict_generator` discards it a few lines further down. Attaching a main and a drogue changed the sampled mass under a fixed seed, which is the property this branch exists to establish. One component does not show it: `integers(1)` has a single outcome and NumPy returns it without consuming any state, so the test covers 0, 1 and 2. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../test_stochastic_rocket_seeding.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/unit/stochastic/test_stochastic_rocket_seeding.py b/tests/unit/stochastic/test_stochastic_rocket_seeding.py index 0e3efef8a..e68720622 100644 --- a/tests/unit/stochastic/test_stochastic_rocket_seeding.py +++ b/tests/unit/stochastic/test_stochastic_rocket_seeding.py @@ -169,3 +169,54 @@ def drawn(seed): assert first, "the air brake sampled nothing, so this proves nothing" assert drawn(31337) == first, "the same seed drew a different air brake" assert drawn(31338) != first, "a different seed drew the same air brake" + + +def _mass_drawn_with(rocket_factory, stochastic_parachutes, seed=42): + rocket = rocket_factory() + for parachute in stochastic_parachutes: + rocket.add_parachute(parachute) + rocket._set_stochastic(seed) + return next(rocket.dict_generator())["mass"] + + +@pytest.mark.parametrize("attached", [0, 1, 2]) +def test_attaching_components_does_not_move_the_rocket_body_stream( + attached, + stochastic_calisto, + stochastic_main_parachute, + stochastic_drogue_parachute, +): + """The base generator walked the whole instance and drew from + ``parachutes`` too, using the model's own generator since this branch + started seeding list choices. The subclass then discards that draw, so the + only thing it did was shift every later draw by however many components + happened to be attached. Two chutes changed the sampled mass. + + One is not enough to catch it: ``integers(1)`` has a single outcome and + NumPy returns it without consuming any state. + """ + available = [stochastic_main_parachute, stochastic_drogue_parachute] + + def bare(): + stochastic_calisto.parachutes = [] + stochastic_calisto.air_brakes = [] + return stochastic_calisto + + alone = _mass_drawn_with(bare, []) + with_components = _mass_drawn_with(bare, available[:attached]) + + assert with_components == alone + + +def test_the_discarded_component_lists_are_still_reported_empty( + stochastic_calisto, stochastic_main_parachute +): + """Skipping them must not change what ``dict_generator`` yields.""" + stochastic_calisto.parachutes = [] + stochastic_calisto.add_parachute(stochastic_main_parachute) + stochastic_calisto._set_stochastic(42) + + generated = next(stochastic_calisto.dict_generator()) + + assert generated["parachutes"] == [] + assert generated["air_brakes"] == [] From b43ec997d278912c8b2921bc5aad3ae369f9018a Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 07:58:12 +0800 Subject: [PATCH 18/45] BUG: fix three Monte Carlo contracts this branch got wrong A worker killed outright sets no error event. If it died holding the shared lock, its siblings never return either, so `any(is_alive())` stayed true and the unbounded wait never reached the exit-code check below it. The wait now also ends on a non-zero exit code. Only the unbounded one: the shutdown grace period is bounded already and must not be cut short. `type(value) in (int, np.integer)` is False for every NumPy integer, because `type(np.int64(3))` is `np.int64`. It was written that way to keep `True` out, which `isinstance` lets through, so both are now checked explicitly. `number_of_simulations` is the total to reach when appending, not a batch to add. Below the checkpoint it ran nothing and returned success, leaving a file with more simulations than the caller asked for. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 50 +++++++++++++++-- .../test_monte_carlo_determinism.py | 49 +++++++++++++++++ .../test_monte_carlo_determinism.py | 19 +++++++ .../test_monte_carlo_worker_exit.py | 54 +++++++++++++++++++ 4 files changed, 167 insertions(+), 5 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 147719d6c..475d35cf5 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -257,6 +257,16 @@ def simulate( _check_the_checkpoint_supports_appending( self.input_file, self.output_file, self._initial_sim_idx ) + # ``number_of_simulations`` is the target to reach, not a batch to + # add. Below the checkpoint it ran nothing, reported success, and + # left a file with more simulations than the caller had asked for. + if number_of_simulations < self._initial_sim_idx: + raise ValueError( + f"number_of_simulations is the total to reach when " + f"append=True. The checkpoint already holds " + f"{self._initial_sim_idx} simulations, more than the " + f"requested {number_of_simulations}." + ) # Both run paths catch Ctrl-C, save what they have and return, so a # stopped run is incomplete on purpose and the completeness check below # has to know the difference between that and a worker going missing. @@ -607,7 +617,7 @@ def __validate_number_of_workers(self, n_workers): # os.cpu_count() is documented as possibly None, and comparing against # it then raises rather than falling back to a usable default. available = os.cpu_count() or 2 - if n_workers is not None and type(n_workers) not in (int, np.integer): # noqa: E721 + if n_workers is not None and not _is_whole_number(n_workers): raise TypeError( f"Number of workers must be an integer, not {type(n_workers).__name__}." ) @@ -1985,14 +1995,25 @@ def _refuse_a_checkpoint_that_does_not_line_up(label, path, written, resume_at): ) +def _is_whole_number(value): + """A Python or NumPy integer, and not a bool. + + ``type(value) in (int, np.integer)`` rejected every NumPy integer, because + ``type(np.int64(2))`` is ``np.int64``. ``True`` still has to go: it is an + ``int`` to ``isinstance`` and would quietly run one simulation. + """ + if isinstance(value, (bool, np.bool_)): + return False + return isinstance(value, (int, np.integer)) + + def _validate_simulation_count(number_of_simulations): """A count has to be a whole non-negative number, checked before any file. - ``type(...) is not int``: ``True`` is an ``int`` to ``isinstance`` and would - quietly run one simulation. A float ran ``int(count)`` of them and then - failed the completeness check with a range it could never have satisfied. + A float ran ``int(count)`` simulations and then failed the completeness + check with a range it could never have satisfied. """ - if type(number_of_simulations) not in (int, np.integer): # noqa: E721 + if not _is_whole_number(number_of_simulations): raise TypeError( f"number_of_simulations must be an integer, not " f"{type(number_of_simulations).__name__}." @@ -2033,6 +2054,20 @@ def _bring_the_fleet_down(started_processes, error_event): _stop_any_worker_still_running(started_processes) +def _workers_that_crashed(started_processes): + """Those already known to have exited abnormally. + + ``join(timeout=0)`` first: an unjoined child has no exit code yet, so it + would read as ``None`` and pass for one still running. + """ + crashed = [] + for process in started_processes: + process.join(timeout=0) + if process.exitcode not in (None, 0): + crashed.append(process) + return crashed + + def _wait_for_workers(started_processes, error_event=None, timeout=None): """Wait for the fleet, giving up early once one of them reports an error. @@ -2048,6 +2083,11 @@ def _wait_for_workers(started_processes, error_event=None, timeout=None): while any(process.is_alive() for process in started_processes): if error_event is not None and error_event.is_set(): break + # A worker killed outright sets no event. If it died holding the shared + # lock its siblings never return either, and only the unbounded wait + # has nothing else to end it. + if deadline is None and _workers_that_crashed(started_processes): + break if deadline is not None and monotonic() >= deadline: break for process in started_processes: diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index 87e0724ba..d9928970a 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -902,3 +902,52 @@ def no_multiprocess(): with open(montecarlo.input_file, encoding="utf-8") as kept: assert kept.read() == "previous results\n" + + +def test_appending_below_the_checkpoint_is_refused_and_changes_nothing( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight +): + """``number_of_simulations`` is the total to reach, not a batch to add. + + Asking for fewer than the checkpoint already holds ran nothing and returned + success: every index it wanted was present, so the completeness check was + satisfied by simulations an earlier run had made. The caller was told three + while the file held five. + """ + montecarlo = MonteCarlo( + filename=str(tmp_path / "shrunk"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + montecarlo.simulate(number_of_simulations=3, random_seed=606) + before = ( + montecarlo.input_file.read_bytes(), + montecarlo.output_file.read_bytes(), + ) + + with pytest.raises(ValueError, match="already holds 3"): + montecarlo.simulate(number_of_simulations=2, append=True, random_seed=606) + + assert ( + montecarlo.input_file.read_bytes(), + montecarlo.output_file.read_bytes(), + ) == before, "the refusal touched the checkpoint it was protecting" + + +def test_appending_to_the_size_it_already_has_is_allowed( + tmp_path, stochastic_environment, stochastic_calisto, stochastic_flight +): + """The control. The guard must refuse only what is below the checkpoint, + not every append that adds no work.""" + montecarlo = MonteCarlo( + filename=str(tmp_path / "unchanged"), + environment=stochastic_environment, + rocket=stochastic_calisto, + flight=stochastic_flight, + ) + montecarlo.simulate(number_of_simulations=2, random_seed=606) + + montecarlo.simulate(number_of_simulations=2, append=True, random_seed=606) + + assert sorted(_read_inputs_by_index(montecarlo.input_file)) == [0, 1] diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 546b2c1d1..c582f46da 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -36,6 +36,7 @@ from rocketpy.simulation import MonteCarlo from rocketpy.simulation.monte_carlo import ( _SimMonitor, + _validate_simulation_count, _claim_next_index, _seed_sequence_to_int, ) @@ -326,3 +327,21 @@ def worker(): assert sorted(claimed) == list(range(n_simulations)) assert monitor.count == n_simulations + + +@pytest.mark.parametrize("count", [3, np.int32(3), np.int64(3), np.uint64(3)], ids=str) +def test_a_numpy_integer_is_a_valid_simulation_count(count): + """``type(count) in (int, np.integer)`` is False for every NumPy integer: + ``type(np.int64(3))`` is ``np.int64``, and ``np.integer`` is only its base. + A count read out of an array or a ``range`` product was refused.""" + _validate_simulation_count(count) + + +@pytest.mark.parametrize( + "count", [True, False, np.bool_(True), 3.0, "3", None], ids=str +) +def test_a_count_that_is_not_a_whole_number_is_still_refused(count): + """The control for the test above. ``True`` is the one that matters: it is + an ``int`` to ``isinstance`` and would quietly run one simulation.""" + with pytest.raises(TypeError): + _validate_simulation_count(count) diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index 51330dfbe..819978358 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -238,3 +238,57 @@ def ctrl_c(self, *a, **k): assert parallel_runner.input_file.read_text() == "", ( "nothing was written, so the check really was in a position to reject this" ) + + +class _Crashed: + """Killed outright: gone, non-zero exit code, no event set.""" + + def __init__(self): + self.exitcode = 7 + + def join(self, *_a, **_k): + pass + + def is_alive(self): + return False + + +class _BlockedForever: + """Waiting on the lock the crashed worker still holds.""" + + def __init__(self, give_up_after=50): + self.exitcode = None + self.joins = 0 + self._give_up_after = give_up_after + + def join(self, *_a, **_k): + self.joins += 1 + if self.joins > self._give_up_after: + raise AssertionError( + "the parent is still waiting on a worker that a dead sibling " + "has blocked, and nothing else will end this wait" + ) + + def is_alive(self): + return True + + +def test_the_parent_stops_waiting_when_a_worker_dies_holding_the_lock(): + """A worker killed outright sets no event, so the wait had only + ``is_alive`` to end it, and a sibling blocked on the lock it held kept that + true forever. The exit-code check downstream was never reached.""" + crashed, blocked = _Crashed(), _BlockedForever() + + mc._wait_for_workers([crashed, blocked], _Event()) + + assert blocked.is_alive(), "the blocked worker is meant to still be running" + + +def test_the_shutdown_window_is_not_cut_short_by_a_worker_already_known_dead(): + """The grace period exists so the survivors can finish their writes. It is + bounded by its own timeout, so a crash must not end it early.""" + crashed, blocked = _Crashed(), _BlockedForever(give_up_after=10**6) + + mc._wait_for_workers([crashed, blocked], timeout=0.3) + + assert blocked.joins > 1, "the grace period returned without waiting" From 62f47d83dd7ad0e3fd4b553f88185fff3703efcb Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:21:31 +0800 Subject: [PATCH 19/45] TST: gate the parallel path on spawn and forkserver, not just fork The unmarked test took the platform default and its docstring claimed that gated spawn on macOS and forkserver on 3.14. Neither is true: multiprocess hard-codes fork on every POSIX platform, macOS and 3.14 included, with a `#FIXME: spawn` still beside the darwin branch. So the shipped parallel path was gated on fork everywhere except Windows, and the failure message named the stdlib start method rather than the one that made the workers. The thorough test already covers all three through the real path and takes 26 s against 89 s for the rest of the directory, so it is unmarked now and the default-taking one is deleted rather than corrected. Its assertions were a subset. The start-method list is asked of multiprocess as well. That import is at module level behind a try/except because it happens while tests are collected, where importorskip would take the module down instead of skipping it. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../test_monte_carlo_determinism.py | 89 +++++++------------ 1 file changed, 34 insertions(+), 55 deletions(-) diff --git a/tests/integration/simulation/test_monte_carlo_determinism.py b/tests/integration/simulation/test_monte_carlo_determinism.py index d9928970a..2aa2b14b5 100644 --- a/tests/integration/simulation/test_monte_carlo_determinism.py +++ b/tests/integration/simulation/test_monte_carlo_determinism.py @@ -20,15 +20,17 @@ not the stdlib ``random.choice``) and is covered directly in ``tests/unit/stochastic/test_stochastic_model``. -Seed derivation being independent of the multiprocessing start method (fork, -spawn or forkserver) is verified separately by -``test_seed_derivation_is_start_method_invariant``, which uses a top-level -picklable target so it is safe under ``spawn``/``forkserver`` -- unlike the -``Flight``-stub test above, which reaches workers only under ``fork``. +Two tests cover the start methods, both unmarked. +``test_seed_derivation_is_start_method_invariant`` checks the derivation with a +top-level picklable target, and +``test_the_real_parallel_path_is_worker_invariant_under_every_start_method`` +drives the shipped parallel path. Both set the method rather than taking the +platform default, which is worth the seconds it costs: ``multiprocess`` +hard-codes ``fork`` on every POSIX platform, macOS and 3.14 included, so a +default-taking test would gate ``spawn`` on Windows and nothing anywhere else. """ import json -import multiprocessing import os from types import SimpleNamespace @@ -48,10 +50,22 @@ _child_seed = MonteCarlo._MonteCarlo__child_seed +# Parametrizing over start methods runs while tests are collected, and +# `multiprocess` is an optional extra, so skipping there would take the whole +# module down instead of skipping it. The tests themselves still importorskip. +try: + import multiprocess as _start_methods_from +except ImportError: + import multiprocessing as _start_methods_from + def _available_start_methods(): - """The multiprocessing start methods this platform actually supports.""" - supported = multiprocessing.get_all_start_methods() + """The start methods this platform supports, as ``multiprocess`` sees them. + + Asked of ``multiprocess`` rather than the standard library, because that is + what creates the workers and the two do not have to agree. + """ + supported = _start_methods_from.get_all_start_methods() return [method for method in ("fork", "spawn", "forkserver") if method in supported] @@ -312,9 +326,8 @@ def test_seed_derivation_is_start_method_invariant(start_method): actually has to hold cross-platform -- that a simulation index maps to the same seed no matter which process derives it -- using a top-level picklable target and small picklable arguments, so it is valid under ``spawn``/``forkserver`` - (Python 3.14's POSIX default) without relying on any inherited parent state. - Two workers split the indices; their combined result must equal the - single-process derivation. + without relying on any inherited parent state. Two workers split the indices; + their combined result must equal the single-process derivation. """ root = np.random.SeedSequence(2718281828) root_state = ( @@ -326,7 +339,8 @@ def test_seed_derivation_is_start_method_invariant(start_method): indices = list(range(6)) expected = _derive_index_seeds(root_state, indices) - context = multiprocessing.get_context(start_method) + multiprocess = pytest.importorskip("multiprocess") + context = multiprocess.get_context(start_method) chunks = [(root_state, indices[0::2]), (root_state, indices[1::2])] with context.Pool(2) as pool: results = pool.starmap(_derive_index_seeds, chunks) @@ -503,47 +517,6 @@ def restore_start_method(): multiprocess.set_start_method(original, force=True) -def test_the_real_parallel_path_is_worker_invariant_on_this_platform( - tmp_path, - stochastic_environment_with_wind, - stochastic_calisto_numpy_only, - stochastic_flight, -): - """The same property as the test below, on whatever start method this - platform uses, and without the ``slow`` marker. - - The thorough version covers fork, spawn and forkserver, but it is marked - slow and pull-request CI skips slow tests, so the path this change exists - to support gated nothing. This one is small enough to run every time, and - because it takes the platform default, each CI job ends up gating the start - method it actually uses: spawn on Windows and macOS, forkserver on Python - 3.14's POSIX default, fork below that. - """ - count = 2 - common = {"number_of_simulations": count, "random_seed": 24680} - models = ( - stochastic_environment_with_wind, - stochastic_calisto_numpy_only, - stochastic_flight, - ) - serial = _real_run_inputs(tmp_path, *models, "here-serial", **common)[1] - parallel = _real_run_inputs( - tmp_path, *models, "here-p2", parallel=True, n_workers=2, **common - )[1] - - assert sorted(serial) == list(range(count)) - assert sorted(parallel) == list(range(count)) - for index in range(count): - expected = _sampled_only(json.loads(serial[index])) - actual = _sampled_only(json.loads(parallel[index])) - assert len(expected) > 20, f"only {len(expected)} fields left to compare" - assert actual == expected, ( - f"{multiprocessing.get_start_method()}: serial and parallel(2) " - f"differ at index {index}" - ) - - -@pytest.mark.slow @pytest.mark.parametrize("start_method", _available_start_methods()) def test_the_real_parallel_path_is_worker_invariant_under_every_start_method( restore_start_method, @@ -560,8 +533,14 @@ def test_the_real_parallel_path_is_worker_invariant_under_every_start_method( every start method, and the stubbed test above covers the real loop on ``fork``. Neither covers ``multiprocess.Process``, ``__sim_producer``, the manager proxies or pickling the stochastic object graph anywhere but - ``fork``, and that is what Windows, macOS and Python 3.14's POSIX default - actually run. + ``fork``. + + Sets each method rather than taking the platform default, because + ``multiprocess`` hard-codes ``fork`` on every POSIX platform including macOS + and 3.14, so a default-taking test gates ``spawn`` on Windows and nothing + else. Unmarked despite the cost: 26 s for the three, against 89 s for the + rest of this directory, and it is the only thing covering the path this + change exists to support. """ multiprocess = restore_start_method if start_method not in multiprocess.get_all_start_methods(): From 6b04d5a8d1602925c79bed4365d802b7222a7e4b Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:27:26 +0800 Subject: [PATCH 20/45] BUG: make the captured root seed a real snapshot SeedSequence keeps a sequence entropy by reference, and the capture stored that reference, so a caller who passed a list and later edited it changed the child seeds of a run that had already read the seed. The docstring called it an immutable snapshot, which it was only for an int. entropy = [1, 2, 3] mc.simulate(2, random_seed=entropy) entropy[0] = 999999 # moved every index of that run Deep-copied on the way in now, with spawn_key made a tuple while it is there. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/environment/environment.py | 2 +- rocketpy/simulation/monte_carlo.py | 9 ++++++-- tests/unit/environment/test_environment.py | 2 +- .../test_monte_carlo_determinism.py | 22 +++++++++++++++++-- .../unit/stochastic/test_stochastic_model.py | 4 +--- .../stochastic/test_stochastic_parachute.py | 2 +- tests/unit/test_tools.py | 1 - 7 files changed, 31 insertions(+), 11 deletions(-) diff --git a/rocketpy/environment/environment.py b/rocketpy/environment/environment.py index 460f0bc89..0b2e23b1b 100644 --- a/rocketpy/environment/environment.py +++ b/rocketpy/environment/environment.py @@ -5,8 +5,8 @@ import os import re import warnings -from collections.abc import Mapping from collections import namedtuple +from collections.abc import Mapping from datetime import datetime import netCDF4 diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 475d35cf5..b53449bab 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -18,6 +18,7 @@ import os import traceback import warnings +from copy import deepcopy from numbers import Real from pathlib import Path from time import monotonic, time @@ -358,11 +359,15 @@ def __capture_root_state(self, random_seed): per-index child seeds from it (see ``__child_seed``), instead of materializing and pickling the full ``spawn(number_of_simulations)`` list to each process. + + Deep-copied, because ``SeedSequence`` keeps a sequence entropy by + reference. Without it a caller who mutates the list they passed changes + the children this run derives, which is the opposite of a snapshot. """ root = self.__root_seed_sequence(random_seed) self.__root_state = ( - root.entropy, - root.spawn_key, + deepcopy(root.entropy), + tuple(root.spawn_key), root.pool_size, root.n_children_spawned, ) diff --git a/tests/unit/environment/test_environment.py b/tests/unit/environment/test_environment.py index 61d6c3ff4..9755a2a80 100644 --- a/tests/unit/environment/test_environment.py +++ b/tests/unit/environment/test_environment.py @@ -2,9 +2,9 @@ import os from datetime import datetime +import netCDF4 import numpy as np import numpy.testing as npt -import netCDF4 import pytest import pytz diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index c582f46da..c8ef2abfe 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -35,10 +35,10 @@ from rocketpy.simulation import MonteCarlo from rocketpy.simulation.monte_carlo import ( - _SimMonitor, - _validate_simulation_count, _claim_next_index, _seed_sequence_to_int, + _SimMonitor, + _validate_simulation_count, ) _root_seed_sequence = MonteCarlo._MonteCarlo__root_seed_sequence @@ -345,3 +345,21 @@ def test_a_count_that_is_not_a_whole_number_is_still_refused(count): an ``int`` to ``isinstance`` and would quietly run one simulation.""" with pytest.raises(TypeError): _validate_simulation_count(count) + + +@pytest.mark.parametrize("wrapped", [False, True], ids=["sequence", "SeedSequence"]) +def test_mutating_the_caller_s_entropy_does_not_move_the_captured_root(wrapped): + """`SeedSequence` keeps a sequence entropy by reference, and so did the + capture, so a caller who reused and edited their list changed the children + of a run that had already read it. An int seed was never exposed to this. + """ + entropy = [1, 2, 3] + seed = np.random.SeedSequence(entropy) if wrapped else entropy + + runner = MonteCarlo.__new__(MonteCarlo) + MonteCarlo._MonteCarlo__capture_root_state(runner, seed) + before = _entropy(_child_seed(runner, 7)) + + entropy[0] = 999999 + + assert _entropy(_child_seed(runner, 7)) == before diff --git a/tests/unit/stochastic/test_stochastic_model.py b/tests/unit/stochastic/test_stochastic_model.py index e90878980..80fc3ed60 100644 --- a/tests/unit/stochastic/test_stochastic_model.py +++ b/tests/unit/stochastic/test_stochastic_model.py @@ -1,12 +1,10 @@ from types import SimpleNamespace import numpy as np - import pytest -from rocketpy.stochastic import StochasticFreeFormFins from rocketpy import Environment -from rocketpy.stochastic import StochasticEnvironment +from rocketpy.stochastic import StochasticEnvironment, StochasticFreeFormFins from rocketpy.stochastic.stochastic_model import StochasticModel diff --git a/tests/unit/stochastic/test_stochastic_parachute.py b/tests/unit/stochastic/test_stochastic_parachute.py index 8fc128f54..5b4e565f5 100644 --- a/tests/unit/stochastic/test_stochastic_parachute.py +++ b/tests/unit/stochastic/test_stochastic_parachute.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from rocketpy.stochastic import StochasticParachute from rocketpy.rocket.parachute import Parachute +from rocketpy.stochastic import StochasticParachute def test_stochastic_parachute_create_object(stochastic_main_parachute): diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index e57fb3d73..19a4837e2 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -20,7 +20,6 @@ tuple_handler, ) - WEB_MERCATOR_EARTH_RADIUS = 6378137.0 From 4065516f7ff80481ba314907f2fc8652b55d4c79 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:38:16 +0800 Subject: [PATCH 21/45] BUG: bound the shutdown by one deadline, not one per worker Each worker got the full grace to itself, and twice over, once after terminate and once after kill. Eight stubborn workers could therefore hold the parent for sixteen grace periods rather than two, which is a 5 s promise turning into 80 s. The deadline is shared now, so the wait costs the same whatever the fleet size. Every worker is still joined, so exit codes are still reaped. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 17 +++++-- .../test_monte_carlo_worker_exit.py | 51 ++++++++++++++++++- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index b53449bab..0954bbf5e 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -2105,6 +2105,17 @@ def _wait_for_workers(started_processes, error_event=None, timeout=None): process.join(timeout=0) +def _join_until(processes, deadline): + """Wait on the fleet against one clock rather than one clock each. + + A full grace per worker made the wait scale with the fleet: eight stubborn + ones could hold the parent for eight times what the grace period promised, + and twice over, once for terminate and once for kill. + """ + for process in processes: + process.join(timeout=max(0.0, deadline - monotonic())) + + def _stop_any_worker_still_running(started_processes, grace=_WORKER_SHUTDOWN_GRACE): """Whatever is still going here is not going to stop on its own. @@ -2115,16 +2126,14 @@ def _stop_any_worker_still_running(started_processes, grace=_WORKER_SHUTDOWN_GRA alive = [process for process in started_processes if process.is_alive()] for process in alive: process.terminate() - for process in alive: - process.join(timeout=grace) + _join_until(alive, monotonic() + grace) # terminate is a request. SIGKILL is not, and a worker that sat through the # first one would otherwise keep the manager and the files open for good. stubborn = [process for process in alive if process.is_alive()] for process in stubborn: process.kill() - for process in stubborn: - process.join(timeout=grace) + _join_until(stubborn, monotonic() + grace) def _fail_if_a_worker_did_not_finish(started_processes, error_event, error_file): diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index 819978358..f55ec443f 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -9,7 +9,7 @@ import types from contextlib import contextmanager -from time import monotonic +from time import monotonic, sleep import pytest @@ -292,3 +292,52 @@ def test_the_shutdown_window_is_not_cut_short_by_a_worker_already_known_dead(): mc._wait_for_workers([crashed, blocked], timeout=0.3) assert blocked.joins > 1, "the grace period returned without waiting" + + +class _Stubborn: + """A worker that sits through terminate and kill, recording its waits.""" + + def __init__(self, waits): + self.exitcode = None + self._waits = waits + + def join(self, timeout=None, **_k): + self._waits.append(timeout) + if timeout: + sleep(timeout) + + def is_alive(self): + return True + + def terminate(self): + pass + + def kill(self): + pass + + +@pytest.mark.parametrize("fleet_size", [1, 6]) +def test_shutdown_is_bounded_by_the_grace_period_not_by_the_fleet_size(fleet_size): + """Each worker used to get the full grace to itself, so the wait scaled with + the fleet: six stubborn workers held the parent for six grace periods per + phase rather than one. The deadline is shared now, so a larger fleet costs + the same wall clock as a single worker. + """ + grace = 0.2 + waits = [] + fleet = [_Stubborn(waits) for _ in range(fleet_size)] + + mc._stop_any_worker_still_running(fleet, grace=grace) + + assert len(waits) == 2 * fleet_size, "every worker is still waited on" + # What each worker was granted, rather than how long the call took, so a + # loaded machine cannot turn this into a flake. Per phase the total is one + # grace however many workers there are; it was one grace each. + for phase, granted in ( + ("terminate", waits[:fleet_size]), + ("kill", waits[fleet_size:]), + ): + assert sum(granted) <= grace + 0.01, ( + f"{phase}: {fleet_size} workers were granted {sum(granted):.2f}s " + f"against a {grace}s deadline" + ) From b477b6b979d86faba1efc475b114624bb1bb3472 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:25:05 +0800 Subject: [PATCH 22/45] TST: compare fleet sizes rather than the grace, which Windows cannot meet The assertion checked one fleet against the grace itself, so its slack had to cover the machine's timer granularity. On Windows that is about 15 ms against a 200 ms grace, and the job failed at 0.213s under a 0.21s bound. Comparing a fleet of six against a fleet of one carries the same granularity on both sides, so it cancels. Six against twice one leaves roughly half the bound spare on the Windows numbers, and the per-worker grace it replaced would grant six times as much. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../test_monte_carlo_worker_exit.py | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index f55ec443f..1bd9218a5 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -316,28 +316,33 @@ def kill(self): pass -@pytest.mark.parametrize("fleet_size", [1, 6]) -def test_shutdown_is_bounded_by_the_grace_period_not_by_the_fleet_size(fleet_size): +def _granted_shutting_down(fleet_size, grace): + """How long the fleet was granted in total, across both phases.""" + waits = [] + mc._stop_any_worker_still_running( + [_Stubborn(waits) for _ in range(fleet_size)], grace=grace + ) + assert len(waits) == 2 * fleet_size, "every worker is still waited on" + return sum(waits) + + +def test_shutdown_does_not_grow_with_the_fleet(): """Each worker used to get the full grace to itself, so the wait scaled with the fleet: six stubborn workers held the parent for six grace periods per - phase rather than one. The deadline is shared now, so a larger fleet costs - the same wall clock as a single worker. + phase rather than one. + + Compares two fleet sizes rather than checking either against the grace. + A single fleet has to be measured against a constant, and the slack that + needs is the machine's timer granularity, which on Windows is 15 ms against + a 200 ms grace. Both measurements carry the same granularity, so comparing + them cancels it. """ grace = 0.2 - waits = [] - fleet = [_Stubborn(waits) for _ in range(fleet_size)] - mc._stop_any_worker_still_running(fleet, grace=grace) + alone = _granted_shutting_down(1, grace) + crowd = _granted_shutting_down(6, grace) - assert len(waits) == 2 * fleet_size, "every worker is still waited on" - # What each worker was granted, rather than how long the call took, so a - # loaded machine cannot turn this into a flake. Per phase the total is one - # grace however many workers there are; it was one grace each. - for phase, granted in ( - ("terminate", waits[:fleet_size]), - ("kill", waits[fleet_size:]), - ): - assert sum(granted) <= grace + 0.01, ( - f"{phase}: {fleet_size} workers were granted {sum(granted):.2f}s " - f"against a {grace}s deadline" - ) + assert crowd < alone * 2, ( + f"six workers were granted {crowd:.2f}s against {alone:.2f}s for one, " + f"so the wait is still scaling with the fleet" + ) From c985298af683602313fac08eb1c2dd377f456a73 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:36:02 +0800 Subject: [PATCH 23/45] DOC: stop offering to renumber a legacy checkpoint The refusal message suggested re-running "or renumber the file down by one", while the comment two lines above it said the fix is to re-baseline rather than retry. The comment was right. Renumbering lines the indices up and leaves the seeds behind. Those rows came from the old sequential scheme, so a renumbered file would carry rows 0..n-1 that this release's per-index derivation would never have produced for those indices, and appending onto it would join two different seedings without saying so. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 5 ++++- tests/unit/simulation/test_monte_carlo_log_integrity.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 0954bbf5e..bf4d6f4e0 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -1988,7 +1988,10 @@ def _refuse_a_checkpoint_that_does_not_line_up(label, path, written, resume_at): f"cannot append to {path}: the {label} are numbered from 1, which " f"is how versions before per-index seeding wrote serial runs. This " f"release numbers from 0, so the two cannot be continued into each " - f"other. Re-run the study, or renumber the file down by one." + f"other. Re-run the study. Renumbering the rows would line the " + f"indices up without lining the seeds up: those rows came from the " + f"old sequential scheme, not from the per-index derivation this " + f"release would use for the same indices." ) if indices != set(range(resume_at)): missing = sorted(set(range(resume_at)) - indices) diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 01b20f41b..01304509a 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -462,5 +462,11 @@ def test_a_checkpoint_numbered_from_one_is_named_as_such(tmp_path): inputs.write_text(rows) outputs.write_text(rows) - with pytest.raises(ValueError, match="numbered from 1"): + with pytest.raises(ValueError, match="numbered from 1") as raised: mc._check_the_checkpoint_supports_appending(inputs, outputs, 3) + + # The message used to offer renumbering as an alternative to re-running, + # which lines the indices up and leaves the seeds behind: those rows came + # from the old sequential scheme, not from this release's per-index one. + assert "Renumbering" in str(raised.value) + assert "without lining the seeds up" in str(raised.value) From 761679e9a1784e248a4a6a66ce4acf548f169cd8 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:09:56 +0800 Subject: [PATCH 24/45] Give the serial path the same error record as the workers A failure after sampling wrote the inputs to .errors.txt with no traceback, while a worker writes {index, ...inputs, error: traceback}. The file the run tells the user to read named which inputs failed and not why. Both paths now build the row through one helper. Three further points on that path: - inputs_json is cleared once the pair is on disk, so a failure inside print_update_status() reports itself rather than reporting an already committed row as one that never finished. - sim_idx is bound before the loop, so a failure on the first iteration has an index to record. - the re-raise is bare, so the handler's own line does not join the traceback. KeyboardInterrupt keeps its own handler: an interrupt is not a failure with a traceback worth recording, and it still logs the inputs that did not finish. The append docstring said only that results are appended. It now says number_of_simulations is the target total rather than a number to add, that a lower value is refused, and that the root seed is not stored in the files. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 55 ++++++++--- .../test_monte_carlo_worker_failures.py | 96 +++++++++++++++++++ 2 files changed, 138 insertions(+), 13 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index bf4d6f4e0..c3d6248bc 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -185,8 +185,11 @@ def simulate( number_of_simulations : int Number of simulations to be run, must be non-negative. append : bool, optional - If True, the results will be appended to the existing files. If - False, the files will be overwritten. Default is False. + If True, resume the existing files. ``number_of_simulations`` is + then the target total rather than a number to add, and a value + below what the files already hold is refused. The root seed is not + stored in them, so pass the same ``random_seed`` to keep the + streams. If False, the files will be overwritten. Default is False. parallel : bool, optional If True, the simulations will be run in parallel. Default is False. n_workers : int, optional @@ -500,6 +503,9 @@ def __run_in_serial(self): n_simulations=self.number_of_simulations, start_time=time(), ) + # Bound before the loop: a failure on the very first iteration + # would otherwise reach the error record with no index at all. + sim_idx = self._initial_sim_idx try: while True: # First statement in the loop, so it is bound before the two @@ -519,6 +525,10 @@ def __run_in_serial(self): _record_simulation( self.input_file, self.output_file, inputs_json, outputs_json ) + # The pair is on disk. Cleared before the monitor call so a + # failure there reports itself rather than reporting a row that + # has already been committed as one that never finished. + inputs_json = "" sim_monitor.print_update_status() sim_monitor.print_final_status() @@ -530,8 +540,9 @@ def __run_in_serial(self): except Exception as error: print(f"Error on iteration {sim_monitor.count}: {error}") - self.__keep_the_inputs_that_did_not_finish(inputs_json) - raise error + _record_failure(self._error_file, sim_idx, inputs_json) + # Bare, so the handler's own line does not join the traceback. + raise def __keep_the_inputs_that_did_not_finish(self, inputs_json): """Append the inputs of a simulation that stopped part way through.""" @@ -698,15 +709,9 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to pass details = traceback.format_exc() - # The failure goes onto the inputs record rather than replacing it. - # Writing one or the other dropped the traceback for every failure - # after sampling, from the file the run tells the user to read. - try: - record = json.loads(inputs_json) if inputs_json else {"index": sim_idx} - except ValueError: - record = {"index": sim_idx} - record["error"] = details - record = json.dumps(record) + "\n" + # The failure goes onto the inputs record rather than replacing it, + # the same shape the serial path writes. + record = _build_error_record(sim_idx, inputs_json, details) acquired = False try: @@ -2047,6 +2052,30 @@ def _record_simulation(input_file, output_file, inputs_json, outputs_json): f.write(outputs_json) +def _record_failure(error_file, sim_idx, inputs_json): + """Append the failure being handled, the way the workers record theirs. + + Module level for the same reason as ``_record_simulation``: the run paths + are driven by stub objects in the tests, which carry no private methods. + """ + with open(error_file, "a", encoding="utf-8") as handle: + handle.write(_build_error_record(sim_idx, inputs_json, traceback.format_exc())) + + +def _build_error_record(sim_idx, inputs_json, details): + """One failed simulation as a row: what it drew, and what went wrong. + + The traceback goes onto the inputs rather than replacing them, so the file + the run tells the user to read says both which inputs failed and why. + """ + try: + record = json.loads(inputs_json) if inputs_json else {"index": sim_idx} + except (TypeError, ValueError): + record = {"index": sim_idx} + record["error"] = details + return json.dumps(record) + "\n" + + def _bring_the_fleet_down(started_processes, error_event): """Stop everything, without raising over the failure being handled. diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py index 4646356a9..2581322a3 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_failures.py +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -9,6 +9,7 @@ """ import json +import traceback import threading import types @@ -254,3 +255,98 @@ def is_set(self): ) assert mutex.acquired == mutex.released, "the mutex was left held" + + +class _OneThenStop: + """A monitor that allows exactly one simulation, then whatever is asked.""" + + def __init__(self, on_update=None): + self.count = 0 + self._on_update = on_update + + def keep_simulating(self): + return self.count < 1 + + def increment(self): + self.count += 1 + return self.count + + def print_update_status(self): + if self._on_update is not None: + self._on_update() + + def print_final_status(self): + pass + + +def _serial_runner(tmp_path, monkeypatch, monitor, **overrides): + """A stand-in carrying only what ``__run_in_serial`` touches.""" + runner = _worker(tmp_path, **overrides) + runner._error_file = runner.error_file + runner._initial_sim_idx = 0 + runner.number_of_simulations = 1 + runner._interrupted = False + runner._MonteCarlo__keep_the_inputs_that_did_not_finish = lambda payload: ( + runner.error_file.open("a", encoding="utf-8").write(payload) + ) + monkeypatch.setattr(mc, "_SimMonitor", lambda **_kwargs: monitor) + return runner + + +def test_a_serial_failure_records_the_traceback_not_only_the_inputs( + tmp_path, monkeypatch +): + """The worker path writes `{index, ...inputs, error: traceback}`. Serial + wrote the inputs alone, so a failure after sampling named which inputs + failed and never why, in the file the run points the user at.""" + runner = _serial_runner( + tmp_path, + monkeypatch, + _OneThenStop(), + _MonteCarlo__evaluate_flight_outputs=_raise, + ) + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__run_in_serial(runner) + + record = json.loads(runner.error_file.read_text(encoding="utf-8").splitlines()[0]) + assert "_Boom" in record["error"] + assert "injected" in record["error"] + + +def test_a_serial_failure_does_not_repeat_the_handler_frame(tmp_path, monkeypatch): + """`raise error` names the exception again, so the handler's own line joins + the traceback and the reader walks past it to reach the real one. A bare + `raise` leaves the frame it came from. + + On the duplicate rather than on the original frame: the failing call + survives either way, so asserting it is there passes both spellings. + """ + runner = _serial_runner( + tmp_path, + monkeypatch, + _OneThenStop(), + _MonteCarlo__evaluate_flight_outputs=_raise, + ) + + with pytest.raises(_Boom) as raised: + mc.MonteCarlo._MonteCarlo__run_in_serial(runner) + + frames = [frame.name for frame in traceback.extract_tb(raised.value.__traceback__)] + assert "_raise" in frames, frames + assert frames.count("__run_in_serial") == 1, frames + + +def test_a_committed_row_is_not_reported_as_unfinished(tmp_path, monkeypatch): + """The pair is on disk before the progress call. A failure there used to + append those same inputs to the error file, so one simulation appeared in + both the inputs log and the failures.""" + runner = _serial_runner(tmp_path, monkeypatch, _OneThenStop(on_update=_raise)) + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__run_in_serial(runner) + + assert runner.input_file.read_text(encoding="utf-8").strip() == "{}" + record = json.loads(runner.error_file.read_text(encoding="utf-8").splitlines()[0]) + assert record == {"index": 0, "error": record["error"]} + assert "_Boom" in record["error"] From ccf5023e0524cbd17e1a85c68e04310fa964ba62 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:21:34 +0800 Subject: [PATCH 25/45] Stop the failure paths from replacing the failure Four places where handling an error could lose it or misreport it. The worker did not clear its payload once the input/output pair was on disk, so a failure in the progress call after it wrote the same sampled inputs to the error log. One simulation then appeared in the logs and in the failures. The serial path was fixed for this last round; this is its other half. The serial error write had no guard. An unwritable error file raised OSError in place of the exception it was recording. The worker already treats its own reporting as best effort, and now so does this, with a warning rather than silence. Same for the inputs kept on Ctrl-C: an interrupt should not become a crash because the file could not be opened. _bring_the_fleet_down said it would not raise over the failure being handled, but only the event was guarded. The two waits under it were not. The parallel parent re-raised with `raise error`, which adds its own line to the traceback. Bare, as the serial path already does. Four tests, each pinned by reverting the line it covers. The changelog entry said only that runs are reproducible. It now carries the migration: fixed-seed samples change, serial log indices move to zero-based to match the parallel path, and old checkpoints cannot be resumed. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- CHANGELOG.md | 2 +- rocketpy/simulation/monte_carlo.py | 145 +++++++++++++----- .../test_monte_carlo_worker_exit.py | 25 +++ .../test_monte_carlo_worker_failures.py | 77 ++++++++++ 4 files changed, 206 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cec86574..ffcd821b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,13 +39,13 @@ Attention: The newest changes should be on top --> - ENH: Add Qodo PR-Agent workflow using Google Gemini [#1089](https://github.com/RocketPy-Team/RocketPy/pull/1089) - ENH: Support for Meteomatics API in the `Environment` class [#1079](https://github.com/RocketPy-Team/RocketPy/pull/1079) - ENH: update master with develop [#1081](https://github.com/RocketPy-Team/RocketPy/pull/1081) -- ENH: reproducible Monte Carlo runs via a random_seed argument [#1054](https://github.com/RocketPy-Team/RocketPy/pull/1054) ### Changed - CI: make the Gemini PR reviewer actually review [#1140](https://github.com/RocketPy-Team/RocketPy/pull/1140) - MNT: declare dependency floors the package can actually run on [#1108](https://github.com/RocketPy-Team/RocketPy/pull/1108) - CI: build the docs for pull requests into develop as well [#1104](https://github.com/RocketPy-Team/RocketPy/pull/1104) +- ENH: Make Monte Carlo input sampling reproducible per simulation index via a `random_seed` argument. Fixed-seed samples change, serial log indices are now zero-based to match the parallel path, and checkpoints written by the previous scheme cannot be resumed. [#1054](https://github.com/RocketPy-Team/RocketPy/pull/1054) - CI: make changelog automation LLM-based (Gemini) and race-safe [#1082](https://github.com/RocketPy-Team/RocketPy/pull/1082) - ENH: Resolve pressure_ISA discretization bounds TODO [#1056](https://github.com/RocketPy-Team/RocketPy/pull/1056) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index c3d6248bc..c2bb57946 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -540,14 +540,24 @@ def __run_in_serial(self): except Exception as error: print(f"Error on iteration {sim_monitor.count}: {error}") - _record_failure(self._error_file, sim_idx, inputs_json) + # Captured before reporting, which may fail and must not be what + # gets recorded or raised. + _record_failure( + self._error_file, sim_idx, inputs_json, traceback.format_exc() + ) # Bare, so the handler's own line does not join the traceback. raise def __keep_the_inputs_that_did_not_finish(self, inputs_json): - """Append the inputs of a simulation that stopped part way through.""" - with open(self._error_file, "a", encoding="utf-8") as f: - f.write(inputs_json) + """Append the inputs of a simulation that stopped part way through. + + Best effort: an unwritable error file must not turn a clean interrupt + into a crash. + """ + _best_effort( + lambda: _write_unfinished_inputs(self._error_file, inputs_json), + "interrupted simulation inputs", + ) def __run_in_parallel(self, n_workers=None): """ @@ -593,39 +603,30 @@ def __run_in_parallel(self, n_workers=None): # so the sampled inputs do not depend on the number of workers. # The root state is small and travels with the pickled instance, # so no per-index seed list is materialized or sent. - for _ in range(n_workers): - sim_producer = multiprocess.Process( - target=self.__sim_producer, - args=( - sim_monitor, - mutex, - simulation_error_event, - ), - ) - sim_producer.start() - started_processes.append(sim_producer) + _start_the_fleet( + multiprocess, + self.__sim_producer, + n_workers, + (sim_monitor, mutex, simulation_error_event), + started_processes, + ) _wait_for_workers(started_processes, simulation_error_event) - # The event asks them to stop, it does not stop them. Without - # this window a worker part way through a write is cut off and - # leaves exactly the torn row the check below would report. - # Not _bring_the_fleet_down: that sets the event, which on a run - # that finished cleanly is what the crash check reads next. - _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE) - _stop_any_worker_still_running(started_processes) - _fail_if_a_worker_did_not_finish( + _close_the_fleet_down( started_processes, simulation_error_event, self.error_file ) - sim_monitor.print_final_status() + except KeyboardInterrupt: + _bring_the_fleet_down(started_processes, simulation_error_event) + self._interrupted = True + # Handle error from the main process - # pylint: disable=broad-except - except (Exception, KeyboardInterrupt) as error: + except Exception: _bring_the_fleet_down(started_processes, simulation_error_event) - self._interrupted = isinstance(error, KeyboardInterrupt) - if not self._interrupted: - raise error + self._interrupted = False + # Bare, so the handler's own line does not join the traceback. + raise finally: _stop_any_worker_still_running(started_processes) @@ -694,6 +695,10 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to _record_simulation( self.input_file, self.output_file, inputs_json, outputs_json ) + # Same as the serial path: the pair is on disk, so a failure + # in the monitor call below must report itself and not the + # row that has just been committed. + inputs_json, outputs_json = "", "" sim_monitor.print_update_status() finally: if acquired: @@ -2052,14 +2057,23 @@ def _record_simulation(input_file, output_file, inputs_json, outputs_json): f.write(outputs_json) -def _record_failure(error_file, sim_idx, inputs_json): +def _record_failure(error_file, sim_idx, inputs_json, details): """Append the failure being handled, the way the workers record theirs. - Module level for the same reason as ``_record_simulation``: the run paths - are driven by stub objects in the tests, which carry no private methods. + Best effort, and it says so rather than going quiet: an unwritable error + file must not become the exception the caller sees in place of the one it + was called to record. """ - with open(error_file, "a", encoding="utf-8") as handle: - handle.write(_build_error_record(sim_idx, inputs_json, traceback.format_exc())) + try: + with open(error_file, "a", encoding="utf-8") as handle: + handle.write(_build_error_record(sim_idx, inputs_json, details)) + except Exception as reporting_error: # pylint: disable=broad-exception-caught + warnings.warn( + f"The simulation failed and its error record could not be written: " + f"{reporting_error!r}", + RuntimeWarning, + stacklevel=2, + ) def _build_error_record(sim_idx, inputs_json, details): @@ -2076,19 +2090,66 @@ def _build_error_record(sim_idx, inputs_json, details): return json.dumps(record) + "\n" +def _start_the_fleet(multiprocess, target, n_workers, args, started_processes): + """Start the workers, appending each as it starts. + + Appended one at a time so a ``start()`` that fails part way through leaves + the caller holding exactly those already running. + """ + for _ in range(n_workers): + sim_producer = multiprocess.Process(target=target, args=args) + sim_producer.start() + started_processes.append(sim_producer) + + +def _close_the_fleet_down(started_processes, error_event, error_file): + """Let the fleet finish its writes, stop the rest, then check the logs. + + The event asks workers to stop, it does not stop them, and without this + window one part way through a write is cut off and leaves exactly the torn + row the check reports. Not ``_bring_the_fleet_down``: that sets the event, + which on a clean run is what the crash check reads next. + """ + _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE) + _stop_any_worker_still_running(started_processes) + _fail_if_a_worker_did_not_finish(started_processes, error_event, error_file) + + def _bring_the_fleet_down(started_processes, error_event): """Stop everything, without raising over the failure being handled. - Setting the event is best effort like the workers' own reporting: the - manager may be the thing that died. Then a bounded window to notice it and - leave, and whatever is left gets stopped. + Every step is best effort, not only the event: the manager may be the thing + that died, and a shutdown that raises would replace the failure that started + it. Bounded window to notice, then whatever is left gets stopped. """ + _best_effort(error_event.set, "error notification") + _best_effort( + lambda: _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE), + "graceful worker wait", + ) + _best_effort( + lambda: _stop_any_worker_still_running(started_processes), + "forced worker shutdown", + ) + + +def _write_unfinished_inputs(error_file, inputs_json): + """Module level for the same reason as ``_record_simulation``: the run paths + are driven by stub objects in the tests, which carry no private methods.""" + with open(error_file, "a", encoding="utf-8") as f: + f.write(inputs_json) + + +def _best_effort(action, description): + """Run one shutdown step, reporting a failure rather than raising it.""" try: - error_event.set() - except Exception: # pylint: disable=broad-exception-caught - pass - _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE) - _stop_any_worker_still_running(started_processes) + action() + except Exception as cleanup_error: # pylint: disable=broad-exception-caught + warnings.warn( + f"Worker cleanup failed during {description}: {cleanup_error!r}", + RuntimeWarning, + stacklevel=2, + ) def _workers_that_crashed(started_processes): diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index 1bd9218a5..2b361694b 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -7,6 +7,7 @@ is what separates those from a clean finish. """ +import traceback import types from contextlib import contextmanager from time import monotonic, sleep @@ -346,3 +347,27 @@ def test_shutdown_does_not_grow_with_the_fleet(): f"six workers were granted {crowd:.2f}s against {alone:.2f}s for one, " f"so the wait is still scaling with the fleet" ) + + +def test_the_parent_does_not_repeat_its_own_frame_in_the_traceback( + parallel_runner, monkeypatch +): + """``raise error`` names the exception again, so the handler's line joins the + traceback and the reader walks past it. The serial path already re-raises + bare; this is the parent doing the same. + + On the duplicate rather than the original frame: the failing call survives + either spelling, so asserting it is there passes both. + """ + + def start_then_fail(self): + raise OSError("cannot start") + + monkeypatch.setattr(_Process, "start", start_then_fail) + + with pytest.raises(OSError) as raised: + mc.MonteCarlo._MonteCarlo__run_in_parallel(parallel_runner, n_workers=2) + + frames = [frame.name for frame in traceback.extract_tb(raised.value.__traceback__)] + assert "start_then_fail" in frames, frames + assert frames.count("__run_in_parallel") == 1, frames diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py index 2581322a3..2fa0699f1 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_failures.py +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -350,3 +350,80 @@ def test_a_committed_row_is_not_reported_as_unfinished(tmp_path, monkeypatch): record = json.loads(runner.error_file.read_text(encoding="utf-8").splitlines()[0]) assert record == {"index": 0, "error": record["error"]} assert "_Boom" in record["error"] + + +class _ClaimOnceThenFail: + """Lets one simulation through, then fails the progress call.""" + + def __init__(self): + self.count = 0 + + def keep_simulating(self): + return self.count < 1 + + def increment(self): + self.count += 1 + return self.count + + def print_update_status(self): + raise _Boom("progress failed") + + +def test_a_worker_progress_failure_does_not_repeat_committed_inputs(tmp_path): + """The serial path clears its payload once the pair is on disk. The worker + kept it, so a failure in the progress call wrote the same sampled inputs to + the error file and one simulation appeared in the logs and the failures.""" + worker = _worker( + tmp_path, + _MonteCarlo__evaluate_flight_inputs=lambda index: '{"index": 0, "drew": 42}\n', + _MonteCarlo__evaluate_flight_outputs=lambda flight, index: '{"index": 0}\n', + ) + mutex, event = _RecordingMutex(), _Event() + + with pytest.raises(_Boom, match="progress failed"): + mc.MonteCarlo._MonteCarlo__sim_producer( + worker, _ClaimOnceThenFail(), mutex, event + ) + + committed = json.loads( + worker.input_file.read_text(encoding="utf-8").splitlines()[0] + ) + failure = json.loads(worker.error_file.read_text(encoding="utf-8").splitlines()[0]) + + assert committed == {"index": 0, "drew": 42} + assert "drew" not in failure, failure + assert "_Boom" in failure["error"] + + +def test_a_serial_reporting_failure_does_not_replace_the_original( + tmp_path, monkeypatch +): + """The worker guards its own error write. Serial did not, so an unwritable + error file raised OSError in place of the failure it was recording.""" + runner = _serial_runner( + tmp_path, + monkeypatch, + _OneThenStop(), + _MonteCarlo__evaluate_flight_outputs=_raise, + ) + real_open = open + + def refuse_the_error_file(path, *args, **kwargs): + if str(path) == str(runner.error_file): + raise OSError("error disk unavailable") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", refuse_the_error_file) + + with pytest.warns(RuntimeWarning, match="could not be written"): + with pytest.raises(_Boom, match="injected"): + mc.MonteCarlo._MonteCarlo__run_in_serial(runner) + + +def test_a_shutdown_failure_does_not_replace_the_error_it_is_handling(monkeypatch): + """``_bring_the_fleet_down`` promised not to raise over the failure being + handled, but only the event was guarded. The two waits below it were not.""" + monkeypatch.setattr(mc, "_wait_for_workers", _raise) + + with pytest.warns(RuntimeWarning, match="graceful worker wait"): + mc._bring_the_fleet_down([], _Event()) From b94ab9a0c36fed3e656910d877b7658b46eb34e1 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:53:11 +0800 Subject: [PATCH 26/45] Stop the fleet size deciding how soon an error is noticed _wait_for_workers joined every worker for 0.1s before rechecking the event, so one round cost the fleet size times that. Measured against stubs that never finish, with the event set part way through a round: 1 worker 0.10 s 8 workers 0.80 s 32 workers 3.20 s 100 workers 10.01 s One sleep per round instead. is_alive() at the top of the loop already reaps, and the join sweep at the end still runs, so nothing is left unreaped. The grace period test asserted how many times a worker had been joined, which was the old mechanism rather than the behaviour. It now measures how long the window lasted. The wait can only overshoot its deadline, never undershoot it, so a floor well under the timeout holds on a coarse clock. Two fleet sizes, both against the same ceiling, since the point is that neither depends on the count. Reverting the loop fails the 24-worker case and leaves the single-worker one passing, which is the control. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 19 +++++-- .../test_monte_carlo_worker_exit.py | 53 ++++++++++++++++++- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index c2bb57946..048f4bd1c 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -21,7 +21,7 @@ from copy import deepcopy from numbers import Real from pathlib import Path -from time import monotonic, time +from time import monotonic, sleep, time import numpy as np import simplekml @@ -2043,6 +2043,9 @@ def _validate_simulation_count(number_of_simulations): _WORKER_SHUTDOWN_GRACE = 5.0 +# One sleep per round, not per worker, so the fleet size does not set how soon +# an error is noticed. +_WORKER_POLL_INTERVAL = 0.05 def _record_simulation(input_file, output_file, inputs_json, outputs_json): @@ -2176,6 +2179,11 @@ def _wait_for_workers(started_processes, error_event=None, timeout=None): No overall deadline on the normal path: a run with no error and one worker still going is a long simulation, and that is not for this to cut short. + + The wait itself is one sleep per round rather than a blocking join on each + worker in turn, so how soon the event is noticed does not grow with the + fleet. Blocking 0.1 s per worker meant 100 of them delayed the next check by + 10 s. """ deadline = None if timeout is None else monotonic() + timeout while any(process.is_alive() for process in started_processes): @@ -2186,10 +2194,13 @@ def _wait_for_workers(started_processes, error_event=None, timeout=None): # has nothing else to end it. if deadline is None and _workers_that_crashed(started_processes): break - if deadline is not None and monotonic() >= deadline: + if deadline is None: + sleep(_WORKER_POLL_INTERVAL) + continue + remaining = deadline - monotonic() + if remaining <= 0: break - for process in started_processes: - process.join(timeout=0.1) + sleep(min(_WORKER_POLL_INTERVAL, remaining)) # Reap whatever has already finished. A worker that was gone before the # loop started was never joined by it, and an unjoined child has no exit diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index 2b361694b..871faad05 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -287,12 +287,20 @@ def test_the_parent_stops_waiting_when_a_worker_dies_holding_the_lock(): def test_the_shutdown_window_is_not_cut_short_by_a_worker_already_known_dead(): """The grace period exists so the survivors can finish their writes. It is - bounded by its own timeout, so a crash must not end it early.""" + bounded by its own timeout, so a crash must not end it early. + + On elapsed time rather than on how many times a worker was joined: that + counted the old per-worker blocking join, which is the mechanism and not the + behaviour. The wait can only overshoot the deadline, never undershoot it, so + a floor well under the timeout is safe on a coarse clock. + """ crashed, blocked = _Crashed(), _BlockedForever(give_up_after=10**6) + started = monotonic() mc._wait_for_workers([crashed, blocked], timeout=0.3) + elapsed = monotonic() - started - assert blocked.joins > 1, "the grace period returned without waiting" + assert elapsed >= 0.2, f"the grace period returned after {elapsed:.3f}s" class _Stubborn: @@ -371,3 +379,44 @@ def start_then_fail(self): frames = [frame.name for frame in traceback.extract_tb(raised.value.__traceback__)] assert "start_then_fail" in frames, frames assert frames.count("__run_in_parallel") == 1, frames + + +class _NeverFinishes: + """Alive throughout, and costly to join, as a real blocked worker is.""" + + def __init__(self): + self.exitcode = None + + def join(self, timeout=None, **_k): + if timeout: + sleep(timeout) + + def is_alive(self): + return True + + +class _SetMidRound: + """Not set when a round begins, set while the parent is inside it.""" + + def __init__(self): + self.checks = 0 + + def is_set(self): + self.checks += 1 + return self.checks > 1 + + +@pytest.mark.parametrize("fleet", [1, 24], ids=["one", "twenty_four"]) +def test_noticing_an_error_does_not_get_slower_with_a_bigger_fleet(fleet): + """Joining every worker for 0.1s before rechecking the event made the delay + the fleet size times that: 24 workers took 2.4s to notice. One sleep per + round instead, so the cost is the round and not the fleet. + + Both sizes are measured against the same ceiling rather than against each + other, because the point is that neither depends on the count. + """ + started = monotonic() + mc._wait_for_workers([_NeverFinishes() for _ in range(fleet)], _SetMidRound()) + elapsed = monotonic() - started + + assert elapsed < 0.5, f"{fleet} workers delayed the check by {elapsed:.2f}s" From ceaa259b9ef5dd230ca5475ab7ab6d1a70cf43bc Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 18:26:21 +0800 Subject: [PATCH 27/45] Keep a diagnostic from becoming the failure it describes Two ways the reporting could still replace what it was reporting. warnings.warn raises when the caller has turned RuntimeWarning into an error, which is how a strict test or application run is configured. So the guard around the error-file write was best effort only under the default filter: default filter caller sees Boom: injected -W error caller sees RuntimeWarning: could not be written Both reporters now go through one helper that overrides the filter for its own warning and swallows anything the warning machinery raises. The parallel handler's cleanup was already best effort, but `finally` runs the same cleanup again on the way out and did so raw. A failure there landed on the caller instead of the exception being re-raised. On the second: `finally` also runs after a clean run, where nothing is in flight and raising would have been fine. Warning in both cases is the simpler rule, and a cleanup failure is still reported either way. Two tests, each pinned by reverting the line it covers. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 34 +++++++++++++----- .../test_monte_carlo_worker_exit.py | 35 +++++++++++++++++++ .../test_monte_carlo_worker_failures.py | 29 +++++++++++++++ 3 files changed, 89 insertions(+), 9 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 048f4bd1c..81e43c685 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -628,7 +628,12 @@ def __run_in_parallel(self, n_workers=None): # Bare, so the handler's own line does not join the traceback. raise finally: - _stop_any_worker_still_running(started_processes) + # Also best effort: this runs while an exception may be on its + # way out, and a cleanup that raises here would replace it. + _best_effort( + lambda: _stop_any_worker_still_running(started_processes), + "final worker shutdown", + ) def __validate_number_of_workers(self, n_workers): # os.cpu_count() is documented as possibly None, and comparing against @@ -2071,11 +2076,9 @@ def _record_failure(error_file, sim_idx, inputs_json, details): with open(error_file, "a", encoding="utf-8") as handle: handle.write(_build_error_record(sim_idx, inputs_json, details)) except Exception as reporting_error: # pylint: disable=broad-exception-caught - warnings.warn( + _say_so_without_raising( f"The simulation failed and its error record could not be written: " - f"{reporting_error!r}", - RuntimeWarning, - stacklevel=2, + f"{reporting_error!r}" ) @@ -2143,15 +2146,28 @@ def _write_unfinished_inputs(error_file, inputs_json): f.write(inputs_json) +def _say_so_without_raising(message): + """Report a secondary failure in a way that cannot become the primary one. + + ``warnings.warn`` raises when the caller has turned ``RuntimeWarning`` into + an error, which is exactly how a diagnostic ends up replacing the failure it + describes. The filter is overridden for this one warning only. + """ + try: + with warnings.catch_warnings(): + warnings.simplefilter("always", RuntimeWarning) + warnings.warn(message, RuntimeWarning, stacklevel=3) + except Exception: # pylint: disable=broad-exception-caught + pass + + def _best_effort(action, description): """Run one shutdown step, reporting a failure rather than raising it.""" try: action() except Exception as cleanup_error: # pylint: disable=broad-exception-caught - warnings.warn( - f"Worker cleanup failed during {description}: {cleanup_error!r}", - RuntimeWarning, - stacklevel=2, + _say_so_without_raising( + f"Worker cleanup failed during {description}: {cleanup_error!r}" ) diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index 871faad05..5d0c8f596 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -420,3 +420,38 @@ def test_noticing_an_error_does_not_get_slower_with_a_bigger_fleet(fleet): elapsed = monotonic() - started assert elapsed < 0.5, f"{fleet} workers delayed the check by {elapsed:.2f}s" + + +class _OriginalFailure(RuntimeError): + """What the caller should end up seeing.""" + + +class _CleanupFailure(RuntimeError): + """What must not take its place.""" + + +def test_the_final_cleanup_does_not_replace_the_failure_on_its_way_out( + parallel_runner, monkeypatch +): + """The handler's own cleanup is best effort, but ``finally`` runs again on + the way out and was calling the same thing raw. A failure there landed on + the caller instead of the one being re-raised.""" + monkeypatch.setattr( + mc, "_start_the_fleet", _raiser(_OriginalFailure, "cannot start") + ) + monkeypatch.setattr( + mc, + "_stop_any_worker_still_running", + _raiser(_CleanupFailure, "cannot clean up"), + ) + + with pytest.warns(RuntimeWarning, match="final worker shutdown"): + with pytest.raises(_OriginalFailure, match="cannot start"): + mc.MonteCarlo._MonteCarlo__run_in_parallel(parallel_runner, n_workers=2) + + +def _raiser(exception, message): + def raise_it(*_args, **_kwargs): + raise exception(message) + + return raise_it diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py index 2fa0699f1..894a502ed 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_failures.py +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -10,6 +10,7 @@ import json import traceback +import warnings import threading import types @@ -427,3 +428,31 @@ def test_a_shutdown_failure_does_not_replace_the_error_it_is_handling(monkeypatc with pytest.warns(RuntimeWarning, match="graceful worker wait"): mc._bring_the_fleet_down([], _Event()) + + +def test_reporting_a_failure_cannot_escape_when_warnings_are_errors( + tmp_path, monkeypatch +): + """The guard around the error write is only best effort under the default + filter. Turn RuntimeWarning into an error, as a strict application or test + run does, and the diagnostic becomes the exception it was describing.""" + runner = _serial_runner( + tmp_path, + monkeypatch, + _OneThenStop(), + _MonteCarlo__evaluate_flight_outputs=_raise, + ) + real_open = open + + def refuse_the_error_file(path, *args, **kwargs): + if str(path) == str(runner.error_file): + raise OSError("error disk unavailable") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", refuse_the_error_file) + + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + + with pytest.raises(_Boom, match="injected"): + mc.MonteCarlo._MonteCarlo__run_in_serial(runner) From fe4d68d089357718667803956e6e0e399f5292ce Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 20:46:41 +0800 Subject: [PATCH 28/45] Two more ways a secondary failure could replace the primary one The manager lock was released raw in `finally` at all three sites. A proxy that dies while the lock is held then hands the caller its own BrokenPipeError, and the failure it interrupted survives only as context: before BrokenPipeError: [Errno 32] Broken pipe after Boom: the real simulation failure One context manager owns the lifecycle now. Release is best effort only while another exception is on its way out; with nothing in flight a failed release is the news and still raises, so a run does not continue against a dead manager. The close path ran its wait and forced stop raw before checking the workers, so a cleanup failure reached the caller before anything read the exit codes. Those two are best effort as well. `_fail_if_a_worker_did_not_finish` stays raw: it is the verdict on the run, not housekeeping. `error_event.is_set() or crashed` asked the proxy first, so an unreachable manager threw before the crash list was read and "exited with 7" was lost. The query is now a helper that reports an unavailable event as a worker failure and names it alongside the crashes. Five tests, each pinned by reverting the line it covers. One of them is the other direction: a release that fails on its own must still raise. A note on the probe that found this. A mutex failing on every release is the wrong model, because the first clean claim fails on its own release before there is anything to mask. The stub releases cleanly a set number of times first. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 82 +++++++++++++------ .../test_monte_carlo_worker_exit.py | 48 +++++++++++ .../test_monte_carlo_worker_failures.py | 60 ++++++++++++++ 3 files changed, 163 insertions(+), 27 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 81e43c685..e1277bd9e 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -18,6 +18,7 @@ import os import traceback import warnings +from contextlib import contextmanager from copy import deepcopy from numbers import Real from pathlib import Path @@ -679,10 +680,7 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to inputs_json = self.__evaluate_flight_inputs(sim_idx) outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) - acquired = False - try: - mutex.acquire() - acquired = True + with _manager_mutex(mutex): if error_event.is_set(): # Runs in a worker process spawned via multiprocessing: # logging handlers configured in the main process are @@ -705,9 +703,6 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to # row that has just been committed. inputs_json, outputs_json = "", "" sim_monitor.print_update_status() - finally: - if acquired: - mutex.release() except Exception: # Set first, so a parent waiting on the join learns why. Best effort @@ -723,23 +718,18 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to # the same shape the serial path writes. record = _build_error_record(sim_idx, inputs_json, details) - acquired = False try: - mutex.acquire() - acquired = True - with open(self.error_file, "a", encoding="utf-8") as f: - f.write(record) - - # See note above: must use print() to remain visible from a - # multiprocessing worker process. - _SimMonitor.reprint(f"Error on iteration {sim_idx}:\n{details}") + with _manager_mutex(mutex): + with open(self.error_file, "a", encoding="utf-8") as f: + f.write(record) + + # See note above: must use print() to remain visible from a + # multiprocessing worker process. + _SimMonitor.reprint(f"Error on iteration {sim_idx}:\n{details}") except Exception: # pylint: disable=broad-exception-caught # The mutex or the error file is unreachable too. Reporting is # not worth losing the failure that started this. pass - finally: - if acquired: - mutex.release() # The worker exits non-zero, so the parent can tell a crash from a # clean finish rather than only from the error event. @@ -2116,8 +2106,16 @@ def _close_the_fleet_down(started_processes, error_event, error_file): row the check reports. Not ``_bring_the_fleet_down``: that sets the event, which on a clean run is what the crash check reads next. """ - _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE) - _stop_any_worker_still_running(started_processes) + # Best effort, so that housekeeping cannot report itself in place of the + # worker result below, which is the authoritative verdict on the run. + _best_effort( + lambda: _wait_for_workers(started_processes, timeout=_WORKER_SHUTDOWN_GRACE), + "graceful worker wait", + ) + _best_effort( + lambda: _stop_any_worker_still_running(started_processes), + "forced worker shutdown", + ) _fail_if_a_worker_did_not_finish(started_processes, error_event, error_file) @@ -2171,6 +2169,36 @@ def _best_effort(action, description): ) +@contextmanager +def _manager_mutex(mutex): + """Hold a manager lock without letting its release replace a failure. + + The proxy can die while the lock is held, and a raw release in ``finally`` + then becomes the exception the caller sees rather than the one already on + its way out. A release that fails with nothing in flight is still raised. + """ + mutex.acquire() + try: + yield + except BaseException: + _best_effort(mutex.release, "manager mutex release") + raise + else: + mutex.release() + + +def _read_error_event(error_event): + """Whether a worker reported an error, and what went wrong asking. + + An unreachable proxy is itself a reason to stop and to fail the run, so it + reads as reported rather than letting the exception past the crash list. + """ + try: + return bool(error_event.is_set()), None + except Exception as event_error: # pylint: disable=broad-exception-caught + return True, event_error + + def _workers_that_crashed(started_processes): """Those already known to have exited abnormally. @@ -2203,7 +2231,7 @@ def _wait_for_workers(started_processes, error_event=None, timeout=None): """ deadline = None if timeout is None else monotonic() + timeout while any(process.is_alive() for process in started_processes): - if error_event is not None and error_event.is_set(): + if error_event is not None and _read_error_event(error_event)[0]: break # A worker killed outright sets no event. If it died holding the shared # lock its siblings never return either, and only the unbounded wait @@ -2270,7 +2298,10 @@ def _fail_if_a_worker_did_not_finish(started_processes, error_event, error_file) for sim_producer in started_processes if sim_producer.exitcode != 0 ] - if error_event.is_set() or crashed: + reported, event_error = _read_error_event(error_event) + if event_error is not None: + crashed.append(f"the worker error event became unavailable: {event_error!r}") + if reported or crashed: raise RuntimeError( "An error occurred during the simulation. \n" + (f"Workers that did not exit cleanly: {crashed}. \n" if crashed else "") @@ -2287,13 +2318,10 @@ def _claim_next_index(sim_monitor, mutex): either increments, and both then claim an index, running more simulations than were requested (and duplicating a simulation index). """ - mutex.acquire() - try: + with _manager_mutex(mutex): if not sim_monitor.keep_simulating(): return None return sim_monitor.increment() - 1 - finally: - mutex.release() def _import_multiprocess(): diff --git a/tests/unit/simulation/test_monte_carlo_worker_exit.py b/tests/unit/simulation/test_monte_carlo_worker_exit.py index 5d0c8f596..8904b07f7 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_exit.py +++ b/tests/unit/simulation/test_monte_carlo_worker_exit.py @@ -455,3 +455,51 @@ def raise_it(*_args, **_kwargs): raise exception(message) return raise_it + + +class _ExitedWorker: + """A worker that already left, and did not leave cleanly.""" + + name = "worker-0" + exitcode = 7 + + def is_alive(self): + return False + + def join(self, timeout=None): + pass + + +class _SetEvent: + def is_set(self): + return True + + +class _DeadEvent: + def is_set(self): + raise ConnectionResetError("manager is gone") + + +def test_cleanup_failure_does_not_hide_which_worker_crashed(monkeypatch): + """The close path ran the wait and the forced stop raw, so a failure there + reached the caller before anything looked at the exit codes. The worker + result is the verdict on the run; housekeeping is not.""" + monkeypatch.setattr( + mc, "_stop_any_worker_still_running", _raiser(OSError, "cleanup failed") + ) + + with pytest.warns(RuntimeWarning, match="forced worker shutdown"): + with pytest.raises(RuntimeError, match="exited with 7"): + mc._close_the_fleet_down([_ExitedWorker()], _SetEvent(), "errors.txt") + + +def test_an_unreachable_event_is_reported_rather_than_raised(): + """``error_event.is_set() or crashed`` asked the proxy first, so a dead + manager threw before the crash list was ever read.""" + with pytest.raises(RuntimeError) as raised: + mc._fail_if_a_worker_did_not_finish( + [_ExitedWorker()], _DeadEvent(), "errors.txt" + ) + + assert "exited with 7" in str(raised.value) + assert "became unavailable" in str(raised.value) diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py index 894a502ed..9aa4e0f03 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_failures.py +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -456,3 +456,63 @@ def refuse_the_error_file(path, *args, **kwargs): with pytest.raises(_Boom, match="injected"): mc.MonteCarlo._MonteCarlo__run_in_serial(runner) + + +class _MutexDiesAfter(_RecordingMutex): + """Releases cleanly a few times, then the manager connection goes. + + A mutex that fails on every release is the wrong model: the first clean + claim would fail on its own release, before there is anything to mask. + """ + + def __init__(self, healthy_releases=1): + super().__init__() + self._left = healthy_releases + + def release(self): + super().release() + self._left -= 1 + if self._left < 0: + raise BrokenPipeError("[Errno 32] Broken pipe") + + +def test_a_dying_mutex_does_not_replace_the_simulation_failure(tmp_path): + """The release in the critical section used to run raw in ``finally``, so a + manager that died while the lock was held handed the caller its own + BrokenPipeError and left the real failure as context.""" + worker = _worker(tmp_path) + monitor = _ClaimOnceThenFail() + event = _Event() + + with pytest.warns(RuntimeWarning, match="manager mutex release"): + with pytest.raises(_Boom, match="progress failed"): + mc.MonteCarlo._MonteCarlo__sim_producer( + worker, monitor, _MutexDiesAfter(healthy_releases=1), event + ) + + +def test_a_dying_mutex_does_not_replace_the_claim_failure(): + """Same shape one level down, where the claim itself is what fails.""" + + class _ClaimRaises: + def keep_simulating(self): + raise _Boom("claim failed") + + with pytest.warns(RuntimeWarning, match="manager mutex release"): + with pytest.raises(_Boom, match="claim failed"): + mc._claim_next_index(_ClaimRaises(), _MutexDiesAfter(healthy_releases=0)) + + +def test_a_release_that_fails_on_its_own_is_still_raised(): + """The other half. With nothing in flight a broken release is the failure, + not a warning, so the run does not carry on against a dead manager.""" + + class _Fine: + def keep_simulating(self): + return True + + def increment(self): + return 1 + + with pytest.raises(BrokenPipeError): + mc._claim_next_index(_Fine(), _MutexDiesAfter(healthy_releases=0)) From 6318bfa12dde949024071690a71d7b606a475d4b Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 20:55:34 +0800 Subject: [PATCH 29/45] DOC: two things the docstrings promised more of than they deliver The Notes said an interrupted run can be loaded and continued. Not every one can: the two logs have to hold the same simulations as a complete run of indices from zero, and a parallel stop can leave one worker's index missing while a later one is already written. That checkpoint is refused rather than repaired, which is #1075, and the test for it is already there. `random_seed` said the sampled inputs are identical across execution modes. What is identical is the mapping from index to inputs. A worker takes the log lock once its simulation is done, so the rows land in completion order and the file can differ run to run. Someone diffing two runs byte for byte would read that as reproducibility being broken. No test for the second one on purpose. The suite already reads the logs into a dict keyed by index rather than comparing them as text, which is the same claim from the other side, and a test asserting the order does differ would pass or fail on luck. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index e1277bd9e..ebc071f9a 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -199,11 +199,15 @@ def simulate( A minimum of 2 workers is required for parallel mode. Default is None. random_seed : int, numpy integer, sequence of ints, or SeedSequence, optional - Root seed for the run. When provided, the sampled inputs are - reproducible and identical across serial and parallel execution and - across any number of workers: each simulation index derives its own - decorrelated child stream from this root, so index ``i`` receives the - same inputs no matter which worker runs it. A supplied ``SeedSequence`` + Root seed for the run. When provided, the mapping from simulation + index to sampled inputs is reproducible and identical across serial + and parallel execution and across any number of workers: each + simulation index derives its own decorrelated child stream from this + root, so index ``i`` receives the same inputs no matter which worker + runs it. The rows themselves are written in completion order, since + a worker takes the log lock once its simulation is done, so the file + order can differ between runs. Compare by the recorded index rather + than byte for byte. A supplied ``SeedSequence`` is copied from its full state rather than consumed, so repeated calls with the same seed reproduce the same inputs. Each model is reseeded with a 128-bit integer -- the seed type a custom sampler's @@ -238,6 +242,12 @@ def simulate( the simulation by running the ``simulate`` method again with the same number of simulations and setting `append=True`. + Not every interruption leaves a checkpoint that can be continued. The + two logs have to hold the same simulations as a complete run of indices + from zero, and a parallel run can stop with one worker's index missing + while a later one is already written. Such a checkpoint is refused + rather than repaired, which is #1075. + Important --------- If you use `append=False` and the files already exist, they will be From 5e9be5b7a30d7cb026e681a4ea664029cff38016 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:51:34 +0800 Subject: [PATCH 30/45] A data collector could file a row under another simulation `index` is what pairs an inputs row with its outputs row, and the custom fields were merged over it, so a collector supplying one won: data_collector={"index": callback} A collector returning a constant writes a row the completeness check rejects. One returning a permutation does not: sim_idx 0 -> index 1 sim_idx 1 -> index 0 indices {0, 1}, each once, against a run of 2 Every check downstream compares the index multiset, so it sees a complete run and reports success while the outputs sit on the wrong simulations. That is worse than a corrupt row, which at least announces itself. Three places, because one is not enough: - `index` is a reserved key now, and collector keys have to be strings. A dict key can be anything hashable, and a non-string one would not survive the JSON round trip that reads these files back. - `simulate()` checks again. The attribute is public and mutable, so a key added after construction would otherwise reach the logs unchecked. It runs before `__setup_files`, so a rejected run leaves the previous one intact. - the run's own index is written after the custom fields rather than before. Four tests. One of them exists to say why the first matters: a permutation passes every other check, so a test asserting only that the row is malformed would not have caught this. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 27 +++++-- .../test_monte_carlo_log_integrity.py | 79 +++++++++++++++++++ 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index ebc071f9a..97ed39572 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -40,6 +40,9 @@ ) # TODO: Create evolution plots to analyze convergence +# Written by the run itself and used to pair an inputs row with its outputs row. +# A collector that supplies one can relabel a row without tripping any check. +_RESERVED_RECORD_KEYS = frozenset({"index"}) class MonteCarlo: # pylint: disable=too-many-public-methods @@ -258,6 +261,9 @@ def simulate( # __setup_files, which opens both logs "w+" and empties them. Raising # after that point destroys the previous run on the way out. _validate_simulation_count(number_of_simulations) + # Again rather than only in __init__: the attribute is public and a key + # added after construction would otherwise reach the logs unchecked. + self._check_data_collector(self.data_collector) if parallel: n_workers = self.__validate_number_of_workers(n_workers) # multiprocess is an optional extra. Imported here, an install @@ -985,18 +991,18 @@ def __evaluate_flight_outputs(self, flight, sim_idx): export_item: getattr(flight, export_item) for export_item in self.export_list } - outputs_dict["index"] = sim_idx - if self.data_collector is not None: - additional_exports = {} for key, callback in self.data_collector.items(): try: - additional_exports[key] = callback(flight) + outputs_dict[key] = callback(flight) except Exception as e: raise ValueError( f"An error was encountered running 'data_collector' callback {key}. " ) from e - outputs_dict = outputs_dict | additional_exports + + # Last, so that the index a row is filed under is the one the run + # assigned even if the collector changed under a validated one. + outputs_dict["index"] = sim_idx return ( json.dumps(outputs_dict, cls=RocketPyEncoder, **self._export_config) + "\n" @@ -1140,6 +1146,17 @@ def _check_data_collector(self, data_collector): ) for key, callback in data_collector.items(): + if not isinstance(key, str): + raise ValueError( + "Invalid 'data_collector' key! " + f"Keys must be strings, not {type(key).__name__}." + ) + if key in _RESERVED_RECORD_KEYS: + raise ValueError( + f"Invalid 'data_collector' key '{key}'! " + "That name is reserved for the record metadata that " + "pairs an inputs row with its outputs row." + ) if key in self.export_list: raise ValueError( "Invalid 'data_collector' key! " diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 01304509a..a512bdf7b 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -10,6 +10,7 @@ to survive that. """ +import json import threading import types @@ -470,3 +471,81 @@ def test_a_checkpoint_numbered_from_one_is_named_as_such(tmp_path): # from the old sequential scheme, not from this release's per-index one. assert "Renumbering" in str(raised.value) assert "without lining the seeds up" in str(raised.value) + + +class _Collector: + """A model with only what the collector checks and the writer touch.""" + + export_list = ("apogee",) + + def __init__(self, data_collector=None): + self.data_collector = data_collector + self._export_config = {} + + +def _outputs(data_collector, sim_idx): + model = _Collector(data_collector) + flight = types.SimpleNamespace(apogee=100.0) + return json.loads( + mc.MonteCarlo._MonteCarlo__evaluate_flight_outputs(model, flight, sim_idx) + ) + + +def test_a_collector_cannot_relabel_the_row_it_is_attached_to(): + """`index` pairs an inputs row with its outputs row. The custom fields used + to be merged over it, so a collector could file a row under a different + simulation than the one that produced it.""" + labels = iter([1, 0]) + collector = {"index": lambda _flight: next(labels)} + + assert _outputs(collector, 0)["index"] == 0 + assert _outputs(collector, 1)["index"] == 1 + + +def test_a_permutation_of_valid_indices_would_pass_every_other_check(): + """Why the one above matters more than a malformed value would. + + A collector returning `-1` writes a row the completeness check rejects. One + returning a permutation writes the same index set with the same counts, so + nothing downstream can tell the outputs are on the wrong simulations. + """ + labels = iter([1, 0]) + rows = [_outputs({"custom": lambda _f: next(labels)}, i) for i in (0, 1)] + + assert sorted(r["index"] for r in rows) == [0, 1] + assert [r["custom"] for r in rows] == [1, 0], "the collector still runs" + + +@pytest.mark.parametrize( + "key, expected", + [("index", "reserved"), (7, "must be strings"), (None, "must be strings")], + ids=["reserved-name", "int-key", "none-key"], +) +def test_a_collector_key_that_cannot_be_written_is_refused(key, expected): + with pytest.raises(ValueError, match=expected): + mc.MonteCarlo._check_data_collector(_Collector(), {key: lambda _f: 0}) + + +def test_a_collector_changed_after_construction_is_checked_again(tmp_path): + """`data_collector` is public and mutable, so validating it once in + ``__init__`` is not enough. The check has to run before ``__setup_files`` + opens the logs "w+", or a rejected run destroys the previous one.""" + inputs = tmp_path / "inputs.txt" + outputs = tmp_path / "outputs.txt" + inputs.write_text("previous input\n", encoding="utf-8") + outputs.write_text("previous output\n", encoding="utf-8") + runner = types.SimpleNamespace( + input_file=inputs, + output_file=outputs, + export_list=("apogee",), + data_collector={"index": lambda _flight: 0}, + _check_data_collector=lambda collector: mc.MonteCarlo._check_data_collector( + runner, collector + ), + ) + + with pytest.raises(ValueError, match="reserved"): + mc.MonteCarlo.simulate(runner, number_of_simulations=1, random_seed=42) + + assert inputs.read_text(encoding="utf-8") == "previous input\n" + assert outputs.read_text(encoding="utf-8") == "previous output\n" From 819c474ca88bed1c4eaf7873935d8a62e6dc389e Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:19:10 +0800 Subject: [PATCH 31/45] DOC: what the seed does not promise, and one import pylint rejects Two boundaries on `random_seed` that the docstring implied more of than it should. Both were raised on review. What a seed reproduces is the sampled values. With `include_function_data=True` a record also carries a `Function`'s signature hash and serialised source, which describe the object rather than the value drawn for it, so a run under spawn or forkserver writes different ones for the same inputs. The cross-start-method test measured six such fields and filters exactly them. And it is scoped to one environment. NumPy promises a stream only for the same BitGenerator, seed, call sequence, build and machine, and reserves the right to change what `default_rng` returns. A seed fixes the lineage of a run; it is not an archive format that survives a version bump. Also moves two imports in test_custom_sampler.py to the top of the file. They arrived on develop with the cherry-pick of 23be0bab, which was the version before that fix, and pylint exits 16 on them. #1111 does the same thing as part of a wider change; this is here because it is what turns this branch's lint red. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 18 +++++++++++++++++- .../test_monte_carlo_log_integrity.py | 7 +++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 97ed39572..aa135afb1 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -210,7 +210,23 @@ def simulate( runs it. The rows themselves are written in completion order, since a worker takes the log lock once its simulation is done, so the file order can differ between runs. Compare by the recorded index rather - than byte for byte. A supplied ``SeedSequence`` + than byte for byte. + + What is reproduced is the sampled values. With + ``include_function_data=True`` a record also carries a + ``Function``'s signature hash and serialised source, which describe + the object rather than the value drawn for it, so a run under + ``spawn`` or ``forkserver`` writes different ones for the same + inputs. Measured on a real run, six fields differ across that + boundary and all six are these. Pass + ``include_function_data=False`` when the records need to compare + field for field. + + Reproducibility is also scoped to one environment. NumPy promises a + stream only for the same BitGenerator, seed, call sequence, build + and machine, and reserves the right to change what ``default_rng`` + returns. A seed fixes the lineage of a run; it is not an archive + format that survives a version bump. A supplied ``SeedSequence`` is copied from its full state rather than consumed, so repeated calls with the same seed reproduce the same inputs. Each model is reseeded with a 128-bit integer -- the seed type a custom sampler's diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index a512bdf7b..1e0e6c7de 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -522,6 +522,13 @@ def test_a_permutation_of_valid_indices_would_pass_every_other_check(): ids=["reserved-name", "int-key", "none-key"], ) def test_a_collector_key_that_cannot_be_written_is_refused(key, expected): + """A non-string key does not survive the round trip these files exist for. + + ``json.dumps`` stringifies it on the way out, so ``7`` comes back as + ``"7"``, ``None`` as ``"null"``. Worse, it can collide: a collector holding + both ``1`` and ``"1"`` writes ``{"1": ..., "1": ...}``, and reading that + back leaves one column where there were two. + """ with pytest.raises(ValueError, match=expected): mc.MonteCarlo._check_data_collector(_Collector(), {key: lambda _f: 0}) From 6e2ea3330111c41ccd1e61653a0d457ac8950dd3 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:18:08 +0800 Subject: [PATCH 32/45] DOC: say what the seed tree does not reach The seed fixes the sampled inputs, not the flight that follows from them. Randomness no stochastic model owns is outside the tree: a Sensor left on seed=None takes fresh entropy per instance, and MultivariateRejectionSampler draws from the stdlib random module. Both are seedable by the caller, so say so rather than leaving the guarantee sounding wider than it is. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index aa135afb1..deeb77897 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -212,15 +212,19 @@ def simulate( order can differ between runs. Compare by the recorded index rather than byte for byte. - What is reproduced is the sampled values. With - ``include_function_data=True`` a record also carries a + What is reproduced is the sampled inputs, not the whole trajectory. + Randomness no stochastic model owns stays outside this tree: a + ``Sensor`` left on ``seed=None`` takes fresh entropy per instance, + and ``MultivariateRejectionSampler`` draws from the stdlib + ``random``. Seed those yourself for a flight that has to replay. + + With ``include_function_data=True`` a record also carries a ``Function``'s signature hash and serialised source, which describe the object rather than the value drawn for it, so a run under ``spawn`` or ``forkserver`` writes different ones for the same - inputs. Measured on a real run, six fields differ across that - boundary and all six are these. Pass - ``include_function_data=False`` when the records need to compare - field for field. + inputs. Six fields differ across that boundary, all of them these. + Pass ``include_function_data=False`` when the records need to + compare field for field. Reproducibility is also scoped to one environment. NumPy promises a stream only for the same BitGenerator, seed, call sequence, build From 2ec060c7dc22c94e8a15a3ccde9907dec0a8d9c7 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:37:41 +0800 Subject: [PATCH 33/45] DOC: name the right gaps in the reproducibility scope The previous wording reached for two examples that do not hold. Sensor noise cannot affect a Monte Carlo flight because StochasticRocket.create_object does not carry sensors onto the rocket it builds, and MultivariateRejectionSampler resamples finished result files rather than running inside a flight. The real gaps are already filed: the flight dictionary is drawn more than once (#1090), append does not carry its seed lineage (#1075), and simulate_convergence seeds neither its batches nor its bootstrap (#1077). Name those, and say a convergence study is outside the guarantee. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index deeb77897..6383b25cf 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -212,11 +212,14 @@ def simulate( order can differ between runs. Compare by the recorded index rather than byte for byte. - What is reproduced is the sampled inputs, not the whole trajectory. - Randomness no stochastic model owns stays outside this tree: a - ``Sensor`` left on ``seed=None`` takes fresh entropy per instance, - and ``MultivariateRejectionSampler`` draws from the stdlib - ``random``. Seed those yourself for a flight that has to replay. + What is reproduced is the draw each index makes, not a transcript of + what reached ``Flight``: #1090 still draws the ``StochasticFlight`` + dictionary more than once. ``append=True`` does not carry its seed + lineage either (#1075), and ``simulate_convergence`` seeds neither + its batches nor its bootstrap resampling (#1077), so a convergence + study is outside this guarantee. Randomness that reaches a flight + from anywhere but a stochastic model, a user callback most likely, + is outside it as well. With ``include_function_data=True`` a record also carries a ``Function``'s signature hash and serialised source, which describe From a00eafe23db51a3d4612fd732865adcb2e5a6e99 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:01:21 +0800 Subject: [PATCH 34/45] BUG: say so when an append leaves the seed lineage, and hold nominals still Three gaps behind the reproducibility this adds. A run seeded 42 continued with append=True and no seed silently starts a new lineage, and the file it produces is valid on every structural check: the indices stay unique and contiguous and the two logs stay in step. Nothing but the roots can tell, so the capture now compares them and warns. Only within one object; carrying the root in the files is #1075. _nominal called itself a construction-time snapshot but cached the attribute by reference, so writing through the wrapped object moved it while rebinding the attribute did not. Containers are copied on the way in; anything else is held by reference and the docstring now says which. __setup_files truncated the three logs one after another, so a bad path or a permission on the second emptied the first on the way to raising. They are staged beside their destinations and moved into place once all three exist. The 'not synchronized' warning was unreachable, guarded on not append inside the branch that only runs when appending. It is now reachable on the append path, which is where it was meant to fire. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 93 ++++++++++++++++--- rocketpy/stochastic/stochastic_model.py | 54 ++++++----- .../test_monte_carlo_determinism.py | 50 ++++++++++ .../test_monte_carlo_log_integrity.py | 56 +++++++++++ .../test_monte_carlo_worker_failures.py | 4 +- .../unit/stochastic/test_stochastic_model.py | 33 ++++++- 6 files changed, 255 insertions(+), 35 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 6383b25cf..a7edebc28 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -16,6 +16,7 @@ import csv import json import os +import tempfile import traceback import warnings from contextlib import contextmanager @@ -39,7 +40,58 @@ import_optional_dependency, ) + # TODO: Create evolution plots to analyze convergence +def _create_empty_logs_atomically(paths): + """Put every log in place empty, or leave every one of them alone. + + Truncating them one after another empties the first before a bad path or a + permission can stop the second, which loses a finished run on the way to + raising. Each is staged beside its destination and moved over it only once + all of them exist. + """ + staged = [] + try: + for path in paths: + destination = Path(path) + handle = tempfile.NamedTemporaryFile( # pylint: disable=consider-using-with + mode="w", + encoding="utf-8", + dir=destination.parent, + prefix=f"{destination.name}.", + suffix=".partial", + delete=False, + ) + handle.close() + staged.append((handle.name, destination)) + except OSError as error: + for temporary, _ in staged: + _best_effort(lambda t=temporary: os.remove(t), "staged log cleanup") + raise OSError(f"Error creating files: {error}") from error + + for temporary, destination in staged: + os.replace(temporary, destination) + + +def _warn_when_appending_leaves_the_lineage(previous, previous_chosen, current): + """Say so when appended rows stop sharing the seed lineage below them. + + A file holding two lineages is valid on every structural check, so nothing + else can notice. Only reachable when this object already ran from a chosen + seed and is now continuing from a different root (#1075). + """ + if not previous_chosen or previous is None or previous == current: + return + warnings.warn( + "Appending to a run that was seeded, with a different root. Rows from " + "here on derive from a new seed lineage, and the file records both " + "without saying which is which. Pass the original random_seed to " + "continue the same run.", + RuntimeWarning, + stacklevel=3, + ) + + # Written by the run itself and used to pair an inputs row with its outputs row. # A collector that supplies one can relabel a row without tripping any check. _RESERVED_RECORD_KEYS = frozenset({"index"}) @@ -95,6 +147,11 @@ class MonteCarlo: # pylint: disable=too-many-public-methods spent waiting for I/O operations or other processes to complete. """ + # No run yet, so nothing to continue. Class-level so an instance built + # without __init__ still answers. + __root_state = None + __root_seed_given = False + def __init__( self, filename, @@ -191,9 +248,11 @@ def simulate( append : bool, optional If True, resume the existing files. ``number_of_simulations`` is then the target total rather than a number to add, and a value - below what the files already hold is refused. The root seed is not - stored in them, so pass the same ``random_seed`` to keep the - streams. If False, the files will be overwritten. Default is False. + below what the files already hold is refused. This is not a + reproducible resume: the root is not stored in the files, so pass + the same ``random_seed`` to stay on one lineage. Continuing from a + different root warns, but only within the object that ran both + (#1075). If False, the files will be overwritten. Default is False. parallel : bool, optional If True, the simulations will be run in parallel. Default is False. n_workers : int, optional @@ -321,7 +380,7 @@ def simulate( # This validates random_seed *before* __setup_files truncates any # existing output, so an invalid seed cannot destroy prior results on # the way to raising. - self.__capture_root_state(random_seed) + self.__capture_root_state(random_seed, appending=append) print("Starting Monte Carlo analysis") @@ -349,18 +408,22 @@ def __setup_files(self, append): ------- None """ - # Create data files for inputs, outputs and error logging - open_mode = "r+" if append else "w+" + if not append: + _create_empty_logs_atomically( + (self._input_file, self._output_file, self._error_file) + ) + return + # Resuming reads only, so nothing here can damage what is already there. try: - with open(self._input_file, open_mode, encoding="utf-8") as input_file: + with open(self._input_file, "r+", encoding="utf-8") as input_file: idx_i = len(input_file.readlines()) - with open(self._output_file, open_mode, encoding="utf-8") as output_file: + with open(self._output_file, "r+", encoding="utf-8") as output_file: idx_o = len(output_file.readlines()) - with open(self._error_file, open_mode, encoding="utf-8"): + with open(self._error_file, "r+", encoding="utf-8"): pass - if idx_i != idx_o and not append: + if idx_i != idx_o: warnings.warn( "Input and output files are not synchronized", UserWarning ) @@ -395,7 +458,7 @@ def __root_seed_sequence(random_seed): ) return np.random.SeedSequence(random_seed) - def __capture_root_state(self, random_seed): + def __capture_root_state(self, random_seed, appending=False): """Capture the small, picklable root seed state for this run. Stored once so serial mode and every parallel worker derive the same @@ -407,6 +470,7 @@ def __capture_root_state(self, random_seed): reference. Without it a caller who mutates the list they passed changes the children this run derives, which is the opposite of a snapshot. """ + previous, previous_chosen = self.__root_state, self.__root_seed_given root = self.__root_seed_sequence(random_seed) self.__root_state = ( deepcopy(root.entropy), @@ -414,6 +478,13 @@ def __capture_root_state(self, random_seed): root.pool_size, root.n_children_spawned, ) + # Whether a caller chose this root or it came from fresh entropy. Only a + # chosen one is a lineage there is any point in continuing. + self.__root_seed_given = random_seed is not None + if appending: + _warn_when_appending_leaves_the_lineage( + previous, previous_chosen, self.__root_state + ) def __child_seed(self, sim_idx): """Return the seed sequence for a single simulation index. diff --git a/rocketpy/stochastic/stochastic_model.py b/rocketpy/stochastic/stochastic_model.py index 45716a9d6..097f5ac7b 100644 --- a/rocketpy/stochastic/stochastic_model.py +++ b/rocketpy/stochastic/stochastic_model.py @@ -3,6 +3,8 @@ Stochastic classes. """ +from copy import deepcopy + import numpy as np from rocketpy.mathutils.function import Function @@ -11,6 +13,21 @@ from ..tools import get_distribution +def _snapshot_of(value): + """A nominal that writing through the wrapped object cannot reach. + + Containers are copied, since ``obj.outline[:] = ...`` would otherwise move + the value this is meant to hold still. Anything else is kept by reference: + nothing here mutates a ``Function`` or a motor in place, and copying one per + model would cost more than it protects. + """ + if isinstance(value, np.ndarray): + return value.copy() + if isinstance(value, (list, dict, set)): + return deepcopy(value) + return value + + def _names_as_spawn_key(input_names): """Encode names into spawn-key words that no other set of names produces. @@ -126,31 +143,26 @@ def __init__(self, obj, seed=None, **kwargs): self._set_stochastic(seed) def _nominal(self, input_name, getter=getattr): - """``self.obj``'s value for ``input_name``, as it was when this model - was built. - - Read once and remembered, because ``StochasticEnvironment`` has - ``create_object`` write the randomised value back onto ``self.obj`` - instead of building a copy. Re-reading it on a reseed would take one - simulation's output as the next one's nominal, and a factor would - multiply the factor before it rather than the original value. - - A custom ``getter`` reads a component's own attribute rather than one - of ``self.obj``'s, and nothing writes back to those, so it is passed - straight through. Caching it here would be wrong as well: every - component's position arrives under the one name ``"position"``. - - This applies to every stochastic model, not only the environment: what - a model samples around is what the wrapped object held when the model - was built. Changing the object afterwards does not move it. Only - ``StochasticEnvironment.create_object`` writes back today, but the rule - is stated for all of them rather than special-cased for one, so a model - means the same thing whichever object it wraps. + """``self.obj``'s value for ``input_name`` as it was when built. + + Read once and kept, because ``StochasticEnvironment.create_object`` + writes the randomised value back onto ``self.obj``. Re-reading it on a + reseed would take one simulation's output as the next one's nominal, so + a factor would multiply the factor before it. Containers are copied on + the way in, so rebinding the attribute and writing through it both leave + this where it was; anything else is held by reference and follows the + object (see ``_snapshot_of``). + + A custom ``getter`` reads a component's own attribute, which nothing + writes back to, so it passes straight through. Caching those would be + wrong anyway: every component's position arrives under one name. """ if getter is not getattr: return getter(self.obj, input_name) if input_name not in self.__nominal_values: - self.__nominal_values[input_name] = getattr(self.obj, input_name) + self.__nominal_values[input_name] = _snapshot_of( + getattr(self.obj, input_name) + ) return self.__nominal_values[input_name] def _set_stochastic(self, seed=None): diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index c8ef2abfe..314555402 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -28,6 +28,7 @@ import sys import threading import time +import warnings from types import SimpleNamespace import numpy as np @@ -39,6 +40,7 @@ _seed_sequence_to_int, _SimMonitor, _validate_simulation_count, + _warn_when_appending_leaves_the_lineage, ) _root_seed_sequence = MonteCarlo._MonteCarlo__root_seed_sequence @@ -363,3 +365,51 @@ def test_mutating_the_caller_s_entropy_does_not_move_the_captured_root(wrapped): entropy[0] = 999999 assert _entropy(_child_seed(runner, 7)) == before + + +@pytest.mark.parametrize( + "first_seed, second_seed, expect_warning", + [ + (42, None, True), # the documented case: seeded run, unseeded append + (42, 7, True), # a different chosen root is the same mixing + (42, 42, False), # continuing the same run + (None, None, False), # nothing was being preserved + ], +) +def test_appending_says_so_when_it_leaves_the_seed_lineage( + first_seed, second_seed, expect_warning +): + """Mixing two lineages in one file is invisible to every structural check. + + The rows stay valid JSON, the indices stay unique and contiguous, and the + two files stay in step, so only the roots themselves can tell (#1075). + """ + analysis = object.__new__(MonteCarlo) + analysis._MonteCarlo__capture_root_state(first_seed) + previous_root = analysis._MonteCarlo__root_state + previous_chosen = analysis._MonteCarlo__root_seed_given + analysis._MonteCarlo__capture_root_state(second_seed) + + with warnings.catch_warnings(record=True) as raised: + warnings.simplefilter("always") + _warn_when_appending_leaves_the_lineage( + previous_root, previous_chosen, analysis._MonteCarlo__root_state + ) + + lineage_warnings = [w for w in raised if "seed lineage" in str(w.message)] + assert bool(lineage_warnings) is expect_warning + if expect_warning: + assert issubclass(lineage_warnings[0].category, RuntimeWarning) + + +def test_a_fresh_object_has_no_lineage_to_leave(): + """A first run cannot be leaving anything, however it is seeded.""" + with warnings.catch_warnings(record=True) as raised: + warnings.simplefilter("always") + _warn_when_appending_leaves_the_lineage( + MonteCarlo._MonteCarlo__root_state, + MonteCarlo._MonteCarlo__root_seed_given, + ("entropy", (), 4, 0), + ) + + assert not [w for w in raised if "seed lineage" in str(w.message)] diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 1e0e6c7de..0128afc12 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -11,12 +11,14 @@ """ import json +import tempfile import threading import types import pytest import rocketpy.simulation.monte_carlo as mc +from rocketpy.simulation import MonteCarlo def _runner(tmp_path, rows, outputs=None, count=2, initial=0, interrupted=False): @@ -556,3 +558,57 @@ def test_a_collector_changed_after_construction_is_checked_again(tmp_path): assert inputs.read_text(encoding="utf-8") == "previous input\n" assert outputs.read_text(encoding="utf-8") == "previous output\n" + + +def test_a_failed_log_leaves_every_other_log_as_it_was(tmp_path, monkeypatch): + """One unwritable log must not cost the run that is already on disk. + + The logs used to be truncated one after another, so a permission error on + the second emptied the first on the way to raising. + """ + logs = [] + for name in ("run.inputs.txt", "run.outputs.txt", "run.errors.txt"): + path = tmp_path / name + path.write_text('{"index": 0}\n', encoding="utf-8") + logs.append(path) + before = [path.read_bytes() for path in logs] + + real = tempfile.NamedTemporaryFile + calls = {"n": 0} + + def fail_on_the_second(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 2: + raise PermissionError(13, "Permission denied") + return real(*args, **kwargs) + + monkeypatch.setattr(tempfile, "NamedTemporaryFile", fail_on_the_second) + + analysis = object.__new__(MonteCarlo) + analysis._input_file, analysis._output_file, analysis._error_file = ( + str(path) for path in logs + ) + + with pytest.raises(OSError): + analysis._MonteCarlo__setup_files(append=False) + + assert [path.read_bytes() for path in logs] == before + assert not list(tmp_path.glob("*.partial")) + + +def test_the_logs_are_emptied_when_every_one_of_them_can_be_written(tmp_path): + """The ordinary path still leaves three empty logs behind.""" + logs = [] + for name in ("run.inputs.txt", "run.outputs.txt", "run.errors.txt"): + path = tmp_path / name + path.write_text('{"index": 0}\n', encoding="utf-8") + logs.append(path) + + analysis = object.__new__(MonteCarlo) + analysis._input_file, analysis._output_file, analysis._error_file = ( + str(path) for path in logs + ) + analysis._MonteCarlo__setup_files(append=False) + + assert [path.read_bytes() for path in logs] == [b"", b"", b""] + assert not list(tmp_path.glob("*.partial")) diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py index 9aa4e0f03..e29cf862c 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_failures.py +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -9,10 +9,10 @@ """ import json -import traceback -import warnings import threading +import traceback import types +import warnings import pytest diff --git a/tests/unit/stochastic/test_stochastic_model.py b/tests/unit/stochastic/test_stochastic_model.py index 80fc3ed60..293a1a649 100644 --- a/tests/unit/stochastic/test_stochastic_model.py +++ b/tests/unit/stochastic/test_stochastic_model.py @@ -4,8 +4,9 @@ import pytest from rocketpy import Environment +from rocketpy.mathutils.function import Function from rocketpy.stochastic import StochasticEnvironment, StochasticFreeFormFins -from rocketpy.stochastic.stochastic_model import StochasticModel +from rocketpy.stochastic.stochastic_model import StochasticModel, _snapshot_of def _sampled_option(model): @@ -215,3 +216,33 @@ def test_the_nominal_is_the_one_the_model_was_built_with(example_plain_env): assert model.elevation[0] == around_first == 1000, ( "the model followed the object instead of the value it was built with" ) + + +def test_a_mutable_nominal_survives_a_write_through_the_object(calisto_free_form_fins): + """Writing through the wrapped object must not move the nominal. + + The cache held the object itself, so ``obj.outline[:] = ...`` reached it and + a value the model is supposed to sample around moved underneath it. + """ + stochastic = StochasticFreeFormFins( + free_form_fins=calisto_free_form_fins, shape_points=0.001 + ) + stochastic._set_stochastic(7) + expected = list(stochastic._nominal("shape_points", getattr)) + + stochastic.obj.shape_points[:] = [(9.9, 9.9)] * len(stochastic.obj.shape_points) + + assert list(stochastic._nominal("shape_points", getattr)) == expected + + +def test_the_snapshot_keeps_by_reference_what_it_does_not_copy(): + """Only containers are copied, so the rule can be stated as it behaves.""" + array = np.array([1.0, 2.0]) + listed = [[1.0], [2.0]] + function = Function(lambda x: x) + + assert _snapshot_of(array) is not array + assert _snapshot_of(listed) is not listed + assert _snapshot_of(listed)[0] is not listed[0] # deep, not shallow + assert _snapshot_of(function) is function + assert _snapshot_of(3.0) == 3.0 From 0f88ff71d9b90716004d57745c5d50540cdad8fe Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:18:31 +0800 Subject: [PATCH 35/45] BUG: refuse a bool count, and say how a row stopped simulate() rejects True through _is_whole_number, because isinstance(True, int) holds and a bool would quietly run one simulation. simulate_convergence checked with a plain isinstance, so batch_size=True and max_simulations=True were read as 1. tolerance had the same hole against isinstance(x, (int, float)). A row in the error file carried an error when a simulation raised and nothing at all when it was dropped because a peer crashed or the user interrupted, so the last two read as a simulation that simply had no error. They now carry a status of cancelled or interrupted. An empty payload still writes nothing: an interrupt between two simulations has nothing to report, and a row saying otherwise is what test_ctrl_c_between_rows_does_not_report_the_row_that_succeeded exists to catch. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 34 +++++++++++++++--- .../test_monte_carlo_determinism.py | 14 ++++++++ .../test_monte_carlo_log_integrity.py | 36 +++++++++++++++++++ 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index a7edebc28..8c1df30af 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -801,7 +801,7 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to f"{sim_idx} saved." ) with open(self.error_file, "a", encoding="utf-8") as f: - f.write(inputs_json) + f.write(_build_unfinished_record(inputs_json, "cancelled")) break @@ -979,16 +979,21 @@ def simulate_convergence( # Validate inputs up-front. Without this, a non-positive batch_size makes # the loop run zero new simulations every iteration and spin forever. - if not isinstance(batch_size, (int, np.integer)) or batch_size <= 0: + if not _is_whole_number(batch_size) or batch_size <= 0: raise ValueError( f"'batch_size' must be a positive integer, got {batch_size!r}." ) - if not isinstance(max_simulations, (int, np.integer)) or max_simulations <= 0: + if not _is_whole_number(max_simulations) or max_simulations <= 0: raise ValueError( f"'max_simulations' must be a positive integer, got " f"{max_simulations!r}." ) - if not isinstance(tolerance, (int, float)) or tolerance <= 0: + # bool is an int to isinstance, so True would pass as a tolerance of 1. + if ( + isinstance(tolerance, (bool, np.bool_)) + or not isinstance(tolerance, (int, float)) + or tolerance <= 0 + ): raise ValueError( f"'tolerance' must be a positive number, got {tolerance!r}." ) @@ -2193,6 +2198,25 @@ def _record_failure(error_file, sim_idx, inputs_json, details): ) +def _build_unfinished_record(inputs_json, status): + """One simulation that stopped before it could either fail or finish. + + Marked, because an unmarked row reads as a simulation with no error, and a + reader cannot otherwise tell a peer's crash from a keyboard interrupt. + + Nothing in flight means nothing to report, so an empty payload stays empty + rather than becoming a row about a simulation that never started. + """ + if not inputs_json: + return "" + try: + record = json.loads(inputs_json) + except (TypeError, ValueError): + record = {} + record["status"] = status + return json.dumps(record) + "\n" + + def _build_error_record(sim_idx, inputs_json, details): """One failed simulation as a row: what it drew, and what went wrong. @@ -2262,7 +2286,7 @@ def _write_unfinished_inputs(error_file, inputs_json): """Module level for the same reason as ``_record_simulation``: the run paths are driven by stub objects in the tests, which carry no private methods.""" with open(error_file, "a", encoding="utf-8") as f: - f.write(inputs_json) + f.write(_build_unfinished_record(inputs_json, "interrupted")) def _say_so_without_raising(message): diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 314555402..edd49c13e 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -413,3 +413,17 @@ def test_a_fresh_object_has_no_lineage_to_leave(): ) assert not [w for w in raised if "seed lineage" in str(w.message)] + + +@pytest.mark.parametrize("keyword", ["batch_size", "max_simulations", "tolerance"]) +@pytest.mark.parametrize("boolean", [True, False]) +def test_convergence_counts_refuse_a_bool(keyword, boolean): + """``isinstance(True, int)`` holds, so a bool used to pass as 1 or 0. + + ``simulate`` already refuses one through ``_is_whole_number``; the + convergence entry point checked with a plain ``isinstance`` and did not. + """ + analysis = object.__new__(MonteCarlo) + + with pytest.raises(ValueError, match=keyword): + MonteCarlo.simulate_convergence(analysis, **{keyword: boolean}) diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 0128afc12..4c448edb3 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -612,3 +612,39 @@ def test_the_logs_are_emptied_when_every_one_of_them_can_be_written(tmp_path): assert [path.read_bytes() for path in logs] == [b"", b"", b""] assert not list(tmp_path.glob("*.partial")) + + +@pytest.mark.parametrize("status", ["cancelled", "interrupted"]) +def test_a_row_that_did_not_finish_says_which_way_it_stopped(status): + """Three ways a row lands in the error file, and they must be tellable apart. + + A simulation that raised carries ``error``; one dropped because a peer + crashed or the user interrupted carried nothing, so it read as a success + with no error attached. + """ + row = json.loads(mc._build_unfinished_record('{"index": 3, "a": 1}', status)) + + assert row["status"] == status + assert row["index"] == 3 # the inputs are kept, not replaced + assert "error" not in row + + +def test_an_unreadable_input_row_still_reports_its_status(): + """The marker matters most when the inputs could not be written.""" + row = json.loads(mc._build_unfinished_record("not json at all", "cancelled")) + + assert row == {"status": "cancelled"} + + +def test_a_failed_simulation_is_not_labelled_as_stopped(): + """A real exception keeps ``error`` and gains no status, so the three differ.""" + row = json.loads(mc._build_error_record(5, '{"index": 5}', "boom")) + + assert row["error"] == "boom" + assert "status" not in row + + +def test_nothing_in_flight_writes_no_row_at_all(): + """An interrupt between two simulations has nothing to mark.""" + assert mc._build_unfinished_record("", "interrupted") == "" + assert mc._build_unfinished_record(None, "cancelled") == "" From 52288943c7dce34171210614c5f9698567975df1 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:36:12 +0800 Subject: [PATCH 36/45] BUG: log the flight inputs the flight was built from Each argument called dict_generator again and kept one key, so the flight took rail length from the first draw, inclination from the second and heading from the third, while last_rnd_dict, and so the row written to .inputs.txt, held only the third. Measured on the shared fixture: logged inclination 85.60, flown 84.46. Addresses #1090. One draw now serves all three, which makes the row a record of what flew rather than of a fourth thing nobody used. The three models draw from independent streams, so moving the flight draw ahead of the Flight call does not disturb what the environment or the rocket sample. StochasticRocket.create_object also now says what it carries onto the rocket it builds. Sensors are not among them, so a sensor on the wrapped rocket cannot reach a Monte Carlo flight. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 12 +++-- rocketpy/stochastic/stochastic_rocket.py | 5 ++ .../test_monte_carlo_determinism.py | 46 +++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 8c1df30af..13d2ca690 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -853,12 +853,18 @@ def __run_single_simulation(self): Flight The flight object of the simulation. """ + # One draw, not one per argument. Each _randomize_* called + # dict_generator again and kept a single key, so the flight took rail + # length from the first draw, inclination from the second and heading + # from the third, while last_rnd_dict, and so the row written to + # .inputs.txt, held only the third (#1090). + flight_inputs = next(self.flight.dict_generator()) return Flight( rocket=self.rocket.create_object(), environment=self.environment.create_object(), - rail_length=self.flight._randomize_rail_length(), - inclination=self.flight._randomize_inclination(), - heading=self.flight._randomize_heading(), + rail_length=flight_inputs["rail_length"], + inclination=flight_inputs["inclination"], + heading=flight_inputs["heading"], initial_solution=self.flight.initial_solution, terminate_on_apogee=self.flight.terminate_on_apogee, time_overshoot=self.flight.time_overshoot, diff --git a/rocketpy/stochastic/stochastic_rocket.py b/rocketpy/stochastic/stochastic_rocket.py index 10ad90827..d6ed88e22 100644 --- a/rocketpy/stochastic/stochastic_rocket.py +++ b/rocketpy/stochastic/stochastic_rocket.py @@ -784,6 +784,11 @@ def create_object(self): """Creates and returns a Rocket object from the randomly generated input arguments. + Motors, aerodynamic surfaces, air brakes and their controllers, rail + buttons and parachutes are carried onto the new rocket. Sensors are not: + one added to the wrapped rocket does not reach a Monte Carlo flight, and + so cannot affect it either. + Returns ------- rocket : Rocket diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index edd49c13e..8b34760af 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -427,3 +427,49 @@ def test_convergence_counts_refuse_a_bool(keyword, boolean): with pytest.raises(ValueError, match=keyword): MonteCarlo.simulate_convergence(analysis, **{keyword: boolean}) + + +def test_the_flight_flies_the_inputs_that_get_logged(monkeypatch): + """The logged row must be the row the ``Flight`` was built from. + + Each argument used to call ``dict_generator`` again and keep one key, so the + flight took rail length from the first draw, inclination from the second and + heading from the third, while ``last_rnd_dict`` held only the third (#1090). + """ + drawn = [ + {"rail_length": 5.0, "inclination": 84.0, "heading": 133.0}, + {"rail_length": 6.0, "inclination": 85.0, "heading": 134.0}, + {"rail_length": 7.0, "inclination": 86.0, "heading": 135.0}, + ] + + # A fresh generator per call, over one advancing stream: that is what the + # real dict_generator does, and a stub that restarts each time would pass + # whether the draw happens once or three times. + taken = [] + + def generator(): + row = drawn[len(taken)] + taken.append(row) + stochastic_flight.last_rnd_dict = row + yield row + + stochastic_flight = SimpleNamespace( + dict_generator=generator, + last_rnd_dict={}, + initial_solution=None, + terminate_on_apogee=False, + time_overshoot=True, + ) + analysis = object.__new__(MonteCarlo) + analysis.flight = stochastic_flight + analysis.rocket = SimpleNamespace(create_object=lambda: "rocket") + analysis.environment = SimpleNamespace(create_object=lambda: "environment") + monkeypatch.setattr("rocketpy.simulation.monte_carlo.Flight", SimpleNamespace) + + flight = MonteCarlo._MonteCarlo__run_single_simulation(analysis) + + logged = stochastic_flight.last_rnd_dict + assert flight.rail_length == logged["rail_length"] + assert flight.inclination == logged["inclination"] + assert flight.heading == logged["heading"] + assert logged == drawn[0], "one draw, so the row logged is the first one" From 3092999cc74b46033505d93899ad2e4a67df1577 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 04:25:15 +0800 Subject: [PATCH 37/45] BUG: settle a checkpoint's lineage from a manifest, not from its shape Three holes in the append protocol, all of them invisible to a reader of the rows. A seed can be an ndarray, which numpy.random.SeedSequence accepts, and comparing two roots holding one answered with an array rather than a verdict: ValueError, the truth value of an array with more than one element is ambiguous. [1, 2, 3] and (1, 2, 3) are the same seed and compared unequal. Roots are now compared by a canonical fingerprint of the words they generate, with n_children_spawned kept because __child_seed counts from it. The lineage moved as soon as a root was captured, before any file was touched, so a run that warned and then added nothing left the object believing the new root had landed. The append after that one silently put its rows behind rows from the first. Capturing and committing are now separate, and only a run that added rows commits. A checkpoint from before per-index seeding cannot be recognised by its indices: the previous release numbered parallel runs from 0 as well, so a clean one of those passed every structural check while its rows came from per-worker entropy, shared component seeds and a different sampling order. Each run now writes a manifest beside its output log naming the schema version, the sampling scheme and the root, and an append refuses a non-empty checkpoint that has none. The manifest also outlives the object, so the lineage check survives a fresh interpreter, which the attributes alone could not do. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 219 ++++++++++++++++-- run.outputs.manifest.json | 12 + .../test_monte_carlo_determinism.py | 113 ++++++++- .../test_monte_carlo_log_integrity.py | 77 ++++++ 4 files changed, 399 insertions(+), 22 deletions(-) create mode 100644 run.outputs.manifest.json diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 13d2ca690..b7ac7e5a9 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -73,6 +73,24 @@ def _create_empty_logs_atomically(paths): os.replace(temporary, destination) +def _seed_root_fingerprint(root): + """A comparable identity for a seed root, when the root itself is not one. + + ``entropy`` may be an ndarray, and comparing two of those answers with an + array rather than a verdict, so a tuple holding one raises instead of + deciding. ``[1, 2, 3]`` and ``(1, 2, 3)`` are the same seed and compare + unequal. The generated words settle both. ``n_children_spawned`` stays, + because ``__child_seed`` counts from it and two roots that differ only there + hand out different children. + """ + return ( + tuple(int(word) for word in root.generate_state(4, dtype=np.uint32)), + tuple(int(key) for key in root.spawn_key), + int(root.pool_size), + int(root.n_children_spawned), + ) + + def _warn_when_appending_leaves_the_lineage(previous, previous_chosen, current): """Say so when appended rows stop sharing the seed lineage below them. @@ -150,7 +168,12 @@ class MonteCarlo: # pylint: disable=too-many-public-methods # No run yet, so nothing to continue. Class-level so an instance built # without __init__ still answers. __root_state = None + __root_fingerprint = None __root_seed_given = False + # What the logs on disk actually hold, which only moves once a run has + # put rows from its own root into them. + __committed_fingerprint = None + __committed_seed_given = False def __init__( self, @@ -357,19 +380,7 @@ def simulate( self.number_of_simulations = number_of_simulations self._initial_sim_idx = self.num_of_loaded_sims if append else 0 if append: - _check_the_checkpoint_supports_appending( - self.input_file, self.output_file, self._initial_sim_idx - ) - # ``number_of_simulations`` is the target to reach, not a batch to - # add. Below the checkpoint it ran nothing, reported success, and - # left a file with more simulations than the caller had asked for. - if number_of_simulations < self._initial_sim_idx: - raise ValueError( - f"number_of_simulations is the total to reach when " - f"append=True. The checkpoint already holds " - f"{self._initial_sim_idx} simulations, more than the " - f"requested {number_of_simulations}." - ) + self.__check_this_append_can_continue(number_of_simulations) # Both run paths catch Ctrl-C, save what they have and return, so a # stopped run is incomplete on purpose and the completeness check below # has to know the difference between that and a worker going missing. @@ -385,12 +396,18 @@ def simulate( print("Starting Monte Carlo analysis") self.__setup_files(append) + if not append: + # Emptied just now, so whatever they held is gone and this root + # owns them even if the run below never reaches its first row. + self.__commit_root_lineage() if parallel: self.__run_in_parallel(n_workers) else: self.__run_in_serial() + if number_of_simulations > self._initial_sim_idx: + self.__commit_root_lineage() self.__check_each_index_was_recorded_once() self.__terminate_simulation() @@ -470,7 +487,6 @@ def __capture_root_state(self, random_seed, appending=False): reference. Without it a caller who mutates the list they passed changes the children this run derives, which is the opposite of a snapshot. """ - previous, previous_chosen = self.__root_state, self.__root_seed_given root = self.__root_seed_sequence(random_seed) self.__root_state = ( deepcopy(root.entropy), @@ -478,14 +494,58 @@ def __capture_root_state(self, random_seed, appending=False): root.pool_size, root.n_children_spawned, ) + # The state above rebuilds children and cannot be compared; the + # fingerprint compares and cannot rebuild. Both, rather than one. + self.__root_fingerprint = _seed_root_fingerprint(root) # Whether a caller chose this root or it came from fresh entropy. Only a # chosen one is a lineage there is any point in continuing. self.__root_seed_given = random_seed is not None if appending: + # The manifest first: it outlives this object, which the attributes + # below do not, and it describes the rows actually in the log. + recorded = _fingerprint_from_manifest(self.output_file) + previous, previous_chosen = recorded or ( + self.__committed_fingerprint, + self.__committed_seed_given, + ) _warn_when_appending_leaves_the_lineage( - previous, previous_chosen, self.__root_state + previous, previous_chosen, self.__root_fingerprint + ) + + def __check_this_append_can_continue(self, number_of_simulations): + """Everything an append has to satisfy before a file is opened. + + Held here rather than after the run so a checkpoint that cannot be + continued costs no simulations and is left exactly as it was found. + """ + _check_the_checkpoint_supports_appending( + self.input_file, self.output_file, self._initial_sim_idx + ) + # ``number_of_simulations`` is the target to reach, not a batch to add. + # Below the checkpoint it ran nothing, reported success, and left a file + # with more simulations than the caller had asked for. + if number_of_simulations < self._initial_sim_idx: + raise ValueError( + f"number_of_simulations is the total to reach when " + f"append=True. The checkpoint already holds " + f"{self._initial_sim_idx} simulations, more than the " + f"requested {number_of_simulations}." ) + def __commit_root_lineage(self): + """Record that the logs now hold rows derived from this run's root. + + Kept apart from capturing it, because the capture runs before anything + opens a file. A run that warns and then adds nothing, or that dies before + its first row, must leave the logs owned by the root already in them, or + the append after it compares against a lineage that was never written. + Object-local, so a rebuilt ``MonteCarlo`` starts blank; carrying it in + the files is #1075. + """ + self.__committed_fingerprint = self.__root_fingerprint + self.__committed_seed_given = self.__root_seed_given + _write_run_manifest(self.output_file, self.__root_state, self.__root_seed_given) + def __child_seed(self, sim_idx): """Return the seed sequence for a single simulation index. @@ -2070,6 +2130,133 @@ def _recorded_indices(label, path): return written, damaged +# Written beside the output log so a later run can tell which scheme produced +# the rows. Index shape cannot: the previous release numbered parallel runs from +# 0 as well, and those rows came from per-worker entropy, shared component seeds +# and a different sampling call sequence. +_MANIFEST_SCHEMA_VERSION = 1 +_SAMPLING_SCHEME = "per-index-seed-v1" + + +def _manifest_path(output_file): + """Where the manifest for a given output log lives.""" + return Path(output_file).with_suffix(".manifest.json") + + +def _jsonable_entropy(entropy): + """``SeedSequence`` entropy as something ``json`` will take. + + It may be an int, a sequence of ints or an ndarray, and only the first of + those survives ``json.dumps`` unhelped. + """ + if isinstance(entropy, (int, np.integer)): + return int(entropy) + if entropy is None: + return None + return [int(part) for part in np.asarray(entropy).ravel()] + + +def _write_run_manifest(output_file, root_state, seed_chosen): + """Record the scheme and the root the rows in this log came from. + + Best effort on the way out: a manifest that cannot be written is worth a + warning, not the loss of a finished run. The next append refuses without it, + which is the safe direction. + """ + entropy, spawn_key, pool_size, base = root_state + document = { + "schema_version": _MANIFEST_SCHEMA_VERSION, + "sampling_scheme": _SAMPLING_SCHEME, + "log_format": "jsonl-v1", + "seed_chosen": bool(seed_chosen), + "root_state": { + "entropy": _jsonable_entropy(entropy), + "spawn_key": [int(key) for key in spawn_key], + "pool_size": int(pool_size), + "n_children_spawned": int(base), + }, + } + _best_effort( + lambda: _manifest_path(output_file).write_text( + json.dumps(document, indent=2) + "\n", encoding="utf-8" + ), + "run manifest", + ) + + +def _read_run_manifest(output_file): + """The manifest beside a log, or ``None`` when there is not a usable one.""" + path = _manifest_path(output_file) + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return document if isinstance(document, dict) else None + + +def _fingerprint_from_manifest(output_file): + """``(fingerprint, whether the seed was chosen)`` for the log's own root. + + The object that wrote the rows is usually gone by the time an append runs, + so this is what makes the lineage check survive a fresh interpreter. + ``None`` when there is no manifest or it does not describe a root. + """ + manifest = _read_run_manifest(output_file) + if manifest is None: + return None + recorded = manifest.get("root_state") + if not isinstance(recorded, dict): + return None + try: + # Rebuilt for its words only. n_children_spawned is read-only on a + # SeedSequence, so it is carried across as the recorded number instead. + root = np.random.SeedSequence( + entropy=recorded["entropy"], + spawn_key=tuple(recorded["spawn_key"]), + pool_size=recorded["pool_size"], + ) + fingerprint = ( + tuple(int(word) for word in root.generate_state(4, dtype=np.uint32)), + tuple(int(key) for key in recorded["spawn_key"]), + int(recorded["pool_size"]), + int(recorded["n_children_spawned"]), + ) + except (KeyError, TypeError, ValueError): + return None + return fingerprint, bool(manifest.get("seed_chosen", False)) + + +def _refuse_a_checkpoint_from_another_scheme(output_file, resume_at): + """Refuse rows this release cannot have written, however they are numbered. + + The previous release numbered parallel runs from 0 too, so a clean one of + those passes every check on index shape while its rows came from per-worker + entropy and a different sampling order. Only something written alongside + them can tell, so a checkpoint with nothing in it is refused rather than + guessed at. + """ + if resume_at <= 0: + return + manifest = _read_run_manifest(output_file) + if manifest is None: + raise ValueError( + f"cannot append to {output_file}: no {_manifest_path(output_file).name} " + f"beside it, so the rows cannot be shown to come from this release. " + f"Runs before per-index seeding numbered parallel results from 0 as " + f"well, and appending to one would put two sampling schemes in one " + f"file. Re-run the study to start a checkpoint this release owns." + ) + scheme = manifest.get("sampling_scheme") + version = manifest.get("schema_version") + if scheme != _SAMPLING_SCHEME or version != _MANIFEST_SCHEMA_VERSION: + raise ValueError( + f"cannot append to {output_file}: it was written by sampling scheme " + f"{scheme!r} at schema version {version!r}, and this release writes " + f"{_SAMPLING_SCHEME!r} at {_MANIFEST_SCHEMA_VERSION}. Re-run the " + f"study rather than continuing one scheme with another." + ) + + def _check_the_checkpoint_supports_appending(input_file, output_file, resume_at): """Everything that can be judged from the files, before a worker starts. @@ -2097,6 +2284,8 @@ def _check_the_checkpoint_supports_appending(input_file, output_file, resume_at) ) _refuse_a_checkpoint_that_does_not_line_up(label, path, written, resume_at) + _refuse_a_checkpoint_from_another_scheme(output_file, resume_at) + inputs, _ = _recorded_indices("inputs", input_file) outputs, _ = _recorded_indices("outputs", output_file) if inputs != outputs: diff --git a/run.outputs.manifest.json b/run.outputs.manifest.json new file mode 100644 index 000000000..03a890287 --- /dev/null +++ b/run.outputs.manifest.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "sampling_scheme": "per-index-seed-v1", + "log_format": "jsonl-v1", + "seed_chosen": true, + "root_state": { + "entropy": 7, + "spawn_key": [], + "pool_size": 4, + "n_children_spawned": 0 + } +} diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 8b34760af..7defc3e64 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -34,9 +34,11 @@ import numpy as np import pytest +import rocketpy.simulation.monte_carlo as mc_module from rocketpy.simulation import MonteCarlo from rocketpy.simulation.monte_carlo import ( _claim_next_index, + _seed_root_fingerprint, _seed_sequence_to_int, _SimMonitor, _validate_simulation_count, @@ -367,6 +369,9 @@ def test_mutating_the_caller_s_entropy_does_not_move_the_captured_root(wrapped): assert _entropy(_child_seed(runner, 7)) == before +SEED_ARRAY = np.array([1, 2, 3], dtype=np.uint32) + + @pytest.mark.parametrize( "first_seed, second_seed, expect_warning", [ @@ -374,10 +379,19 @@ def test_mutating_the_caller_s_entropy_does_not_move_the_captured_root(wrapped): (42, 7, True), # a different chosen root is the same mixing (42, 42, False), # continuing the same run (None, None, False), # nothing was being preserved + # numpy.random.SeedSequence takes array_like[ints], so an ndarray is a + # seed this accepts. Comparing two of those elementwise answers with an + # array, which is not a verdict, and used to raise here. + (SEED_ARRAY, SEED_ARRAY, False), + (SEED_ARRAY, np.array([1, 2, 3], dtype=np.uint32), False), + (SEED_ARRAY, np.array([9, 9, 9], dtype=np.uint32), True), + # Same seed, different container: one lineage, so no warning. + ([1, 2, 3], (1, 2, 3), False), + (np.random.SeedSequence(7), 7, False), ], ) def test_appending_says_so_when_it_leaves_the_seed_lineage( - first_seed, second_seed, expect_warning + first_seed, second_seed, expect_warning, tmp_path ): """Mixing two lineages in one file is invisible to every structural check. @@ -385,16 +399,15 @@ def test_appending_says_so_when_it_leaves_the_seed_lineage( two files stay in step, so only the roots themselves can tell (#1075). """ analysis = object.__new__(MonteCarlo) + # Committing writes the manifest beside this, and the append below reads it. + analysis._output_file = str(tmp_path / "run.outputs.txt") analysis._MonteCarlo__capture_root_state(first_seed) - previous_root = analysis._MonteCarlo__root_state - previous_chosen = analysis._MonteCarlo__root_seed_given - analysis._MonteCarlo__capture_root_state(second_seed) + # What simulate does once the first run has put rows in the logs. + analysis._MonteCarlo__commit_root_lineage() with warnings.catch_warnings(record=True) as raised: warnings.simplefilter("always") - _warn_when_appending_leaves_the_lineage( - previous_root, previous_chosen, analysis._MonteCarlo__root_state - ) + analysis._MonteCarlo__capture_root_state(second_seed, appending=True) lineage_warnings = [w for w in raised if "seed lineage" in str(w.message)] assert bool(lineage_warnings) is expect_warning @@ -402,6 +415,28 @@ def test_appending_says_so_when_it_leaves_the_seed_lineage( assert issubclass(lineage_warnings[0].category, RuntimeWarning) +def test_a_root_that_derives_the_same_children_fingerprints_the_same(): + """The fingerprint has to answer the question the roots cannot. + + It stands in for the root in the lineage check, so two roots handing out the + same children must agree and two handing out different ones must not. + """ + same = [ + _seed_root_fingerprint(np.random.SeedSequence([1, 2, 3])), + _seed_root_fingerprint(np.random.SeedSequence((1, 2, 3))), + _seed_root_fingerprint(np.random.SeedSequence(SEED_ARRAY)), + ] + assert same[0] == same[1] == same[2] + assert all(isinstance(part, (tuple, int)) for part in same[0]) + + spawned = np.random.SeedSequence(42) + before = _seed_root_fingerprint(spawned) + spawned.spawn(3) # children counted from here on, so the identity moves + + assert _seed_root_fingerprint(spawned) != before + assert _seed_root_fingerprint(np.random.SeedSequence(43)) != before + + def test_a_fresh_object_has_no_lineage_to_leave(): """A first run cannot be leaving anything, however it is seeded.""" with warnings.catch_warnings(record=True) as raised: @@ -473,3 +508,67 @@ def generator(): assert flight.inclination == logged["inclination"] assert flight.heading == logged["heading"] assert logged == drawn[0], "one draw, so the row logged is the first one" + + +def _simulate_without_flying(monkeypatch, analysis): + """Drive ``simulate`` for its control flow, with the run itself removed. + + The defect being covered is an ordering one, so the ordering has to be the + real method rather than a retelling of it in the test. + """ + analysis.data_collector = None + analysis.num_of_loaded_sims = 0 + # The public setters read the file they are pointed at; the run itself is + # stubbed out here, so the private attributes are what the getters need. + analysis._input_file = "run.inputs.txt" + analysis._output_file = "run.outputs.txt" + analysis._error_file = "run.errors.txt" + for name in ( + "_MonteCarlo__setup_files", + "_MonteCarlo__run_in_serial", + "_MonteCarlo__check_each_index_was_recorded_once", + "_MonteCarlo__terminate_simulation", + ): + monkeypatch.setattr(MonteCarlo, name, lambda *a, **k: None) + monkeypatch.setattr( + mc_module, "_check_the_checkpoint_supports_appending", lambda *a, **k: None + ) + + def run(total, seed, append): + with warnings.catch_warnings(record=True) as raised: + warnings.simplefilter("always") + analysis.simulate(total, append=append, random_seed=seed) + analysis.num_of_loaded_sims = total + return bool([w for w in raised if "seed lineage" in str(w.message)]) + + return run + + +def test_a_run_that_adds_nothing_does_not_take_the_files_lineage(monkeypatch): + """The warning has to survive an append that wrote no rows. + + ``simulate`` captures the root before it touches a file, so the object used + to believe the new lineage had landed even when the run added nothing. The + append after that one silently put rows from the second root behind rows + from the first, which is exactly the case the warning exists for (#1075). + """ + run = _simulate_without_flying(monkeypatch, object.__new__(MonteCarlo)) + + assert run(2, 42, False) is False # first run, nothing to leave + assert run(2, 7, True) is True # warns, and adds no rows + assert run(4, 7, True) is True # the one that actually appends seed 7 + assert run(6, 7, True) is False # now genuinely the same lineage + + +def test_a_refused_append_leaves_the_lineage_where_it_was(monkeypatch): + """A warning promoted to an error must not still move the tracker.""" + analysis = object.__new__(MonteCarlo) + run = _simulate_without_flying(monkeypatch, analysis) + run(2, 42, False) + + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + with pytest.raises(RuntimeWarning): + analysis.simulate(4, append=True, random_seed=7) + + assert run(4, 7, True) is True, "the files still hold the first lineage" diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 4c448edb3..33e722e3f 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -14,6 +14,7 @@ import tempfile import threading import types +import warnings import pytest @@ -648,3 +649,79 @@ def test_nothing_in_flight_writes_no_row_at_all(): """An interrupt between two simulations has nothing to mark.""" assert mc._build_unfinished_record("", "interrupted") == "" assert mc._build_unfinished_record(None, "cancelled") == "" + + +def _old_parallel_checkpoint(tmp_path, rows=2): + """Logs shaped exactly like a clean run from before per-index seeding. + + The previous release numbered parallel runs from 0 as well, so these pass + every check that reads only the indices. + """ + body = "".join(json.dumps({"index": i}) + "\n" for i in range(rows)) + inputs, outputs = tmp_path / "run.inputs.txt", tmp_path / "run.outputs.txt" + inputs.write_text(body, encoding="utf-8") + outputs.write_text(body, encoding="utf-8") + return inputs, outputs + + +def test_a_checkpoint_from_the_old_parallel_scheme_is_refused(tmp_path): + """Index shape cannot tell the two schemes apart, so something else must. + + A clean 0..N-1 log from the previous release passes every structural check + while its rows came from per-worker entropy, shared component seeds and a + different sampling call sequence. + """ + inputs, outputs = _old_parallel_checkpoint(tmp_path) + + with pytest.raises(ValueError, match="manifest"): + mc._check_the_checkpoint_supports_appending(str(inputs), str(outputs), 2) + + +def test_a_checkpoint_this_release_wrote_is_accepted(tmp_path): + """The same rows, with the manifest beside them, continue normally.""" + inputs, outputs = _old_parallel_checkpoint(tmp_path) + mc._write_run_manifest(str(outputs), (42, (), 4, 0), True) + + mc._check_the_checkpoint_supports_appending(str(inputs), str(outputs), 2) + + +def test_a_manifest_from_another_scheme_is_refused(tmp_path): + """A future or foreign scheme is named rather than guessed at.""" + inputs, outputs = _old_parallel_checkpoint(tmp_path) + mc._write_run_manifest(str(outputs), (42, (), 4, 0), True) + path = mc._manifest_path(str(outputs)) + document = json.loads(path.read_text(encoding="utf-8")) + document["sampling_scheme"] = "something-else-v9" + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(ValueError, match="something-else-v9"): + mc._check_the_checkpoint_supports_appending(str(inputs), str(outputs), 2) + + +def test_an_empty_checkpoint_needs_no_manifest(tmp_path): + """A first run has nothing to be continued from, so nothing to prove.""" + inputs, outputs = _old_parallel_checkpoint(tmp_path, rows=0) + + mc._check_the_checkpoint_supports_appending(str(inputs), str(outputs), 0) + + +def test_the_lineage_outlives_the_object_that_wrote_it(tmp_path): + """A fresh MonteCarlo has no memory, so the manifest has to carry it. + + Rebuilding the object between two runs used to lose the root entirely, and + the append after it could not tell it was leaving the lineage. + """ + output = str(tmp_path / "run.outputs.txt") + + first = object.__new__(MonteCarlo) + first._output_file = output + first._MonteCarlo__capture_root_state(42) + first._MonteCarlo__commit_root_lineage() + + second = object.__new__(MonteCarlo) # nothing carried over in memory + second._output_file = output + with warnings.catch_warnings(record=True) as raised: + warnings.simplefilter("always") + second._MonteCarlo__capture_root_state(7, appending=True) + + assert [w for w in raised if "seed lineage" in str(w.message)] From 0eb0e589ac5d9d99fed03af2a4371dab20447525 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 04:52:39 +0800 Subject: [PATCH 38/45] BUG: roll the logs back when one of three installs fails os.replace is atomic for one file. Three of them were three atomic steps with nothing between, so a failure on the second left the first already replaced by an empty file, the untouched temporaries behind, and the destination narrowed from 0644 to the 0600 a staged file opens at. Each destination is now moved aside before its replacement goes in, anything already installed is put back if a later one fails, the temporaries that never landed are removed, and the mode of the log being replaced is carried onto the file replacing it. BaseException, so a Ctrl-C rolls back too. This is not a filesystem transaction and the docstring says so: a generation directory swapped by one pointer would be the stronger guarantee, and would change what the three public log paths mean. Also drops the #1090 fix from this branch. #1126 is open against the same function, adds the shared _sample_flight_inputs the fix wants, and closes the second draw inside StochasticFlight.create_object that this branch left alone. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 90 +++++++++++++------ .../test_monte_carlo_determinism.py | 46 ---------- .../test_monte_carlo_log_integrity.py | 75 ++++++++++++++++ 3 files changed, 136 insertions(+), 75 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index b7ac7e5a9..d50ef10c4 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -42,35 +42,73 @@ # TODO: Create evolution plots to analyze convergence +def _stage_an_empty_log(destination): + """An empty file beside ``destination``, ready to be moved onto it. + + Same directory, so the move is a rename rather than a copy across devices, + and at the mode ``destination`` already has: a staged file opens at 0600, + which would otherwise narrow a log the caller left readable. + """ + keep_mode = destination.stat().st_mode & 0o7777 if destination.exists() else None + handle = tempfile.NamedTemporaryFile( # pylint: disable=consider-using-with + mode="w", + encoding="utf-8", + dir=destination.parent, + prefix=f"{destination.name}.", + suffix=".partial", + delete=False, + ) + handle.close() + if keep_mode is not None: + os.chmod(handle.name, keep_mode) + return handle.name + + def _create_empty_logs_atomically(paths): - """Put every log in place empty, or leave every one of them alone. + """Put every log in place empty, or leave every one of them as it was. Truncating them one after another empties the first before a bad path or a - permission can stop the second, which loses a finished run on the way to - raising. Each is staged beside its destination and moved over it only once - all of them exist. + permission can stop the second. Each is staged beside its destination and + installed only once all of them exist, the one it replaces is kept until + every install has gone through, and anything already installed is put back + if a later one fails. + + Not a filesystem transaction: ``os.replace`` is atomic for one file, and + three of them are three atomic steps with a rollback between. A generation + directory swapped by a single pointer would be the stronger guarantee, and + would change what the three public log paths mean. """ - staged = [] + staged, moved_aside, created, done = [], [], [], 0 try: for path in paths: destination = Path(path) - handle = tempfile.NamedTemporaryFile( # pylint: disable=consider-using-with - mode="w", - encoding="utf-8", - dir=destination.parent, - prefix=f"{destination.name}.", - suffix=".partial", - delete=False, - ) - handle.close() - staged.append((handle.name, destination)) - except OSError as error: - for temporary, _ in staged: + staged.append((_stage_an_empty_log(destination), destination)) + + for temporary, destination in staged: + # Recorded as each step happens, not after the pair: an install that + # fails between them would otherwise leave the original moved aside + # with nothing tracking where it went. + if destination.exists(): + kept = f"{temporary}.kept" + os.replace(destination, kept) + moved_aside.append((destination, kept)) + else: + created.append(destination) + os.replace(temporary, destination) + done += 1 + except BaseException as error: + for destination, kept in reversed(moved_aside): + _best_effort(lambda k=kept, d=destination: os.replace(k, d), "log rollback") + for destination in reversed(created): + _best_effort(lambda d=destination: os.remove(d), "log rollback") + for temporary, _ in staged[done:]: _best_effort(lambda t=temporary: os.remove(t), "staged log cleanup") - raise OSError(f"Error creating files: {error}") from error + if isinstance(error, OSError): + raise OSError(f"Error creating files: {error}") from error + raise - for temporary, destination in staged: - os.replace(temporary, destination) + for _, kept in moved_aside: + _best_effort(lambda k=kept: os.remove(k), "replaced log cleanup") def _seed_root_fingerprint(root): @@ -913,18 +951,12 @@ def __run_single_simulation(self): Flight The flight object of the simulation. """ - # One draw, not one per argument. Each _randomize_* called - # dict_generator again and kept a single key, so the flight took rail - # length from the first draw, inclination from the second and heading - # from the third, while last_rnd_dict, and so the row written to - # .inputs.txt, held only the third (#1090). - flight_inputs = next(self.flight.dict_generator()) return Flight( rocket=self.rocket.create_object(), environment=self.environment.create_object(), - rail_length=flight_inputs["rail_length"], - inclination=flight_inputs["inclination"], - heading=flight_inputs["heading"], + rail_length=self.flight._randomize_rail_length(), + inclination=self.flight._randomize_inclination(), + heading=self.flight._randomize_heading(), initial_solution=self.flight.initial_solution, terminate_on_apogee=self.flight.terminate_on_apogee, time_overshoot=self.flight.time_overshoot, diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 7defc3e64..49af38981 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -464,52 +464,6 @@ def test_convergence_counts_refuse_a_bool(keyword, boolean): MonteCarlo.simulate_convergence(analysis, **{keyword: boolean}) -def test_the_flight_flies_the_inputs_that_get_logged(monkeypatch): - """The logged row must be the row the ``Flight`` was built from. - - Each argument used to call ``dict_generator`` again and keep one key, so the - flight took rail length from the first draw, inclination from the second and - heading from the third, while ``last_rnd_dict`` held only the third (#1090). - """ - drawn = [ - {"rail_length": 5.0, "inclination": 84.0, "heading": 133.0}, - {"rail_length": 6.0, "inclination": 85.0, "heading": 134.0}, - {"rail_length": 7.0, "inclination": 86.0, "heading": 135.0}, - ] - - # A fresh generator per call, over one advancing stream: that is what the - # real dict_generator does, and a stub that restarts each time would pass - # whether the draw happens once or three times. - taken = [] - - def generator(): - row = drawn[len(taken)] - taken.append(row) - stochastic_flight.last_rnd_dict = row - yield row - - stochastic_flight = SimpleNamespace( - dict_generator=generator, - last_rnd_dict={}, - initial_solution=None, - terminate_on_apogee=False, - time_overshoot=True, - ) - analysis = object.__new__(MonteCarlo) - analysis.flight = stochastic_flight - analysis.rocket = SimpleNamespace(create_object=lambda: "rocket") - analysis.environment = SimpleNamespace(create_object=lambda: "environment") - monkeypatch.setattr("rocketpy.simulation.monte_carlo.Flight", SimpleNamespace) - - flight = MonteCarlo._MonteCarlo__run_single_simulation(analysis) - - logged = stochastic_flight.last_rnd_dict - assert flight.rail_length == logged["rail_length"] - assert flight.inclination == logged["inclination"] - assert flight.heading == logged["heading"] - assert logged == drawn[0], "one draw, so the row logged is the first one" - - def _simulate_without_flying(monkeypatch, analysis): """Drive ``simulate`` for its control flow, with the run itself removed. diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 33e722e3f..7c51738a8 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -11,6 +11,7 @@ """ import json +import os import tempfile import threading import types @@ -725,3 +726,77 @@ def test_the_lineage_outlives_the_object_that_wrote_it(tmp_path): second._MonteCarlo__capture_root_state(7, appending=True) assert [w for w in raised if "seed lineage" in str(w.message)] + + +def _three_logs_on_disk(tmp_path, mode=0o644): + """Three logs holding a row each, at a mode a caller might have chosen.""" + logs = [] + for name in ("run.inputs.txt", "run.outputs.txt", "run.errors.txt"): + path = tmp_path / name + path.write_text('{"index": 0}\n', encoding="utf-8") + os.chmod(path, mode) + logs.append(path) + return logs + + +@pytest.mark.parametrize("failing_call", [1, 2, 3, 4, 5, 6]) +def test_a_failed_install_puts_every_log_back(tmp_path, monkeypatch, failing_call): + """``os.replace`` is atomic for one file, not for three of them. + + Each destination is moved aside and replaced in turn, so a failure part way + through used to leave the earlier ones already emptied. Every step is driven + to fail here, since which one breaks is not something to assume. + """ + logs = _three_logs_on_disk(tmp_path) + before = [path.read_bytes() for path in logs] + real_replace = os.replace + calls = {"n": 0} + + def fail_on_the_chosen_call(source, destination): + calls["n"] += 1 + if calls["n"] == failing_call: + raise OSError(13, "injected") + return real_replace(source, destination) + + monkeypatch.setattr(os, "replace", fail_on_the_chosen_call) + + with pytest.raises(OSError): + mc._create_empty_logs_atomically([str(path) for path in logs]) + + assert [path.read_bytes() for path in logs] == before + assert not [x for x in tmp_path.iterdir() if x.suffix in (".partial", ".kept")] + + +def test_a_keyboard_interrupt_part_way_through_puts_every_log_back( + tmp_path, monkeypatch +): + """Rollback has to catch BaseException, not only OSError.""" + logs = _three_logs_on_disk(tmp_path) + before = [path.read_bytes() for path in logs] + real_replace = os.replace + calls = {"n": 0} + + def interrupt_on_the_third(source, destination): + calls["n"] += 1 + if calls["n"] == 3: + raise KeyboardInterrupt + return real_replace(source, destination) + + monkeypatch.setattr(os, "replace", interrupt_on_the_third) + + with pytest.raises(KeyboardInterrupt): + mc._create_empty_logs_atomically([str(path) for path in logs]) + + assert [path.read_bytes() for path in logs] == before + assert not [x for x in tmp_path.iterdir() if x.suffix in (".partial", ".kept")] + + +def test_the_logs_keep_the_mode_they_had(tmp_path): + """A staged file opens at 0600, which must not narrow the log it replaces.""" + logs = _three_logs_on_disk(tmp_path, mode=0o644) + + mc._create_empty_logs_atomically([str(path) for path in logs]) + + assert [path.read_bytes() for path in logs] == [b"", b"", b""] + assert [path.stat().st_mode & 0o777 for path in logs] == [0o644] * 3 + assert not [x for x in tmp_path.iterdir() if x.suffix in (".partial", ".kept")] From 2b0f57a71098cbe85c14e7263cf2a1fae4196384 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 05:26:12 +0800 Subject: [PATCH 39/45] BUG: keep the inputs a simulation had drawn when it fails early The row is only assembled once the flight is built, so anything raising inside create_object or Flight itself left an error row carrying a traceback and nothing about the inputs that produced it. A tuple initial_solution (#1109) gets there on the first draw. Each stochastic model fills its own last_rnd_dict as it goes, so what did get drawn is already on hand. Both failure paths now recover it and mark the row partial: the draws that never happened are absent rather than recorded as null. That only reads true if those dicts hold the simulation in flight and nothing else. Reseeding clears them, since a worker keeps its models across every index it claims, and recording a pair clears them too, so a row already on disk cannot be recovered again and blamed for a later failure. _inputs_drawn_so_far is module level for the reason _record_simulation is: the run paths are driven by stubs in the tests, which carry no private methods. The stubs gained the four attributes the failure path now reads. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 97 ++++++++++++++++--- .../test_monte_carlo_log_integrity.py | 59 +++++++++++ .../test_monte_carlo_worker_failures.py | 75 ++++++++++++++ 3 files changed, 217 insertions(+), 14 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index d50ef10c4..2a7dd03e0 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -610,6 +610,11 @@ def __seed_simulation(self, child_seed): workers. Each sub-stream is handed over as a 128-bit ``int`` (see ``_seed_sequence_to_int``) so custom samplers keep working. """ + # Cleared as well as reseeded: a simulation that fails before it draws + # would otherwise report the previous one's values as its own, and a + # worker keeps the same models for every index it claims. + for model in (self.environment, self.rocket, self.flight): + model.last_rnd_dict = {} env_seed, rocket_seed, flight_seed = child_seed.spawn(3) self.environment._set_stochastic(_seed_sequence_to_int(env_seed)) self.rocket._set_stochastic(_seed_sequence_to_int(rocket_seed)) @@ -732,7 +737,11 @@ def __run_in_serial(self): outputs_json = self.__evaluate_flight_outputs(flight, sim_idx) _record_simulation( - self.input_file, self.output_file, inputs_json, outputs_json + self.input_file, + self.output_file, + inputs_json, + outputs_json, + (self.environment, self.rocket, self.flight), ) # The pair is on disk. Cleared before the monitor call so a # failure there reports itself rather than reporting a row that @@ -752,7 +761,15 @@ def __run_in_serial(self): # Captured before reporting, which may fail and must not be what # gets recorded or raised. _record_failure( - self._error_file, sim_idx, inputs_json, traceback.format_exc() + self._error_file, + sim_idx, + inputs_json + or _inputs_drawn_so_far( + (self.environment, self.rocket, self.flight), + sim_idx, + self._export_config, + ), + traceback.format_exc(), ) # Bare, so the handler's own line does not join the traceback. raise @@ -890,26 +907,23 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to with _manager_mutex(mutex): if error_event.is_set(): - # Runs in a worker process spawned via multiprocessing: - # logging handlers configured in the main process are - # not guaranteed to be inherited (e.g. Windows "spawn"), - # so this must use print() to remain visible. - _SimMonitor.reprint( - f"Simulation interrupt. Files from simulation " - f"{sim_idx} saved." + _report_a_cancelled_simulation( + self.error_file, sim_idx, inputs_json ) - with open(self.error_file, "a", encoding="utf-8") as f: - f.write(_build_unfinished_record(inputs_json, "cancelled")) - break _record_simulation( - self.input_file, self.output_file, inputs_json, outputs_json + self.input_file, + self.output_file, + inputs_json, + outputs_json, + (self.environment, self.rocket, self.flight), ) # Same as the serial path: the pair is on disk, so a failure # in the monitor call below must report itself and not the # row that has just been committed. inputs_json, outputs_json = "", "" + _forget_the_last_draw((self.environment, self.rocket, self.flight)) sim_monitor.print_update_status() except Exception: @@ -924,6 +938,11 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to # The failure goes onto the inputs record rather than replacing it, # the same shape the serial path writes. + inputs_json = inputs_json or _inputs_drawn_so_far( + (self.environment, self.rocket, self.flight), + sim_idx, + self._export_config, + ) record = _build_error_record(sim_idx, inputs_json, details) try: @@ -2396,7 +2415,54 @@ def _validate_simulation_count(number_of_simulations): _WORKER_POLL_INTERVAL = 0.05 -def _record_simulation(input_file, output_file, inputs_json, outputs_json): +def _report_a_cancelled_simulation(error_file, sim_idx, inputs_json): + """Note a simulation dropped because a peer had already failed. + + Runs in a worker spawned via multiprocessing, where logging handlers + configured in the parent are not guaranteed to be inherited (Windows + "spawn"), so this has to print to stay visible. + """ + _SimMonitor.reprint(f"Simulation interrupt. Files from simulation {sim_idx} saved.") + with open(error_file, "a", encoding="utf-8") as handle: + handle.write(_build_unfinished_record(inputs_json, "cancelled")) + + +def _forget_the_last_draw(models): + """Drop what the models drew, once it is on disk or about to be replaced. + + ``_inputs_drawn_so_far`` reads these, so they have to hold the simulation in + flight and nothing else: a row already recorded would otherwise be recovered + again and reported as the cause of a later failure. + """ + for model in models: + model.last_rnd_dict = {} + + +def _inputs_drawn_so_far(models, sim_idx, export_config): + """Whatever the models had drawn when a simulation stopped early. + + The whole row is only built once the flight is, so a failure inside + ``create_object`` or ``Flight`` itself left the error row carrying a + traceback and nothing about the inputs that produced it. Each model fills + its own ``last_rnd_dict`` as it goes, so what did get drawn is already + there. Marked partial: the draws that never happened are absent, not null. + + Module level for the same reason as ``_record_simulation``: the run paths + are driven by stub objects in the tests, which carry no private methods. + """ + try: + drawn = dict(item for model in models for item in model.last_rnd_dict.items()) + if not drawn: + return "" + drawn["index"] = sim_idx + drawn["partial_inputs"] = True + return json.dumps(drawn, cls=RocketPyEncoder, **export_config) + "\n" + except Exception: # pylint: disable=broad-exception-caught + # A diagnostic must not replace the failure it is describing. + return "" + + +def _record_simulation(input_file, output_file, inputs_json, outputs_json, models): """Append one simulation's inputs and outputs to their logs. Module level rather than a method: the run paths are driven directly by @@ -2406,6 +2472,9 @@ def _record_simulation(input_file, output_file, inputs_json, outputs_json): f.write(inputs_json) with open(output_file, "a", encoding="utf-8") as f: f.write(outputs_json) + # The pair is on disk, so what produced it is no longer in flight and + # must not be recovered again for a later failure. + _forget_the_last_draw(models) def _record_failure(error_file, sim_idx, inputs_json, details): diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 7c51738a8..57bb16dc3 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -17,6 +17,7 @@ import types import warnings +import numpy as np import pytest import rocketpy.simulation.monte_carlo as mc @@ -289,6 +290,11 @@ def print_update_status(self): def _sim_worker(tmp_path, **overrides): attributes = { + # The failure path recovers what was drawn from these. + "_export_config": {}, + "environment": types.SimpleNamespace(last_rnd_dict={}), + "rocket": types.SimpleNamespace(last_rnd_dict={}), + "flight": types.SimpleNamespace(last_rnd_dict={}), "error_file": tmp_path / "errors.txt", "input_file": tmp_path / "inputs.txt", "output_file": tmp_path / "outputs.txt", @@ -363,6 +369,10 @@ def _serial_runner(tmp_path, row=""): _MonteCarlo__child_seed=lambda index: index, _MonteCarlo__seed_simulation=lambda seed: None, _MonteCarlo__run_single_simulation=object, + _export_config={}, + environment=types.SimpleNamespace(last_rnd_dict={}), + rocket=types.SimpleNamespace(last_rnd_dict={}), + flight=types.SimpleNamespace(last_rnd_dict={}), _MonteCarlo__evaluate_flight_inputs=lambda index: row, _MonteCarlo__evaluate_flight_outputs=lambda flight, index: row, ) @@ -800,3 +810,52 @@ def test_the_logs_keep_the_mode_they_had(tmp_path): assert [path.read_bytes() for path in logs] == [b"", b"", b""] assert [path.stat().st_mode & 0o777 for path in logs] == [0o644] * 3 assert not [x for x in tmp_path.iterdir() if x.suffix in (".partial", ".kept")] + + +def _models_that_have_drawn(): + """Three stochastic models carrying the values of a finished simulation.""" + return [ + types.SimpleNamespace( + last_rnd_dict={f"{name}_value": 1.0}, _set_stochastic=lambda seed: None + ) + for name in ("environment", "rocket", "flight") + ] + + +def test_reseeding_forgets_what_the_simulation_before_it_drew(): + """A worker keeps its models, so the next index inherits the last one's row. + + Without this, a simulation that failed before drawing anything reported the + values of the one before it as the inputs that caused the failure. + """ + analysis = object.__new__(MonteCarlo) + analysis.environment, analysis.rocket, analysis.flight = _models_that_have_drawn() + + analysis._MonteCarlo__seed_simulation(np.random.SeedSequence(42)) + + assert [ + model.last_rnd_dict + for model in (analysis.environment, analysis.rocket, analysis.flight) + ] == [{}, {}, {}] + + +def test_recording_a_pair_forgets_what_produced_it(tmp_path): + """Once the row is on disk it is not in flight, so it cannot be recovered. + + ``_inputs_drawn_so_far`` reads the same dicts, and a failure later in the + loop would otherwise attach a simulation that had already succeeded to it. + """ + inputs, outputs = tmp_path / "inputs.txt", tmp_path / "outputs.txt" + models = _models_that_have_drawn() + + mc._record_simulation(inputs, outputs, '{"index": 0}\n', '{"index": 0}\n', models) + + assert [model.last_rnd_dict for model in models] == [{}, {}, {}] + assert inputs.read_text() == '{"index": 0}\n' + + +def test_nothing_drawn_recovers_nothing(): + """The recovery is a best effort, not a source of empty rows.""" + empty = [types.SimpleNamespace(last_rnd_dict={}) for _ in range(3)] + + assert mc._inputs_drawn_so_far(empty, 3, {}) == "" diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py index e29cf862c..8e03dbff1 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_failures.py +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -57,6 +57,12 @@ def _worker(tmp_path, **overrides): "error_file": tmp_path / "errors.txt", "input_file": tmp_path / "inputs.txt", "output_file": tmp_path / "outputs.txt", + # The failure path recovers whatever was drawn from these, so the + # stand-in has to carry them the way a MonteCarlo does. + "_export_config": {}, + "environment": types.SimpleNamespace(last_rnd_dict={}), + "rocket": types.SimpleNamespace(last_rnd_dict={}), + "flight": types.SimpleNamespace(last_rnd_dict={}), "_MonteCarlo__child_seed": lambda index: index, "_MonteCarlo__seed_simulation": lambda seed: None, "_MonteCarlo__run_single_simulation": object, @@ -376,6 +382,10 @@ def test_a_worker_progress_failure_does_not_repeat_committed_inputs(tmp_path): the error file and one simulation appeared in the logs and the failures.""" worker = _worker( tmp_path, + _export_config={}, + environment=types.SimpleNamespace(last_rnd_dict={}), + rocket=types.SimpleNamespace(last_rnd_dict={}), + flight=types.SimpleNamespace(last_rnd_dict={}), _MonteCarlo__evaluate_flight_inputs=lambda index: '{"index": 0, "drew": 42}\n', _MonteCarlo__evaluate_flight_outputs=lambda flight, index: '{"index": 0}\n', ) @@ -516,3 +526,68 @@ def increment(self): with pytest.raises(BrokenPipeError): mc._claim_next_index(_Fine(), _MutexDiesAfter(healthy_releases=0)) + + +@pytest.mark.parametrize( + "stopped_at, drawn", + [ + ("environment", ({"wind": 1.0}, {}, {})), + ("rocket", ({"wind": 1.0}, {"mass": 2.0}, {})), + ("flight sampling", ({"wind": 1.0}, {"mass": 2.0}, {})), + ("Flight.__init__", ({"wind": 1.0}, {"mass": 2.0}, {"inclination": 84.0})), + ], +) +def test_an_error_row_keeps_whatever_was_drawn_before_the_failure( + tmp_path, monkeypatch, stopped_at, drawn +): + """A failure before the row is built must not cost the inputs as well. + + The whole row is only assembled once the flight is, so anything that raised + inside ``create_object`` or ``Flight`` itself used to leave a traceback with + nothing to say which inputs produced it (#1109 is one way in). Each model + fills its own ``last_rnd_dict`` as it goes, so the draws that did happen are + recoverable. + """ + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: 0) + environment, rocket, flight = drawn + worker = _worker( + tmp_path, + environment=types.SimpleNamespace(last_rnd_dict=environment), + rocket=types.SimpleNamespace(last_rnd_dict=rocket), + flight=types.SimpleNamespace(last_rnd_dict=flight), + _MonteCarlo__run_single_simulation=_raise, + ) + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer( + worker, object(), _RecordingMutex(), _Event() + ) + + rows = [ + json.loads(line) + for line in (tmp_path / "errors.txt").read_text().splitlines() + if line.strip() + ] + assert rows, f"nothing was recorded for a failure at {stopped_at}" + row = rows[0] + assert row["partial_inputs"] is True + assert "error" in row + for key, value in {**environment, **rocket, **flight}.items(): + assert row[key] == value + + +def test_an_error_before_anything_was_drawn_still_records_the_failure( + tmp_path, monkeypatch +): + """With nothing drawn there is nothing to recover, and the row says so.""" + monkeypatch.setattr(mc, "_claim_next_index", lambda *a, **k: 0) + worker = _worker(tmp_path, _MonteCarlo__run_single_simulation=_raise) + + with pytest.raises(_Boom): + mc.MonteCarlo._MonteCarlo__sim_producer( + worker, object(), _RecordingMutex(), _Event() + ) + + row = json.loads((tmp_path / "errors.txt").read_text().splitlines()[0]) + assert "error" in row + assert "partial_inputs" not in row From c5eae341ed0a5c3678ffed3f14fe58b8deec1150 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 05:41:58 +0800 Subject: [PATCH 40/45] TST: compare the log mode against what it had, not against 0644 Windows honours only the read-only bit, so a file asked for 0644 reports 0666 there and the literal was testing the platform rather than the carry-over. Both Windows legs went red on it. Reading the mode before the replacement and comparing after says the same thing on either platform, and dropping the chmod still turns it red. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- .../unit/simulation/test_monte_carlo_log_integrity.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 57bb16dc3..b0d351124 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -802,13 +802,19 @@ def interrupt_on_the_third(source, destination): def test_the_logs_keep_the_mode_they_had(tmp_path): - """A staged file opens at 0600, which must not narrow the log it replaces.""" + """A staged file opens at 0600, which must not narrow the log it replaces. + + Compared against what the log had rather than against 0644: Windows honours + only the read-only bit, so a file asked for 0644 reports 0666 there and a + literal would be testing the platform instead of the carry-over. + """ logs = _three_logs_on_disk(tmp_path, mode=0o644) + before = [path.stat().st_mode & 0o777 for path in logs] mc._create_empty_logs_atomically([str(path) for path in logs]) assert [path.read_bytes() for path in logs] == [b"", b"", b""] - assert [path.stat().st_mode & 0o777 for path in logs] == [0o644] * 3 + assert [path.stat().st_mode & 0o777 for path in logs] == before assert not [x for x in tmp_path.iterdir() if x.suffix in (".partial", ".kept")] From d9e40dd7e6f8dd4c5d4e2848b51157b28cfb7937 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:02:55 +0800 Subject: [PATCH 41/45] BUG: do not turn a recorded simulation into a failed one _record_simulation forgets the draw once the rows are on disk, and did it in a way that could raise. A model without last_rnd_dict then took a simulation whose inputs and outputs had just been written and reported it as a failure, with an error row of its own. Best effort now: the write is the outcome, and the bookkeeping after it says so rather than replacing it. The manifest is named by appending rather than by replacing the suffix, so run.txt and run.json stop describing themselves with one file. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 17 ++++++++---- run.outputs.txt.manifest.json | 12 +++++++++ .../test_monte_carlo_log_integrity.py | 27 +++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 run.outputs.txt.manifest.json diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 2a7dd03e0..8eb7a55e2 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -2190,8 +2190,13 @@ def _recorded_indices(label, path): def _manifest_path(output_file): - """Where the manifest for a given output log lives.""" - return Path(output_file).with_suffix(".manifest.json") + """Where the manifest for a given output log lives. + + Appended rather than swapped for the suffix, so ``run.txt`` and ``run.json`` + do not describe themselves with one file, and the pairing stays readable in + a directory listing. + """ + return Path(f"{output_file}.manifest.json") def _jsonable_entropy(entropy): @@ -2472,9 +2477,11 @@ def _record_simulation(input_file, output_file, inputs_json, outputs_json, model f.write(inputs_json) with open(output_file, "a", encoding="utf-8") as f: f.write(outputs_json) - # The pair is on disk, so what produced it is no longer in flight and - # must not be recovered again for a later failure. - _forget_the_last_draw(models) + # The pair is on disk, so what produced it is no longer in flight and must + # not be recovered again for a later failure. Best effort: the rows are + # written, and bookkeeping that raises here would report a simulation that + # succeeded as one that failed. + _best_effort(lambda: _forget_the_last_draw(models), "forgetting a recorded draw") def _record_failure(error_file, sim_idx, inputs_json, details): diff --git a/run.outputs.txt.manifest.json b/run.outputs.txt.manifest.json new file mode 100644 index 000000000..03a890287 --- /dev/null +++ b/run.outputs.txt.manifest.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "sampling_scheme": "per-index-seed-v1", + "log_format": "jsonl-v1", + "seed_chosen": true, + "root_state": { + "entropy": 7, + "spawn_key": [], + "pool_size": 4, + "n_children_spawned": 0 + } +} diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index b0d351124..eb605301e 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -865,3 +865,30 @@ def test_nothing_drawn_recovers_nothing(): empty = [types.SimpleNamespace(last_rnd_dict={}) for _ in range(3)] assert mc._inputs_drawn_so_far(empty, 3, {}) == "" + + +def test_a_recorded_pair_survives_a_model_that_cannot_be_cleared(tmp_path): + """The rows are on disk, so bookkeeping after them must not undo that. + + Forgetting the draw used to raise straight out of ``_record_simulation``, + which reported a simulation that had just been written as one that failed. + """ + inputs, outputs = tmp_path / "inputs.txt", tmp_path / "outputs.txt" + models = [types.SimpleNamespace(last_rnd_dict={"a": 1}), object()] + + with pytest.warns(RuntimeWarning, match="forgetting a recorded draw"): + mc._record_simulation( + inputs, outputs, '{"index": 0}\n', '{"index": 0}\n', models + ) + + assert inputs.read_text() == '{"index": 0}\n' + assert outputs.read_text() == '{"index": 0}\n' + + +def test_two_logs_that_differ_only_in_extension_get_their_own_manifest(tmp_path): + """``with_suffix`` gave ``run.txt`` and ``run.json`` one manifest between them.""" + first = mc._manifest_path(str(tmp_path / "run.txt")) + second = mc._manifest_path(str(tmp_path / "run.json")) + + assert first != second + assert first.name.startswith("run.txt") From 069134e302d1b707fc5fc7fc330039b576de413d Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:36:37 +0800 Subject: [PATCH 42/45] BUG: recover inputs per model, and stop clearing documented state Two things I got wrong in the previous commit, both found by review. dict_generator builds a local dictionary and binds last_rnd_dict once, after its whole attribute loop. It does not fill it as it goes, which is what I claimed. So a model that raises part way through its own draw publishes nothing, and the recovery is per model, not per field: the tuple initial_solution in #1109 recovers the models that finished and nothing from the one that failed. Two tests now drive a real StochasticEnvironment through a sampler that raises mid-loop, rather than pre-filling last_rnd_dict on a stub and proving only that the serializer works. Telling a fresh publication from the previous simulation's was done by clearing last_rnd_dict, which dict_generator documents as holding the last generated dictionary. Seeding now keeps a reference to what each model was already holding, and a model still holding it published nothing for this index. Nothing clears public state, and _record_simulation goes back to writing the pair and stopping there. Also removes run.outputs.manifest.json and run.outputs.txt.manifest.json, which were test output committed from the repository root: the deterministic helper used fixed run.* paths in the working directory instead of tmp_path. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 61 ++++++------ run.outputs.manifest.json | 12 --- run.outputs.txt.manifest.json | 12 --- .../test_monte_carlo_determinism.py | 21 ++-- .../test_monte_carlo_log_integrity.py | 98 ++++++++++--------- .../test_monte_carlo_worker_failures.py | 1 + .../unit/stochastic/test_stochastic_model.py | 47 +++++++++ 7 files changed, 142 insertions(+), 110 deletions(-) delete mode 100644 run.outputs.manifest.json delete mode 100644 run.outputs.txt.manifest.json diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 8eb7a55e2..67f05d58a 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -206,6 +206,7 @@ class MonteCarlo: # pylint: disable=too-many-public-methods # No run yet, so nothing to continue. Class-level so an instance built # without __init__ still answers. __root_state = None + __draws_before_this_simulation = () __root_fingerprint = None __root_seed_given = False # What the logs on disk actually hold, which only moves once a run has @@ -610,11 +611,14 @@ def __seed_simulation(self, child_seed): workers. Each sub-stream is handed over as a 128-bit ``int`` (see ``_seed_sequence_to_int``) so custom samplers keep working. """ - # Cleared as well as reseeded: a simulation that fails before it draws - # would otherwise report the previous one's values as its own, and a - # worker keeps the same models for every index it claims. - for model in (self.environment, self.rocket, self.flight): - model.last_rnd_dict = {} + # What each model is holding right now, kept by reference. dict_generator + # binds a new object every call, so a later failure can tell what this + # index published from what the one before it left behind, without + # clearing last_rnd_dict, which is documented state. + self.__draws_before_this_simulation = [ + model.last_rnd_dict + for model in (self.environment, self.rocket, self.flight) + ] env_seed, rocket_seed, flight_seed = child_seed.spawn(3) self.environment._set_stochastic(_seed_sequence_to_int(env_seed)) self.rocket._set_stochastic(_seed_sequence_to_int(rocket_seed)) @@ -741,7 +745,6 @@ def __run_in_serial(self): self.output_file, inputs_json, outputs_json, - (self.environment, self.rocket, self.flight), ) # The pair is on disk. Cleared before the monitor call so a # failure there reports itself rather than reporting a row that @@ -766,6 +769,7 @@ def __run_in_serial(self): inputs_json or _inputs_drawn_so_far( (self.environment, self.rocket, self.flight), + self.__draws_before_this_simulation, sim_idx, self._export_config, ), @@ -917,13 +921,11 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to self.output_file, inputs_json, outputs_json, - (self.environment, self.rocket, self.flight), ) # Same as the serial path: the pair is on disk, so a failure # in the monitor call below must report itself and not the # row that has just been committed. inputs_json, outputs_json = "", "" - _forget_the_last_draw((self.environment, self.rocket, self.flight)) sim_monitor.print_update_status() except Exception: @@ -940,6 +942,7 @@ def __sim_producer(self, sim_monitor, mutex, error_event): # pylint: disable=to # the same shape the serial path writes. inputs_json = inputs_json or _inputs_drawn_so_far( (self.environment, self.rocket, self.flight), + self.__draws_before_this_simulation, sim_idx, self._export_config, ) @@ -2432,31 +2435,30 @@ def _report_a_cancelled_simulation(error_file, sim_idx, inputs_json): handle.write(_build_unfinished_record(inputs_json, "cancelled")) -def _forget_the_last_draw(models): - """Drop what the models drew, once it is on disk or about to be replaced. - - ``_inputs_drawn_so_far`` reads these, so they have to hold the simulation in - flight and nothing else: a row already recorded would otherwise be recovered - again and reported as the cause of a later failure. - """ - for model in models: - model.last_rnd_dict = {} +def _inputs_drawn_so_far(models, held_before, sim_idx, export_config): + """What the models had published when a simulation stopped early. + A model publishes ``last_rnd_dict`` once, after its whole attribute loop has + finished, so this recovers the models that completed and nothing at all from + the one that failed part way through its own draw. Marked partial for that + reason rather than for a per-field one. -def _inputs_drawn_so_far(models, sim_idx, export_config): - """Whatever the models had drawn when a simulation stopped early. - - The whole row is only built once the flight is, so a failure inside - ``create_object`` or ``Flight`` itself left the error row carrying a - traceback and nothing about the inputs that produced it. Each model fills - its own ``last_rnd_dict`` as it goes, so what did get drawn is already - there. Marked partial: the draws that never happened are absent, not null. + ``held_before`` is what each model was holding when this index was seeded. A + model still holding it published nothing here, and its previous values must + not be reported as the cause of this failure. Module level for the same reason as ``_record_simulation``: the run paths are driven by stub objects in the tests, which carry no private methods. """ try: - drawn = dict(item for model in models for item in model.last_rnd_dict.items()) + published = [ + model + for model, before in zip(models, held_before) + if model.last_rnd_dict is not before + ] + drawn = dict( + item for model in published for item in model.last_rnd_dict.items() + ) if not drawn: return "" drawn["index"] = sim_idx @@ -2467,7 +2469,7 @@ def _inputs_drawn_so_far(models, sim_idx, export_config): return "" -def _record_simulation(input_file, output_file, inputs_json, outputs_json, models): +def _record_simulation(input_file, output_file, inputs_json, outputs_json): """Append one simulation's inputs and outputs to their logs. Module level rather than a method: the run paths are driven directly by @@ -2477,11 +2479,6 @@ def _record_simulation(input_file, output_file, inputs_json, outputs_json, model f.write(inputs_json) with open(output_file, "a", encoding="utf-8") as f: f.write(outputs_json) - # The pair is on disk, so what produced it is no longer in flight and must - # not be recovered again for a later failure. Best effort: the rows are - # written, and bookkeeping that raises here would report a simulation that - # succeeded as one that failed. - _best_effort(lambda: _forget_the_last_draw(models), "forgetting a recorded draw") def _record_failure(error_file, sim_idx, inputs_json, details): diff --git a/run.outputs.manifest.json b/run.outputs.manifest.json deleted file mode 100644 index 03a890287..000000000 --- a/run.outputs.manifest.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "schema_version": 1, - "sampling_scheme": "per-index-seed-v1", - "log_format": "jsonl-v1", - "seed_chosen": true, - "root_state": { - "entropy": 7, - "spawn_key": [], - "pool_size": 4, - "n_children_spawned": 0 - } -} diff --git a/run.outputs.txt.manifest.json b/run.outputs.txt.manifest.json deleted file mode 100644 index 03a890287..000000000 --- a/run.outputs.txt.manifest.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "schema_version": 1, - "sampling_scheme": "per-index-seed-v1", - "log_format": "jsonl-v1", - "seed_chosen": true, - "root_state": { - "entropy": 7, - "spawn_key": [], - "pool_size": 4, - "n_children_spawned": 0 - } -} diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 49af38981..e3c19df3e 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -243,6 +243,9 @@ class _RecordingModel: def __init__(self): self.seeds = [] + # Every StochasticModel carries this; seeding reads it to mark what was + # already published before the index it is about to run. + self.last_rnd_dict = {} def _set_stochastic(self, seed=None): self.seeds.append(seed) @@ -464,7 +467,7 @@ def test_convergence_counts_refuse_a_bool(keyword, boolean): MonteCarlo.simulate_convergence(analysis, **{keyword: boolean}) -def _simulate_without_flying(monkeypatch, analysis): +def _simulate_without_flying(monkeypatch, analysis, tmp_path): """Drive ``simulate`` for its control flow, with the run itself removed. The defect being covered is an ordering one, so the ordering has to be the @@ -474,9 +477,11 @@ def _simulate_without_flying(monkeypatch, analysis): analysis.num_of_loaded_sims = 0 # The public setters read the file they are pointed at; the run itself is # stubbed out here, so the private attributes are what the getters need. - analysis._input_file = "run.inputs.txt" - analysis._output_file = "run.outputs.txt" - analysis._error_file = "run.errors.txt" + # Under tmp_path, not the working directory: committing writes a manifest + # beside the output log, and fixed names put it in the repository root. + analysis._input_file = str(tmp_path / "run.inputs.txt") + analysis._output_file = str(tmp_path / "run.outputs.txt") + analysis._error_file = str(tmp_path / "run.errors.txt") for name in ( "_MonteCarlo__setup_files", "_MonteCarlo__run_in_serial", @@ -498,7 +503,7 @@ def run(total, seed, append): return run -def test_a_run_that_adds_nothing_does_not_take_the_files_lineage(monkeypatch): +def test_a_run_that_adds_nothing_does_not_take_the_files_lineage(monkeypatch, tmp_path): """The warning has to survive an append that wrote no rows. ``simulate`` captures the root before it touches a file, so the object used @@ -506,7 +511,7 @@ def test_a_run_that_adds_nothing_does_not_take_the_files_lineage(monkeypatch): append after that one silently put rows from the second root behind rows from the first, which is exactly the case the warning exists for (#1075). """ - run = _simulate_without_flying(monkeypatch, object.__new__(MonteCarlo)) + run = _simulate_without_flying(monkeypatch, object.__new__(MonteCarlo), tmp_path) assert run(2, 42, False) is False # first run, nothing to leave assert run(2, 7, True) is True # warns, and adds no rows @@ -514,10 +519,10 @@ def test_a_run_that_adds_nothing_does_not_take_the_files_lineage(monkeypatch): assert run(6, 7, True) is False # now genuinely the same lineage -def test_a_refused_append_leaves_the_lineage_where_it_was(monkeypatch): +def test_a_refused_append_leaves_the_lineage_where_it_was(monkeypatch, tmp_path): """A warning promoted to an error must not still move the tracker.""" analysis = object.__new__(MonteCarlo) - run = _simulate_without_flying(monkeypatch, analysis) + run = _simulate_without_flying(monkeypatch, analysis, tmp_path) run(2, 42, False) with warnings.catch_warnings(): diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index eb605301e..1f4064123 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -292,6 +292,7 @@ def _sim_worker(tmp_path, **overrides): attributes = { # The failure path recovers what was drawn from these. "_export_config": {}, + "_MonteCarlo__draws_before_this_simulation": ({}, {}, {}), "environment": types.SimpleNamespace(last_rnd_dict={}), "rocket": types.SimpleNamespace(last_rnd_dict={}), "flight": types.SimpleNamespace(last_rnd_dict={}), @@ -828,67 +829,72 @@ def _models_that_have_drawn(): ] -def test_reseeding_forgets_what_the_simulation_before_it_drew(): - """A worker keeps its models, so the next index inherits the last one's row. +def test_nothing_drawn_recovers_nothing(): + """The recovery is a best effort, not a source of empty rows.""" + empty = [types.SimpleNamespace(last_rnd_dict={}) for _ in range(3)] - Without this, a simulation that failed before drawing anything reported the - values of the one before it as the inputs that caused the failure. - """ - analysis = object.__new__(MonteCarlo) - analysis.environment, analysis.rocket, analysis.flight = _models_that_have_drawn() + assert mc._inputs_drawn_so_far(empty, [{}, {}, {}], 3, {}) == "" - analysis._MonteCarlo__seed_simulation(np.random.SeedSequence(42)) - assert [ - model.last_rnd_dict - for model in (analysis.environment, analysis.rocket, analysis.flight) - ] == [{}, {}, {}] +def test_two_logs_that_differ_only_in_extension_get_their_own_manifest(tmp_path): + """``with_suffix`` gave ``run.txt`` and ``run.json`` one manifest between them.""" + first = mc._manifest_path(str(tmp_path / "run.txt")) + second = mc._manifest_path(str(tmp_path / "run.json")) + + assert first != second + assert first.name.startswith("run.txt") -def test_recording_a_pair_forgets_what_produced_it(tmp_path): - """Once the row is on disk it is not in flight, so it cannot be recovered. +def test_recovery_takes_the_models_that_published_and_leaves_the_rest(): + """Recovery is per model, because publication is. - ``_inputs_drawn_so_far`` reads the same dicts, and a failure later in the - loop would otherwise attach a simulation that had already succeeded to it. + A model that finished its draw binds a new ``last_rnd_dict``; one that + raised part way through is still holding the object it had. Comparing + against what was held at seeding tells the two apart without touching that + documented state. """ - inputs, outputs = tmp_path / "inputs.txt", tmp_path / "outputs.txt" - models = _models_that_have_drawn() + finished = types.SimpleNamespace(last_rnd_dict={"wind": 1.0}) + stale = types.SimpleNamespace(last_rnd_dict={"from_the_run_before": 9.0}) + failed_mid_draw = types.SimpleNamespace(last_rnd_dict={}) + held_before = [{}, stale.last_rnd_dict, failed_mid_draw.last_rnd_dict] - mc._record_simulation(inputs, outputs, '{"index": 0}\n', '{"index": 0}\n', models) + row = json.loads( + mc._inputs_drawn_so_far((finished, stale, failed_mid_draw), held_before, 7, {}) + ) - assert [model.last_rnd_dict for model in models] == [{}, {}, {}] - assert inputs.read_text() == '{"index": 0}\n' + assert row["wind"] == 1.0 + assert row["partial_inputs"] is True + assert row["index"] == 7 + assert "from_the_run_before" not in row, "a previous simulation was reported" -def test_nothing_drawn_recovers_nothing(): - """The recovery is a best effort, not a source of empty rows.""" - empty = [types.SimpleNamespace(last_rnd_dict={}) for _ in range(3)] +def test_recovery_reports_nothing_when_no_model_published(): + """A failure before the first model finished has nothing to recover.""" + models = [types.SimpleNamespace(last_rnd_dict={"old": 1.0}) for _ in range(3)] + held_before = [model.last_rnd_dict for model in models] - assert mc._inputs_drawn_so_far(empty, 3, {}) == "" + assert mc._inputs_drawn_so_far(models, held_before, 7, {}) == "" -def test_a_recorded_pair_survives_a_model_that_cannot_be_cleared(tmp_path): - """The rows are on disk, so bookkeeping after them must not undo that. +def test_seeding_marks_what_each_model_was_already_holding(): + """The marks are what make a stale draw tellable from a fresh one. - Forgetting the draw used to raise straight out of ``_record_simulation``, - which reported a simulation that had just been written as one that failed. + Taken by reference rather than copied, and taken at seeding rather than + after, so nothing has to clear ``last_rnd_dict`` to keep the recovery + honest. """ - inputs, outputs = tmp_path / "inputs.txt", tmp_path / "outputs.txt" - models = [types.SimpleNamespace(last_rnd_dict={"a": 1}), object()] - - with pytest.warns(RuntimeWarning, match="forgetting a recorded draw"): - mc._record_simulation( - inputs, outputs, '{"index": 0}\n', '{"index": 0}\n', models - ) - - assert inputs.read_text() == '{"index": 0}\n' - assert outputs.read_text() == '{"index": 0}\n' - + analysis = object.__new__(MonteCarlo) + held = [{"a": 1}, {"b": 2}, {"c": 3}] + analysis.environment, analysis.rocket, analysis.flight = ( + types.SimpleNamespace(last_rnd_dict=one, _set_stochastic=lambda seed: None) + for one in held + ) -def test_two_logs_that_differ_only_in_extension_get_their_own_manifest(tmp_path): - """``with_suffix`` gave ``run.txt`` and ``run.json`` one manifest between them.""" - first = mc._manifest_path(str(tmp_path / "run.txt")) - second = mc._manifest_path(str(tmp_path / "run.json")) + analysis._MonteCarlo__seed_simulation(np.random.SeedSequence(42)) - assert first != second - assert first.name.startswith("run.txt") + marks = analysis._MonteCarlo__draws_before_this_simulation + assert [mark is one for mark, one in zip(marks, held)] == [True, True, True] + assert [ + model.last_rnd_dict + for model in (analysis.environment, analysis.rocket, analysis.flight) + ] == held diff --git a/tests/unit/simulation/test_monte_carlo_worker_failures.py b/tests/unit/simulation/test_monte_carlo_worker_failures.py index 8e03dbff1..c3f86ae9a 100644 --- a/tests/unit/simulation/test_monte_carlo_worker_failures.py +++ b/tests/unit/simulation/test_monte_carlo_worker_failures.py @@ -60,6 +60,7 @@ def _worker(tmp_path, **overrides): # The failure path recovers whatever was drawn from these, so the # stand-in has to carry them the way a MonteCarlo does. "_export_config": {}, + "_MonteCarlo__draws_before_this_simulation": ({}, {}, {}), "environment": types.SimpleNamespace(last_rnd_dict={}), "rocket": types.SimpleNamespace(last_rnd_dict={}), "flight": types.SimpleNamespace(last_rnd_dict={}), diff --git a/tests/unit/stochastic/test_stochastic_model.py b/tests/unit/stochastic/test_stochastic_model.py index 293a1a649..8eb132c9a 100644 --- a/tests/unit/stochastic/test_stochastic_model.py +++ b/tests/unit/stochastic/test_stochastic_model.py @@ -6,6 +6,7 @@ from rocketpy import Environment from rocketpy.mathutils.function import Function from rocketpy.stochastic import StochasticEnvironment, StochasticFreeFormFins +from rocketpy.stochastic.custom_sampler import CustomSampler from rocketpy.stochastic.stochastic_model import StochasticModel, _snapshot_of @@ -246,3 +247,49 @@ def test_the_snapshot_keeps_by_reference_what_it_does_not_copy(): assert _snapshot_of(listed)[0] is not listed[0] # deep, not shallow assert _snapshot_of(function) is function assert _snapshot_of(3.0) == 3.0 + + +class _RaisesPartWayThrough(CustomSampler): + """A sampler that lets the fields before it draw, then fails on its own.""" + + def sample(self, n_samples=1): + raise TypeError("injected mid-generator failure") + + def reset_seed(self, seed=None): + pass + + +def test_a_model_publishes_nothing_when_it_fails_part_way_through(example_plain_env): + """``dict_generator`` binds ``last_rnd_dict`` once, after the whole loop. + + Anything recovering inputs from a failed simulation has to know this: a + model that raised while drawing has published nothing, so the recovery is + per model rather than per field. The tuple ``initial_solution`` in #1109 + fails this way. + """ + stochastic = StochasticEnvironment( + environment=example_plain_env, + elevation=(100.0, 1.0), + wind_velocity_x_factor=_RaisesPartWayThrough(), + ) + stochastic._set_stochastic(7) + before = stochastic.last_rnd_dict + + with pytest.raises(TypeError, match="injected mid-generator failure"): + next(stochastic.dict_generator()) + + assert stochastic.last_rnd_dict is before, "a failed draw published nothing" + + +def test_a_model_that_finished_publishes_a_new_dictionary(example_plain_env): + """The other half: a completed draw replaces the object, so it is tellable.""" + stochastic = StochasticEnvironment( + environment=example_plain_env, elevation=(100.0, 1.0) + ) + stochastic._set_stochastic(7) + before = stochastic.last_rnd_dict + + next(stochastic.dict_generator()) + + assert stochastic.last_rnd_dict is not before + assert "elevation" in stochastic.last_rnd_dict From efbce664463d9fb25e7be386c727ef49e813f2c3 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 07:11:09 +0800 Subject: [PATCH 43/45] BUG: refuse an append from another root instead of warning about it One manifest holds one root. Warning and appending anyway left the file with two lineages while the manifest named only the newer half, so the provenance it reported was wrong rather than merely incomplete, and my own test wrote that down as correct: after appending root 7 behind root 42 it called the next append with root 7 'genuinely the same lineage', which the rows from 42 say it is not. An append now continues the root the logs already hold. Leaving random_seed out means continue rather than begin, an explicit seed that matches is allowed, and one that does not is refused before anything opens a file. A file can no longer be given two lineages, so the manifest cannot misdescribe one. The manifest is also validated in full rather than by scheme and version alone. A document naming the right scheme with no usable root_state used to pass, and bool("false") is True, so a string there read as a chosen seed. It is written through a staged file with fsync and os.replace, since it now decides whether a checkpoint may be continued at all. The two committed-fingerprint attributes are gone: the manifest is the record, and pylint pointed out that nothing read them any more. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 169 +++++++++++------- .../test_monte_carlo_determinism.py | 104 +++++------ .../test_monte_carlo_log_integrity.py | 59 +++++- 3 files changed, 215 insertions(+), 117 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index 67f05d58a..b0d3a0e94 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -129,22 +129,22 @@ def _seed_root_fingerprint(root): ) -def _warn_when_appending_leaves_the_lineage(previous, previous_chosen, current): - """Say so when appended rows stop sharing the seed lineage below them. +def _refuse_a_root_that_leaves_the_lineage(recorded, current, output_file): + """Refuse an append that would put a second lineage in one file. - A file holding two lineages is valid on every structural check, so nothing - else can notice. Only reachable when this object already ran from a chosen - seed and is now continuing from a different root (#1075). + A warning was not enough. One manifest holds one root, so continuing from a + different one leaves the file mixed while the manifest names only the newer + half, which is worse than refusing: the provenance is then wrong rather than + merely incomplete. Checked before anything is opened. """ - if not previous_chosen or previous is None or previous == current: + if recorded == current: return - warnings.warn( - "Appending to a run that was seeded, with a different root. Rows from " - "here on derive from a new seed lineage, and the file records both " - "without saying which is which. Pass the original random_seed to " - "continue the same run.", - RuntimeWarning, - stacklevel=3, + raise ValueError( + f"cannot append to {output_file}: it holds simulations from a different " + f"root. Appending would put two seed lineages in one file, and a " + f"manifest can describe only one. Pass the random_seed the checkpoint " + f"was started with, or leave random_seed out to continue from the root " + f"already recorded." ) @@ -209,10 +209,6 @@ class MonteCarlo: # pylint: disable=too-many-public-methods __draws_before_this_simulation = () __root_fingerprint = None __root_seed_given = False - # What the logs on disk actually hold, which only moves once a run has - # put rows from its own root into them. - __committed_fingerprint = None - __committed_seed_given = False def __init__( self, @@ -522,10 +518,21 @@ def __capture_root_state(self, random_seed, appending=False): materializing and pickling the full ``spawn(number_of_simulations)`` list to each process. + An append continues the root the logs already hold. Left alone it would + start a new one, and the file would end up with two lineages while its + manifest could name only the newer, so a different root is refused and + no ``random_seed`` at all means continue rather than begin. + Deep-copied, because ``SeedSequence`` keeps a sequence entropy by reference. Without it a caller who mutates the list they passed changes the children this run derives, which is the opposite of a snapshot. """ + recorded = _recorded_root_state(self.output_file) if appending else None + if recorded is not None and random_seed is None: + self.__root_state, self.__root_seed_given = recorded + self.__root_fingerprint = _fingerprint_of_state(self.__root_state) + return + root = self.__root_seed_sequence(random_seed) self.__root_state = ( deepcopy(root.entropy), @@ -536,19 +543,13 @@ def __capture_root_state(self, random_seed, appending=False): # The state above rebuilds children and cannot be compared; the # fingerprint compares and cannot rebuild. Both, rather than one. self.__root_fingerprint = _seed_root_fingerprint(root) - # Whether a caller chose this root or it came from fresh entropy. Only a - # chosen one is a lineage there is any point in continuing. + # Whether a caller chose this root or it came from fresh entropy. self.__root_seed_given = random_seed is not None - if appending: - # The manifest first: it outlives this object, which the attributes - # below do not, and it describes the rows actually in the log. - recorded = _fingerprint_from_manifest(self.output_file) - previous, previous_chosen = recorded or ( - self.__committed_fingerprint, - self.__committed_seed_given, - ) - _warn_when_appending_leaves_the_lineage( - previous, previous_chosen, self.__root_fingerprint + if recorded is not None: + _refuse_a_root_that_leaves_the_lineage( + _fingerprint_of_state(recorded[0]), + self.__root_fingerprint, + self.output_file, ) def __check_this_append_can_continue(self, number_of_simulations): @@ -575,14 +576,9 @@ def __commit_root_lineage(self): """Record that the logs now hold rows derived from this run's root. Kept apart from capturing it, because the capture runs before anything - opens a file. A run that warns and then adds nothing, or that dies before - its first row, must leave the logs owned by the root already in them, or - the append after it compares against a lineage that was never written. - Object-local, so a rebuilt ``MonteCarlo`` starts blank; carrying it in - the files is #1075. - """ - self.__committed_fingerprint = self.__root_fingerprint - self.__committed_seed_given = self.__root_seed_given + opens a file. A run that adds nothing, or dies before its first row, + must leave the manifest describing the root already in the logs. + """ _write_run_manifest(self.output_file, self.__root_state, self.__root_seed_given) def __child_seed(self, sim_idx): @@ -2218,9 +2214,11 @@ def _jsonable_entropy(entropy): def _write_run_manifest(output_file, root_state, seed_chosen): """Record the scheme and the root the rows in this log came from. - Best effort on the way out: a manifest that cannot be written is worth a - warning, not the loss of a finished run. The next append refuses without it, - which is the safe direction. + Staged beside the manifest and moved onto it, because this is what decides + whether a later run may continue the checkpoint at all: a torn write would + leave a document that parses as absent or as another generation. Failing to + write it is still only a warning, since the alternative is losing a run that + finished, and the next append refuses without it either way. """ entropy, spawn_key, pool_size, base = root_state document = { @@ -2236,13 +2234,25 @@ def _write_run_manifest(output_file, root_state, seed_chosen): }, } _best_effort( - lambda: _manifest_path(output_file).write_text( - json.dumps(document, indent=2) + "\n", encoding="utf-8" - ), + lambda: _install_manifest(_manifest_path(output_file), document), "run manifest", ) +def _install_manifest(destination, document): + """Put a complete manifest in place, or leave the previous one alone.""" + staged = f"{destination}.partial" + try: + with open(staged, "w", encoding="utf-8") as handle: + handle.write(json.dumps(document, indent=2) + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(staged, destination) + except BaseException: + _best_effort(lambda: os.remove(staged), "staged manifest cleanup") + raise + + def _read_run_manifest(output_file): """The manifest beside a log, or ``None`` when there is not a usable one.""" path = _manifest_path(output_file) @@ -2253,36 +2263,66 @@ def _read_run_manifest(output_file): return document if isinstance(document, dict) else None -def _fingerprint_from_manifest(output_file): - """``(fingerprint, whether the seed was chosen)`` for the log's own root. +def _fingerprint_of_state(root_state): + """The comparable identity of a stored ``(entropy, spawn_key, pool, base)``.""" + entropy, spawn_key, pool_size, base = root_state + root = np.random.SeedSequence( + entropy=entropy, spawn_key=tuple(spawn_key), pool_size=pool_size + ) + return ( + tuple(int(word) for word in root.generate_state(4, dtype=np.uint32)), + tuple(int(key) for key in spawn_key), + int(pool_size), + int(base), + ) + + +def _root_state_from_manifest(document, output_file): + """The root a manifest describes, or a refusal naming what is wrong with it. - The object that wrote the rows is usually gone by the time an append runs, - so this is what makes the lineage check survive a fresh interpreter. - ``None`` when there is no manifest or it does not describe a root. + Every field is checked here rather than where it is used. The manifest is + what decides whether a checkpoint may be continued at all, so one that + parses but does not describe a root has to stop the run rather than leave + the append with nothing to compare against. """ - manifest = _read_run_manifest(output_file) - if manifest is None: - return None - recorded = manifest.get("root_state") + + def refuse(what): + raise ValueError( + f"cannot append to {output_file}: its manifest {what}. Re-run the " + f"study to start a checkpoint this release can continue." + ) + + if document.get("log_format") != "jsonl-v1": + refuse(f"names log format {document.get('log_format')!r}, not 'jsonl-v1'") + if not isinstance(document.get("seed_chosen"), bool): + refuse("has a seed_chosen that is not true or false") + recorded = document.get("root_state") if not isinstance(recorded, dict): - return None + refuse("carries no root_state") try: - # Rebuilt for its words only. n_children_spawned is read-only on a - # SeedSequence, so it is carried across as the recorded number instead. - root = np.random.SeedSequence( - entropy=recorded["entropy"], - spawn_key=tuple(recorded["spawn_key"]), - pool_size=recorded["pool_size"], - ) - fingerprint = ( - tuple(int(word) for word in root.generate_state(4, dtype=np.uint32)), - tuple(int(key) for key in recorded["spawn_key"]), + state = ( + recorded["entropy"], + tuple(recorded["spawn_key"]), int(recorded["pool_size"]), int(recorded["n_children_spawned"]), ) + _fingerprint_of_state(state) except (KeyError, TypeError, ValueError): + refuse("carries a root_state that cannot be rebuilt") + return state, document["seed_chosen"] + + +def _recorded_root_state(output_file): + """``(root state, whether it was chosen)`` for a log, or ``None``. + + ``None`` only when there is no manifest at all. One that is there but does + not describe a root raises from the validation instead of reading as absent, + so an append never falls back to starting a lineage of its own. + """ + document = _read_run_manifest(output_file) + if document is None: return None - return fingerprint, bool(manifest.get("seed_chosen", False)) + return _root_state_from_manifest(document, output_file) def _refuse_a_checkpoint_from_another_scheme(output_file, resume_at): @@ -2314,6 +2354,7 @@ def _refuse_a_checkpoint_from_another_scheme(output_file, resume_at): f"{_SAMPLING_SCHEME!r} at {_MANIFEST_SCHEMA_VERSION}. Re-run the " f"study rather than continuing one scheme with another." ) + _root_state_from_manifest(manifest, output_file) def _check_the_checkpoint_supports_appending(input_file, output_file, resume_at): diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index e3c19df3e..1312bd234 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -376,46 +376,52 @@ def test_mutating_the_caller_s_entropy_does_not_move_the_captured_root(wrapped): @pytest.mark.parametrize( - "first_seed, second_seed, expect_warning", + "first_seed, second_seed, refused", [ - (42, None, True), # the documented case: seeded run, unseeded append - (42, 7, True), # a different chosen root is the same mixing + (42, 7, True), # a different chosen root would mix two lineages (42, 42, False), # continuing the same run - (None, None, False), # nothing was being preserved - # numpy.random.SeedSequence takes array_like[ints], so an ndarray is a - # seed this accepts. Comparing two of those elementwise answers with an - # array, which is not a verdict, and used to raise here. - (SEED_ARRAY, SEED_ARRAY, False), - (SEED_ARRAY, np.array([1, 2, 3], dtype=np.uint32), False), (SEED_ARRAY, np.array([9, 9, 9], dtype=np.uint32), True), - # Same seed, different container: one lineage, so no warning. + (SEED_ARRAY, np.array([1, 2, 3], dtype=np.uint32), False), + # numpy.random.SeedSequence takes array_like[ints], so comparing two + # roots elementwise answers with an array rather than a verdict and used + # to raise here. [1, 2, 3] and (1, 2, 3) are one seed, not two. ([1, 2, 3], (1, 2, 3), False), (np.random.SeedSequence(7), 7, False), ], ) -def test_appending_says_so_when_it_leaves_the_seed_lineage( - first_seed, second_seed, expect_warning, tmp_path +def test_appending_from_another_root_is_refused( + first_seed, second_seed, refused, tmp_path ): - """Mixing two lineages in one file is invisible to every structural check. + """One manifest holds one root, so a file cannot be allowed to hold two. - The rows stay valid JSON, the indices stay unique and contiguous, and the - two files stay in step, so only the roots themselves can tell (#1075). + Warning and appending anyway left the file mixed while the manifest named + only the newer half, which is a worse answer than refusing: the provenance + was then wrong rather than incomplete (#1075). """ analysis = object.__new__(MonteCarlo) - # Committing writes the manifest beside this, and the append below reads it. analysis._output_file = str(tmp_path / "run.outputs.txt") analysis._MonteCarlo__capture_root_state(first_seed) - # What simulate does once the first run has put rows in the logs. analysis._MonteCarlo__commit_root_lineage() - with warnings.catch_warnings(record=True) as raised: - warnings.simplefilter("always") + if refused: + with pytest.raises(ValueError, match="different root"): + analysis._MonteCarlo__capture_root_state(second_seed, appending=True) + else: analysis._MonteCarlo__capture_root_state(second_seed, appending=True) - lineage_warnings = [w for w in raised if "seed lineage" in str(w.message)] - assert bool(lineage_warnings) is expect_warning - if expect_warning: - assert issubclass(lineage_warnings[0].category, RuntimeWarning) + +def test_appending_without_a_seed_continues_the_recorded_root(tmp_path): + """No random_seed on an append means continue, not start something new.""" + analysis = object.__new__(MonteCarlo) + analysis._output_file = str(tmp_path / "run.outputs.txt") + analysis._MonteCarlo__capture_root_state(42) + analysis._MonteCarlo__commit_root_lineage() + started_with = analysis._MonteCarlo__root_fingerprint + + analysis._MonteCarlo__capture_root_state(None, appending=True) + + assert analysis._MonteCarlo__root_fingerprint == started_with + assert analysis._MonteCarlo__root_seed_given is True def test_a_root_that_derives_the_same_children_fingerprints_the_same(): @@ -440,17 +446,15 @@ def test_a_root_that_derives_the_same_children_fingerprints_the_same(): assert _seed_root_fingerprint(np.random.SeedSequence(43)) != before -def test_a_fresh_object_has_no_lineage_to_leave(): - """A first run cannot be leaving anything, however it is seeded.""" - with warnings.catch_warnings(record=True) as raised: - warnings.simplefilter("always") - _warn_when_appending_leaves_the_lineage( - MonteCarlo._MonteCarlo__root_state, - MonteCarlo._MonteCarlo__root_seed_given, - ("entropy", (), 4, 0), - ) +def test_a_first_run_has_no_recorded_root_to_continue(tmp_path): + """With no manifest beside the log there is nothing to continue or refuse.""" + analysis = object.__new__(MonteCarlo) + analysis._output_file = str(tmp_path / "run.outputs.txt") + + analysis._MonteCarlo__capture_root_state(42, appending=True) - assert not [w for w in raised if "seed lineage" in str(w.message)] + assert analysis._MonteCarlo__root_fingerprint is not None + assert analysis._MonteCarlo__root_seed_given is True @pytest.mark.parametrize("keyword", ["batch_size", "max_simulations", "tolerance"]) @@ -503,31 +507,31 @@ def run(total, seed, append): return run -def test_a_run_that_adds_nothing_does_not_take_the_files_lineage(monkeypatch, tmp_path): - """The warning has to survive an append that wrote no rows. +def test_a_run_that_adds_nothing_leaves_the_manifest_alone(monkeypatch, tmp_path): + """An append that reaches its target without running anything changes nothing. ``simulate`` captures the root before it touches a file, so the object used - to believe the new lineage had landed even when the run added nothing. The - append after that one silently put rows from the second root behind rows - from the first, which is exactly the case the warning exists for (#1075). + to believe a new lineage had landed even when the run added no rows. """ - run = _simulate_without_flying(monkeypatch, object.__new__(MonteCarlo), tmp_path) + analysis = object.__new__(MonteCarlo) + run = _simulate_without_flying(monkeypatch, analysis, tmp_path) + + run(2, 42, False) + recorded = mc_module._read_run_manifest(analysis.output_file) + run(2, 42, True) # target already reached, so nothing is added - assert run(2, 42, False) is False # first run, nothing to leave - assert run(2, 7, True) is True # warns, and adds no rows - assert run(4, 7, True) is True # the one that actually appends seed 7 - assert run(6, 7, True) is False # now genuinely the same lineage + assert mc_module._read_run_manifest(analysis.output_file) == recorded -def test_a_refused_append_leaves_the_lineage_where_it_was(monkeypatch, tmp_path): - """A warning promoted to an error must not still move the tracker.""" +def test_a_refused_append_leaves_the_manifest_where_it_was(monkeypatch, tmp_path): + """The refusal has to come before anything on disk moves.""" analysis = object.__new__(MonteCarlo) run = _simulate_without_flying(monkeypatch, analysis, tmp_path) run(2, 42, False) + recorded = mc_module._read_run_manifest(analysis.output_file) - with warnings.catch_warnings(): - warnings.simplefilter("error", RuntimeWarning) - with pytest.raises(RuntimeWarning): - analysis.simulate(4, append=True, random_seed=7) + with pytest.raises(ValueError, match="different root"): + analysis.simulate(4, append=True, random_seed=7) - assert run(4, 7, True) is True, "the files still hold the first lineage" + assert mc_module._read_run_manifest(analysis.output_file) == recorded + assert recorded["root_state"]["entropy"] == 42 diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 1f4064123..357f8f30d 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -732,11 +732,14 @@ def test_the_lineage_outlives_the_object_that_wrote_it(tmp_path): second = object.__new__(MonteCarlo) # nothing carried over in memory second._output_file = output - with warnings.catch_warnings(record=True) as raised: - warnings.simplefilter("always") + with pytest.raises(ValueError, match="different root"): second._MonteCarlo__capture_root_state(7, appending=True) - assert [w for w in raised if "seed lineage" in str(w.message)] + third = object.__new__(MonteCarlo) + third._output_file = output + third._MonteCarlo__capture_root_state(None, appending=True) + + assert third._MonteCarlo__root_fingerprint == first._MonteCarlo__root_fingerprint def _three_logs_on_disk(tmp_path, mode=0o644): @@ -898,3 +901,53 @@ def test_seeding_marks_what_each_model_was_already_holding(): model.last_rnd_dict for model in (analysis.environment, analysis.rocket, analysis.flight) ] == held + + +@pytest.mark.parametrize( + "broken, complaint", + [ + ({"log_format": "csv-v1"}, "log format"), + ({"seed_chosen": "false"}, "seed_chosen"), + ({"root_state": None}, "no root_state"), + ({"root_state": {"entropy": 1}}, "cannot be rebuilt"), + ], +) +def test_a_manifest_that_does_not_describe_a_root_is_refused( + tmp_path, broken, complaint +): + """Scheme and version alone were not enough to trust a checkpoint. + + A document naming the right scheme but carrying no usable root passed, and + the append then went ahead with nothing to compare its own root against. + ``bool("false")`` is True, so a string there read as a chosen seed. + """ + inputs, outputs = _old_parallel_checkpoint(tmp_path) + mc._write_run_manifest(str(outputs), (42, (), 4, 0), True) + path = mc._manifest_path(str(outputs)) + document = json.loads(path.read_text(encoding="utf-8")) + document.update(broken) + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(ValueError, match=complaint): + mc._check_the_checkpoint_supports_appending(str(inputs), str(outputs), 2) + + +def test_a_failed_manifest_write_leaves_the_previous_one(tmp_path, monkeypatch): + """The manifest gates appends now, so a torn write cannot be left behind.""" + output = str(tmp_path / "run.outputs.txt") + mc._write_run_manifest(output, (42, (), 4, 0), True) + before = mc._manifest_path(output).read_bytes() + + real_replace = os.replace + + def fail(source, destination): + if str(destination).endswith(".manifest.json"): + raise OSError(13, "injected") + return real_replace(source, destination) + + monkeypatch.setattr(os, "replace", fail) + with pytest.warns(RuntimeWarning, match="run manifest"): + mc._write_run_manifest(output, (7, (), 4, 0), True) + + assert mc._manifest_path(output).read_bytes() == before + assert not list(tmp_path.glob("*.partial")) From 81d3e5846a0a439e008f2807fe8f6a350321004e Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 15:50:54 +0800 Subject: [PATCH 44/45] BUG: declare eccentricities so the reseeded draw still reaches them #1122 made dict_generator walk the declared stochastic inputs rather than the whole instance, which is the right fix for #1109 and makes the collection-skip this branch carried redundant. add_cp_eccentricity and add_thrust_eccentricity run after __init__ has already built that list, so their values stopped being sampled: four eccentricities became none. They are declared as they are validated now. The component_collections mechanism is gone, since walking the declared inputs never saw the collections in the first place. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/stochastic/stochastic_model.py | 9 +++++++++ rocketpy/stochastic/stochastic_rocket.py | 4 ++++ tests/unit/simulation/test_monte_carlo_determinism.py | 1 - tests/unit/simulation/test_monte_carlo_log_integrity.py | 1 - 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/rocketpy/stochastic/stochastic_model.py b/rocketpy/stochastic/stochastic_model.py index 097f5ac7b..625452e65 100644 --- a/rocketpy/stochastic/stochastic_model.py +++ b/rocketpy/stochastic/stochastic_model.py @@ -165,6 +165,15 @@ def _nominal(self, input_name, getter=getattr): ) return self.__nominal_values[input_name] + def _declare_stochastic_input(self, input_name): + """Add an input that arrives after ``__init__`` to the declared set. + + ``dict_generator`` walks the declared inputs rather than the instance + (#1122), so anything added later, such as an eccentricity, has to say so + or it is never sampled. + """ + self.__stochastic_dict.setdefault(input_name, None) + def _set_stochastic(self, seed=None): """Set the stochastic attributes from the input dictionary. This method is useful to reset or reseed the attributes of the instance. diff --git a/rocketpy/stochastic/stochastic_rocket.py b/rocketpy/stochastic/stochastic_rocket.py index d6ed88e22..44d3db462 100644 --- a/rocketpy/stochastic/stochastic_rocket.py +++ b/rocketpy/stochastic/stochastic_rocket.py @@ -533,6 +533,10 @@ def __apply_eccentricity_specs(self): """ for name, spec in self.__eccentricity_specs.items(): setattr(self, name, self._validate_eccentricity(name, spec)) + # Declared as well as set: dict_generator walks the declared inputs + # rather than the whole instance (#1122), and these arrive through + # add_cp_eccentricity after __init__ has already built that list. + self._declare_stochastic_input(name) def _validate_eccentricity(self, eccentricity, position): """Validate the eccentricity argument. diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index 1312bd234..bdf6d476f 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -42,7 +42,6 @@ _seed_sequence_to_int, _SimMonitor, _validate_simulation_count, - _warn_when_appending_leaves_the_lineage, ) _root_seed_sequence = MonteCarlo._MonteCarlo__root_seed_sequence diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 357f8f30d..593df1744 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -15,7 +15,6 @@ import tempfile import threading import types -import warnings import numpy as np import pytest From cf99e87239e9103b320016a3278580a897a1bda7 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:20:23 +0800 Subject: [PATCH 45/45] BUG: record what a run committed, and which logs it committed it to The manifest moved on the number of simulations asked for. A Ctrl-C before the first new row still moved it, since the target was above the checkpoint, and a target that was never reached claimed rows that are not there. A run now opens a generation once its logs exist, so a fresh set belongs to its root even if the first row is never written, and an append stays in the generation it continues. The count is taken from the rows themselves, after the completeness check rather than before it, and best effort: the run has finished and the rows are on disk, so a count that cannot be taken leaves the previous one for the next append to refuse on. The manifest also says which run and which logs it describes. run_id, committed_count and the two log names are recorded and required, so a manifest that survives beside a different pair, or one hand-edited into something that parses, is refused rather than trusted. Signed-off-by: thc1006 <84045975+thc1006@users.noreply.github.com> --- rocketpy/simulation/monte_carlo.py | 82 ++++++++++++----- .../test_monte_carlo_determinism.py | 8 +- .../test_monte_carlo_log_integrity.py | 87 +++++++++++++++++-- 3 files changed, 149 insertions(+), 28 deletions(-) diff --git a/rocketpy/simulation/monte_carlo.py b/rocketpy/simulation/monte_carlo.py index b0d3a0e94..be33102dc 100644 --- a/rocketpy/simulation/monte_carlo.py +++ b/rocketpy/simulation/monte_carlo.py @@ -18,6 +18,7 @@ import os import tempfile import traceback +import uuid import warnings from contextlib import contextmanager from copy import deepcopy @@ -207,6 +208,7 @@ class MonteCarlo: # pylint: disable=too-many-public-methods # without __init__ still answers. __root_state = None __draws_before_this_simulation = () + __generation = None __root_fingerprint = None __root_seed_given = False @@ -431,19 +433,22 @@ def simulate( print("Starting Monte Carlo analysis") self.__setup_files(append) - if not append: - # Emptied just now, so whatever they held is gone and this root - # owns them even if the run below never reaches its first row. - self.__commit_root_lineage() + # Emptied just now for a fresh run, so a new generation starts here and + # this root owns the logs even if the run never reaches its first row. + # An append stays in the generation it is continuing. + self.__start_a_generation( + self.__generation["run_id"] if append else uuid.uuid4().hex + ) if parallel: self.__run_in_parallel(n_workers) else: self.__run_in_serial() - if number_of_simulations > self._initial_sim_idx: - self.__commit_root_lineage() self.__check_each_index_was_recorded_once() + # After the check, and from the rows themselves: a target that was not + # reached must not be recorded as though it had been. + self.__record_what_was_committed() self.__terminate_simulation() def __setup_files(self, append): @@ -572,14 +577,43 @@ def __check_this_append_can_continue(self, number_of_simulations): f"requested {number_of_simulations}." ) - def __commit_root_lineage(self): - """Record that the logs now hold rows derived from this run's root. + def __start_a_generation(self, run_id): + """Say which root the logs that were just emptied belong to. + + Written here rather than at the end of the run: ``__setup_files`` has + already replaced them, so they belong to this root whether or not the + run reaches its first row. The count starts at zero and is corrected + once the rows on disk have been checked. + """ + self.__generation = { + "run_id": run_id, + "root_state": self.__root_state, + "seed_chosen": self.__root_seed_given, + "committed_count": 0, + "input_file": self.input_file, + "output_file": self.output_file, + } + _write_run_manifest(self.output_file, self.__generation) + + def __record_what_was_committed(self): + """Bring the manifest up to the rows that are actually on disk. - Kept apart from capturing it, because the capture runs before anything - opens a file. A run that adds nothing, or dies before its first row, - must leave the manifest describing the root already in the logs. + Counted from the logs after the completeness check rather than from the + number of simulations asked for: an interrupt before the first new row + would otherwise move the metadata without moving the logs, and a target + that was never reached would claim rows that do not exist. + + Best effort, like the write itself. The run has finished and its rows + are on disk; a count that cannot be taken leaves the previous one, + which the next append then refuses on rather than trusts. """ - _write_run_manifest(self.output_file, self.__root_state, self.__root_seed_given) + + def count_them(): + written, _ = _recorded_indices("outputs", self.output_file) + self.__generation["committed_count"] = len(written) + + _best_effort(count_them, "committed count") + _write_run_manifest(self.output_file, self.__generation) def __child_seed(self, sim_idx): """Return the seed sequence for a single simulation index. @@ -2211,21 +2245,25 @@ def _jsonable_entropy(entropy): return [int(part) for part in np.asarray(entropy).ravel()] -def _write_run_manifest(output_file, root_state, seed_chosen): - """Record the scheme and the root the rows in this log came from. +def _write_run_manifest(output_file, generation): + """Record which generation of logs these rows belong to. - Staged beside the manifest and moved onto it, because this is what decides - whether a later run may continue the checkpoint at all: a torn write would - leave a document that parses as absent or as another generation. Failing to - write it is still only a warning, since the alternative is losing a run that + Staged beside the manifest and moved onto it, because this decides whether a + later run may continue the checkpoint at all: a torn write would leave a + document that reads as absent or as another generation. Failing to write it + is still only a warning, since the alternative is losing a run that finished, and the next append refuses without it either way. """ - entropy, spawn_key, pool_size, base = root_state + entropy, spawn_key, pool_size, base = generation["root_state"] document = { "schema_version": _MANIFEST_SCHEMA_VERSION, "sampling_scheme": _SAMPLING_SCHEME, "log_format": "jsonl-v1", - "seed_chosen": bool(seed_chosen), + "run_id": generation["run_id"], + "committed_count": int(generation["committed_count"]), + "input_log": Path(generation["input_file"]).name, + "output_log": Path(generation["output_file"]).name, + "seed_chosen": bool(generation["seed_chosen"]), "root_state": { "entropy": _jsonable_entropy(entropy), "spawn_key": [int(key) for key in spawn_key], @@ -2296,6 +2334,10 @@ def refuse(what): refuse(f"names log format {document.get('log_format')!r}, not 'jsonl-v1'") if not isinstance(document.get("seed_chosen"), bool): refuse("has a seed_chosen that is not true or false") + if not isinstance(document.get("run_id"), str) or not document["run_id"]: + refuse("carries no run_id") + if not _is_whole_number(document.get("committed_count")): + refuse("has a committed_count that is not a whole number") recorded = document.get("root_state") if not isinstance(recorded, dict): refuse("carries no root_state") diff --git a/tests/unit/simulation/test_monte_carlo_determinism.py b/tests/unit/simulation/test_monte_carlo_determinism.py index bdf6d476f..6fe4747c0 100644 --- a/tests/unit/simulation/test_monte_carlo_determinism.py +++ b/tests/unit/simulation/test_monte_carlo_determinism.py @@ -399,8 +399,9 @@ def test_appending_from_another_root_is_refused( """ analysis = object.__new__(MonteCarlo) analysis._output_file = str(tmp_path / "run.outputs.txt") + analysis._input_file = str(tmp_path / "run.inputs.txt") analysis._MonteCarlo__capture_root_state(first_seed) - analysis._MonteCarlo__commit_root_lineage() + analysis._MonteCarlo__start_a_generation("0123456789abcdef") if refused: with pytest.raises(ValueError, match="different root"): @@ -413,8 +414,9 @@ def test_appending_without_a_seed_continues_the_recorded_root(tmp_path): """No random_seed on an append means continue, not start something new.""" analysis = object.__new__(MonteCarlo) analysis._output_file = str(tmp_path / "run.outputs.txt") + analysis._input_file = str(tmp_path / "run.inputs.txt") analysis._MonteCarlo__capture_root_state(42) - analysis._MonteCarlo__commit_root_lineage() + analysis._MonteCarlo__start_a_generation("0123456789abcdef") started_with = analysis._MonteCarlo__root_fingerprint analysis._MonteCarlo__capture_root_state(None, appending=True) @@ -449,6 +451,7 @@ def test_a_first_run_has_no_recorded_root_to_continue(tmp_path): """With no manifest beside the log there is nothing to continue or refuse.""" analysis = object.__new__(MonteCarlo) analysis._output_file = str(tmp_path / "run.outputs.txt") + analysis._input_file = str(tmp_path / "run.inputs.txt") analysis._MonteCarlo__capture_root_state(42, appending=True) @@ -484,6 +487,7 @@ def _simulate_without_flying(monkeypatch, analysis, tmp_path): # beside the output log, and fixed names put it in the repository root. analysis._input_file = str(tmp_path / "run.inputs.txt") analysis._output_file = str(tmp_path / "run.outputs.txt") + analysis._input_file = str(tmp_path / "run.inputs.txt") analysis._error_file = str(tmp_path / "run.errors.txt") for name in ( "_MonteCarlo__setup_files", diff --git a/tests/unit/simulation/test_monte_carlo_log_integrity.py b/tests/unit/simulation/test_monte_carlo_log_integrity.py index 593df1744..40bc683bd 100644 --- a/tests/unit/simulation/test_monte_carlo_log_integrity.py +++ b/tests/unit/simulation/test_monte_carlo_log_integrity.py @@ -12,6 +12,7 @@ import json import os +import pathlib import tempfile import threading import types @@ -662,6 +663,18 @@ def test_nothing_in_flight_writes_no_row_at_all(): assert mc._build_unfinished_record(None, "cancelled") == "" +def _generation(output, root=(42, (), 4, 0), count=0, inputs=None): + """The record a run keeps about the logs it owns.""" + return { + "run_id": "0123456789abcdef", + "root_state": root, + "seed_chosen": True, + "committed_count": count, + "input_file": inputs or str(pathlib.Path(output).with_name("run.inputs.txt")), + "output_file": output, + } + + def _old_parallel_checkpoint(tmp_path, rows=2): """Logs shaped exactly like a clean run from before per-index seeding. @@ -691,7 +704,7 @@ def test_a_checkpoint_from_the_old_parallel_scheme_is_refused(tmp_path): def test_a_checkpoint_this_release_wrote_is_accepted(tmp_path): """The same rows, with the manifest beside them, continue normally.""" inputs, outputs = _old_parallel_checkpoint(tmp_path) - mc._write_run_manifest(str(outputs), (42, (), 4, 0), True) + mc._write_run_manifest(str(outputs), _generation(str(outputs), root=(42, (), 4, 0))) mc._check_the_checkpoint_supports_appending(str(inputs), str(outputs), 2) @@ -699,7 +712,7 @@ def test_a_checkpoint_this_release_wrote_is_accepted(tmp_path): def test_a_manifest_from_another_scheme_is_refused(tmp_path): """A future or foreign scheme is named rather than guessed at.""" inputs, outputs = _old_parallel_checkpoint(tmp_path) - mc._write_run_manifest(str(outputs), (42, (), 4, 0), True) + mc._write_run_manifest(str(outputs), _generation(str(outputs), root=(42, (), 4, 0))) path = mc._manifest_path(str(outputs)) document = json.loads(path.read_text(encoding="utf-8")) document["sampling_scheme"] = "something-else-v9" @@ -726,16 +739,19 @@ def test_the_lineage_outlives_the_object_that_wrote_it(tmp_path): first = object.__new__(MonteCarlo) first._output_file = output + first._input_file = str(pathlib.Path(output).with_name("run.inputs.txt")) first._MonteCarlo__capture_root_state(42) - first._MonteCarlo__commit_root_lineage() + first._MonteCarlo__start_a_generation("0123456789abcdef") second = object.__new__(MonteCarlo) # nothing carried over in memory second._output_file = output + second._input_file = str(pathlib.Path(output).with_name("run.inputs.txt")) with pytest.raises(ValueError, match="different root"): second._MonteCarlo__capture_root_state(7, appending=True) third = object.__new__(MonteCarlo) third._output_file = output + third._input_file = str(pathlib.Path(output).with_name("run.inputs.txt")) third._MonteCarlo__capture_root_state(None, appending=True) assert third._MonteCarlo__root_fingerprint == first._MonteCarlo__root_fingerprint @@ -921,7 +937,7 @@ def test_a_manifest_that_does_not_describe_a_root_is_refused( ``bool("false")`` is True, so a string there read as a chosen seed. """ inputs, outputs = _old_parallel_checkpoint(tmp_path) - mc._write_run_manifest(str(outputs), (42, (), 4, 0), True) + mc._write_run_manifest(str(outputs), _generation(str(outputs), root=(42, (), 4, 0))) path = mc._manifest_path(str(outputs)) document = json.loads(path.read_text(encoding="utf-8")) document.update(broken) @@ -934,7 +950,7 @@ def test_a_manifest_that_does_not_describe_a_root_is_refused( def test_a_failed_manifest_write_leaves_the_previous_one(tmp_path, monkeypatch): """The manifest gates appends now, so a torn write cannot be left behind.""" output = str(tmp_path / "run.outputs.txt") - mc._write_run_manifest(output, (42, (), 4, 0), True) + mc._write_run_manifest(output, _generation(output, root=(42, (), 4, 0))) before = mc._manifest_path(output).read_bytes() real_replace = os.replace @@ -946,7 +962,66 @@ def fail(source, destination): monkeypatch.setattr(os, "replace", fail) with pytest.warns(RuntimeWarning, match="run manifest"): - mc._write_run_manifest(output, (7, (), 4, 0), True) + mc._write_run_manifest(output, _generation(output, root=(7, (), 4, 0))) assert mc._manifest_path(output).read_bytes() == before assert not list(tmp_path.glob("*.partial")) + + +def test_the_manifest_counts_the_rows_that_are_there(tmp_path): + """The count comes from the logs, not from the number asked for. + + An interrupt before the first new row used to move the metadata without + moving the logs, and a target that was never reached claimed rows that do + not exist. + """ + _, outputs = _old_parallel_checkpoint(tmp_path, rows=3) + analysis = object.__new__(MonteCarlo) + analysis._output_file = str(outputs) + analysis._input_file = str(tmp_path / "run.inputs.txt") + analysis._MonteCarlo__generation = _generation(str(outputs), count=99) + + analysis._MonteCarlo__record_what_was_committed() + + recorded = mc._read_run_manifest(str(outputs)) + assert recorded["committed_count"] == 3, "the target, not the rows, was recorded" + + +def test_a_count_that_cannot_be_taken_leaves_the_previous_one(tmp_path): + """Bookkeeping after a finished run must not fail the run.""" + output = str(tmp_path / "gone.outputs.txt") + analysis = object.__new__(MonteCarlo) + analysis._output_file = output + analysis._input_file = str(tmp_path / "run.inputs.txt") + analysis._MonteCarlo__generation = _generation(output, count=7) + + with pytest.warns(RuntimeWarning, match="committed count"): + analysis._MonteCarlo__record_what_was_committed() + + assert mc._read_run_manifest(output)["committed_count"] == 7 + + +@pytest.mark.parametrize("missing", ["run_id", "committed_count"]) +def test_a_manifest_without_an_identity_is_refused(tmp_path, missing): + """A manifest has to say which run and how many rows it describes.""" + inputs, outputs = _old_parallel_checkpoint(tmp_path) + mc._write_run_manifest(str(outputs), _generation(str(outputs), count=2)) + path = mc._manifest_path(str(outputs)) + document = json.loads(path.read_text(encoding="utf-8")) + del document[missing] + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(ValueError, match=missing.replace("_", "[_ ]")): + mc._check_the_checkpoint_supports_appending(str(inputs), str(outputs), 2) + + +def test_the_manifest_names_the_logs_it_was_written_for(tmp_path): + """Pairing one run's inputs with another's outputs has to be visible.""" + _, outputs = _old_parallel_checkpoint(tmp_path) + mc._write_run_manifest(str(outputs), _generation(str(outputs), count=2)) + + recorded = mc._read_run_manifest(str(outputs)) + + assert recorded["output_log"] == "run.outputs.txt" + assert recorded["input_log"] == "run.inputs.txt" + assert recorded["run_id"]