diff --git a/src/ensemble/ensemble.py b/src/ensemble/ensemble.py index b1488d3..356cd66 100644 --- a/src/ensemble/ensemble.py +++ b/src/ensemble/ensemble.py @@ -277,7 +277,7 @@ def calc_prediction(self, enX=None, save_prediction=None): # Parallelization on HPC using SLURM elif self.sim.input_dict.get('hpc', False): # Run prediction in parallel on hpc - en_pred = self.run_on_HPC(enX, batch_size=nparallel) + en_pred = self.run_on_HPC(enX, batch_size=nparallel, save_prediction=save_prediction) # Parallelization on local machine using p_map else: @@ -412,6 +412,11 @@ def run_on_HPC(self, enX, batch_size=None, **kwargs): else: en_pred.append(False) self.sim.remove_folder(member_i) + + # Store partial batch results if needed + part_pred_data = dtools.en_pred_to_pred_data(en_pred) + if 'save_prediction' in kwargs and kwargs['save_prediction']: + np.savez(kwargs['save_prediction'] +'.npz', **{'pred_data': part_pred_data}) return en_pred diff --git a/src/popt/loop/ensemble_base.py b/src/popt/loop/ensemble_base.py index d0d3881..3415b4b 100644 --- a/src/popt/loop/ensemble_base.py +++ b/src/popt/loop/ensemble_base.py @@ -139,7 +139,7 @@ def function(self, x, *args, **kwargs): else: func_values = np.inf # the simulations have crashed - if len(x.shape) == 1: + if len(x.shape) == 1 and func_values != np.inf: self.stateF = func_values else: self.enF = func_values diff --git a/src/popt/loop/optimize.py b/src/popt/loop/optimize.py index c04dfa0..dc6260f 100644 --- a/src/popt/loop/optimize.py +++ b/src/popt/loop/optimize.py @@ -191,7 +191,9 @@ def run_loop(self): self.logger(f'─────> EPF-EnOpt: maximum epf iterations reached') # print epf info break #p = np.abs(previous_state-self.xk) / (np.abs(previous_state) + 1.0e-9) - p = np.mean(self.epf['penalty'] / self.epf['r']) # the penalty term (without r) + if self.epf['penalty'].size == 0: + raise ValueError('EPF penalty is empty; cannot compute convergence criterion.') + p = np.mean(self.epf['penalty']) / self.epf['r'] # the penalty term (without r) conv_crit = self.epf['conv_crit'] if p > conv_crit: epf_not_converged = True @@ -208,7 +210,7 @@ def run_loop(self): r = self.epf['r'] self.logger(f'─────> EPF-EnOpt: {self.epf_iteration}, {r} (outer iteration, penalty factor)') # print epf info else: - self.logger(f'─────> EPF-EnOpt: converged, no variables changed more than {conv_crit*100} %') # print epf info + self.logger(f'─────> EPF-EnOpt: converged, penalty term larger than {conv_crit}') # print epf info final_obj_no_penalty = str( round( float( np.mean(self.fun(self.xk)) ),4) ) self.logger(f'─────> EPF-EnOpt: objective value without penalty = {final_obj_no_penalty}') # print epf info def save(self):