fix(gate): resolve issue #100

skillopt/config.py

@@ -125,6 +125,9 @@
     "evaluation.use_gate": "use_gate",
     "evaluation.gate_metric": "gate_metric",
     "evaluation.gate_mixed_weight": "gate_mixed_weight",
+    "evaluation.use_semantic_density": "use_semantic_density",
+    "evaluation.semantic_density_weight": "semantic_density_weight",
+    "evaluation.leading_words": "leading_words",
     "evaluation.sel_env_num": "sel_env_num",
     "evaluation.test_env_num": "test_env_num",
     "evaluation.eval_test": "eval_test",

skillopt/engine/trainer.py

@@ -709,26 +709,26 @@ def _build_eval_env(split: str, env_num: int, seed: int):
             effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")),
             max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384),
         )
-        configure_qwen_chat(
-            base_url=cfg.get("qwen_chat_base_url") or None,
-            api_key=cfg.get("qwen_chat_api_key") or None,
-            temperature=cfg.get("qwen_chat_temperature"),
-            timeout_seconds=cfg.get("qwen_chat_timeout_seconds"),
-            max_tokens=cfg.get("qwen_chat_max_tokens"),
-            enable_thinking=cfg.get("qwen_chat_enable_thinking"),
-            optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None,
-            optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None,
-            optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"),
-            optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"),
-            optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"),
-            optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"),
-            target_base_url=cfg.get("target_qwen_chat_base_url") or None,
-            target_api_key=cfg.get("target_qwen_chat_api_key") or None,
-            target_temperature=cfg.get("target_qwen_chat_temperature"),
-            target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"),
-            target_max_tokens=cfg.get("target_qwen_chat_max_tokens"),
-            target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"),
-        )
+        configure_qwen_chat(
+            base_url=cfg.get("qwen_chat_base_url") or None,
+            api_key=cfg.get("qwen_chat_api_key") or None,
+            temperature=cfg.get("qwen_chat_temperature"),
+            timeout_seconds=cfg.get("qwen_chat_timeout_seconds"),
+            max_tokens=cfg.get("qwen_chat_max_tokens"),
+            enable_thinking=cfg.get("qwen_chat_enable_thinking"),
+            optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None,
+            optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None,
+            optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"),
+            optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"),
+            optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"),
+            optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"),
+            target_base_url=cfg.get("target_qwen_chat_base_url") or None,
+            target_api_key=cfg.get("target_qwen_chat_api_key") or None,
+            target_temperature=cfg.get("target_qwen_chat_temperature"),
+            target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"),
+            target_max_tokens=cfg.get("target_qwen_chat_max_tokens"),
+            target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"),
+        )
         configure_minimax_chat(
             base_url=cfg.get("minimax_base_url") or None,
             api_key=cfg.get("minimax_api_key") or None,
@@ -964,6 +964,15 @@ def _persist_runtime_state(last_completed_step: int) -> None:
                 f"got {gate_metric!r}"
             )
         gate_mixed_weight = float(cfg.get("gate_mixed_weight", 0.5))
+        use_semantic_density = bool(cfg.get("use_semantic_density", False))
+        semantic_density_weight = float(cfg.get("semantic_density_weight", 0.05))
+        leading_words_raw = cfg.get("leading_words", None)
+        leading_words = None
+        if leading_words_raw is not None:
+            if isinstance(leading_words_raw, str):
+                leading_words = [w.strip() for w in leading_words_raw.split(",") if w.strip()]
+            else:
+                leading_words = list(leading_words_raw)
         if not 0.0 <= gate_mixed_weight <= 1.0:
             raise ValueError(
                 f"evaluation.gate_mixed_weight must be in [0, 1], "
@@ -1003,6 +1012,10 @@ def _persist_runtime_state(last_completed_step: int) -> None:
             baseline_hard, baseline_soft = compute_score(baseline_results)
             current_score = select_gate_score(
                 baseline_hard, baseline_soft, gate_metric, gate_mixed_weight,
+                skill_content=skill_init,
+                use_semantic_density=use_semantic_density,
+                semantic_density_weight=semantic_density_weight,
+                leading_words=leading_words,
             )
             best_score = current_score
             sh = skill_hash(skill_init)
@@ -1450,9 +1463,16 @@ def _persist_runtime_state(last_completed_step: int) -> None:
                     cand_soft=cand_soft,
                     metric=gate_metric,
                     mixed_weight=gate_mixed_weight,
+                    use_semantic_density=use_semantic_density,
+                    semantic_density_weight=semantic_density_weight,
+                    leading_words=leading_words,
                 ) if use_gate else None
                 cand_gate_score = select_gate_score(
                     cand_hard, cand_soft, gate_metric, gate_mixed_weight,
+                    skill_content=candidate_skill,
+                    use_semantic_density=use_semantic_density,
+                    semantic_density_weight=semantic_density_weight,
+                    leading_words=leading_words,
                 )
                 if not use_gate:
                     # Validation ran (scores recorded above) but the gate is
@@ -1844,6 +1864,9 @@ def _persist_runtime_state(last_completed_step: int) -> None:
                                 cand_soft=slow_sel_soft,
                                 metric=gate_metric,
                                 mixed_weight=gate_mixed_weight,
+                                use_semantic_density=use_semantic_density,
+                                semantic_density_weight=semantic_density_weight,
+                                leading_words=leading_words,
                             )
                             slow_result["selection_hard"] = slow_sel_hard
                             slow_result["selection_soft"] = slow_sel_soft
@@ -2093,6 +2116,10 @@ def _persist_runtime_state(last_completed_step: int) -> None:
                     final_gate_score = select_gate_score(
                         final_selection_hard, final_selection_soft,
                         gate_metric, gate_mixed_weight,
+                        skill_content=current_skill,
+                        use_semantic_density=use_semantic_density,
+                        semantic_density_weight=semantic_density_weight,
+                        leading_words=leading_words,
                     )
                     print(
                         f"\n  [final skill val] items={fval_n} "

skillopt/evaluation/gate.py

@@ -43,11 +43,55 @@ class GateResult:
     best_step: int
 
 
+def compute_semantic_density(
+    skill_content: str,
+    leading_words: list[str] | None = None,
+) -> float:
+    """Compute the semantic density of leading words in a skill document."""
+    if not skill_content or not skill_content.strip():
+        return 0.0
+    if leading_words is None:
+        leading_words = [
+            "MUST", "ALWAYS", "NEVER", "ONLY", "CRITICAL", "IMPORTANT",
+            "RESOLVE", "PREFER", "ENSURE", "STRICT", "VERIFY"
+        ]
+
+    # Strip metadata comments to focus purely on instruction text
+    skill = skill_content
+    for start, end in [
+        ("<!-- SLOW_UPDATE_START -->", "<!-- SLOW_UPDATE_END -->"),
+        ("<!-- APPENDIX_START -->", "<!-- APPENDIX_END -->")
+    ]:
+        while True:
+            s_idx = skill.find(start)
+            if s_idx == -1:
+                break
+            e_idx = skill.find(end, s_idx)
+            if e_idx == -1:
+                skill = skill[:s_idx] + skill[s_idx + len(start):]
+                break
+            skill = skill[:s_idx] + skill[e_idx + len(end):]
+
+    import re
+    words = re.findall(r'[a-zA-Z0-9]+', skill.lower())
+    if not words:
+        return 0.0
+
+    leading_set = {w.lower() for w in leading_words}
+    leading_count = sum(1 for w in words if w in leading_set)
+    return leading_count / len(words)
+
+
 def select_gate_score(
     hard: float,
     soft: float,
     metric: GateMetric = "hard",
     mixed_weight: float = 0.5,
+    *,
+    skill_content: str = "",
+    use_semantic_density: bool = False,
+    semantic_density_weight: float = 0.05,
+    leading_words: list[str] | None = None,
 ) -> float:
     """Project (hard, soft) onto a single comparison metric.
 
@@ -60,17 +104,32 @@ def select_gate_score(
     mixed_weight
         For ``"mixed"``: weight given to ``soft``. Must be in ``[0, 1]``.
         Ignored for ``"hard"`` / ``"soft"``.
+    skill_content
+        The raw skill document content.
+    use_semantic_density
+        Whether to adjust the score based on semantic density of leading words.
+    semantic_density_weight
+        Scaling weight for the semantic density bonus.
+    leading_words
+        Optional custom list of high-influence words to prioritize.
     """
     if metric == "hard":
-        return float(hard)
-    if metric == "soft":
-        return float(soft)
-    if metric == "mixed":
+        score = float(hard)
+    elif metric == "soft":
+        score = float(soft)
+    elif metric == "mixed":
         w = max(0.0, min(1.0, float(mixed_weight)))
-        return (1.0 - w) * float(hard) + w * float(soft)
-    raise ValueError(
-        f"unknown gate metric {metric!r}; expected 'hard', 'soft', or 'mixed'"
-    )
+        score = (1.0 - w) * float(hard) + w * float(soft)
+    else:
+        raise ValueError(
+            f"unknown gate metric {metric!r}; expected 'hard', 'soft', or 'mixed'"
+        )
+
+    if use_semantic_density:
+        density = compute_semantic_density(skill_content, leading_words)
+        score += float(semantic_density_weight) * density
+
+    return score
 
 
 def evaluate_gate(
@@ -86,6 +145,9 @@ def evaluate_gate(
     cand_soft: float = 0.0,
     metric: GateMetric = "hard",
     mixed_weight: float = 0.5,
+    use_semantic_density: bool = False,
+    semantic_density_weight: float = 0.05,
+    leading_words: list[str] | None = None,
 ) -> GateResult:
     """Pure gate decision: compare candidate score to current/best.
 
@@ -111,14 +173,29 @@ def evaluate_gate(
         the original gate behavior.
     mixed_weight
         Weight on ``soft`` when ``metric == "mixed"``.
+    use_semantic_density
+        Whether to adjust the score based on semantic density of leading words.
+    semantic_density_weight
+        Scaling weight for the semantic density bonus.
+    leading_words
+        Optional custom list of high-influence words to prioritize.
 
     Returns
     -------
     GateResult
         Updated state; the caller decides what to do with it (print,
         mutate trainer state, log, etc.).
     """
-    cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight)
+    cand_score = select_gate_score(
+        cand_hard,
+        cand_soft,
+        metric,
+        mixed_weight,
+        skill_content=candidate_skill,
+        use_semantic_density=use_semantic_density,
+        semantic_density_weight=semantic_density_weight,
+        leading_words=leading_words,
+    )
 
     if cand_score > current_score:
         if cand_score > best_score:

tests/test_semantic_density.py

@@ -0,0 +1,124 @@
+"""Tests for semantic density heuristic in the validation gate."""
+from __future__ import annotations
+
+import unittest
+from skillopt.evaluation.gate import (
+    compute_semantic_density,
+    select_gate_score,
+    evaluate_gate,
+)
+
+
+class TestSemanticDensity(unittest.TestCase):
+    """Test suite for semantic density scoring and gating decisions."""
+
+    def test_compute_semantic_density_basic(self) -> None:
+        """Verify basic compute_semantic_density behaviour with default words."""
+        # 10 words, 2 leading words ("always", "never") -> 0.2 density
+        skill = "Always check the inputs and never mix up proxy values."
+        density = compute_semantic_density(skill)
+        self.assertAlmostEqual(density, 0.2)
+
+        # Empty skill should have 0 density
+        self.assertEqual(compute_semantic_density(""), 0.0)
+        self.assertEqual(compute_semantic_density("   \n  "), 0.0)
+
+    def test_compute_semantic_density_custom_leading_words(self) -> None:
+        """Verify compute_semantic_density with custom leading words."""
+        skill = "Check the inputs carefully and resolve the equation."
+        leading = ["check", "resolve"]
+        # 8 words, 2 custom leading words -> 0.25 density
+        density = compute_semantic_density(skill, leading_words=leading)
+        self.assertAlmostEqual(density, 0.25)
+
+    def test_compute_semantic_density_with_protected_regions(self) -> None:
+        """Verify protected comments are excluded from density calculation."""
+        skill = (
+            "Always check inputs.\n"
+            "<!-- SLOW_UPDATE_START -->\n"
+            "This contains many words that should not count towards density "
+            "always and never and only.\n"
+            "<!-- SLOW_UPDATE_END -->\n"
+            "<!-- APPENDIX_START -->\n"
+            "More excluded words.\n"
+            "<!-- APPENDIX_END -->\n"
+        )
+        # Without stripping, there would be many more words and a different density.
+        # Stripped text: "Always check inputs." -> 3 words, 1 leading word ("always") -> 1/3 density
+        density = compute_semantic_density(skill)
+        self.assertAlmostEqual(density, 1.0 / 3.0)
+
+    def test_select_gate_score_no_density(self) -> None:
+        """Verify select_gate_score without semantic density adjustment."""
+        # Default behavior: no semantic density adjustment
+        score_hard = select_gate_score(0.8, 0.6, metric="hard")
+        self.assertEqual(score_hard, 0.8)
+
+        score_soft = select_gate_score(0.8, 0.6, metric="soft")
+        self.assertEqual(score_soft, 0.6)
+
+        score_mixed = select_gate_score(0.8, 0.6, metric="mixed", mixed_weight=0.5)
+        self.assertAlmostEqual(score_mixed, 0.7)
+
+    def test_select_gate_score_with_density(self) -> None:
+        """Verify select_gate_score with semantic density adjustment."""
+        # 10 words, 2 leading words ("always", "never") -> 0.2 density
+        skill = "Always check the inputs and never mix up proxy values."
+        # bonus: 0.1 (weight) * 0.2 (density) = 0.02
+        score = select_gate_score(
+            hard=0.8,
+            soft=0.6,
+            metric="hard",
+            skill_content=skill,
+            use_semantic_density=True,
+            semantic_density_weight=0.1,
+        )
+        self.assertAlmostEqual(score, 0.82)
+
+    def test_evaluate_gate_with_density_preference(self) -> None:
+        """Verify evaluate_gate prefers candidates with higher semantic density."""
+        # Baseline/current skill:
+        # "Always do this task step by step and be very careful because errors are bad."
+        # 15 words, 1 leading ("always") -> 1/15 density = ~0.0667
+        current_skill = "Always do this task step by step and be very careful because errors are bad."
+
+        # Candidate skill (shorter/more steerable):
+        # "Always verify outputs. Never mix proxy values."
+        # 7 words, 3 leading ("always", "verify", "never") -> 3/7 density = ~0.4286
+        candidate_skill = "Always verify outputs. Never mix proxy values."
+
+        # Both have same rollout accuracy (hard=0.8, soft=0.8)
+        # Baseline/current score: 0.8 + 0.1 * (1/15) = ~0.8067
+        current_score = select_gate_score(
+            hard=0.8,
+            soft=0.8,
+            metric="hard",
+            skill_content=current_skill,
+            use_semantic_density=True,
+            semantic_density_weight=0.1,
+        )
+
+        # Candidate score: 0.8 + 0.1 * (3/7) = ~0.8429
+        # Even though accuracy is equal, the candidate should be accepted due to higher semantic density
+        res = evaluate_gate(
+            candidate_skill=candidate_skill,
+            cand_hard=0.8,
+            current_skill=current_skill,
+            current_score=current_score,
+            best_skill=current_skill,
+            best_score=current_score,
+            best_step=1,
+            global_step=2,
+            cand_soft=0.8,
+            metric="hard",
+            use_semantic_density=True,
+            semantic_density_weight=0.1,
+        )
+
+        self.assertEqual(res.action, "accept_new_best")
+        self.assertEqual(res.current_skill, candidate_skill)
+        self.assertAlmostEqual(res.current_score, 0.8 + 0.1 * (3.0 / 7.0))
+
+
+if __name__ == "__main__":
+    unittest.main()