Skip to content

Commit d543877

Browse files
bug-opsbug-ops
authored
fix(skills,memory): wire SkillEvaluator, ProactiveExplorer, and PromotionEngine in bootstrap (#3356)
Closes #3355. SkillEvaluator: add skill_evaluator/eval_weights/eval_threshold to SkillState, add with_skill_evaluator() to AgentBuilder, attach evaluator to SkillGenerator in skill_commands.rs, build from config in src/bootstrap/skills.rs, wire in runner.rs. ProactiveExplorer: add apply_proactive_explorer() to agent_setup.rs resolving provider and output dir from config.skills.proactive_exploration, wire after with_learning() in runner.rs (gated on !exec_mode.bare). PromotionEngine: implement GeneratorSkillWriter in src/bootstrap/skills.rs as the SkillWriter bridge between zeph-memory and zeph-skills, add apply_promotion_engine() to agent_setup.rs, wire in runner.rs (gated on !exec_mode.bare). Each feature logs at INFO level when enabled at startup.
1 parent 17320e0 commit d543877

6 files changed

Lines changed: 372 additions & 0 deletions

File tree

‎crates/zeph-core/src/agent/builder.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1838,6 +1838,26 @@ impl<C: Channel> Agent<C> {
18381838
self
18391839
}
18401840

1841+
/// Attach a quality-gate evaluator for generated SKILL.md files (#3319).
1842+
///
1843+
/// When set, every `SkillGenerator` used by the agent (including `/skill create`) scores
1844+
/// generated skills through the critic LLM before writing them to disk. Skills below the
1845+
/// configured threshold are rejected.
1846+
///
1847+
/// Pass `None` to disable (default).
1848+
#[must_use]
1849+
pub fn with_skill_evaluator(
1850+
mut self,
1851+
evaluator: Option<std::sync::Arc<zeph_skills::evaluator::SkillEvaluator>>,
1852+
weights: zeph_skills::evaluator::EvaluationWeights,
1853+
threshold: f32,
1854+
) -> Self {
1855+
self.skill_state.skill_evaluator = evaluator;
1856+
self.skill_state.eval_weights = weights;
1857+
self.skill_state.eval_threshold = threshold;
1858+
self
1859+
}
1860+
18411861
/// Attach a proactive world-knowledge explorer (#3320).
18421862
///
18431863
/// When set, the agent will classify each incoming query and trigger background skill

‎crates/zeph-core/src/agent/learning/skill_commands.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,15 @@ impl<C: Channel> Agent<C> {
128128
let generation_provider =
129129
self.resolve_background_provider(&self.skill_state.generation_provider_name.clone());
130130
let generator = zeph_skills::SkillGenerator::new(generation_provider, output_dir.clone());
131+
let generator = if let Some(ref eval) = self.skill_state.skill_evaluator {
132+
generator.with_evaluator(
133+
std::sync::Arc::clone(eval),
134+
self.skill_state.eval_weights,
135+
self.skill_state.eval_threshold,
136+
)
137+
} else {
138+
generator
139+
};
131140
let request = zeph_skills::SkillGenerationRequest {
132141
description: description.to_owned(),
133142
category: None,

‎crates/zeph-core/src/agent/state/mod.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,15 @@ pub(crate) struct SkillState {
118118
pub(crate) generation_output_dir: Option<std::path::PathBuf>,
119119
/// Provider name for `/skill create` generation. Empty = primary.
120120
pub(crate) generation_provider_name: String,
121+
/// Optional quality-gate evaluator for generated SKILL.md files (#3319).
122+
///
123+
/// When `Some`, the evaluator is attached to every `SkillGenerator` instance so that
124+
/// generated skills are scored before being written to disk.
125+
pub(crate) skill_evaluator: Option<std::sync::Arc<zeph_skills::evaluator::SkillEvaluator>>,
126+
/// Weights for the evaluator composite score — forwarded to `SkillGenerator::with_evaluator`.
127+
pub(crate) eval_weights: zeph_skills::evaluator::EvaluationWeights,
128+
/// Minimum composite score required to accept a generated skill (forwarded to the generator).
129+
pub(crate) eval_threshold: f32,
121130
}
122131

123132
pub(crate) struct McpState {
@@ -948,6 +957,9 @@ impl SkillState {
948957
rl_warmup_updates: 50,
949958
generation_output_dir: None,
950959
generation_provider_name: String::new(),
960+
skill_evaluator: None,
961+
eval_weights: zeph_skills::evaluator::EvaluationWeights::default(),
962+
eval_threshold: 0.60,
951963
}
952964
}
953965
}

‎src/agent_setup.rs‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1326,6 +1326,113 @@ pub(crate) fn apply_mcp_discovery<C: Channel>(
13261326
agent.with_mcp_discovery(strategy, params, discovery_provider)
13271327
}
13281328

1329+
/// Wire a [`zeph_skills::proactive::ProactiveExplorer`] onto the agent from config.
1330+
///
1331+
/// Resolves the generation provider, builds the explorer, and calls
1332+
/// [`Agent::with_proactive_explorer`]. Returns the agent unchanged when
1333+
/// `config.skills.proactive_exploration.enabled = false`.
1334+
pub(crate) fn apply_proactive_explorer<C: zeph_core::channel::Channel>(
1335+
agent: zeph_core::agent::Agent<C>,
1336+
config: &zeph_core::config::Config,
1337+
primary: &zeph_llm::any::AnyProvider,
1338+
evaluator: Option<std::sync::Arc<zeph_skills::evaluator::SkillEvaluator>>,
1339+
skills_paths: &[std::path::PathBuf],
1340+
) -> zeph_core::agent::Agent<C> {
1341+
let exp_cfg = &config.skills.proactive_exploration;
1342+
if !exp_cfg.enabled {
1343+
return agent;
1344+
}
1345+
1346+
let output_dir = if let Some(ref dir) = exp_cfg.output_dir {
1347+
std::path::PathBuf::from(dir)
1348+
} else if let Some(first) = skills_paths.first() {
1349+
first.join("generated")
1350+
} else {
1351+
crate::bootstrap::skills::managed_skills_dir().join("generated")
1352+
};
1353+
1354+
let provider = if exp_cfg.provider.is_empty() {
1355+
primary.clone()
1356+
} else {
1357+
match crate::bootstrap::create_named_provider(&exp_cfg.provider, config) {
1358+
Ok(p) => p,
1359+
Err(e) => {
1360+
tracing::warn!(
1361+
provider = %exp_cfg.provider,
1362+
error = %e,
1363+
"proactive exploration provider resolution failed, falling back to primary"
1364+
);
1365+
primary.clone()
1366+
}
1367+
}
1368+
};
1369+
1370+
let generator = zeph_skills::SkillGenerator::new(provider, output_dir.clone());
1371+
let explorer = zeph_skills::proactive::ProactiveExplorer::new(
1372+
generator,
1373+
evaluator,
1374+
output_dir,
1375+
exp_cfg.max_chars,
1376+
exp_cfg.timeout_ms,
1377+
exp_cfg.excluded_domains.clone(),
1378+
);
1379+
tracing::info!("skills.proactive_exploration: enabled");
1380+
agent.with_proactive_explorer(Some(std::sync::Arc::new(explorer)))
1381+
}
1382+
1383+
/// Wire a [`zeph_memory::compression::promotion::PromotionEngine`] onto the agent from config.
1384+
///
1385+
/// Resolves the output directory and skill writer, then calls
1386+
/// [`Agent::with_promotion_engine`]. Returns the agent unchanged when
1387+
/// `config.memory.compression_spectrum.enabled = false` or when no `SkillWriter`
1388+
/// could be built (missing provider or skills paths).
1389+
pub(crate) fn apply_promotion_engine<C: zeph_core::channel::Channel>(
1390+
agent: zeph_core::agent::Agent<C>,
1391+
config: &zeph_core::config::Config,
1392+
primary: &zeph_llm::any::AnyProvider,
1393+
evaluator: Option<std::sync::Arc<zeph_skills::evaluator::SkillEvaluator>>,
1394+
eval_weights: zeph_skills::evaluator::EvaluationWeights,
1395+
eval_threshold: f32,
1396+
skills_paths: &[std::path::PathBuf],
1397+
) -> zeph_core::agent::Agent<C> {
1398+
let spectrum_cfg = &config.memory.compression_spectrum;
1399+
if !spectrum_cfg.enabled {
1400+
return agent;
1401+
}
1402+
1403+
let output_dir = if let Some(ref dir) = spectrum_cfg.promotion_output_dir {
1404+
std::path::PathBuf::from(dir)
1405+
} else if let Some(first) = skills_paths.first() {
1406+
first.join("promoted")
1407+
} else {
1408+
crate::bootstrap::skills::managed_skills_dir().join("promoted")
1409+
};
1410+
1411+
let Some(writer) = crate::bootstrap::skills::build_skill_writer(
1412+
config,
1413+
primary,
1414+
evaluator,
1415+
eval_weights,
1416+
eval_threshold,
1417+
skills_paths,
1418+
) else {
1419+
return agent;
1420+
};
1421+
1422+
let promotion_config = zeph_memory::compression::promotion::PromotionConfig {
1423+
min_occurrences: spectrum_cfg.min_occurrences,
1424+
min_sessions: spectrum_cfg.min_sessions,
1425+
cluster_threshold: spectrum_cfg.cluster_threshold,
1426+
};
1427+
let engine = zeph_memory::compression::promotion::PromotionEngine::new(
1428+
writer,
1429+
promotion_config,
1430+
output_dir,
1431+
);
1432+
tracing::info!("memory.compression_spectrum: enabled");
1433+
agent.with_promotion_engine(Some(std::sync::Arc::new(engine)))
1434+
}
1435+
13291436
/// Build a `SandboxPolicy` from the TOML `[tools.sandbox]` config section.
13301437
pub(crate) fn sandbox_policy_from_config(
13311438
cfg: &zeph_tools::config::SandboxConfig,

‎src/bootstrap/skills.rs‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
pub use zeph_core::provider_factory::effective_embedding_model;
55

66
use std::path::PathBuf;
7+
use std::pin::Pin;
8+
use std::sync::Arc;
79
use zeph_llm::any::AnyProvider;
810
use zeph_memory::QdrantOps;
911
use zeph_memory::semantic::SemanticMemory;
@@ -106,3 +108,166 @@ pub fn managed_skills_dir() -> PathBuf {
106108
pub fn plugins_dir() -> PathBuf {
107109
zeph_plugins::PluginManager::default_plugins_dir()
108110
}
111+
112+
/// Build a [`zeph_skills::evaluator::SkillEvaluator`] from `[skills.evaluation]` config.
113+
///
114+
/// Returns `None` when `config.skills.evaluation.enabled = false`.
115+
/// On provider resolution failure falls back to `primary` and logs a warning.
116+
///
117+
/// # Examples
118+
///
119+
/// ```rust,no_run
120+
/// # use zeph_llm::any::AnyProvider;
121+
/// # use zeph_core::config::Config;
122+
/// # use std::path::Path;
123+
/// # let config = Config::load(Path::new("/nonexistent")).unwrap();
124+
/// # let provider = AnyProvider::Mock(zeph_llm::mock::MockProvider::default());
125+
/// let evaluator = crate::bootstrap::skills::build_skill_evaluator(&config, &provider);
126+
/// ```
127+
pub fn build_skill_evaluator(
128+
config: &Config,
129+
primary: &AnyProvider,
130+
) -> Option<Arc<zeph_skills::evaluator::SkillEvaluator>> {
131+
let eval_cfg = &config.skills.evaluation;
132+
if !eval_cfg.enabled {
133+
return None;
134+
}
135+
136+
let critic = if eval_cfg.provider.is_empty() {
137+
primary.clone()
138+
} else {
139+
match crate::bootstrap::create_named_provider(&eval_cfg.provider, config) {
140+
Ok(p) => p,
141+
Err(e) => {
142+
tracing::warn!(
143+
provider = %eval_cfg.provider,
144+
error = %e,
145+
"skill evaluator provider resolution failed, falling back to primary"
146+
);
147+
primary.clone()
148+
}
149+
}
150+
};
151+
152+
let weights = zeph_skills::evaluator::EvaluationWeights {
153+
correctness: eval_cfg.weight_correctness,
154+
reusability: eval_cfg.weight_reusability,
155+
specificity: eval_cfg.weight_specificity,
156+
};
157+
158+
Some(Arc::new(zeph_skills::evaluator::SkillEvaluator::new(
159+
critic,
160+
weights,
161+
eval_cfg.quality_threshold,
162+
eval_cfg.fail_open_on_error,
163+
eval_cfg.timeout_ms,
164+
)))
165+
}
166+
167+
/// `SkillWriter` implementation that delegates to a `SkillGenerator`.
168+
///
169+
/// Bridges `zeph-memory`'s `SkillWriter` trait (which cannot depend on `zeph-skills`)
170+
/// to the concrete `SkillGenerator` in `zeph-skills`. Defined in the binary crate to
171+
/// avoid the circular dependency `zeph-memory` ↔ `zeph-skills`.
172+
struct GeneratorSkillWriter {
173+
/// Provider used to build a fresh `SkillGenerator` per call.
174+
provider: AnyProvider,
175+
/// Output directory for generated SKILL.md files.
176+
output_dir: PathBuf,
177+
/// Optional quality gate — forwarded to the generator via `with_evaluator`.
178+
evaluator: Option<Arc<zeph_skills::evaluator::SkillEvaluator>>,
179+
/// Evaluation weights forwarded to `with_evaluator`.
180+
eval_weights: zeph_skills::evaluator::EvaluationWeights,
181+
/// Evaluation threshold forwarded to `with_evaluator`.
182+
eval_threshold: f32,
183+
}
184+
185+
impl zeph_memory::compression::promotion::SkillWriter for GeneratorSkillWriter {
186+
fn write_skill(
187+
&self,
188+
description: String,
189+
signature: String,
190+
) -> Pin<Box<dyn std::future::Future<Output = Result<(), String>> + Send + '_>> {
191+
Box::pin(async move {
192+
let generator =
193+
zeph_skills::SkillGenerator::new(self.provider.clone(), self.output_dir.clone());
194+
let generator = if let Some(ref eval) = self.evaluator {
195+
generator.with_evaluator(Arc::clone(eval), self.eval_weights, self.eval_threshold)
196+
} else {
197+
generator
198+
};
199+
200+
let req = zeph_skills::SkillGenerationRequest {
201+
description: description.clone(),
202+
category: None,
203+
allowed_tools: vec![],
204+
};
205+
let generated = generator.generate(req).await.map_err(|e| e.to_string())?;
206+
207+
// Use the signature as idempotency key: skip write if skill dir already exists.
208+
let skill_dir = self.output_dir.join(format!(
209+
"promoted-pattern-{}",
210+
&signature[..12.min(signature.len())]
211+
));
212+
if skill_dir.exists() {
213+
return Ok(());
214+
}
215+
216+
generator
217+
.approve_and_save(&generated)
218+
.await
219+
.map(|_| ())
220+
.map_err(|e| e.to_string())
221+
})
222+
}
223+
}
224+
225+
/// Build an `Arc<dyn SkillWriter>` backed by a `SkillGenerator`.
226+
///
227+
/// Returns `None` when the promotion engine is disabled or the output directory cannot be
228+
/// determined. On provider resolution failure falls back to `primary`.
229+
pub fn build_skill_writer(
230+
config: &Config,
231+
primary: &AnyProvider,
232+
evaluator: Option<Arc<zeph_skills::evaluator::SkillEvaluator>>,
233+
eval_weights: zeph_skills::evaluator::EvaluationWeights,
234+
eval_threshold: f32,
235+
skills_paths: &[PathBuf],
236+
) -> Option<Arc<dyn zeph_memory::compression::promotion::SkillWriter>> {
237+
let spectrum_cfg = &config.memory.compression_spectrum;
238+
if !spectrum_cfg.enabled {
239+
return None;
240+
}
241+
242+
let output_dir = if let Some(ref dir) = spectrum_cfg.promotion_output_dir {
243+
PathBuf::from(dir)
244+
} else if let Some(first) = skills_paths.first() {
245+
first.join("promoted")
246+
} else {
247+
managed_skills_dir().join("promoted")
248+
};
249+
250+
let provider = if spectrum_cfg.promotion_provider.is_empty() {
251+
primary.clone()
252+
} else {
253+
match crate::bootstrap::create_named_provider(&spectrum_cfg.promotion_provider, config) {
254+
Ok(p) => p,
255+
Err(e) => {
256+
tracing::warn!(
257+
provider = %spectrum_cfg.promotion_provider,
258+
error = %e,
259+
"promotion provider resolution failed, falling back to primary"
260+
);
261+
primary.clone()
262+
}
263+
}
264+
};
265+
266+
Some(Arc::new(GeneratorSkillWriter {
267+
provider,
268+
output_dir,
269+
evaluator,
270+
eval_weights,
271+
eval_threshold,
272+
}))
273+
}

0 commit comments

Comments
 (0)