fix(sandbox): 容器不可用自动降级 + clean_expired 并发回归测试
P1-1 降级机制:
- sandbox 新增 DEGRADED 全局标志,ensure_container 失败时置位
- 新增 ensure_ready_or_degrade: exec 前检测容器运行状态,
不在则重建,重建失败置降级标志并返回 false
- executor 调用 ensure_ready_or_degrade,降级时回退本地执行
(审批已在入口完成,安全边界不受影响)
- spawn_child 保留 spawn 级失败兜底(docker 二进制缺失等)
- 解决: IAS_DOCKER_SANDBOX=1 但 docker 异常时工具不再持续 SpawnFailed
P1-2 并发回归测试:
- approval 新增 create_with_ttl 测试入口(cfg(test))
- clean_expired_only_removes_expired_entries: 验证只删过期条目
- clean_expired_safe_when_no_pending: 空 pending 不 panic
- clean_expired_skips_entries_already_removed_by_reply:
回归保护——clean_expired 第一阶段收集 hash 后、第二阶段删除前,
条目被 handle_reply 提前移除时,按 hash 查找应安全跳过不误删
P2-5 容器崩溃恢复:
- ensure_ready_or_degrade 统一处理容器崩溃重建
- recover_if_crashed 保留供 spawn 级失败兜底调用
P2-6 build_exec_cmd 用 spec.tool 替代 path 文件名提取,消除隐式耦合
验证: 28 测试通过(原25+3),clippy 22<基线23 无新增警告,
端到端验证容器重建/降级/本地三种路径均正确。
This commit is contained in:
+52
-12
@@ -146,20 +146,15 @@ pub async fn execute_loop(
|
||||
|
||||
// ── spawn ──
|
||||
// 执行隔离边界:启用沙箱且工具 profile 非 local 时,通过 docker exec
|
||||
// 在长期容器内执行;否则本地 spawn(env_clear + 白名单注入)
|
||||
let use_docker =
|
||||
crate::tools::sandbox::enabled() && spec.sandbox.profile != "local";
|
||||
let mut cmd = if use_docker {
|
||||
crate::tools::sandbox::build_exec_cmd(spec)
|
||||
// 在长期容器内执行;否则本地 spawn(env_clear + 白名单注入)。
|
||||
// 沙箱降级(ensure 失败或容器崩溃未恢复)时自动回退本地执行。
|
||||
let use_docker = if crate::tools::sandbox::should_use_docker(&spec.sandbox.profile) {
|
||||
// 执行前确保容器就绪:未运行则重建,重建失败则降级
|
||||
crate::tools::sandbox::ensure_ready_or_degrade("tools").await
|
||||
} else {
|
||||
build_local_cmd(spec, binary_path)
|
||||
false
|
||||
};
|
||||
cmd.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
|
||||
let mut child = match cmd.spawn() {
|
||||
let mut child = match spawn_child(spec, binary_path, use_docker).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
error!(target: "ias::err","[spawn] {e}");
|
||||
@@ -370,3 +365,48 @@ fn build_local_cmd(spec: &ToolSpec, binary_path: &str) -> Command {
|
||||
}
|
||||
cmd
|
||||
}
|
||||
|
||||
/// spawn 工具进程。docker exec 模式下若首次 spawn 失败,尝试检测容器崩溃并重建后重试一次;
|
||||
/// 重建失败则降级为本地执行(保留可用性,但仍受审批约束)。
|
||||
async fn spawn_child(
|
||||
spec: &ToolSpec,
|
||||
binary_path: &str,
|
||||
use_docker: bool,
|
||||
) -> Result<tokio::process::Child, std::io::Error> {
|
||||
let make_cmd = || {
|
||||
let mut cmd = if use_docker {
|
||||
crate::tools::sandbox::build_exec_cmd(spec)
|
||||
} else {
|
||||
build_local_cmd(spec, binary_path)
|
||||
};
|
||||
cmd.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
cmd
|
||||
};
|
||||
|
||||
if let Ok(child) = make_cmd().spawn() {
|
||||
return Ok(child);
|
||||
}
|
||||
|
||||
// 首次 spawn 失败。docker 模式下可能是容器崩溃,尝试重建后重试一次
|
||||
if use_docker && crate::tools::sandbox::recover_if_crashed("tools").await {
|
||||
tracing::info!(target: "ias::sandbox", "沙箱容器已重建,重试执行");
|
||||
if let Ok(child) = make_cmd().spawn() {
|
||||
return Ok(child);
|
||||
}
|
||||
// 重建后仍失败,降级本地执行
|
||||
tracing::warn!(target: "ias::sandbox", "重建后 exec 仍失败,降级本地执行");
|
||||
} else if use_docker {
|
||||
tracing::warn!(target: "ias::sandbox", "容器不可用,降级本地执行");
|
||||
}
|
||||
|
||||
// 降级:本地 spawn(审批已在入口完成,安全边界不受影响)
|
||||
build_local_cmd(spec, binary_path)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user