文件
acoustic_algo_scheduler/src/bridges.py
T

544 行
24 KiB
Python
原始文件 Blame 文件历史

此文件含有模棱两可的 Unicode 字符
此文件含有可能会与其他字符混淆的 Unicode 字符。 如果您是想特意这样的,可以安全地忽略该警告。 使用 Escape 按钮显示他们。
from abc import ABC, abstractmethod
from src.models import WaterSoundData
import os
import shutil
import tempfile
from typing import Callable, Dict, List, Any, Optional
class AlgorithmBridge(ABC):
"""桥接插件层抽象基类"""
def __init__(self, name: str, is_core: bool = False):
self.name = name
self.is_core = is_core # 核心算法标志,用于容错策略
@abstractmethod
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
pass
class PythonBridge(AlgorithmBridge):
"""Python 桥接插件"""
def __init__(self, name: str, algo_func: Callable,
input_mapping: Dict[str, str],
output_mapping: Dict[str, str],
preprocessing: Optional[List[Dict[str, Any]]] = None,
is_core: bool = False,
requirements: Optional[Dict[str, Any]] = None,
output_specs: Optional[Dict[str, Any]] = None,
algo_type: str = "python"):
super().__init__(name, is_core)
self.algo_func = algo_func # Python算法核心函数
self.input_mapping = input_mapping # 输入映射 {algo_param: water_data_attr}
self.output_mapping = output_mapping# 输出映射 {algo_return_index: water_data_attr}
self.preprocessing = preprocessing or [] # 预处理算子 [{ "op": "norm", "params": {...} }]
self.requirements = requirements or {}
self.output_specs = output_specs or {}
self.algo_type = algo_type
def _apply_preprocessing(self, data: Any, ops: List[Dict[str, Any]]) -> Any:
"""应用轻量级预处理算子"""
import numpy as np
for op_config in ops:
op = op_config.get("op")
params = op_config.get("params", {})
if op == "normalize" and isinstance(data, np.ndarray):
# 简单归一化示例
if data.size == 0:
continue
min_val = params.get("min", 0)
max_val = params.get("max", 1)
data_min = float(data.min())
data_max = float(data.max())
denom = data_max - data_min
if denom == 0:
data = np.full_like(data, float(min_val), dtype=float)
else:
data = (data.astype(float) - data_min) / denom * (float(max_val) - float(min_val)) + float(min_val)
elif op == "scale" and isinstance(data, (int, float, np.ndarray)):
data = data * params.get("factor", 1.0)
return data
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
try:
water_data.add_log(self.name, "Start Execution")
# 1. 提取并映射输入,同时应用预处理
algo_inputs = {}
for param, attr in self.input_mapping.items():
if hasattr(water_data, attr):
val = getattr(water_data, attr)
# 查找是否有针对该字段的预处理
relevant_ops = [op for op in self.preprocessing if op.get("field") == attr]
if relevant_ops:
val = self._apply_preprocessing(val, relevant_ops)
algo_inputs[param] = val
else:
raise AttributeError(f"Field {attr} not found in WaterSoundData")
# 2. 调用算法
results = self.algo_func(**algo_inputs)
# 3. 映射结果回 standardization 数据
if not isinstance(results, (list, tuple)):
results = [results]
for key, attr in self.output_mapping.items():
# key can be an index (int) or a key if results is dict
if isinstance(results, dict):
val = results[key]
else:
idx = int(key) if isinstance(key, str) and key.isdigit() else key
val = results[idx]
setattr(water_data, attr, val)
water_data.add_log(self.name, "Finished Execution")
except Exception as e:
water_data.set_error(f"Algorithm {self.name} failed: {str(e)}")
return water_data
import ctypes
import numpy as np
class CppBridge(AlgorithmBridge):
"""C++ 桥接插件 (ctypes 实现)"""
def __init__(self, name: str, lib_path: str, function_name: str,
input_mapping: Dict[str, str],
output_mapping: Dict[str, str],
preprocessing: Optional[List[Dict[str, Any]]] = None,
requirements: Optional[Dict[str, Any]] = None,
output_specs: Optional[Dict[str, Any]] = None,
expected_sha256: Optional[str] = None,
isolate: bool = False,
is_core: bool = False):
super().__init__(name, is_core)
self.lib_path = lib_path
self.function_name = function_name
self.input_mapping = input_mapping
self.output_mapping = output_mapping
self.preprocessing = preprocessing or []
self.requirements = requirements or {}
self.output_specs = output_specs or {}
self.algo_type = "cpp"
self._lib = None
self._func = None
self.expected_sha256 = expected_sha256
self.isolate = isolate
def _validate_path(self, path: str) -> str:
"""Security Whitelisting: Ensure path is within the project's 'src' or 'data' directory"""
base_dir = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # project root
abs_path = os.path.realpath(path if os.path.isabs(path) else os.path.join(base_dir, path))
allowed_dirs = [
os.path.join(base_dir, "src"),
os.path.join(base_dir, "data"),
]
if not any(os.path.commonpath([abs_path, d]) == d for d in allowed_dirs):
raise PermissionError(f"Security Block: Path {path} is outside allowed directories (src/data).")
return abs_path
def _load_lib(self):
"""延迟加载动态库并安全配置 ABI"""
if self._lib is None:
abs_lib_path = self._validate_path(self.lib_path)
if not os.path.exists(abs_lib_path):
raise FileNotFoundError(f"C++ library not found: {abs_lib_path}")
if self.expected_sha256:
from src.integrity import ensure_file_sha256
ensure_file_sha256(abs_lib_path, self.expected_sha256, label=f"cpp library {self.name}")
self._lib = ctypes.CDLL(abs_lib_path)
self._func = getattr(self._lib, self.function_name)
# Phase 5: ABI Hardening
# 默认签名(统一约定):void (double*, int, double*, int)
self._func.argtypes = [
ctypes.POINTER(ctypes.c_double),
ctypes.c_int,
ctypes.POINTER(ctypes.c_double),
ctypes.c_int,
]
self._func.restype = None
# 如果 requirements 中定义了签名,则应用它
if "argtypes" in self.requirements:
# 简单映射: 目前支持 "double_ptr", "int"
mapping = {"double_ptr": ctypes.POINTER(ctypes.c_double), "int": ctypes.c_int}
self._func.argtypes = [mapping.get(t, ctypes.c_void_p) for t in self.requirements["argtypes"]]
if "restype" in self.requirements:
mapping = {"int": ctypes.c_int, "void": None}
self._func.restype = mapping.get(self.requirements["restype"], None)
def _call_isolated(self, in_data: np.ndarray) -> np.ndarray:
import json
import subprocess
import sys
abs_lib_path = self._validate_path(self.lib_path)
if self.expected_sha256:
from src.integrity import ensure_file_sha256
ensure_file_sha256(abs_lib_path, self.expected_sha256, label=f"cpp library {self.name}")
cfg = {
"lib_path": abs_lib_path,
"function_name": self.function_name,
"expected_sha256": self.expected_sha256,
}
if isinstance(self.requirements, dict):
if isinstance(self.requirements.get("argtypes"), list):
cfg["argtypes"] = self.requirements.get("argtypes")
if isinstance(self.requirements.get("restype"), str):
cfg["restype"] = self.requirements.get("restype")
base_dir = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
env = os.environ.copy()
env["PYTHONPATH"] = base_dir + os.pathsep + env.get("PYTHONPATH", "")
with tempfile.TemporaryDirectory(prefix="cpp_bridge_") as tmp_dir:
cfg_path = os.path.join(tmp_dir, "config.json")
in_path = os.path.join(tmp_dir, "in.npy")
out_path = os.path.join(tmp_dir, "out.npy")
with open(cfg_path, "w", encoding="utf-8") as f:
json.dump(cfg, f, ensure_ascii=False)
np.save(in_path, np.ascontiguousarray(in_data, dtype=np.float64), allow_pickle=False)
cmd = [sys.executable, "-m", "src.cpp_isolated_runner", "--config", cfg_path, "--input", in_path, "--output", out_path]
proc = subprocess.run(cmd, cwd=base_dir, env=env, capture_output=True, text=True)
if proc.returncode != 0:
stderr = (proc.stderr or "").strip()
stdout = (proc.stdout or "").strip()
details = stderr or stdout or f"returncode={proc.returncode}"
raise RuntimeError(f"isolated C++ call failed: {details}")
if not os.path.exists(out_path):
raise RuntimeError("isolated C++ call produced no output file")
return np.load(out_path, allow_pickle=False)
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
try:
water_data.add_log(self.name, "Start C++ Execution")
# 1. 准备输入
# 为了简化 Phase 3.3,我们假设目前只支持 1进1出 (double array)
# 实际生产中会根据 input_mapping 构造更复杂的结构
in_attr = list(self.input_mapping.values())[0]
in_data = getattr(water_data, in_attr)
if not isinstance(in_data, np.ndarray):
raise TypeError(f"C++ bridge expected numpy array for {in_attr}")
# 确保数据连续且为 float64 (double)
in_data = np.ascontiguousarray(in_data, dtype=np.float64)
if self.isolate:
out_data = self._call_isolated(in_data)
else:
self._load_lib()
in_ptr = in_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
in_len = ctypes.c_int(in_data.size)
# 2. 准备输出空间
# 同样简化为与输入等长
out_data = np.zeros_like(in_data)
out_ptr = out_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
out_len = ctypes.c_int(out_data.size)
# 3. 调用 C++ 函数
# 默认签名: void (double*, int, double*, int)
self._func(in_ptr, in_len, out_ptr, out_len)
# 4. 映射结果回 standardization 数据
out_attr = list(self.output_mapping.values())[0]
setattr(water_data, out_attr, out_data)
water_data.add_log(self.name, "Finished C++ Execution")
except Exception as e:
water_data.set_error(f"C++ Algorithm {self.name} failed: {str(e)}")
return water_data
class MatlabBridge(AlgorithmBridge):
"""MATLAB/Octave 桥接插件 (支持 Engine/Oct2py/File 降级)"""
def __init__(self, name: str, m_path: str, function_name: str,
input_mapping: Dict[str, str],
output_mapping: Dict[str, str],
preprocessing: Optional[List[Dict[str, Any]]] = None,
requirements: Optional[Dict[str, Any]] = None,
output_specs: Optional[Dict[str, Any]] = None,
expected_sha256: Optional[str] = None,
is_core: bool = False):
super().__init__(name, is_core)
self.m_path = m_path # .m 文件所在目录
self.function_name = function_name
self.input_mapping = input_mapping
self.output_mapping = output_mapping
self.preprocessing = preprocessing or []
self.requirements = requirements or {}
self.output_specs = output_specs or {}
self.algo_type = "matlab"
self._engine = None
self.expected_sha256 = expected_sha256
def _validate_path(self, path: str) -> str:
"""Security Whitelisting: Ensure MATLAB path is within the project's 'src' or 'data' directory"""
base_dir = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # project root
abs_path = os.path.realpath(path if os.path.isabs(path) else os.path.join(base_dir, path))
allowed_dirs = [
os.path.join(base_dir, "src"),
os.path.join(base_dir, "data"),
]
if not any(os.path.commonpath([abs_path, d]) == d for d in allowed_dirs):
raise PermissionError(f"Security Block: MATLAB path {path} is outside allowed directories.")
return abs_path
def _verify_entrypoint_integrity(self):
if not self.expected_sha256:
return
m_dir = self._validate_path(self.m_path)
entry_file = self._validate_path(os.path.join(m_dir, f"{self.function_name}.m"))
if not os.path.exists(entry_file):
raise FileNotFoundError(f"MATLAB entry file not found: {entry_file}")
from src.integrity import ensure_file_sha256
ensure_file_sha256(entry_file, self.expected_sha256, label=f"matlab entry {self.name}")
def _call_via_engine(self, inputs: Dict[str, Any]) -> Any:
"""方式 1: 使用官方 matlab.engine"""
import matlab.engine
import matlab
m_dir = self._validate_path(self.m_path)
if self._engine is None:
self._engine = matlab.engine.start_matlab()
self._engine.addpath(m_dir)
func = getattr(self._engine, self.function_name)
# 将 numpy 数组转为 matlab 类型 (这里简化处理)
m_inputs = [matlab.double(v.tolist()) if isinstance(v, np.ndarray) else v for v in inputs.values()]
return func(*m_inputs, nargout=len(self.output_mapping))
def _call_via_oct2py(self, inputs: Dict[str, Any]) -> Any:
"""方式 2: 使用开源 oct2py (Octave)"""
from oct2py import octave
m_dir = self._validate_path(self.m_path)
octave.addpath(m_dir)
func = getattr(octave, self.function_name)
return func(*inputs.values())
def _call_via_file(self, inputs: Dict[str, Any]) -> Any:
"""方式 3: 文件兜底 (使用 .mat 文件交换)"""
from scipy.io import savemat, loadmat
import subprocess
m_dir = self._validate_path(self.m_path)
with tempfile.TemporaryDirectory(prefix="matlab_bridge_") as tmp_dir:
in_file = os.path.join(tmp_dir, "tmp_in.mat")
out_file = os.path.join(tmp_dir, "tmp_out.mat")
savemat(in_file, inputs)
# 构造执行脚本:加入 m_path,加载输入,调用函数,保存输出
m_path = os.path.abspath(m_dir).replace("'", "''")
in_file_escaped = in_file.replace("'", "''")
out_file_escaped = out_file.replace("'", "''")
arglist = ",".join(inputs.keys())
script = (
f"addpath('{m_path}'); "
f"load('{in_file_escaped}'); "
f"results = {self.function_name}({arglist}); "
f"save('-mat', '{out_file_escaped}', 'results'); "
"exit;"
)
octave = shutil.which("octave") or shutil.which("octave-cli")
matlab = shutil.which("matlab")
if octave:
cmd = [octave, "--quiet", "--eval", script]
elif matlab:
cmd = [matlab, "-batch", script]
else:
raise RuntimeError("No MATLAB/Octave CLI found for file-based execution.")
subprocess.run(cmd, check=True, capture_output=True, text=True)
res = loadmat(out_file)
return res.get("results")
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
try:
water_data.add_log(self.name, "Start MATLAB Execution")
self._verify_entrypoint_integrity()
# 准备输入
algo_inputs = {}
for param, attr in self.input_mapping.items():
algo_inputs[param] = getattr(water_data, attr)
# 自动寻优/降级逻辑
from src.env_detect import collect_env_info
env = collect_env_info()
if env["matlab"]["engine_available"]:
results = self._call_via_engine(algo_inputs)
elif env["matlab"]["oct2py_available"]:
results = self._call_via_oct2py(algo_inputs)
elif env["matlab"]["octave_cli"]:
results = self._call_via_file(algo_inputs)
else:
raise RuntimeError("No MATLAB/Octave environment found for execution.")
# 结果回写 (支持单出/多出)
if not isinstance(results, (list, tuple)):
results = [results]
for key, attr in self.output_mapping.items():
idx = int(key) if key.isdigit() else 0
setattr(water_data, attr, results[idx])
water_data.add_log(self.name, "Finished MATLAB Execution")
except Exception as e:
water_data.set_error(f"MATLAB Algorithm {self.name} failed: {str(e)}")
return water_data
import requests
class WebBridge(AlgorithmBridge):
"""HTTP/HTTPS 远程服务桥接(通用工作流组件)"""
def __init__(self, name: str, endpoint_url: str,
input_mapping: Dict[str, str], # WSD 字段路径 -> HTTP 请求字段
output_mapping: Dict[str, str], # HTTP 响应字段 -> WSD 字段路径
method: str = "POST",
timeout_sec: int = 300,
slot_key: Optional[str] = None, # 资源槽位 Key(同 Key 串行执行)
fixed_params: Optional[Dict[str, Any]] = None, # 固定参数(如 modelType)
sanitize_output: bool = True, # 是否清洗输出参数
is_core: bool = False):
super().__init__(name, is_core)
self.endpoint_url = endpoint_url
self.input_mapping = input_mapping
self.output_mapping = output_mapping
self.method = method.upper()
self.timeout_sec = timeout_sec
self.slot_key = slot_key
self.fixed_params = fixed_params or {}
self.sanitize_output = sanitize_output
self.algo_type = "web"
def _get_nested(self, obj: Any, path: str) -> Any:
"""支持 'sea_env.modeling.freq_hz' 这种嵌套取值"""
for key in path.split('.'):
if obj is None:
return None
if isinstance(obj, dict):
obj = obj.get(key)
else:
obj = getattr(obj, key, None)
return obj
def _set_nested(self, obj: Any, path: str, value: Any) -> None:
"""支持 'vis_data.tl_result_path' 这种嵌套写入"""
keys = path.split('.')
for key in keys[:-1]:
if isinstance(obj, dict):
if key not in obj:
obj[key] = {}
obj = obj[key]
else:
if not hasattr(obj, key):
setattr(obj, key, {})
next_obj = getattr(obj, key)
if not isinstance(next_obj, dict):
setattr(obj, key, {})
next_obj = getattr(obj, key)
obj = next_obj
final_key = keys[-1]
if isinstance(obj, dict):
obj[final_key] = value
else:
setattr(obj, final_key, value)
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
from src.resource_slots import slot_context
from src.security import (
validate_endpoint_host, sanitize_output_params,
generate_run_id, version_tracker, validate_critical_enums
)
try:
water_data.add_log(self.name, "Start Web Execution", f"Endpoint: {self.endpoint_url}")
# 1. 安全检查:Host 白名单
validate_endpoint_host(self.endpoint_url)
# 2. 构建请求参数
request_params = dict(self.fixed_params)
for http_field, wsd_path in self.input_mapping.items():
value = self._get_nested(water_data, wsd_path)
if value is not None:
request_params[http_field] = value
# 3. 业务逻辑检查:关键参数枚举校验
validate_critical_enums(request_params)
# 3. 清洗输出参数(生成唯一 run_id)
run_id = generate_run_id()
if self.sanitize_output:
request_params = sanitize_output_params(request_params, run_id)
water_data.add_log(self.name, "Request Prepared", f"run_id={run_id}")
# 4. 获取资源槽位锁并发起请求
with slot_context(self.slot_key):
if self.method == "POST":
response = requests.post(
self.endpoint_url,
json=request_params,
timeout=self.timeout_sec
)
elif self.method == "GET":
response = requests.get(
self.endpoint_url,
params=request_params,
timeout=self.timeout_sec
)
else:
raise ValueError(f"Unsupported HTTP method: {self.method}")
response.raise_for_status()
result = response.json()
# 5. 记录服务版本
if "version" in result:
version_tracker.record(self.endpoint_url, result["version"])
water_data.add_log(self.name, "Service Version", result["version"])
# 6. 检查执行结果
if not result.get("success", True):
error_msg = result.get("error", result.get("message", "Unknown error"))
raise RuntimeError(f"Remote service returned failure: {error_msg}")
# 7. 映射响应到 WSD
response_data = result.get("data", result)
for http_field, wsd_path in self.output_mapping.items():
value = self._get_nested(response_data, http_field)
if value is not None:
self._set_nested(water_data, wsd_path, value)
water_data.add_log(self.name, "Finished Web Execution")
except requests.exceptions.Timeout:
water_data.set_error(f"WebBridge {self.name} timeout after {self.timeout_sec}s")
except requests.exceptions.RequestException as e:
water_data.set_error(f"WebBridge {self.name} request failed: {str(e)}")
except Exception as e:
water_data.set_error(f"WebBridge {self.name} failed: {str(e)}")
return water_data