from abc import ABC, abstractmethod from src.models import WaterSoundData import os import shutil import tempfile from typing import Callable, Dict, List, Any, Optional class AlgorithmBridge(ABC): """桥接插件层抽象基类""" def __init__(self, name: str, is_core: bool = False): self.name = name self.is_core = is_core # 核心算法标志，用于容错策略 @abstractmethod def call_algo(self, water_data: WaterSoundData) -> WaterSoundData: pass class PythonBridge(AlgorithmBridge): """Python 桥接插件""" def __init__(self, name: str, algo_func: Callable, input_mapping: Dict[str, str], output_mapping: Dict[str, str], preprocessing: Optional[List[Dict[str, Any]]] = None, is_core: bool = False, requirements: Optional[Dict[str, Any]] = None, output_specs: Optional[Dict[str, Any]] = None, algo_type: str = "python"): super().__init__(name, is_core) self.algo_func = algo_func # Python算法核心函数 self.input_mapping = input_mapping # 输入映射 {algo_param: water_data_attr} self.output_mapping = output_mapping# 输出映射 {algo_return_index: water_data_attr} self.preprocessing = preprocessing or [] # 预处理算子 [{ "op": "norm", "params": {...} }] self.requirements = requirements or {} self.output_specs = output_specs or {} self.algo_type = algo_type def _apply_preprocessing(self, data: Any, ops: List[Dict[str, Any]]) -> Any: """应用轻量级预处理算子""" import numpy as np for op_config in ops: op = op_config.get("op") params = op_config.get("params", {}) if op == "normalize" and isinstance(data, np.ndarray): # 简单归一化示例 if data.size == 0: continue min_val = params.get("min", 0) max_val = params.get("max", 1) data_min = float(data.min()) data_max = float(data.max()) denom = data_max - data_min if denom == 0: data = np.full_like(data, float(min_val), dtype=float) else: data = (data.astype(float) - data_min) / denom * (float(max_val) - float(min_val)) + float(min_val) elif op == "scale" and isinstance(data, (int, float, np.ndarray)): data = data * params.get("factor", 1.0) return data def call_algo(self, water_data: WaterSoundData) -> WaterSoundData: try: water_data.add_log(self.name, "Start Execution") # 1. 提取并映射输入，同时应用预处理 algo_inputs = {} for param, attr in self.input_mapping.items(): if hasattr(water_data, attr): val = getattr(water_data, attr) # 查找是否有针对该字段的预处理 relevant_ops = [op for op in self.preprocessing if op.get("field") == attr] if relevant_ops: val = self._apply_preprocessing(val, relevant_ops) algo_inputs[param] = val else: raise AttributeError(f"Field {attr} not found in WaterSoundData") # 2. 调用算法 results = self.algo_func(**algo_inputs) # 3. 映射结果回 standardization 数据 if not isinstance(results, (list, tuple)): results = [results] for key, attr in self.output_mapping.items(): # key can be an index (int) or a key if results is dict if isinstance(results, dict): val = results[key] else: idx = int(key) if isinstance(key, str) and key.isdigit() else key val = results[idx] setattr(water_data, attr, val) water_data.add_log(self.name, "Finished Execution") except Exception as e: water_data.set_error(f"Algorithm {self.name} failed: {str(e)}") return water_data import ctypes import numpy as np class CppBridge(AlgorithmBridge): """C++ 桥接插件 (ctypes 实现)""" def __init__(self, name: str, lib_path: str, function_name: str, input_mapping: Dict[str, str], output_mapping: Dict[str, str], preprocessing: Optional[List[Dict[str, Any]]] = None, requirements: Optional[Dict[str, Any]] = None, output_specs: Optional[Dict[str, Any]] = None, expected_sha256: Optional[str] = None, isolate: bool = False, is_core: bool = False): super().__init__(name, is_core) self.lib_path = lib_path self.function_name = function_name self.input_mapping = input_mapping self.output_mapping = output_mapping self.preprocessing = preprocessing or [] self.requirements = requirements or {} self.output_specs = output_specs or {} self.algo_type = "cpp" self._lib = None self._func = None self.expected_sha256 = expected_sha256 self.isolate = isolate def _validate_path(self, path: str) -> str: """Security Whitelisting: Ensure path is within the project's 'src' or 'data' directory""" base_dir = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # project root abs_path = os.path.realpath(path if os.path.isabs(path) else os.path.join(base_dir, path)) allowed_dirs = [ os.path.join(base_dir, "src"), os.path.join(base_dir, "data"), ] if not any(os.path.commonpath([abs_path, d]) == d for d in allowed_dirs): raise PermissionError(f"Security Block: Path {path} is outside allowed directories (src/data).") return abs_path def _load_lib(self): """延迟加载动态库并安全配置 ABI""" if self._lib is None: abs_lib_path = self._validate_path(self.lib_path) if not os.path.exists(abs_lib_path): raise FileNotFoundError(f"C++ library not found: {abs_lib_path}") if self.expected_sha256: from src.integrity import ensure_file_sha256 ensure_file_sha256(abs_lib_path, self.expected_sha256, label=f"cpp library {self.name}") self._lib = ctypes.CDLL(abs_lib_path) self._func = getattr(self._lib, self.function_name) # Phase 5: ABI Hardening # 默认签名（统一约定）：void (double*, int, double*, int) self._func.argtypes = [ ctypes.POINTER(ctypes.c_double), ctypes.c_int, ctypes.POINTER(ctypes.c_double), ctypes.c_int, ] self._func.restype = None # 如果 requirements 中定义了签名，则应用它 if "argtypes" in self.requirements: # 简单映射: 目前支持 "double_ptr", "int" mapping = {"double_ptr": ctypes.POINTER(ctypes.c_double), "int": ctypes.c_int} self._func.argtypes = [mapping.get(t, ctypes.c_void_p) for t in self.requirements["argtypes"]] if "restype" in self.requirements: mapping = {"int": ctypes.c_int, "void": None} self._func.restype = mapping.get(self.requirements["restype"], None) def _call_isolated(self, in_data: np.ndarray) -> np.ndarray: import json import subprocess import sys abs_lib_path = self._validate_path(self.lib_path) if self.expected_sha256: from src.integrity import ensure_file_sha256 ensure_file_sha256(abs_lib_path, self.expected_sha256, label=f"cpp library {self.name}") cfg = { "lib_path": abs_lib_path, "function_name": self.function_name, "expected_sha256": self.expected_sha256, } if isinstance(self.requirements, dict): if isinstance(self.requirements.get("argtypes"), list): cfg["argtypes"] = self.requirements.get("argtypes") if isinstance(self.requirements.get("restype"), str): cfg["restype"] = self.requirements.get("restype") base_dir = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) env = os.environ.copy() env["PYTHONPATH"] = base_dir + os.pathsep + env.get("PYTHONPATH", "") with tempfile.TemporaryDirectory(prefix="cpp_bridge_") as tmp_dir: cfg_path = os.path.join(tmp_dir, "config.json") in_path = os.path.join(tmp_dir, "in.npy") out_path = os.path.join(tmp_dir, "out.npy") with open(cfg_path, "w", encoding="utf-8") as f: json.dump(cfg, f, ensure_ascii=False) np.save(in_path, np.ascontiguousarray(in_data, dtype=np.float64), allow_pickle=False) cmd = [sys.executable, "-m", "src.cpp_isolated_runner", "--config", cfg_path, "--input", in_path, "--output", out_path] proc = subprocess.run(cmd, cwd=base_dir, env=env, capture_output=True, text=True) if proc.returncode != 0: stderr = (proc.stderr or "").strip() stdout = (proc.stdout or "").strip() details = stderr or stdout or f"returncode={proc.returncode}" raise RuntimeError(f"isolated C++ call failed: {details}") if not os.path.exists(out_path): raise RuntimeError("isolated C++ call produced no output file") return np.load(out_path, allow_pickle=False) def call_algo(self, water_data: WaterSoundData) -> WaterSoundData: try: water_data.add_log(self.name, "Start C++ Execution") # 1. 准备输入 # 为了简化 Phase 3.3，我们假设目前只支持 1进1出 (double array) # 实际生产中会根据 input_mapping 构造更复杂的结构 in_attr = list(self.input_mapping.values())[0] in_data = getattr(water_data, in_attr) if not isinstance(in_data, np.ndarray): raise TypeError(f"C++ bridge expected numpy array for {in_attr}") # 确保数据连续且为 float64 (double) in_data = np.ascontiguousarray(in_data, dtype=np.float64) if self.isolate: out_data = self._call_isolated(in_data) else: self._load_lib() in_ptr = in_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double)) in_len = ctypes.c_int(in_data.size) # 2. 准备输出空间 # 同样简化为与输入等长 out_data = np.zeros_like(in_data) out_ptr = out_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double)) out_len = ctypes.c_int(out_data.size) # 3. 调用 C++ 函数 # 默认签名: void (double*, int, double*, int) self._func(in_ptr, in_len, out_ptr, out_len) # 4. 映射结果回 standardization 数据 out_attr = list(self.output_mapping.values())[0] setattr(water_data, out_attr, out_data) water_data.add_log(self.name, "Finished C++ Execution") except Exception as e: water_data.set_error(f"C++ Algorithm {self.name} failed: {str(e)}") return water_data class MatlabBridge(AlgorithmBridge): """MATLAB/Octave 桥接插件 (支持 Engine/Oct2py/File 降级)""" def __init__(self, name: str, m_path: str, function_name: str, input_mapping: Dict[str, str], output_mapping: Dict[str, str], preprocessing: Optional[List[Dict[str, Any]]] = None, requirements: Optional[Dict[str, Any]] = None, output_specs: Optional[Dict[str, Any]] = None, expected_sha256: Optional[str] = None, is_core: bool = False): super().__init__(name, is_core) self.m_path = m_path # .m 文件所在目录 self.function_name = function_name self.input_mapping = input_mapping self.output_mapping = output_mapping self.preprocessing = preprocessing or [] self.requirements = requirements or {} self.output_specs = output_specs or {} self.algo_type = "matlab" self._engine = None self.expected_sha256 = expected_sha256 def _validate_path(self, path: str) -> str: """Security Whitelisting: Ensure MATLAB path is within the project's 'src' or 'data' directory""" base_dir = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # project root abs_path = os.path.realpath(path if os.path.isabs(path) else os.path.join(base_dir, path)) allowed_dirs = [ os.path.join(base_dir, "src"), os.path.join(base_dir, "data"), ] if not any(os.path.commonpath([abs_path, d]) == d for d in allowed_dirs): raise PermissionError(f"Security Block: MATLAB path {path} is outside allowed directories.") return abs_path def _verify_entrypoint_integrity(self): if not self.expected_sha256: return m_dir = self._validate_path(self.m_path) entry_file = self._validate_path(os.path.join(m_dir, f"{self.function_name}.m")) if not os.path.exists(entry_file): raise FileNotFoundError(f"MATLAB entry file not found: {entry_file}") from src.integrity import ensure_file_sha256 ensure_file_sha256(entry_file, self.expected_sha256, label=f"matlab entry {self.name}") def _call_via_engine(self, inputs: Dict[str, Any]) -> Any: """方式 1: 使用官方 matlab.engine""" import matlab.engine import matlab m_dir = self._validate_path(self.m_path) if self._engine is None: self._engine = matlab.engine.start_matlab() self._engine.addpath(m_dir) func = getattr(self._engine, self.function_name) # 将 numpy 数组转为 matlab 类型 (这里简化处理) m_inputs = [matlab.double(v.tolist()) if isinstance(v, np.ndarray) else v for v in inputs.values()] return func(*m_inputs, nargout=len(self.output_mapping)) def _call_via_oct2py(self, inputs: Dict[str, Any]) -> Any: """方式 2: 使用开源 oct2py (Octave)""" from oct2py import octave m_dir = self._validate_path(self.m_path) octave.addpath(m_dir) func = getattr(octave, self.function_name) return func(*inputs.values()) def _call_via_file(self, inputs: Dict[str, Any]) -> Any: """方式 3: 文件兜底 (使用 .mat 文件交换)""" from scipy.io import savemat, loadmat import subprocess m_dir = self._validate_path(self.m_path) with tempfile.TemporaryDirectory(prefix="matlab_bridge_") as tmp_dir: in_file = os.path.join(tmp_dir, "tmp_in.mat") out_file = os.path.join(tmp_dir, "tmp_out.mat") savemat(in_file, inputs) # 构造执行脚本：加入 m_path，加载输入，调用函数，保存输出 m_path = os.path.abspath(m_dir).replace("'", "''") in_file_escaped = in_file.replace("'", "''") out_file_escaped = out_file.replace("'", "''") arglist = ",".join(inputs.keys()) script = ( f"addpath('{m_path}'); " f"load('{in_file_escaped}'); " f"results = {self.function_name}({arglist}); " f"save('-mat', '{out_file_escaped}', 'results'); " "exit;" ) octave = shutil.which("octave") or shutil.which("octave-cli") matlab = shutil.which("matlab") if octave: cmd = [octave, "--quiet", "--eval", script] elif matlab: cmd = [matlab, "-batch", script] else: raise RuntimeError("No MATLAB/Octave CLI found for file-based execution.") subprocess.run(cmd, check=True, capture_output=True, text=True) res = loadmat(out_file) return res.get("results") def call_algo(self, water_data: WaterSoundData) -> WaterSoundData: try: water_data.add_log(self.name, "Start MATLAB Execution") self._verify_entrypoint_integrity() # 准备输入 algo_inputs = {} for param, attr in self.input_mapping.items(): algo_inputs[param] = getattr(water_data, attr) # 自动寻优/降级逻辑 from src.env_detect import collect_env_info env = collect_env_info() if env["matlab"]["engine_available"]: results = self._call_via_engine(algo_inputs) elif env["matlab"]["oct2py_available"]: results = self._call_via_oct2py(algo_inputs) elif env["matlab"]["octave_cli"]: results = self._call_via_file(algo_inputs) else: raise RuntimeError("No MATLAB/Octave environment found for execution.") # 结果回写 (支持单出/多出) if not isinstance(results, (list, tuple)): results = [results] for key, attr in self.output_mapping.items(): idx = int(key) if key.isdigit() else 0 setattr(water_data, attr, results[idx]) water_data.add_log(self.name, "Finished MATLAB Execution") except Exception as e: water_data.set_error(f"MATLAB Algorithm {self.name} failed: {str(e)}") return water_data import requests class WebBridge(AlgorithmBridge): """HTTP/HTTPS 远程服务桥接（通用工作流组件）""" def __init__(self, name: str, endpoint_url: str, input_mapping: Dict[str, str], # WSD 字段路径 -> HTTP 请求字段 output_mapping: Dict[str, str], # HTTP 响应字段 -> WSD 字段路径 method: str = "POST", timeout_sec: int = 300, slot_key: Optional[str] = None, # 资源槽位 Key（同 Key 串行执行） fixed_params: Optional[Dict[str, Any]] = None, # 固定参数（如 modelType） sanitize_output: bool = True, # 是否清洗输出参数 is_core: bool = False): super().__init__(name, is_core) self.endpoint_url = endpoint_url self.input_mapping = input_mapping self.output_mapping = output_mapping self.method = method.upper() self.timeout_sec = timeout_sec self.slot_key = slot_key self.fixed_params = fixed_params or {} self.sanitize_output = sanitize_output self.algo_type = "web" def _get_nested(self, obj: Any, path: str) -> Any: """支持 'sea_env.modeling.freq_hz' 这种嵌套取值""" for key in path.split('.'): if obj is None: return None if isinstance(obj, dict): obj = obj.get(key) else: obj = getattr(obj, key, None) return obj def _set_nested(self, obj: Any, path: str, value: Any) -> None: """支持 'vis_data.tl_result_path' 这种嵌套写入""" keys = path.split('.') for key in keys[:-1]: if isinstance(obj, dict): if key not in obj: obj[key] = {} obj = obj[key] else: if not hasattr(obj, key): setattr(obj, key, {}) next_obj = getattr(obj, key) if not isinstance(next_obj, dict): setattr(obj, key, {}) next_obj = getattr(obj, key) obj = next_obj final_key = keys[-1] if isinstance(obj, dict): obj[final_key] = value else: setattr(obj, final_key, value) def call_algo(self, water_data: WaterSoundData) -> WaterSoundData: from src.resource_slots import slot_context from src.security import ( validate_endpoint_host, sanitize_output_params, generate_run_id, version_tracker, validate_critical_enums ) try: water_data.add_log(self.name, "Start Web Execution", f"Endpoint: {self.endpoint_url}") # 1. 安全检查：Host 白名单 validate_endpoint_host(self.endpoint_url) # 2. 构建请求参数 request_params = dict(self.fixed_params) for http_field, wsd_path in self.input_mapping.items(): value = self._get_nested(water_data, wsd_path) if value is not None: request_params[http_field] = value # 3. 业务逻辑检查：关键参数枚举校验 validate_critical_enums(request_params) # 3. 清洗输出参数（生成唯一 run_id） run_id = generate_run_id() if self.sanitize_output: request_params = sanitize_output_params(request_params, run_id) water_data.add_log(self.name, "Request Prepared", f"run_id={run_id}") # 4. 获取资源槽位锁并发起请求 with slot_context(self.slot_key): if self.method == "POST": response = requests.post( self.endpoint_url, json=request_params, timeout=self.timeout_sec ) elif self.method == "GET": response = requests.get( self.endpoint_url, params=request_params, timeout=self.timeout_sec ) else: raise ValueError(f"Unsupported HTTP method: {self.method}") response.raise_for_status() result = response.json() # 5. 记录服务版本 if "version" in result: version_tracker.record(self.endpoint_url, result["version"]) water_data.add_log(self.name, "Service Version", result["version"]) # 6. 检查执行结果 if not result.get("success", True): error_msg = result.get("error", result.get("message", "Unknown error")) raise RuntimeError(f"Remote service returned failure: {error_msg}") # 7. 映射响应到 WSD response_data = result.get("data", result) for http_field, wsd_path in self.output_mapping.items(): value = self._get_nested(response_data, http_field) if value is not None: self._set_nested(water_data, wsd_path, value) water_data.add_log(self.name, "Finished Web Execution") except requests.exceptions.Timeout: water_data.set_error(f"WebBridge {self.name} timeout after {self.timeout_sec}s") except requests.exceptions.RequestException as e: water_data.set_error(f"WebBridge {self.name} request failed: {str(e)}") except Exception as e: water_data.set_error(f"WebBridge {self.name} failed: {str(e)}") return water_data