feat: complete phase6 webbridge with integrity/isolation modules, requests dep, and docs
这个提交包含在:
@@ -53,6 +53,7 @@ graph TD
|
|||||||
| **PythonBridge** | 动态 `importlib` 加载 | 原生性能,易于调试 |
|
| **PythonBridge** | 动态 `importlib` 加载 | 原生性能,易于调试 |
|
||||||
| **CppBridge** | `ctypes` 指针深度透传 / 可选子进程隔离(`src/cpp_isolated_runner.py`) | 计算密集型算子首选,零拷贝;隔离模式可避免段错误拖垮主进程 |
|
| **CppBridge** | `ctypes` 指针深度透传 / 可选子进程隔离(`src/cpp_isolated_runner.py`) | 计算密集型算子首选,零拷贝;隔离模式可避免段错误拖垮主进程 |
|
||||||
| **MatlabBridge** | 三级降级 (Engine/Oct2py/CLI) | 跨平台及离线环境的最大兼容 |
|
| **MatlabBridge** | 三级降级 (Engine/Oct2py/CLI) | 跨平台及离线环境的最大兼容 |
|
||||||
|
| **WebBridge** | HTTP/HTTPS 调用 + slot 串行 + 安全策略(`src/security.py`) | 依赖隔离,远程重型服务可被资产化并自由编排 |
|
||||||
|
|
||||||
### D. 数据模型层 (src/models.py)
|
### D. 数据模型层 (src/models.py)
|
||||||
- **WaterSoundData (WSD)**:系统唯一的标准数据交换格式(核心为 `raw_data` + `meta`)。
|
- **WaterSoundData (WSD)**:系统唯一的标准数据交换格式(核心为 `raw_data` + `meta`)。
|
||||||
|
|||||||
@@ -9,12 +9,14 @@
|
|||||||
|
|
||||||
## 2) 多协议桥接
|
## 2) 多协议桥接
|
||||||
- Python:动态加载并执行(`src/algo_loader.py`、`src/bridges.py`)
|
- Python:动态加载并执行(`src/algo_loader.py`、`src/bridges.py`)
|
||||||
- C++:`.so/.dll` 动态库加载 + `ctypes` 调用(`src/bridges.py`、示例 `src/sample_algos_cpp/`)
|
- C++:`.so/.dll` 动态库加载 + `ctypes` 调用;可选 `isolate=true` 子进程隔离(`src/bridges.py`、`src/cpp_isolated_runner.py`、示例 `src/sample_algos_cpp/`)
|
||||||
- MATLAB/Octave:Engine → oct2py → CLI+matfile 三级降级(`src/bridges.py`、示例 `src/sample_algos_matlab/`)
|
- MATLAB/Octave:Engine → oct2py → CLI+matfile 三级降级(`src/bridges.py`、示例 `src/sample_algos_matlab/`)
|
||||||
|
- WebBridge:HTTP/HTTPS 远程服务桥接(`src/bridges.py`、`src/security.py`、`src/resource_slots.py`)
|
||||||
|
|
||||||
## 3) 工业级自检与五维校验
|
## 3) 工业级自检与五维校验
|
||||||
- 环境自检:编译器、MATLAB/Octave 能力检测(`src/env_detect.py`)
|
- 环境自检:区分 C++ 运行时能力与编译能力,并检测 MATLAB/Octave 能力(`src/env_detect.py`)
|
||||||
- 五维校验:字段/类型/维度/完整性/环境兼容(`src/validator.py`、调度前拦截 `src/scheduler.py`)
|
- 五维校验:字段/类型/维度/完整性/环境兼容(`src/validator.py`、调度前拦截 `src/scheduler.py`)
|
||||||
|
- 插件加固:动态 Python 模块与跨语言插件可启用 `src/data` 白名单 + SHA256 校验(`src/integrity.py`、`src/algo_loader.py`、`src/bridges.py`)
|
||||||
|
|
||||||
## 4) 海试专属与异常中心
|
## 4) 海试专属与异常中心
|
||||||
- 海试一键巡检:内置链路 + CSV 报告导出(`main.py`)
|
- 海试一键巡检:内置链路 + CSV 报告导出(`main.py`)
|
||||||
@@ -30,4 +32,3 @@ python tests/test_core.py
|
|||||||
```bash
|
```bash
|
||||||
streamlit run main.py
|
streamlit run main.py
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -390,6 +390,7 @@ def show_scheduling():
|
|||||||
scheduler = CoreScheduler()
|
scheduler = CoreScheduler()
|
||||||
bridges = []
|
bridges = []
|
||||||
for algo in st.session_state.pipeline:
|
for algo in st.session_state.pipeline:
|
||||||
|
algo_type = algo.get("type", "python")
|
||||||
tpl = None
|
tpl = None
|
||||||
if auto_apply_template:
|
if auto_apply_template:
|
||||||
try:
|
try:
|
||||||
@@ -421,7 +422,6 @@ def show_scheduling():
|
|||||||
preprocessing = []
|
preprocessing = []
|
||||||
|
|
||||||
from src.bridges import PythonBridge, CppBridge
|
from src.bridges import PythonBridge, CppBridge
|
||||||
algo_type = algo.get("type", "python")
|
|
||||||
|
|
||||||
if algo_type == "python":
|
if algo_type == "python":
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
streamlit>=1.28.0
|
streamlit>=1.28.0
|
||||||
numpy>=1.21.0
|
numpy>=1.21.0
|
||||||
scipy>=1.7.0
|
scipy>=1.7.0
|
||||||
|
requests>=2.0.0
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ setup(
|
|||||||
"streamlit>=1.28.0",
|
"streamlit>=1.28.0",
|
||||||
"numpy>=1.21.0",
|
"numpy>=1.21.0",
|
||||||
"scipy>=1.7.0",
|
"scipy>=1.7.0",
|
||||||
|
"requests>=2.0.0",
|
||||||
],
|
],
|
||||||
entry_points={
|
entry_points={
|
||||||
"console_scripts": [
|
"console_scripts": [
|
||||||
|
|||||||
+16
-3
@@ -2,15 +2,29 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from importlib import import_module
|
from importlib import import_module
|
||||||
from typing import Any, Callable
|
from typing import Any, Callable, Optional
|
||||||
|
|
||||||
|
from src.integrity import ensure_file_sha256, ensure_within_allowed_dirs
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=128)
|
@lru_cache(maxsize=128)
|
||||||
def load_algo_func(module_path: str, function_name: str) -> Callable[..., Any]:
|
def load_algo_func(
|
||||||
|
module_path: str,
|
||||||
|
function_name: str,
|
||||||
|
expected_sha256: Optional[str] = None,
|
||||||
|
) -> Callable[..., Any]:
|
||||||
if not module_path or not function_name:
|
if not module_path or not function_name:
|
||||||
raise ValueError("module_path/function_name 不能为空")
|
raise ValueError("module_path/function_name 不能为空")
|
||||||
|
|
||||||
module = import_module(module_path)
|
module = import_module(module_path)
|
||||||
|
module_file = getattr(module, "__file__", None)
|
||||||
|
if module_file:
|
||||||
|
ensure_within_allowed_dirs(module_file)
|
||||||
|
if expected_sha256:
|
||||||
|
ensure_file_sha256(module_file, expected_sha256, label=f"python module {module_path}")
|
||||||
|
elif expected_sha256:
|
||||||
|
raise PermissionError(f"模块 '{module_path}' 缺少 __file__,无法进行 SHA256 校验")
|
||||||
|
|
||||||
func = getattr(module, function_name, None)
|
func = getattr(module, function_name, None)
|
||||||
if func is None:
|
if func is None:
|
||||||
raise AttributeError(f"模块 '{module_path}' 中未找到函数 '{function_name}'")
|
raise AttributeError(f"模块 '{module_path}' 中未找到函数 '{function_name}'")
|
||||||
@@ -18,4 +32,3 @@ def load_algo_func(module_path: str, function_name: str) -> Callable[..., Any]:
|
|||||||
raise TypeError(f"'{module_path}.{function_name}' 不是可调用对象")
|
raise TypeError(f"'{module_path}.{function_name}' 不是可调用对象")
|
||||||
|
|
||||||
return func
|
return func
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -487,7 +487,7 @@ class WebBridge(AlgorithmBridge):
|
|||||||
# 3. 业务逻辑检查:关键参数枚举校验
|
# 3. 业务逻辑检查:关键参数枚举校验
|
||||||
validate_critical_enums(request_params)
|
validate_critical_enums(request_params)
|
||||||
|
|
||||||
# 3. 清洗输出参数(生成唯一 run_id)
|
# 4. 清洗输出参数(生成唯一 run_id)
|
||||||
run_id = generate_run_id()
|
run_id = generate_run_id()
|
||||||
if self.sanitize_output:
|
if self.sanitize_output:
|
||||||
request_params = sanitize_output_params(request_params, run_id)
|
request_params = sanitize_output_params(request_params, run_id)
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import ctypes
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from src.integrity import ensure_file_sha256, ensure_within_allowed_dirs
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_abi(func: Any, cfg: Dict[str, Any]) -> None:
|
||||||
|
# Default ABI: void (double*, int, double*, int)
|
||||||
|
func.argtypes = [
|
||||||
|
ctypes.POINTER(ctypes.c_double),
|
||||||
|
ctypes.c_int,
|
||||||
|
ctypes.POINTER(ctypes.c_double),
|
||||||
|
ctypes.c_int,
|
||||||
|
]
|
||||||
|
func.restype = None
|
||||||
|
|
||||||
|
argtypes = cfg.get("argtypes")
|
||||||
|
if isinstance(argtypes, list) and argtypes:
|
||||||
|
mapping = {"double_ptr": ctypes.POINTER(ctypes.c_double), "int": ctypes.c_int}
|
||||||
|
func.argtypes = [mapping.get(t, ctypes.c_void_p) for t in argtypes]
|
||||||
|
|
||||||
|
restype = cfg.get("restype")
|
||||||
|
if isinstance(restype, str) and restype:
|
||||||
|
mapping = {"int": ctypes.c_int, "void": None}
|
||||||
|
func.restype = mapping.get(restype, None)
|
||||||
|
|
||||||
|
|
||||||
|
def run(config_path: str, input_path: str, output_path: str) -> int:
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
cfg: Dict[str, Any] = json.load(f)
|
||||||
|
|
||||||
|
lib_path = cfg.get("lib_path") or ""
|
||||||
|
function_name = cfg.get("function_name") or ""
|
||||||
|
expected_sha256 = cfg.get("expected_sha256") or None
|
||||||
|
|
||||||
|
if not lib_path or not function_name:
|
||||||
|
raise ValueError("config 缺少 lib_path/function_name")
|
||||||
|
|
||||||
|
abs_lib_path = ensure_within_allowed_dirs(lib_path)
|
||||||
|
if not os.path.exists(abs_lib_path):
|
||||||
|
raise FileNotFoundError(f"C++ library not found: {abs_lib_path}")
|
||||||
|
|
||||||
|
if expected_sha256:
|
||||||
|
ensure_file_sha256(abs_lib_path, expected_sha256, label="cpp library (isolated)")
|
||||||
|
|
||||||
|
in_data = np.load(input_path, allow_pickle=False)
|
||||||
|
if not isinstance(in_data, np.ndarray):
|
||||||
|
raise TypeError("input_npy 不是 ndarray")
|
||||||
|
in_data = np.ascontiguousarray(in_data, dtype=np.float64)
|
||||||
|
|
||||||
|
out_data = np.zeros_like(in_data)
|
||||||
|
|
||||||
|
lib = ctypes.CDLL(abs_lib_path)
|
||||||
|
func = getattr(lib, function_name)
|
||||||
|
_apply_abi(func, cfg)
|
||||||
|
|
||||||
|
in_ptr = in_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
|
||||||
|
out_ptr = out_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
|
||||||
|
in_len = ctypes.c_int(in_data.size)
|
||||||
|
out_len = ctypes.c_int(out_data.size)
|
||||||
|
|
||||||
|
func(in_ptr, in_len, out_ptr, out_len)
|
||||||
|
|
||||||
|
np.save(output_path, out_data, allow_pickle=False)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Optional[List[str]] = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Run C++ bridge call in isolated subprocess")
|
||||||
|
parser.add_argument("--config", required=True, help="Path to JSON config")
|
||||||
|
parser.add_argument("--input", required=True, help="Path to input .npy")
|
||||||
|
parser.add_argument("--output", required=True, help="Path to output .npy")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
try:
|
||||||
|
return run(args.config, args.input, args.output)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[cpp_isolated_runner] error: {e}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
+1
-1
@@ -5,7 +5,7 @@ import os
|
|||||||
import platform
|
import platform
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List, Tuple
|
||||||
|
|
||||||
|
|
||||||
def _has_module(module_name: str) -> bool:
|
def _has_module(module_name: str) -> bool:
|
||||||
|
|||||||
+59
@@ -0,0 +1,59 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
from typing import Tuple
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_ALLOWED_DIRS: Tuple[str, ...] = ("src", "data")
|
||||||
|
|
||||||
|
|
||||||
|
def project_root() -> str:
|
||||||
|
return os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_project_path(path: str) -> str:
|
||||||
|
if not path:
|
||||||
|
raise ValueError("path 不能为空")
|
||||||
|
|
||||||
|
if os.path.isabs(path):
|
||||||
|
return os.path.realpath(path)
|
||||||
|
return os.path.realpath(os.path.join(project_root(), path))
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_within_allowed_dirs(path: str, allowed_dirs: Tuple[str, ...] = DEFAULT_ALLOWED_DIRS) -> str:
|
||||||
|
abs_path = resolve_project_path(path)
|
||||||
|
root = project_root()
|
||||||
|
allowed_abs = [os.path.realpath(os.path.join(root, d)) for d in allowed_dirs]
|
||||||
|
|
||||||
|
if not any(os.path.commonpath([abs_path, d]) == d for d in allowed_abs):
|
||||||
|
raise PermissionError(
|
||||||
|
f"Security Block: Path {abs_path} is outside allowed directories: {', '.join(allowed_dirs)}"
|
||||||
|
)
|
||||||
|
return abs_path
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: str) -> str:
|
||||||
|
abs_path = resolve_project_path(path)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with open(abs_path, "rb") as f:
|
||||||
|
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_file_sha256(path: str, expected_sha256: str, label: str = "file") -> str:
|
||||||
|
if not expected_sha256:
|
||||||
|
return sha256_file(path)
|
||||||
|
|
||||||
|
expected = expected_sha256.strip().lower()
|
||||||
|
if len(expected) != 64 or any(c not in "0123456789abcdef" for c in expected):
|
||||||
|
raise ValueError(f"{label} expected_sha256 非法: {expected_sha256!r}")
|
||||||
|
|
||||||
|
actual = sha256_file(path).lower()
|
||||||
|
if actual != expected:
|
||||||
|
raise PermissionError(
|
||||||
|
f"SHA256 mismatch for {label}: expected {expected}, got {actual} (path={resolve_project_path(path)})"
|
||||||
|
)
|
||||||
|
return actual
|
||||||
|
|
||||||
在新工单中引用
屏蔽一个用户