This commit is contained in:
2026-08-31 18:01:34 +08:00
parent 3ea59551db
commit bee439cf34
46 changed files with 1884 additions and 1898 deletions
+5 -1
View File
@@ -1,6 +1,7 @@
from fastapi import APIRouter
import importlib
from shared.config.settings import settings
from shared.utils.logger import get_logger
logger = get_logger(__name__)
@@ -24,7 +25,10 @@ _safe_include("moldinsight.api.upload_router", "上传")
_safe_include("moldinsight.api.batch_router", "批量")
_safe_include("moldinsight.api.task_router", "任务")
_safe_include("moldinsight.api.history_router", "历史")
_safe_include("moldinsight.api.debug_router", "调试")
_safe_include("moldinsight.api.cam_router", "CAM")
_safe_include("moldinsight.api.advanced_router", "高级")
_safe_include("moldinsight.api.aluminum_price_routes", "铝价")
# 调试端点会 dump 全量任务数据,仅 DEBUG 模式注册(双重防线:还需登录)
if settings.DEBUG:
_safe_include("moldinsight.api.debug_router", "调试")
+59 -13
View File
@@ -65,10 +65,6 @@ def _get_cached_import(key: str):
return None
async def _get_task_data(task_id: str) -> dict:
return await redis_task_manager.get_task(task_id)
async def _ensure_task_access(
db_session: AsyncSession,
task_id: str,
@@ -85,7 +81,8 @@ async def _ensure_task_access(
_, stp_file = row
owner_id = getattr(stp_file, "user_id", None)
if owner_id is not None and owner_id != user_id:
if owner_id != user_id:
# 无主历史数据(owner_id is None)同样拒绝:无主不等于公共
raise HTTPException(403, "无权访问该任务的导出文件")
return row
@@ -283,21 +280,32 @@ async def design_complete_mold_system(
async def detect_undercuts(
request: Request,
current_user: User = Depends(get_current_active_user),
db_session: AsyncSession = Depends(get_db_session),
):
body = await request.json()
task_id = body.get("task_id")
parting_direction = body.get("parting_direction", [0, 0, 1])
mold_size = body.get("mold_size", {"length": 300, "width": 300, "height": 200})
if not task_id:
raise HTTPException(404, "缺少 task_id")
task_data = await _get_task_data(task_id)
if not task_data:
raise HTTPException(404, "任务不存在")
raise HTTPException(400, "缺少 task_id")
await _ensure_task_access(db_session, task_id, current_user.id)
sd = _get_cached_import("side_action_designer")
if not sd:
raise HTTPException(503, "服务不可用:核心模块未加载")
result = sd.analyze_and_design(
shape=None, parting_direction=parting_direction, mold_size=mold_size,
# 从持久化 STP 原件重建几何(此前传 shape=None 会被兜底吞掉,永远返回"无倒扣")
from moldinsight.services.shape_loader import get_shape_loader
shape = await get_shape_loader().load_shape_for_task(db_session, task_id)
if shape is None:
raise HTTPException(410, "任务几何不可用:无法从存储重建 STP 形状,请重新上传分析")
result = await processing_service.run_occ(
sd.analyze_and_design,
shape,
parting_direction,
mold_size,
)
return {"status": "success", "data": result}
@@ -306,13 +314,18 @@ async def detect_undercuts(
async def estimate_cost(
request: Request,
current_user: User = Depends(get_current_active_user),
db_session: AsyncSession = Depends(get_db_session),
):
"""模具成本估算:优先使用 LLM,未启用时降级为规则式估算"""
body = await request.json()
task_id = body.get("task_id")
if not task_id:
raise HTTPException(404, "缺少 task_id")
task_data = await _get_task_data(task_id)
raise HTTPException(400, "缺少 task_id")
await _ensure_task_access(db_session, task_id, current_user.id)
# 统一走任务视图:进行中读 Redis,完成态由 PG+RustFS 组装(Redis 大对象已瘦身)
task_data = await TaskQueryService.get_task_view(db_session, task_id)
if not task_data:
raise HTTPException(404, "任务不存在")
analysis_result = task_data.get("analysis_result")
@@ -475,6 +488,38 @@ async def export_mold_results(
filename = task_data.get("filename", f"mold_{task_id}")
if not cavity_shapes:
# 内存 shape 缓存失效(如服务重启):从持久化的单组件 STEP
# 现场转换缺失格式,用户无需重新分析
artifacts = _get_export_artifacts(task_data)
scheme_data = (artifacts.get("schemes") or {}).get(resolved_scheme_id)
if scheme_data:
base_filename = scheme_data.get("base_filename") or Path(filename).stem
regenerated = await processing_service.regenerate_export_from_persisted(
task_id=task_id,
scheme_id=resolved_scheme_id,
formats=formats,
components=_expand_components(components),
base_filename=base_filename,
scheme_files=scheme_data.get("files", []),
)
if regenerated:
regenerated["files"] = _augment_export_files(
task_id, regenerated.get("files", [])
)
# 合并进持久化 manifest,后续请求直接命中持久化路径
merged_artifacts = _merge_export_artifacts(artifacts, regenerated)
await storage_service.update_task_parameters(
db_session,
task_id,
{"export_artifacts": merged_artifacts},
)
await redis_task_manager.update_task(
task_id, {"export_artifacts": merged_artifacts}
)
TaskQueryService.invalidate_task_view(task_id)
return {"status": "success", "data": regenerated}
raise HTTPException(
409,
"导出缓存已失效或任务尚未完成,请重新分析后再导出以保证方案一致性",
@@ -499,6 +544,7 @@ async def export_mold_results(
{"export_artifacts": merged_artifacts},
)
await redis_task_manager.update_task(task_id, {"export_artifacts": merged_artifacts})
TaskQueryService.invalidate_task_view(task_id) # parameters 已变更,缓存视图失效
return {"status": "success", "data": result}
+41 -28
View File
@@ -6,7 +6,7 @@ moldinsight/api/batch_router.py — 批量分析端点
"""
import uuid
from datetime import datetime
from typing import List, Dict, Any
from typing import List, Dict, Any, Optional
from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Depends
from sqlalchemy.ext.asyncio import AsyncSession
@@ -19,13 +19,7 @@ from shared.services.redis_task_manager import redis_task_manager
from shared.utils.file_handler import FileHandler
from shared.utils.logger import get_logger
from moldinsight.services.storage_integration_rustfs import StorageIntegrationService
try:
from celery_tasks import process_stp_task
_use_celery = True
except ImportError:
process_stp_task = None
_use_celery = False
from moldinsight.services.task_dispatcher import dispatch_processing
logger = get_logger(__name__)
@@ -37,6 +31,9 @@ file_handler = FileHandler()
_BATCH_KEY_PREFIX = "batch:"
_BATCH_TTL = 86400 # 24h
# Redis 不可用时的进程内降级存储(同进程内可查,跨进程/重启不可见)
_batch_meta_memory: Dict[str, dict] = {}
def _batch_redis_key(batch_id: str) -> str:
return f"{_BATCH_KEY_PREFIX}{batch_id}"
@@ -111,14 +108,7 @@ async def batch_upload(
await redis_task_manager.set_task(task_id, task_info)
# 调度处理
if _use_celery:
process_stp_task.delay(task_id, str(file_path), stp_file.id, process_params)
else:
import asyncio
from moldinsight.services.processing_service import processing_service
asyncio.create_task(processing_service.process_file_with_storage(
task_id, str(file_path), stp_file.id, process_params
))
dispatch_processing(task_id, str(file_path), stp_file.id, process_params)
tasks.append({
"filename": file.filename,
@@ -139,7 +129,7 @@ async def batch_upload(
"error": str(exc),
})
# 将 batch 元数据写入 Redis
# 将 batch 元数据写入 Redis;Redis 不可用时降级到进程内存储(任务状态本身有内存回退)
batch_meta = {
"batch_id": batch_id,
"user_id": current_user.id,
@@ -148,11 +138,7 @@ async def batch_upload(
"total": len(tasks),
"params": process_params,
}
await redis_task_manager.redis_client.set(
_batch_redis_key(batch_id),
__import__("json").dumps(batch_meta),
ex=_BATCH_TTL,
)
_save_batch_meta(batch_id, batch_meta)
return {
"batch_id": batch_id,
@@ -162,20 +148,47 @@ async def batch_upload(
}
def _save_batch_meta(batch_id: str, batch_meta: dict):
"""批量元数据持久化:优先 Redis(跨进程、带 TTL),降级进程内 dict。"""
import json as _json
if redis_task_manager.is_connected:
try:
redis_task_manager.redis_client.set(
_batch_redis_key(batch_id),
_json.dumps(batch_meta),
ex=_BATCH_TTL,
)
return
except Exception as exc:
logger.warning(f"[BATCH] batch 元数据写 Redis 失败,降级内存: {exc}")
_batch_meta_memory[batch_id] = batch_meta
async def _load_batch_meta(batch_id: str) -> Optional[dict]:
"""读取批量元数据,Redis 优先,内存兜底;不存在返回 None。"""
import json as _json
if redis_task_manager.is_connected:
try:
raw = await redis_task_manager.redis_client.get(_batch_redis_key(batch_id))
if raw:
return _json.loads(raw)
except Exception as exc:
logger.warning(f"[BATCH] batch 元数据读 Redis 失败: {exc}")
return _batch_meta_memory.get(batch_id)
@router.get("/batch/{batch_id}")
async def get_batch_status(
batch_id: str,
current_user: User = Depends(get_current_active_user),
):
"""聚合查询批量任务进度"""
import json
raw = await redis_task_manager.redis_client.get(_batch_redis_key(batch_id))
if not raw:
batch_meta = await _load_batch_meta(batch_id)
if not batch_meta:
raise HTTPException(404, "批量任务不存在或已过期")
batch_meta = json.loads(raw)
# 权限检查
if batch_meta.get("user_id") and batch_meta["user_id"] != current_user.id:
raise HTTPException(403, "无权访问该批量任务")
+2
View File
@@ -96,6 +96,8 @@ async def generate_cam_plan(
}
processing_task.parameters = parameters
await db_session.commit()
# parameters 已变更,任务视图缓存失效
TaskQueryService.invalidate_task_view(task_id)
return {"status": "success", "data": data, "cam_preferences": cam_preferences}
except Exception as exc:
+5 -3
View File
@@ -1,15 +1,17 @@
# api/v1/debug_router.py
from fastapi import APIRouter
from fastapi import APIRouter, Depends
from shared.services.auth_service import get_current_active_user
from shared.services.redis_task_manager import redis_task_manager
from shared.models.database import User
router = APIRouter()
@router.get("/debug/tasks")
@router.post("/debug/tasks")
async def debug_tasks():
"""调试接口:查看所有任务"""
async def debug_tasks(current_user: User = Depends(get_current_active_user)):
"""调试接口:查看所有任务(仅限 DEBUG 模式注册,且需登录)"""
all_tasks = await redis_task_manager.get_all_tasks()
return {
"total_tasks": len(all_tasks),
+21 -6
View File
@@ -4,17 +4,27 @@ import urllib.parse
from moldinsight.services.storage_integration_rustfs import StorageIntegrationService
from shared.database.database import get_db_session
from shared.services.auth_service import get_current_active_user
from shared.models.database import User
from shared.utils.logger import get_logger
from sqlalchemy.ext.asyncio import AsyncSession
logger = get_logger(__name__)
router = APIRouter()
@router.get("/history")
@router.post("/history")
async def get_file_history(db_session: AsyncSession = Depends(get_db_session)):
"""获取按文件名分组的文件历史记录(支持多上传)"""
async def get_file_history(
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_current_active_user),
):
"""获取当前用户按文件名分组的文件历史记录(支持多上传)"""
storage_service = StorageIntegrationService()
file_groups = await storage_service.get_all_file_groups(db_session)
file_groups = await storage_service.get_all_file_groups(
db_session, user_id=current_user.id
)
return {
"total_files": len(file_groups),
@@ -24,14 +34,19 @@ async def get_file_history(db_session: AsyncSession = Depends(get_db_session)):
@router.get("/history/{filename}")
@router.post("/history/{filename}")
async def get_file_records(filename: str, db_session: AsyncSession = Depends(get_db_session)):
"""获取指定文件名的所有上传记录(支持多上传历史)"""
async def get_file_records(
filename: str,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_current_active_user),
):
"""获取当前用户指定文件名的所有上传记录(支持多上传历史)"""
decoded_filename = urllib.parse.unquote(filename)
storage_service = StorageIntegrationService()
file_records = await storage_service.get_file_history_by_filename(
db_session,
decoded_filename
decoded_filename,
user_id=current_user.id,
)
return file_records
+2 -18
View File
@@ -2,11 +2,11 @@
from fastapi import APIRouter, UploadFile, File, HTTPException, Depends, Form
import uuid
from datetime import datetime
from pathlib import Path
from shared.models.schemas import ProcessingStatus, create_task_info
from shared.utils.file_handler import FileHandler
from moldinsight.services.storage_integration_rustfs import StorageIntegrationService
from moldinsight.services.task_dispatcher import dispatch_processing
from shared.services.redis_task_manager import redis_task_manager
from shared.database.database import get_db_session
from shared.utils.logger import get_logger
@@ -14,13 +14,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
from shared.services.auth_service import get_current_active_user
from shared.models.database import User
try:
from celery_tasks import process_stp_task
_use_celery = True
except ImportError:
process_stp_task = None
_use_celery = False
logger = get_logger(__name__)
router = APIRouter()
@@ -96,16 +89,7 @@ async def upload_stp(
task_info["file_hash"] = file_meta["sha256"]
await redis_task_manager.set_task(task_id, task_info)
if _use_celery:
process_stp_task.delay(task_id, str(file_path), stp_file.id, process_params)
logger.info(f"[UPLOAD] Celery 任务已调度: task_id={task_id}")
else:
import asyncio
from moldinsight.services.processing_service import processing_service
asyncio.create_task(processing_service.process_file_with_storage(
task_id, str(file_path), stp_file.id, process_params
))
logger.info(f"[UPLOAD] 直接后台处理: task_id={task_id} (celery 未安装)")
dispatch_processing(task_id, str(file_path), stp_file.id, process_params)
return {
"task_id": task_id,
+140 -8
View File
@@ -38,6 +38,26 @@ logger = get_logger(__name__)
class CADExporter:
"""CAD 文件导出器"""
# 组件 -> (cavity_data 取值键, 显示名)
_SHAPE_MAP = {
"cavity": ("cavity", "型腔"),
"core": ("core", "型芯"),
"parting_surface": ("parting_surface", "分型面"),
"product": ("product", "产品本体"),
"a_plate": ("a_plate", "A板(上模)"),
"b_plate": ("b_plate", "B板(下模)"),
}
_COMPONENT_LABELS = {
"cavity": "型腔",
"core": "型芯",
"parting_surface": "分型面",
"product": "产品本体",
"a_plate": "A板(上模)",
"b_plate": "B板(下模)",
"assembly": "模具装配体",
}
def __init__(self, output_dir: str = "./exports"):
self.output_dir = output_dir
os.makedirs(output_dir, exist_ok=True)
@@ -71,6 +91,125 @@ class CADExporter:
relative = Path(os.path.basename(filepath))
return relative.as_posix()
def build_file_entry(self, component: str, fmt: str, filepath: str,
label: Optional[str] = None) -> Dict[str, Any]:
"""构造统一格式的导出文件条目(供导出/重建流程复用)。"""
file_size = os.path.getsize(filepath) if os.path.exists(filepath) else 0
return {
"component": component,
"component_label": label or self._COMPONENT_LABELS.get(component, component),
"format": fmt,
"filepath": filepath,
"relative_path": self.get_relative_path(filepath),
"filename": os.path.basename(filepath),
"size_bytes": file_size,
"size_readable": self._format_file_size(file_size),
}
def export_persisted_steps(
self,
cavity_data: Dict,
base_filename: str,
components: Optional[List[str]] = None,
task_id: Optional[str] = None,
scheme_id: Optional[str] = None,
) -> Dict[str, Any]:
"""持久化方案几何为 STEP:装配体单文件 + 逐组件单文件。
逐组件 STEP 是服务重启后按需重导出其他格式(STL/IGES/BRep)的
几何来源:读回单组件 STEP 即可现场转换,用户无需重新分析。
"""
if components is None:
components = list(self._SHAPE_MAP.keys())
export_dir = self.build_export_dir(
base_filename=base_filename,
task_id=task_id,
scheme_id=scheme_id,
)
os.makedirs(export_dir, exist_ok=True)
results = {"files": [], "errors": []}
shapes_with_names: List[Tuple[TopoDS_Shape, str]] = []
for comp in components:
data_key, label = self._SHAPE_MAP.get(comp, (comp, comp))
shape = cavity_data.get(data_key)
if shape is None:
results["errors"].append(f"{label}形状不可用")
continue
shapes_with_names.append((shape, label))
# 逐组件单文件 STEP(重启后转换其他格式的几何来源)
filepath = os.path.join(export_dir, f"{base_filename}_{comp}.step")
if self.export_step(shape, filepath):
results["files"].append(
self.build_file_entry(comp, "step", filepath, label)
)
# 装配体 STEP(所有组件写入同一文件)
if shapes_with_names:
filepath = os.path.join(export_dir, f"{base_filename}_mold.step")
if self.export_assembly_step(shapes_with_names, filepath):
results["files"].append(
self.build_file_entry("assembly", "step", filepath)
)
results["total_files"] = len(results["files"])
results["total_errors"] = len(results["errors"])
logger.info(
f"方案 STEP 持久化完成: {results['total_files']} 个文件, "
f"{results['total_errors']} 个错误"
)
return results
def convert_component_step(self, step_path: str, output_path: str, fmt: str) -> bool:
"""读回单组件 STEP,转换导出为其他格式(stl/iges/brep/step)。
用于内存 shape 缓存失效后的按需重导出。
"""
shape = self._read_step_shape(step_path)
if shape is None:
return False
if fmt == "stl":
return self.export_stl(shape, output_path)
if fmt == "iges":
return self.export_iges(shape, output_path)
if fmt == "brep":
return self.export_brep(shape, output_path)
if fmt == "step":
import shutil
try:
shutil.copyfile(step_path, output_path)
return os.path.exists(output_path) and os.path.getsize(output_path) > 0
except Exception as e:
logger.error(f"STEP 复制失败: {e}")
return False
logger.error(f"不支持的转换格式: {fmt}")
return False
def _read_step_shape(self, filepath: str) -> Optional[TopoDS_Shape]:
"""读回 STEP 文件为 OCC 形状"""
try:
from OCC.Core.STEPControl import STEPControl_Reader
from OCC.Core.IFSelect import IFSelect_RetDone
if not os.path.exists(filepath):
logger.error(f"STEP 文件不存在: {filepath}")
return None
reader = STEPControl_Reader()
if reader.ReadFile(filepath) != IFSelect_RetDone:
logger.error(f"STEP 读回失败(状态非 Done): {filepath}")
return None
reader.TransferRoots()
return reader.OneShape()
except Exception as e:
logger.error(f"STEP 读回失败: {filepath}: {e}")
return None
def export_step(self, shape: TopoDS_Shape, filepath: str,
schema: str = "AP214") -> bool:
"""
@@ -262,14 +401,7 @@ class CADExporter:
"errors": [],
}
shape_map = {
"cavity": ("cavity", "型腔"),
"core": ("core", "型芯"),
"parting_surface": ("parting_surface", "分型面"),
"product": ("product", "产品本体"),
"a_plate": ("a_plate", "A板(上模)"),
"b_plate": ("b_plate", "B板(下模)"),
}
shape_map = self._SHAPE_MAP
shapes_to_export: List[Tuple[str, str, TopoDS_Shape]] = []
assembly_shapes: List[Tuple[TopoDS_Shape, str]] = []
+23 -5
View File
@@ -524,11 +524,29 @@ class LLMService:
}
if expect_json:
payload["response_format"] = {"type": "json_object"}
async with httpx.AsyncClient(timeout=self._timeout) as client:
resp = await client.post(url, json=payload, headers=headers)
resp.raise_for_status()
content = resp.json()["choices"][0]["message"]["content"]
return content.strip() if content else None
# 保持 per-call client:celery 每任务经 asyncio.run 新建事件循环,
# 模块级 AsyncClient 绑定旧循环会失效(与 redis_task_manager 同理)。
# 瞬态错误(网络/5xx/429)重试一次,其余直接抛出。
last_exc: Optional[Exception] = None
for attempt in range(2):
try:
async with httpx.AsyncClient(timeout=self._timeout) as client:
resp = await client.post(url, json=payload, headers=headers)
resp.raise_for_status()
content = resp.json()["choices"][0]["message"]["content"]
return content.strip() if content else None
except httpx.TransportError as exc:
last_exc = exc
logger.warning(f"LLM 请求瞬态失败(第 {attempt + 1} 次): {exc}")
except httpx.HTTPStatusError as exc:
status = exc.response.status_code
if status >= 500 or status == 429:
last_exc = exc
logger.warning(f"LLM 服务端错误 {status}(第 {attempt + 1} 次)")
else:
raise
raise last_exc
@staticmethod
def _prioritize_features_for_side_action(
+115 -8
View File
@@ -2,12 +2,14 @@
"""STP 文件处理流程编排器 — 协调解析、网格生成、型腔生成、保存、验证"""
import asyncio
import os
import time
import traceback
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any
from typing import Optional, Dict, Any, List
from sqlalchemy.ext.asyncio import AsyncSession
@@ -40,12 +42,35 @@ class ProcessingService:
self.storage_service = StorageIntegrationService()
self.multi_scheme_planner = MultiSchemeMoldPlanner()
self.cad_exporter = CADExporter()
self._export_shapes_cache: Dict[str, Dict[str, Dict[str, Any]]] = {}
# TopoDS_Shape 为 C++ 原生内存对象,LRU 上限防止长期运行内存只涨不降
self._export_shapes_cache: "OrderedDict[str, Dict[str, Dict[str, Any]]]" = OrderedDict()
self._export_shapes_cache_max = 32
# OCC 非线程安全,max_workers=1 保证所有 OCC 操作序列化执行,避免偶发崩溃
self._occ_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="occ")
# ─── 对外入口 ───
def _reset_occ_executor(self):
"""超时后重建 OCC executor。
asyncio.wait_for 只能取消协程,正在执行 OCC 布尔运算的线程无法中断;
单 worker executor 中一个挂死线程会让后续任务永久排队直至重启。
代价是泄漏 1 个线程,收益是恢复服务可用性。
"""
old = self._occ_executor
self._occ_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="occ")
old.shutdown(wait=False)
logger.warning("OCC executor 已因处理超时重建(放弃等待旧线程,可能泄漏 1 个线程)")
async def run_occ(self, fn, *args):
"""在 OCC 单线程 executor 中执行同步几何操作。
OCC 非线程安全,所有几何计算(解析/布尔/三角化)统一经由本入口串行执行,
避免各调用方自行创建线程池造成并发崩溃。
"""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(self._occ_executor, fn, *args)
async def process_file_with_storage(
self,
task_id: str,
@@ -79,6 +104,8 @@ class ProcessingService:
)
except asyncio.TimeoutError:
logger.error(f"处理超时: {task_id}")
# OCC 线程无法取消:抛弃整个 executor,避免挂死线程堵死后续所有任务
self._reset_occ_executor()
raise Exception(f"处理超时,超过{timeout_seconds}秒未完成")
except Exception as e:
@@ -338,12 +365,12 @@ class ProcessingService:
},
)
# 更新任务缓存状态(仅保留轻量摘要,完整数据由PG+RustFS持久化)
# 更新任务缓存状态(仅保留轻量摘要,完整数据由PG+RustFS持久化;
# 完成态视图由 TaskQueryService 从 PG+RustFS 组装,Redis 不再存
# geometry_data / analysis_result 等 MB 级大对象)
await redis_task_manager.update_task(task_id, {
"status": ProcessingStatus.COMPLETED,
"completed_at": str(datetime.now()),
"geometry_data": geometry_data,
"analysis_result": analysis_result,
"key_info": best_key_info,
"best_scheme_id": detailed_cavity_json.get("best_scheme_id"),
"material": requested_material,
@@ -471,6 +498,10 @@ class ProcessingService:
def _cache_export_shapes(self, task_id: str, export_shapes: Dict[str, Dict[str, Any]]):
self._export_shapes_cache[task_id] = export_shapes
self._export_shapes_cache.move_to_end(task_id)
# 逐出最旧任务的形状缓存(连原生 OCC shape 引用一起释放)
while len(self._export_shapes_cache) > self._export_shapes_cache_max:
self._export_shapes_cache.popitem(last=False)
def _persist_step_exports(
self,
@@ -492,10 +523,11 @@ class ProcessingService:
for scheme_id, cavity_data in export_shapes.items():
try:
result = self.cad_exporter.export_mold_results(
# 持久化装配体 STEP + 逐组件 STEP(后者是重启后按需
# 重导出其他格式的几何来源,见 regenerate_export_from_persisted)
result = self.cad_exporter.export_persisted_steps(
cavity_data=cavity_data,
base_filename=base_filename,
formats=["step"],
components=components,
task_id=task_id,
scheme_id=scheme_id,
@@ -522,13 +554,88 @@ class ProcessingService:
return manifest
def get_export_shapes(self, task_id: str, scheme_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
scheme_map = self._export_shapes_cache.get(task_id, {})
scheme_map = self._export_shapes_cache.get(task_id)
if not scheme_map:
return None
self._export_shapes_cache.move_to_end(task_id) # LRU 热点保活
if scheme_id:
return scheme_map.get(scheme_id)
return next(iter(scheme_map.values()), None)
async def regenerate_export_from_persisted(
self,
task_id: str,
scheme_id: str,
formats: Optional[List[str]],
components: List[str],
base_filename: str,
scheme_files: List[Dict[str, Any]],
) -> Optional[Dict[str, Any]]:
"""内存导出缓存失效(如服务重启)后,从持久化 STEP 重建导出文件。
分析期已为每个方案持久化装配体 + 逐组件 STEP;
缺失格式(STL/IGES/BRep)读回单组件 STEP 现场转换,
用户无需重新分析。所有组件均不可用时返回 None。
"""
format_list = list(dict.fromkeys(formats or ["step", "stl"]))
step_files = {
f.get("component"): f
for f in scheme_files or []
if f.get("format") == "step" and f.get("component")
}
if not step_files:
return None
files: List[Dict[str, Any]] = []
errors: List[str] = []
if "step" in format_list:
assembly = step_files.get("assembly")
if assembly:
files.append(assembly)
else:
errors.append("模具装配体 (step) 不可用")
for comp in components:
comp_file = step_files.get(comp)
if comp_file is None:
errors.append(f"组件 {comp} 的持久化 STEP 不可用")
continue
for fmt in format_list:
if fmt == "step":
files.append(comp_file)
continue
step_path = os.path.join(
self.cad_exporter.output_dir,
str(comp_file.get("relative_path") or "").replace("/", os.sep),
)
out_path = os.path.join(
os.path.dirname(step_path), f"{base_filename}_{comp}.{fmt}"
)
ok = await self.run_occ(
self.cad_exporter.convert_component_step, step_path, out_path, fmt
)
if ok:
files.append(
self.cad_exporter.build_file_entry(comp, fmt, out_path)
)
else:
errors.append(f"组件 {comp} ({fmt}) 转换失败")
if not files:
return None
return {
"base_filename": base_filename,
"task_id": task_id,
"scheme_id": scheme_id,
"files": files,
"errors": errors,
"total_files": len(files),
"total_errors": len(errors),
"source": "regenerated",
}
@staticmethod
def _normalize_process_params(process_params: Optional[Dict[str, Any]]) -> Dict[str, Any]:
payload = dict(process_params or {})
+88
View File
@@ -0,0 +1,88 @@
# services/shape_loader.py
"""按 task_id 从持久化存储重建 OCC 几何形状。
任务完成后 TopoDS_Shape 不驻留内存/Redis(原生内存与体积原因),
需要几何的端点(倒扣检测、按需重导出等)通过 STP 原件重建:
PG(object_key) -> RustFS 下载 -> 临时文件 -> OCC 单线程 executor 解析。
"""
import tempfile
from pathlib import Path
from typing import Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from shared.models.database import ProcessingTask, STPFile
from shared.utils.logger import get_logger
logger = get_logger(__name__)
class ShapeLoader:
"""任务几何重建器"""
def __init__(self):
from moldinsight.core.stp_parser import STPParser
from moldinsight.services.processing_service import processing_service
self._parser = STPParser()
self._processing = processing_service
async def load_shape_for_task(
self, db_session: AsyncSession, task_id: str
) -> Optional["object"]:
"""重建任务的产品几何。任务不存在或 STP 原件不可用时返回 None。"""
result = await db_session.execute(
select(ProcessingTask, STPFile)
.join(STPFile, ProcessingTask.stp_file_id == STPFile.id)
.where(ProcessingTask.task_id == task_id)
)
row = result.first()
if not row:
logger.warning(f"几何重建失败:任务不存在 {task_id}")
return None
_, stp_file = row
if not stp_file.object_key:
logger.warning(f"几何重建失败:任务缺少 object_key {task_id}")
return None
from moldinsight.storage.rustfs_storage import rustfs_manager
try:
data = await rustfs_manager.download_file(
file_type="stp_files", object_key=stp_file.object_key
)
except Exception as exc:
logger.error(f"几何重建失败:STP 原件下载失败 {task_id}: {exc}")
return None
with tempfile.NamedTemporaryFile(suffix=".stp", delete=False) as tmp:
tmp.write(data)
tmp_path = Path(tmp.name)
try:
shape = await self._processing.run_occ(
self._parser.load_step_file, tmp_path
)
return shape
except Exception as exc:
logger.error(f"几何重建失败:STP 解析失败 {task_id}: {exc}")
return None
finally:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
# 惰性单例:__init__ 会实例化 STPParser 并校验 OCC 可用性,
# 延迟到首次真实使用,避免模块导入期失败拖垮路由加载
_shape_loader: Optional[ShapeLoader] = None
def get_shape_loader() -> ShapeLoader:
global _shape_loader
if _shape_loader is None:
_shape_loader = ShapeLoader()
return _shape_loader
@@ -1,376 +0,0 @@
# services/storage_integration.py
"""存储集成服务 - 协调 PostgreSQL 和 MinIO"""
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from pathlib import Path
from typing import Optional, Dict, Any
import json
from shared.models.database import (
STPFile, GeometryData, MoldCavityData,
HTMLFile, ProcessingTask, User,
FeatureDetection, DesignRecommendation,
UserActivity, SystemLog
)
from moldinsight.storage.object_storage import storage_manager
from shared.utils.logger import get_logger
logger = get_logger(__name__)
class StorageIntegrationService:
"""存储集成服务"""
async def save_stp_file(self, session: AsyncSession,
file_path: Path,
original_filename: str,
user_id: Optional[int] = None) -> STPFile:
"""保存STP文件到PostgreSQL元数据 + MinIO对象存储"""
# 1. 上传到MinIO
upload_result = await storage_manager.upload_stp_file(
file_path,
original_filename
)
# 2. 创建PostgreSQL记录
stp_file = STPFile(
user_id=user_id,
object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['stp_files'],
original_filename=original_filename,
file_size=upload_result['file_size'],
file_hash=upload_result['file_hash'],
status="uploaded",
file_path=str(file_path) # 保留本地路径以兼容
)
session.add(stp_file)
await session.commit()
await session.refresh(stp_file)
logger.info(f"STP文件保存成功: {stp_file.id}")
return stp_file
async def save_geometry_data(self, session: AsyncSession,
stp_file_id: int,
geometry_json: Dict[str, Any],
analysis_method: str = "pythonocc") -> GeometryData:
"""保存几何数据到PostgreSQL元数据 + MinIO对象存储"""
# 1. 获取文件哈希
stp_file = await session.get(STPFile, stp_file_id)
file_hash = stp_file.file_hash
# 2. 上传到MinIO
upload_result = await storage_manager.upload_geometry_data(
geometry_json,
file_hash
)
# 3. 创建PostgreSQL记录
geometry_data = GeometryData(
stp_file_id=stp_file_id,
object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['geometry_data'],
analysis_method=analysis_method,
# 提取摘要字段
volume=geometry_json.get('geometry_data', {}).get('volume'),
surface_area=geometry_json.get('geometry_data', {}).get('surface_area'),
bounding_box_min=geometry_json.get('geometry_data', {}).get('bounding_box', {}).get('min'),
bounding_box_max=geometry_json.get('geometry_data', {}).get('bounding_box', {}).get('max'),
center_of_mass=geometry_json.get('geometry_data', {}).get('center_of_mass'),
topology_faces=geometry_json.get('geometry_data', {}).get('topology', {}).get('faces'),
topology_edges=geometry_json.get('geometry_data', {}).get('topology', {}).get('edges'),
topology_vertices=geometry_json.get('geometry_data', {}).get('topology', {}).get('vertices')
)
session.add(geometry_data)
await session.commit()
await session.refresh(geometry_data)
logger.info(f"几何数据保存成功: {geometry_data.id}")
return geometry_data
async def save_mold_cavity_data(self, session: AsyncSession,
stp_file_id: int,
cavity_json: Dict[str, Any]) -> MoldCavityData:
"""保存模具型腔数据到PostgreSQL元数据 + MinIO对象存储"""
# 1. 获取文件哈希
stp_file = await session.get(STPFile, stp_file_id)
file_hash = stp_file.file_hash
# 2. 上传到MinIO
upload_result = await storage_manager.upload_mold_cavity_data(
cavity_json,
file_hash
)
# 3. 提取关键信息
metadata = cavity_json.get('metadata', {})
product_analysis = cavity_json.get('product_analysis', {})
manufacturing_info = cavity_json.get('manufacturing_info', {})
mold_size = manufacturing_info.get('estimated_mold_size', {})
key_info = cavity_json.get('mold_cavities', {}).get('cavity_key_info', {})
# 4. 创建PostgreSQL记录
mold_cavity = MoldCavityData(
stp_file_id=stp_file_id,
detailed_object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['mold_cavities'],
# 模具参数
mold_material=manufacturing_info.get('recommended_material', 'Aluminum Alloy 7075'),
shrinkage_rate=metadata.get('shrinkage_rate', 0.005),
draft_angle=metadata.get('draft_angle', 2.0),
# 提取的摘要字段
cavity_key_info=key_info,
mold_size_length=mold_size.get('length'),
mold_size_width=mold_size.get('width'),
mold_size_height=mold_size.get('height'),
estimated_clamping_force=manufacturing_info.get('estimated_clamping_force'),
product_volume=product_analysis.get('volume'),
# 从key_info中提取(如果存在)
product_weight=key_info.get('geometric_characteristics', {}).get('product_weight'),
wall_thickness_range=key_info.get('geometric_characteristics', {}).get('wall_thickness_range'),
complexity_score=key_info.get('geometric_characteristics', {}).get('complexity_score'),
# 质量评估
weld_line_risk=key_info.get('quality_considerations', {}).get('potential_weld_lines'),
sink_mark_risk=key_info.get('quality_considerations', {}).get('sink_mark_areas'),
warpage_risk=key_info.get('quality_considerations', {}).get('warpage_risk')
)
session.add(mold_cavity)
await session.commit()
await session.refresh(mold_cavity)
logger.info(f"模具型腔数据保存成功: {mold_cavity.id}")
return mold_cavity
async def save_html_file(self, session: AsyncSession,
stp_file_id: int,
html_content: str,
filename: str) -> HTMLFile:
"""保存HTML文件到PostgreSQL元数据 + MinIO对象存储"""
# 1. 获取文件哈希
stp_file = await session.get(STPFile, stp_file_id)
file_hash = stp_file.file_hash
# 2. 上传到MinIO
upload_result = await storage_manager.upload_html_file(
html_content,
filename,
file_hash
)
# 3. 创建PostgreSQL记录
html_file = HTMLFile(
stp_file_id=stp_file_id,
object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['html_files'],
filename=filename,
file_path=str(Path('html_output') / filename), # 保留本地路径
html_content=html_content # 保留内容以兼容
)
session.add(html_file)
await session.commit()
await session.refresh(html_file)
logger.info(f"HTML文件保存成功: {html_file.id}")
return html_file
async def save_features_and_recommendations(
self, session: AsyncSession,
stp_file_id: int,
features: list,
recommendations: list
):
"""保存特征检测结果和设计建议"""
# 1. 保存特征
for feature in features:
feature_record = FeatureDetection(
stp_file_id=stp_file_id,
feature_type=feature.get('feature_type'),
confidence=feature.get('confidence'),
location=feature.get('location'),
dimensions=feature.get('dimensions'),
parameters=feature.get('parameters')
)
session.add(feature_record)
# 2. 保存建议
for rec in recommendations:
rec_record = DesignRecommendation(
stp_file_id=stp_file_id,
rec_type=rec.get('rec_type'),
priority=rec.get('priority'),
description=rec.get('description'),
reason=rec.get('reason'),
parameters=rec.get('parameters')
)
session.add(rec_record)
await session.commit()
logger.info(f"保存了 {len(features)} 个特征和 {len(recommendations)} 个建议")
async def log_user_activity(self, session: AsyncSession,
user_id: int,
activity_type: str,
resource_type: Optional[str] = None,
resource_id: Optional[int] = None,
description: Optional[str] = None,
metadata: Optional[Dict] = None,
ip_address: Optional[str] = None,
user_agent: Optional[str] = None):
"""记录用户活动"""
activity = UserActivity(
user_id=user_id,
activity_type=activity_type,
resource_type=resource_type,
resource_id=resource_id,
description=description,
metadata=metadata,
ip_address=ip_address,
user_agent=user_agent
)
session.add(activity)
await session.commit()
logger.debug(f"用户活动记录: {activity_type} by user {user_id}")
async def get_stp_file_with_data(self, session: AsyncSession,
stp_file_id: int) -> Dict[str, Any]:
"""获取STP文件及其所有关联数据"""
# 1. 获取STP文件记录
stp_file = await session.get(STPFile, stp_file_id)
if not stp_file:
raise ValueError(f"STP文件不存在: {stp_file_id}")
result = {
'metadata': {
'id': stp_file.id,
'original_filename': stp_file.original_filename,
'file_size': stp_file.file_size,
'file_hash': stp_file.file_hash,
'upload_time': stp_file.upload_time.isoformat() if stp_file.upload_time else None,
'status': stp_file.status,
'user_id': stp_file.user_id
},
'geometry_data': None,
'mold_cavity_data': None,
'html_file': None,
'features': [],
'recommendations': []
}
# 2. 从MinIO获取数据
try:
# 几何数据
if stp_file.geometry_data:
geo_data_bytes = await storage_manager.download_file(
'geometry_data',
stp_file.geometry_data.object_key
)
result['geometry_data'] = json.loads(geo_data_bytes.decode('utf-8'))
# 模具型腔数据
if stp_file.mold_cavity_data:
cavity_data_bytes = await storage_manager.download_file(
'mold_cavities',
stp_file.mold_cavity_data.detailed_object_key
)
result['mold_cavity_data'] = json.loads(cavity_data_bytes.decode('utf-8'))
# HTML文件
if stp_file.html_file:
html_bytes = await storage_manager.download_file(
'html_files',
stp_file.html_file.object_key
)
result['html_content'] = html_bytes.decode('utf-8')
except Exception as e:
logger.error(f"从MinIO获取数据失败: {e}")
# 3. 从PostgreSQL获取特征和建议
features = await session.execute(
select(FeatureDetection).where(FeatureDetection.stp_file_id == stp_file_id)
)
result['features'] = [
{
'feature_type': f.feature_type,
'confidence': f.confidence,
'location': f.location,
'dimensions': f.dimensions,
'parameters': f.parameters
}
for f in features.scalars().all()
]
recommendations = await session.execute(
select(DesignRecommendation).where(DesignRecommendation.stp_file_id == stp_file_id)
)
result['recommendations'] = [
{
'rec_type': r.rec_type,
'priority': r.priority,
'description': r.description,
'reason': r.reason,
'parameters': r.parameters
}
for r in recommendations.scalars().all()
]
return result
async def delete_stp_file_cascade(self, session: AsyncSession,
stp_file_id: int):
"""级联删除STP文件及其所有关联数据"""
stp_file = await session.get(STPFile, stp_file_id)
if not stp_file:
raise ValueError(f"STP文件不存在: {stp_file_id}")
# 1. 删除MinIO中的文件
try:
if stp_file.object_key:
await storage_manager.delete_file('stp_files', stp_file.object_key)
except Exception as e:
logger.error(f"删除MinIO文件失败: {e}")
try:
if stp_file.geometry_data:
await storage_manager.delete_file('geometry_data', stp_file.geometry_data.object_key)
except Exception as e:
logger.error(f"删除几何数据失败: {e}")
try:
if stp_file.mold_cavity_data:
await storage_manager.delete_file('mold_cavities', stp_file.mold_cavity_data.detailed_object_key)
except Exception as e:
logger.error(f"删除型腔数据失败: {e}")
try:
if stp_file.html_file:
await storage_manager.delete_file('html_files', stp_file.html_file.object_key)
except Exception as e:
logger.error(f"删除HTML文件失败: {e}")
# 2. 级联删除PostgreSQL记录(通过外键自动处理)
await session.delete(stp_file)
await session.commit()
logger.info(f"STP文件及其关联数据已删除: {stp_file_id}")
# 全局存储集成服务实例
storage_integration = StorageIntegrationService()
@@ -459,7 +459,9 @@ class StorageIntegrationService:
storage_bucket=upload_result['bucket'],
filename=filename,
file_path=file_path, # 保留本地路径
html_content=html_content, # 保留内容以兼容
# 停止双写完整 HTML 进 PG:读取路径走 RustFS(html_json.content),
# PG 仅存对象键与文件名,避免大文本撑爆表
html_content=None,
visualization_type=visualization_type
)
@@ -738,20 +740,24 @@ class StorageIntegrationService:
result = await session.execute(query)
latest_files = result.unique().scalars().all()
# 一次性聚合每个文件名的上传次数(替代逐文件 count 的 N+1 查询)
count_subquery = (
select(STPFile.original_filename, func.count().label("upload_count"))
.group_by(STPFile.original_filename)
)
if user_id:
count_subquery = count_subquery.where(STPFile.user_id == user_id)
count_result = await session.execute(count_subquery)
upload_counts = {
row.original_filename: row.upload_count for row in count_result
}
# 获取每个文件名的上传次数
file_groups = []
for f in latest_files:
count_query = select(func.count()).where(
STPFile.original_filename == f.original_filename
)
if user_id:
count_query = count_query.where(STPFile.user_id == user_id)
count_result = await session.execute(count_query)
upload_count = count_result.scalar()
task_id = f.processing_tasks[0].task_id if f.processing_tasks else None
upload_count = upload_counts.get(f.original_filename, 1)
file_groups.append({
'filename': f.original_filename,
-296
View File
@@ -1,296 +0,0 @@
# services/storage_service.py
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, update
from datetime import datetime
import hashlib
import json
from pathlib import Path
from typing import Optional, Dict, Any
from shared.models.database import STPFile, GeometryData, HTMLFile, ProcessingTask
from shared.utils.logger import get_logger
from shared.models.database import MoldCavityData
logger = get_logger(__name__)
class StorageService:
"""数据存储服务"""
def __init__(self, db_session: AsyncSession):
self.db_session = db_session
async def save_stp_file(
self,
filename: str,
file_path: str,
file_size: int,
file_content: Optional[bytes] = None
) -> STPFile:
"""保存STP文件信息到数据库"""
try:
# 计算文件哈希
file_hash = self._calculate_file_hash(file_path, file_content)
# 检查是否已存在相同文件
existing_file = await self.db_session.execute(
select(STPFile).where(STPFile.file_hash == file_hash)
)
existing_file = existing_file.scalar_one_or_none()
if existing_file:
logger.info(f"文件已存在,跳过保存: {filename}")
return existing_file
# 创建新的STP文件记录
stp_file = STPFile(
filename=filename,
original_filename=filename,
file_path=file_path,
file_size=file_size,
file_hash=file_hash,
file_content=file_content,
upload_time=datetime.now(),
status="pending",
# 必填字段提供默认值
object_key=f"stp_files/{file_hash}",
storage_bucket="default",
object_url=None
)
self.db_session.add(stp_file)
await self.db_session.commit()
await self.db_session.refresh(stp_file)
logger.info(f"STP文件保存成功: {filename} (ID: {stp_file.id})")
return stp_file
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存STP文件失败: {e}")
raise
async def save_geometry_data(
self,
stp_file_id: int,
geometry_json: Dict[str, Any],
analysis_method: str
) -> GeometryData:
"""保存几何数据JSON到数据库"""
try:
# 提取关键几何属性用于快速查询
volume = geometry_json.get("volume")
surface_area = geometry_json.get("surface_area")
bounding_box = geometry_json.get("bounding_box", {})
geometry_data = GeometryData(
stp_file_id=stp_file_id,
analysis_method=analysis_method,
volume=volume,
surface_area=surface_area,
bounding_box_min=bounding_box.get("min"),
bounding_box_max=bounding_box.get("max"),
created_time=datetime.now(),
# 必填字段提供默认值
object_key=f"geometry_data/{stp_file_id}",
storage_bucket="default",
object_url=None
)
self.db_session.add(geometry_data)
await self.db_session.commit()
await self.db_session.refresh(geometry_data)
logger.info(f"几何数据保存成功: STP文件ID {stp_file_id}")
return geometry_data
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存几何数据失败: {e}")
raise
async def save_html_file(
self,
stp_file_id: int,
filename: str,
file_path: str,
html_content: Optional[str] = None,
visualization_type: str = "3d_viewer"
) -> HTMLFile:
"""保存HTML文件信息到数据库"""
try:
html_file = HTMLFile(
stp_file_id=stp_file_id,
filename=filename,
file_path=file_path,
html_content=html_content,
visualization_type=visualization_type,
has_interactive_elements=True,
generated_time=datetime.now(),
# 必填字段提供默认值
object_key=f"html_files/{stp_file_id}",
storage_bucket="default",
object_url=None
)
self.db_session.add(html_file)
await self.db_session.commit()
await self.db_session.refresh(html_file)
logger.info(f"HTML文件保存成功: {filename} (STP文件ID: {stp_file_id})")
return html_file
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存HTML文件失败: {e}")
raise
async def create_processing_task(
self,
task_id: str,
stp_file_id: int,
task_type: str = "stp_parsing"
) -> ProcessingTask:
"""创建处理任务记录"""
try:
task = ProcessingTask(
task_id=task_id,
stp_file_id=stp_file_id,
task_type=task_type,
status="pending",
started_time=datetime.now()
)
self.db_session.add(task)
await self.db_session.commit()
await self.db_session.refresh(task)
logger.info(f"处理任务创建成功: {task_id}")
return task
except Exception as e:
await self.db_session.rollback()
logger.error(f"创建处理任务失败: {e}")
raise
async def update_task_status(
self,
task_id: str,
status: str,
progress: Optional[int] = None,
current_step: Optional[str] = None,
error_message: Optional[str] = None
):
"""更新任务状态"""
try:
update_data = {
"status": status,
"completed_time": datetime.now() if status in ["completed", "failed"] else None,
"error_message": error_message
}
if progress is not None:
update_data["progress"] = progress
if current_step is not None:
update_data["current_step"] = current_step
await self.db_session.execute(
update(ProcessingTask)
.where(ProcessingTask.task_id == task_id)
.values(**update_data)
)
await self.db_session.commit()
logger.info(f"任务状态更新: {task_id} -> {status}")
except Exception as e:
await self.db_session.rollback()
logger.error(f"更新任务状态失败: {e}")
raise
async def update_stp_file_status(self, stp_file_id: int, status: str):
"""更新STP文件状态"""
try:
await self.db_session.execute(
update(STPFile)
.where(STPFile.id == stp_file_id)
.values(
status=status,
processed_time=datetime.now() if status in ["completed", "failed"] else None
)
)
await self.db_session.commit()
logger.info(f"STP文件状态更新: ID {stp_file_id} -> {status}")
except Exception as e:
await self.db_session.rollback()
logger.error(f"更新STP文件状态失败: {e}")
raise
async def get_stp_file_by_id(self, stp_file_id: int) -> Optional[STPFile]:
"""根据ID获取STP文件"""
try:
result = await self.db_session.execute(
select(STPFile).where(STPFile.id == stp_file_id)
)
return result.scalar_one_or_none()
except Exception as e:
logger.error(f"获取STP文件失败: {e}")
return None
async def get_geometry_data_by_stp_file_id(self, stp_file_id: int) -> Optional[GeometryData]:
"""根据STP文件ID获取几何数据"""
try:
result = await self.db_session.execute(
select(GeometryData).where(GeometryData.stp_file_id == stp_file_id)
)
return result.scalar_one_or_none()
except Exception as e:
logger.error(f"获取几何数据失败: {e}")
return None
def _calculate_file_hash(self, file_path: str, file_content: Optional[bytes] = None) -> str:
"""计算文件哈希值"""
sha256_hash = hashlib.sha256()
if file_content:
sha256_hash.update(file_content)
else:
# 从文件路径读取内容计算哈希
with open(file_path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
sha256_hash.update(chunk)
return sha256_hash.hexdigest()
async def save_mold_cavity_data(
self,
stp_file_id: int,
cavity_json: Dict[str, Any],
key_info: Dict[str, Any]
) -> MoldCavityData:
"""保存模具型腔数据"""
try:
mold_data = MoldCavityData(
stp_file_id=stp_file_id,
cavity_key_info=key_info,
shrinkage_rate=cavity_json["metadata"]["shrinkage_rate"],
draft_angle=cavity_json["metadata"]["draft_angle"],
generated_time=datetime.now(),
# 必填字段提供默认值
detailed_object_key=f"mold_cavity/{stp_file_id}",
storage_bucket="default"
)
self.db_session.add(mold_data)
await self.db_session.commit()
await self.db_session.refresh(mold_data)
logger.info(f"模具型腔数据保存成功: STP文件ID {stp_file_id}")
return mold_data
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存模具型腔数据失败: {e}")
raise
@@ -0,0 +1,51 @@
# services/task_dispatcher.py
"""后台处理任务分派器 - 统一 upload/batch 路由的 Celery/asyncio 分派逻辑。
修复两个问题:
1. fire-and-forget:asyncio.create_task 返回值未持有引用,任务可能被 GC 中途回收,
异常也无从浮现(python 官方文档明确警告的模式);
2. 复制粘贴:upload_router 与 batch_router 各自维护一份相同的分派代码,易漂移。
"""
import asyncio
from shared.utils.logger import get_logger
logger = get_logger(__name__)
try:
from celery_tasks import process_stp_task
_use_celery = True
except ImportError:
process_stp_task = None
_use_celery = False
# 持有后台任务强引用,防止被 GC 回收;完成后自动移出
_background_tasks: set = set()
# API 进程内并发处理上限(celery 路径由 worker 并发数控制,不走这里)。
# asyncio.Semaphore 自 3.10 起惰性绑定事件循环,模块级创建安全;
# 本模块仅在 API 进程(单一事件循环)导入使用。
_dispatch_semaphore = asyncio.Semaphore(2)
async def _run_with_limit(task_id: str, file_path: str, stp_file_id: int, process_params: dict):
async with _dispatch_semaphore:
from moldinsight.services.processing_service import processing_service
await processing_service.process_file_with_storage(
task_id, file_path, stp_file_id, process_params
)
def dispatch_processing(task_id: str, file_path: str, stp_file_id: int, process_params: dict):
"""调度 STP 处理任务:优先 Celery(进程隔离),否则 API 进程内 asyncio 后台执行。"""
if _use_celery:
process_stp_task.delay(task_id, file_path, stp_file_id, process_params)
logger.info(f"[DISPATCH] Celery 任务已调度: task_id={task_id}")
return
task = asyncio.create_task(
_run_with_limit(task_id, file_path, stp_file_id, process_params)
)
_background_tasks.add(task)
task.add_done_callback(_background_tasks.discard)
logger.info(f"[DISPATCH] 进程内后台处理: task_id={task_id} (celery 未安装)")
+54 -13
View File
@@ -1,7 +1,9 @@
# services/task_query_service.py
"""任务状态查询服务 — 从 task_router.py 中的持久化任务组装逻辑抽取"""
"""任务状态查询服务 - 从 task_router.py 中的持久化任务组装逻辑抽取"""
from typing import Optional, Dict, Any, List
import time
from collections import OrderedDict
from typing import Optional, Dict, Any, List, Tuple
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -16,12 +18,47 @@ logger = get_logger(__name__)
class TaskQueryService:
"""任务状态查询与视图组装"""
"""任务状态查询与视图组装
完成态任务的 PG+RustFS 组装成本高(全量下载 geometry/型腔/网格/HTML JSON),
而状态轮询高频触发。对 completed/failed 视图加进程内 LRU+TTL 缓存:
- processing 视图不缓存(数据持续变化,且通常由 Redis 直接提供);
- completed/failed 视图不可变(仅 parameters 会被 export/cam 端点更新,
更新方负责调用 invalidate_task_view 显式失效)。
"""
_VIEW_CACHE_TTL_SECONDS = 60.0
_VIEW_CACHE_MAX_ENTRIES = 16 # 视图为 MB 级 dict,上限控制内存占用
_view_cache: "OrderedDict[str, Tuple[float, Dict[str, Any]]]" = OrderedDict()
@classmethod
def _cache_get(cls, task_id: str) -> Optional[Dict[str, Any]]:
entry = cls._view_cache.get(task_id)
if entry is None:
return None
cached_at, view = entry
if time.monotonic() - cached_at > cls._VIEW_CACHE_TTL_SECONDS:
cls._view_cache.pop(task_id, None)
return None
cls._view_cache.move_to_end(task_id)
return view
@classmethod
def _cache_set(cls, task_id: str, view: Dict[str, Any]):
cls._view_cache[task_id] = (time.monotonic(), view)
cls._view_cache.move_to_end(task_id)
while len(cls._view_cache) > cls._VIEW_CACHE_MAX_ENTRIES:
cls._view_cache.popitem(last=False)
@classmethod
def invalidate_task_view(cls, task_id: str):
"""任务 parameters 被更新后调用(export-mold / cam 等),使缓存视图失效。"""
cls._view_cache.pop(task_id, None)
@staticmethod
async def get_task_view(db_session: AsyncSession, task_id: str) -> Optional[Dict[str, Any]]:
"""
获取任务视图 — 优先返回 Redis 缓存,否则从 PostgreSQL + RustFS 组装
获取任务视图 - 优先返回 Redis 缓存,否则从 PostgreSQL + RustFS 组装
Returns:
任务视图字典,如果任务不存在返回 None
@@ -34,7 +71,13 @@ class TaskQueryService:
logger.info(f"返回缓存任务状态:{task_id} - {status}")
return task
# 2. 持久化任务(已完成/失败,或服务重启后的任务)
# 2. 完成态视图缓存(命中则免 RustFS 全量下载)
cached = TaskQueryService._cache_get(task_id)
if cached is not None:
logger.info(f"返回缓存任务视图: {task_id}")
return cached
# 3. 持久化任务(已完成/失败,或服务重启后的任务)
storage_service = StorageIntegrationService()
# 查询任务和文件元数据(预加载 html_file 关联)
@@ -70,15 +113,9 @@ class TaskQueryService:
mesh_summary = await TaskQueryService._get_mesh_summary(db_session, stp_file.id)
# 构造 html_file 路径(与即时分析的 /html/xxx.html 格式保持一致)
# joinedload 已预加载 stp_file.html_file,无需重复单独查询
html_file_url = None
html_file_record = None
try:
html_file_record = await db_session.execute(
select(HTMLFile).where(HTMLFile.stp_file_id == stp_file.id)
)
html_file_record = html_file_record.scalar_one_or_none()
except Exception:
pass
html_file_record = stp_file.html_file
if html_file_record and html_file_record.filename:
html_file_url = f"/html/{html_file_record.filename}"
if cavity_view.get("html_file"):
@@ -133,6 +170,10 @@ class TaskQueryService:
"error": processing_task.error_message or stp_file.error_message or None,
}
# 仅缓存不可变的终态视图(processing 视图持续变化不缓存)
if processing_task.status in ("completed", "failed"):
TaskQueryService._cache_set(task_id, task_view)
logger.info(f"返回持久化任务状态: {task_id} - {processing_task.status}")
return task_view
-361
View File
@@ -1,361 +0,0 @@
# storage/object_storage.py
"""MinIO/S3 对象存储服务"""
from minio import Minio
from minio.error import S3Error
from pathlib import Path
from typing import Optional, BinaryIO
from io import BytesIO
from shared.utils.logger import get_logger
import hashlib
import uuid
logger = get_logger(__name__)
class ObjectStorageManager:
"""对象存储管理器 - MinIO/S3兼容"""
def __init__(self):
self.client: Optional[Minio] = None
self.is_connected = False
# 桶名称
self.buckets = {
'stp_files': 'moldinsight-stp-files', # STP/STEP文件
'geometry_data': 'moldinsight-geometry', # 几何数据JSON
'mold_cavities': 'moldinsight-mold-cavities', # 模具型腔数据
'html_files': 'moldinsight-html', # HTML报告文件
'user_files': 'moldinsight-user-files' # 用户上传的其他文件
}
async def connect(self, endpoint: str, access_key: str, secret_key: str,
secure: bool = False):
"""连接到MinIO/S3服务"""
try:
self.client = Minio(
endpoint,
access_key=access_key,
secret_key=secret_key,
secure=secure
)
# 测试连接
self.client.list_buckets()
self.is_connected = True
logger.info(f"对象存储连接成功: {endpoint}")
# 确保所有桶都存在
await self._ensure_buckets()
except S3Error as e:
logger.error(f"对象存储连接失败: {e}")
self.is_connected = False
raise
async def _ensure_buckets(self):
"""确保所有必要的桶都存在"""
for bucket_name in self.buckets.values():
try:
if not self.client.bucket_exists(bucket_name):
self.client.make_bucket(bucket_name)
logger.info(f"创建存储桶: {bucket_name}")
else:
logger.debug(f"存储桶已存在: {bucket_name}")
except S3Error as e:
logger.error(f"创建存储桶失败 {bucket_name}: {e}")
def _generate_object_key(self, original_filename: str, prefix: str = '') -> str:
"""生成对象存储的唯一键名"""
# 提取文件扩展名
ext = Path(original_filename).suffix
# 生成唯一ID
unique_id = str(uuid.uuid4())
# 生成键名: prefix/unique_id + original_ext
if prefix:
return f"{prefix}/{unique_id}{ext}"
return f"{unique_id}{ext}"
async def upload_stp_file(self, file_path: Path,
original_filename: str) -> dict:
"""上传STP文件到对象存储"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets['stp_files']
# 计算文件哈希
file_hash = self._calculate_file_hash(file_path)
# 检查是否已存在
existing_key = await self._find_file_by_hash(bucket_name, file_hash)
if existing_key:
logger.info(f"文件已存在,跳过上传: {existing_key}")
return {
'object_key': existing_key,
'file_hash': file_hash,
'already_exists': True
}
# 生成唯一键名
object_key = self._generate_object_key(
original_filename,
prefix='stp'
)
# 上传文件
try:
result = self.client.fput_object(
bucket_name,
object_key,
str(file_path),
content_type='application/octet-stream'
)
logger.info(f"STP文件上传成功: {object_key}")
return {
'object_key': object_key,
'file_hash': file_hash,
'file_size': result.size,
'etag': result.etag,
'already_exists': False
}
except S3Error as e:
logger.error(f"STP文件上传失败: {e}")
raise
async def upload_geometry_data(self, geometry_json: dict,
file_hash: str) -> dict:
"""上传几何数据JSON到对象存储"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets['geometry_data']
# 使用文件哈希作为键名的一部分
object_key = f"geometry/{file_hash}.json"
# 转换为字节
import json
json_bytes = json.dumps(geometry_json, ensure_ascii=False).encode('utf-8')
# 上传
try:
result = self.client.put_object(
bucket_name,
object_key,
BytesIO(json_bytes),
length=len(json_bytes),
content_type='application/json'
)
logger.info(f"几何数据上传成功: {object_key}")
return {
'object_key': object_key,
'file_size': result.size,
'etag': result.etag
}
except S3Error as e:
logger.error(f"几何数据上传失败: {e}")
raise
async def upload_mold_cavity_data(self, cavity_json: dict,
file_hash: str) -> dict:
"""上传模具型腔数据到对象存储"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets['mold_cavities']
object_key = f"mold-cavity/{file_hash}.json"
import json
json_bytes = json.dumps(cavity_json, ensure_ascii=False).encode('utf-8')
try:
result = self.client.put_object(
bucket_name,
object_key,
BytesIO(json_bytes),
length=len(json_bytes),
content_type='application/json'
)
logger.info(f"模具型腔数据上传成功: {object_key}")
return {
'object_key': object_key,
'file_size': result.size,
'etag': result.etag
}
except S3Error as e:
logger.error(f"模具型腔数据上传失败: {e}")
raise
async def upload_html_file(self, html_content: str,
original_filename: str,
file_hash: str) -> dict:
"""上传HTML文件到对象存储"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets['html_files']
object_key = f"html/{file_hash}.html"
html_bytes = html_content.encode('utf-8')
try:
result = self.client.put_object(
bucket_name,
object_key,
BytesIO(html_bytes),
length=len(html_bytes),
content_type='text/html; charset=utf-8'
)
logger.info(f"HTML文件上传成功: {object_key}")
return {
'object_key': object_key,
'file_size': result.size,
'etag': result.etag
}
except S3Error as e:
logger.error(f"HTML文件上传失败: {e}")
raise
async def download_file(self, bucket_type: str,
object_key: str) -> bytes:
"""从对象存储下载文件"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets.get(bucket_type)
if not bucket_name:
raise ValueError(f"未知的桶类型: {bucket_type}")
try:
response = self.client.get_object(bucket_name, object_key)
data = response.read()
response.close()
response.release_conn()
logger.debug(f"文件下载成功: {object_key}")
return data
except S3Error as e:
logger.error(f"文件下载失败 {object_key}: {e}")
raise
async def get_presigned_url(self, bucket_type: str,
object_key: str,
expires: int = 3600) -> str:
"""生成预签名URL(临时访问链接)"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets.get(bucket_type)
if not bucket_name:
raise ValueError(f"未知的桶类型: {bucket_type}")
try:
url = self.client.presigned_get_object(
bucket_name,
object_key,
expires=expires
)
return url
except S3Error as e:
logger.error(f"生成预签名URL失败: {e}")
raise
async def delete_file(self, bucket_type: str, object_key: str):
"""删除对象存储中的文件"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets.get(bucket_type)
if not bucket_name:
raise ValueError(f"未知的桶类型: {bucket_type}")
try:
self.client.remove_object(bucket_name, object_key)
logger.info(f"文件删除成功: {object_key}")
except S3Error as e:
logger.error(f"文件删除失败 {object_key}: {e}")
raise
def _calculate_file_hash(self, file_path: Path) -> str:
"""计算文件的SHA256哈希"""
sha256_hash = hashlib.sha256()
with open(file_path, 'rb') as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
async def _find_file_by_hash(self, bucket_name: str,
file_hash: str) -> Optional[str]:
"""根据哈希查找已存在的文件"""
try:
objects = self.client.list_objects(bucket_name, recursive=True)
for obj in objects:
# 从对象键中提取哈希(如果有)
if file_hash in obj.object_name:
return obj.object_name
return None
except S3Error as e:
logger.warning(f"查找文件哈希失败: {e}")
return None
async def get_file_info(self, bucket_type: str,
object_key: str) -> dict:
"""获取文件信息"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets.get(bucket_type)
if not bucket_name:
raise ValueError(f"未知的桶类型: {bucket_type}")
try:
stat = self.client.stat_object(bucket_name, object_key)
return {
'size': stat.size,
'etag': stat.etag,
'content_type': stat.content_type,
'last_modified': stat.last_modified
}
except S3Error as e:
logger.error(f"获取文件信息失败: {e}")
raise
async def list_files(self, bucket_type: str,
prefix: str = '') -> list:
"""列出存储桶中的文件"""
if not self.is_connected:
raise RuntimeError("对象存储未连接")
bucket_name = self.buckets.get(bucket_type)
if not bucket_name:
raise ValueError(f"未知的桶类型: {bucket_type}")
try:
objects = self.client.list_objects(bucket_name, prefix=prefix)
return [
{
'object_key': obj.object_name,
'size': obj.size,
'etag': obj.etag,
'last_modified': obj.last_modified
}
for obj in objects
]
except S3Error as e:
logger.error(f"列出文件失败: {e}")
raise
# 全局对象存储管理器实例
storage_manager = ObjectStorageManager()