This commit is contained in:
2026-08-31 18:01:34 +08:00
parent 3ea59551db
commit bee439cf34
46 changed files with 1884 additions and 1898 deletions
+23 -5
View File
@@ -524,11 +524,29 @@ class LLMService:
}
if expect_json:
payload["response_format"] = {"type": "json_object"}
async with httpx.AsyncClient(timeout=self._timeout) as client:
resp = await client.post(url, json=payload, headers=headers)
resp.raise_for_status()
content = resp.json()["choices"][0]["message"]["content"]
return content.strip() if content else None
# 保持 per-call client:celery 每任务经 asyncio.run 新建事件循环,
# 模块级 AsyncClient 绑定旧循环会失效(与 redis_task_manager 同理)。
# 瞬态错误(网络/5xx/429)重试一次,其余直接抛出。
last_exc: Optional[Exception] = None
for attempt in range(2):
try:
async with httpx.AsyncClient(timeout=self._timeout) as client:
resp = await client.post(url, json=payload, headers=headers)
resp.raise_for_status()
content = resp.json()["choices"][0]["message"]["content"]
return content.strip() if content else None
except httpx.TransportError as exc:
last_exc = exc
logger.warning(f"LLM 请求瞬态失败(第 {attempt + 1} 次): {exc}")
except httpx.HTTPStatusError as exc:
status = exc.response.status_code
if status >= 500 or status == 429:
last_exc = exc
logger.warning(f"LLM 服务端错误 {status}(第 {attempt + 1} 次)")
else:
raise
raise last_exc
@staticmethod
def _prioritize_features_for_side_action(
+115 -8
View File
@@ -2,12 +2,14 @@
"""STP 文件处理流程编排器 — 协调解析、网格生成、型腔生成、保存、验证"""
import asyncio
import os
import time
import traceback
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any
from typing import Optional, Dict, Any, List
from sqlalchemy.ext.asyncio import AsyncSession
@@ -40,12 +42,35 @@ class ProcessingService:
self.storage_service = StorageIntegrationService()
self.multi_scheme_planner = MultiSchemeMoldPlanner()
self.cad_exporter = CADExporter()
self._export_shapes_cache: Dict[str, Dict[str, Dict[str, Any]]] = {}
# TopoDS_Shape 为 C++ 原生内存对象,LRU 上限防止长期运行内存只涨不降
self._export_shapes_cache: "OrderedDict[str, Dict[str, Dict[str, Any]]]" = OrderedDict()
self._export_shapes_cache_max = 32
# OCC 非线程安全,max_workers=1 保证所有 OCC 操作序列化执行,避免偶发崩溃
self._occ_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="occ")
# ─── 对外入口 ───
def _reset_occ_executor(self):
"""超时后重建 OCC executor。
asyncio.wait_for 只能取消协程,正在执行 OCC 布尔运算的线程无法中断;
单 worker executor 中一个挂死线程会让后续任务永久排队直至重启。
代价是泄漏 1 个线程,收益是恢复服务可用性。
"""
old = self._occ_executor
self._occ_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="occ")
old.shutdown(wait=False)
logger.warning("OCC executor 已因处理超时重建(放弃等待旧线程,可能泄漏 1 个线程)")
async def run_occ(self, fn, *args):
"""在 OCC 单线程 executor 中执行同步几何操作。
OCC 非线程安全,所有几何计算(解析/布尔/三角化)统一经由本入口串行执行,
避免各调用方自行创建线程池造成并发崩溃。
"""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(self._occ_executor, fn, *args)
async def process_file_with_storage(
self,
task_id: str,
@@ -79,6 +104,8 @@ class ProcessingService:
)
except asyncio.TimeoutError:
logger.error(f"处理超时: {task_id}")
# OCC 线程无法取消:抛弃整个 executor,避免挂死线程堵死后续所有任务
self._reset_occ_executor()
raise Exception(f"处理超时,超过{timeout_seconds}秒未完成")
except Exception as e:
@@ -338,12 +365,12 @@ class ProcessingService:
},
)
# 更新任务缓存状态(仅保留轻量摘要,完整数据由PG+RustFS持久化)
# 更新任务缓存状态(仅保留轻量摘要,完整数据由PG+RustFS持久化;
# 完成态视图由 TaskQueryService 从 PG+RustFS 组装,Redis 不再存
# geometry_data / analysis_result 等 MB 级大对象)
await redis_task_manager.update_task(task_id, {
"status": ProcessingStatus.COMPLETED,
"completed_at": str(datetime.now()),
"geometry_data": geometry_data,
"analysis_result": analysis_result,
"key_info": best_key_info,
"best_scheme_id": detailed_cavity_json.get("best_scheme_id"),
"material": requested_material,
@@ -471,6 +498,10 @@ class ProcessingService:
def _cache_export_shapes(self, task_id: str, export_shapes: Dict[str, Dict[str, Any]]):
self._export_shapes_cache[task_id] = export_shapes
self._export_shapes_cache.move_to_end(task_id)
# 逐出最旧任务的形状缓存(连原生 OCC shape 引用一起释放)
while len(self._export_shapes_cache) > self._export_shapes_cache_max:
self._export_shapes_cache.popitem(last=False)
def _persist_step_exports(
self,
@@ -492,10 +523,11 @@ class ProcessingService:
for scheme_id, cavity_data in export_shapes.items():
try:
result = self.cad_exporter.export_mold_results(
# 持久化装配体 STEP + 逐组件 STEP(后者是重启后按需
# 重导出其他格式的几何来源,见 regenerate_export_from_persisted)
result = self.cad_exporter.export_persisted_steps(
cavity_data=cavity_data,
base_filename=base_filename,
formats=["step"],
components=components,
task_id=task_id,
scheme_id=scheme_id,
@@ -522,13 +554,88 @@ class ProcessingService:
return manifest
def get_export_shapes(self, task_id: str, scheme_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
scheme_map = self._export_shapes_cache.get(task_id, {})
scheme_map = self._export_shapes_cache.get(task_id)
if not scheme_map:
return None
self._export_shapes_cache.move_to_end(task_id) # LRU 热点保活
if scheme_id:
return scheme_map.get(scheme_id)
return next(iter(scheme_map.values()), None)
async def regenerate_export_from_persisted(
self,
task_id: str,
scheme_id: str,
formats: Optional[List[str]],
components: List[str],
base_filename: str,
scheme_files: List[Dict[str, Any]],
) -> Optional[Dict[str, Any]]:
"""内存导出缓存失效(如服务重启)后,从持久化 STEP 重建导出文件。
分析期已为每个方案持久化装配体 + 逐组件 STEP;
缺失格式(STL/IGES/BRep)读回单组件 STEP 现场转换,
用户无需重新分析。所有组件均不可用时返回 None。
"""
format_list = list(dict.fromkeys(formats or ["step", "stl"]))
step_files = {
f.get("component"): f
for f in scheme_files or []
if f.get("format") == "step" and f.get("component")
}
if not step_files:
return None
files: List[Dict[str, Any]] = []
errors: List[str] = []
if "step" in format_list:
assembly = step_files.get("assembly")
if assembly:
files.append(assembly)
else:
errors.append("模具装配体 (step) 不可用")
for comp in components:
comp_file = step_files.get(comp)
if comp_file is None:
errors.append(f"组件 {comp} 的持久化 STEP 不可用")
continue
for fmt in format_list:
if fmt == "step":
files.append(comp_file)
continue
step_path = os.path.join(
self.cad_exporter.output_dir,
str(comp_file.get("relative_path") or "").replace("/", os.sep),
)
out_path = os.path.join(
os.path.dirname(step_path), f"{base_filename}_{comp}.{fmt}"
)
ok = await self.run_occ(
self.cad_exporter.convert_component_step, step_path, out_path, fmt
)
if ok:
files.append(
self.cad_exporter.build_file_entry(comp, fmt, out_path)
)
else:
errors.append(f"组件 {comp} ({fmt}) 转换失败")
if not files:
return None
return {
"base_filename": base_filename,
"task_id": task_id,
"scheme_id": scheme_id,
"files": files,
"errors": errors,
"total_files": len(files),
"total_errors": len(errors),
"source": "regenerated",
}
@staticmethod
def _normalize_process_params(process_params: Optional[Dict[str, Any]]) -> Dict[str, Any]:
payload = dict(process_params or {})
+88
View File
@@ -0,0 +1,88 @@
# services/shape_loader.py
"""按 task_id 从持久化存储重建 OCC 几何形状。
任务完成后 TopoDS_Shape 不驻留内存/Redis(原生内存与体积原因),
需要几何的端点(倒扣检测、按需重导出等)通过 STP 原件重建:
PG(object_key) -> RustFS 下载 -> 临时文件 -> OCC 单线程 executor 解析。
"""
import tempfile
from pathlib import Path
from typing import Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from shared.models.database import ProcessingTask, STPFile
from shared.utils.logger import get_logger
logger = get_logger(__name__)
class ShapeLoader:
"""任务几何重建器"""
def __init__(self):
from moldinsight.core.stp_parser import STPParser
from moldinsight.services.processing_service import processing_service
self._parser = STPParser()
self._processing = processing_service
async def load_shape_for_task(
self, db_session: AsyncSession, task_id: str
) -> Optional["object"]:
"""重建任务的产品几何。任务不存在或 STP 原件不可用时返回 None。"""
result = await db_session.execute(
select(ProcessingTask, STPFile)
.join(STPFile, ProcessingTask.stp_file_id == STPFile.id)
.where(ProcessingTask.task_id == task_id)
)
row = result.first()
if not row:
logger.warning(f"几何重建失败:任务不存在 {task_id}")
return None
_, stp_file = row
if not stp_file.object_key:
logger.warning(f"几何重建失败:任务缺少 object_key {task_id}")
return None
from moldinsight.storage.rustfs_storage import rustfs_manager
try:
data = await rustfs_manager.download_file(
file_type="stp_files", object_key=stp_file.object_key
)
except Exception as exc:
logger.error(f"几何重建失败:STP 原件下载失败 {task_id}: {exc}")
return None
with tempfile.NamedTemporaryFile(suffix=".stp", delete=False) as tmp:
tmp.write(data)
tmp_path = Path(tmp.name)
try:
shape = await self._processing.run_occ(
self._parser.load_step_file, tmp_path
)
return shape
except Exception as exc:
logger.error(f"几何重建失败:STP 解析失败 {task_id}: {exc}")
return None
finally:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
# 惰性单例:__init__ 会实例化 STPParser 并校验 OCC 可用性,
# 延迟到首次真实使用,避免模块导入期失败拖垮路由加载
_shape_loader: Optional[ShapeLoader] = None
def get_shape_loader() -> ShapeLoader:
global _shape_loader
if _shape_loader is None:
_shape_loader = ShapeLoader()
return _shape_loader
@@ -1,376 +0,0 @@
# services/storage_integration.py
"""存储集成服务 - 协调 PostgreSQL 和 MinIO"""
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from pathlib import Path
from typing import Optional, Dict, Any
import json
from shared.models.database import (
STPFile, GeometryData, MoldCavityData,
HTMLFile, ProcessingTask, User,
FeatureDetection, DesignRecommendation,
UserActivity, SystemLog
)
from moldinsight.storage.object_storage import storage_manager
from shared.utils.logger import get_logger
logger = get_logger(__name__)
class StorageIntegrationService:
"""存储集成服务"""
async def save_stp_file(self, session: AsyncSession,
file_path: Path,
original_filename: str,
user_id: Optional[int] = None) -> STPFile:
"""保存STP文件到PostgreSQL元数据 + MinIO对象存储"""
# 1. 上传到MinIO
upload_result = await storage_manager.upload_stp_file(
file_path,
original_filename
)
# 2. 创建PostgreSQL记录
stp_file = STPFile(
user_id=user_id,
object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['stp_files'],
original_filename=original_filename,
file_size=upload_result['file_size'],
file_hash=upload_result['file_hash'],
status="uploaded",
file_path=str(file_path) # 保留本地路径以兼容
)
session.add(stp_file)
await session.commit()
await session.refresh(stp_file)
logger.info(f"STP文件保存成功: {stp_file.id}")
return stp_file
async def save_geometry_data(self, session: AsyncSession,
stp_file_id: int,
geometry_json: Dict[str, Any],
analysis_method: str = "pythonocc") -> GeometryData:
"""保存几何数据到PostgreSQL元数据 + MinIO对象存储"""
# 1. 获取文件哈希
stp_file = await session.get(STPFile, stp_file_id)
file_hash = stp_file.file_hash
# 2. 上传到MinIO
upload_result = await storage_manager.upload_geometry_data(
geometry_json,
file_hash
)
# 3. 创建PostgreSQL记录
geometry_data = GeometryData(
stp_file_id=stp_file_id,
object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['geometry_data'],
analysis_method=analysis_method,
# 提取摘要字段
volume=geometry_json.get('geometry_data', {}).get('volume'),
surface_area=geometry_json.get('geometry_data', {}).get('surface_area'),
bounding_box_min=geometry_json.get('geometry_data', {}).get('bounding_box', {}).get('min'),
bounding_box_max=geometry_json.get('geometry_data', {}).get('bounding_box', {}).get('max'),
center_of_mass=geometry_json.get('geometry_data', {}).get('center_of_mass'),
topology_faces=geometry_json.get('geometry_data', {}).get('topology', {}).get('faces'),
topology_edges=geometry_json.get('geometry_data', {}).get('topology', {}).get('edges'),
topology_vertices=geometry_json.get('geometry_data', {}).get('topology', {}).get('vertices')
)
session.add(geometry_data)
await session.commit()
await session.refresh(geometry_data)
logger.info(f"几何数据保存成功: {geometry_data.id}")
return geometry_data
async def save_mold_cavity_data(self, session: AsyncSession,
stp_file_id: int,
cavity_json: Dict[str, Any]) -> MoldCavityData:
"""保存模具型腔数据到PostgreSQL元数据 + MinIO对象存储"""
# 1. 获取文件哈希
stp_file = await session.get(STPFile, stp_file_id)
file_hash = stp_file.file_hash
# 2. 上传到MinIO
upload_result = await storage_manager.upload_mold_cavity_data(
cavity_json,
file_hash
)
# 3. 提取关键信息
metadata = cavity_json.get('metadata', {})
product_analysis = cavity_json.get('product_analysis', {})
manufacturing_info = cavity_json.get('manufacturing_info', {})
mold_size = manufacturing_info.get('estimated_mold_size', {})
key_info = cavity_json.get('mold_cavities', {}).get('cavity_key_info', {})
# 4. 创建PostgreSQL记录
mold_cavity = MoldCavityData(
stp_file_id=stp_file_id,
detailed_object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['mold_cavities'],
# 模具参数
mold_material=manufacturing_info.get('recommended_material', 'Aluminum Alloy 7075'),
shrinkage_rate=metadata.get('shrinkage_rate', 0.005),
draft_angle=metadata.get('draft_angle', 2.0),
# 提取的摘要字段
cavity_key_info=key_info,
mold_size_length=mold_size.get('length'),
mold_size_width=mold_size.get('width'),
mold_size_height=mold_size.get('height'),
estimated_clamping_force=manufacturing_info.get('estimated_clamping_force'),
product_volume=product_analysis.get('volume'),
# 从key_info中提取(如果存在)
product_weight=key_info.get('geometric_characteristics', {}).get('product_weight'),
wall_thickness_range=key_info.get('geometric_characteristics', {}).get('wall_thickness_range'),
complexity_score=key_info.get('geometric_characteristics', {}).get('complexity_score'),
# 质量评估
weld_line_risk=key_info.get('quality_considerations', {}).get('potential_weld_lines'),
sink_mark_risk=key_info.get('quality_considerations', {}).get('sink_mark_areas'),
warpage_risk=key_info.get('quality_considerations', {}).get('warpage_risk')
)
session.add(mold_cavity)
await session.commit()
await session.refresh(mold_cavity)
logger.info(f"模具型腔数据保存成功: {mold_cavity.id}")
return mold_cavity
async def save_html_file(self, session: AsyncSession,
stp_file_id: int,
html_content: str,
filename: str) -> HTMLFile:
"""保存HTML文件到PostgreSQL元数据 + MinIO对象存储"""
# 1. 获取文件哈希
stp_file = await session.get(STPFile, stp_file_id)
file_hash = stp_file.file_hash
# 2. 上传到MinIO
upload_result = await storage_manager.upload_html_file(
html_content,
filename,
file_hash
)
# 3. 创建PostgreSQL记录
html_file = HTMLFile(
stp_file_id=stp_file_id,
object_key=upload_result['object_key'],
storage_bucket=storage_manager.buckets['html_files'],
filename=filename,
file_path=str(Path('html_output') / filename), # 保留本地路径
html_content=html_content # 保留内容以兼容
)
session.add(html_file)
await session.commit()
await session.refresh(html_file)
logger.info(f"HTML文件保存成功: {html_file.id}")
return html_file
async def save_features_and_recommendations(
self, session: AsyncSession,
stp_file_id: int,
features: list,
recommendations: list
):
"""保存特征检测结果和设计建议"""
# 1. 保存特征
for feature in features:
feature_record = FeatureDetection(
stp_file_id=stp_file_id,
feature_type=feature.get('feature_type'),
confidence=feature.get('confidence'),
location=feature.get('location'),
dimensions=feature.get('dimensions'),
parameters=feature.get('parameters')
)
session.add(feature_record)
# 2. 保存建议
for rec in recommendations:
rec_record = DesignRecommendation(
stp_file_id=stp_file_id,
rec_type=rec.get('rec_type'),
priority=rec.get('priority'),
description=rec.get('description'),
reason=rec.get('reason'),
parameters=rec.get('parameters')
)
session.add(rec_record)
await session.commit()
logger.info(f"保存了 {len(features)} 个特征和 {len(recommendations)} 个建议")
async def log_user_activity(self, session: AsyncSession,
user_id: int,
activity_type: str,
resource_type: Optional[str] = None,
resource_id: Optional[int] = None,
description: Optional[str] = None,
metadata: Optional[Dict] = None,
ip_address: Optional[str] = None,
user_agent: Optional[str] = None):
"""记录用户活动"""
activity = UserActivity(
user_id=user_id,
activity_type=activity_type,
resource_type=resource_type,
resource_id=resource_id,
description=description,
metadata=metadata,
ip_address=ip_address,
user_agent=user_agent
)
session.add(activity)
await session.commit()
logger.debug(f"用户活动记录: {activity_type} by user {user_id}")
async def get_stp_file_with_data(self, session: AsyncSession,
stp_file_id: int) -> Dict[str, Any]:
"""获取STP文件及其所有关联数据"""
# 1. 获取STP文件记录
stp_file = await session.get(STPFile, stp_file_id)
if not stp_file:
raise ValueError(f"STP文件不存在: {stp_file_id}")
result = {
'metadata': {
'id': stp_file.id,
'original_filename': stp_file.original_filename,
'file_size': stp_file.file_size,
'file_hash': stp_file.file_hash,
'upload_time': stp_file.upload_time.isoformat() if stp_file.upload_time else None,
'status': stp_file.status,
'user_id': stp_file.user_id
},
'geometry_data': None,
'mold_cavity_data': None,
'html_file': None,
'features': [],
'recommendations': []
}
# 2. 从MinIO获取数据
try:
# 几何数据
if stp_file.geometry_data:
geo_data_bytes = await storage_manager.download_file(
'geometry_data',
stp_file.geometry_data.object_key
)
result['geometry_data'] = json.loads(geo_data_bytes.decode('utf-8'))
# 模具型腔数据
if stp_file.mold_cavity_data:
cavity_data_bytes = await storage_manager.download_file(
'mold_cavities',
stp_file.mold_cavity_data.detailed_object_key
)
result['mold_cavity_data'] = json.loads(cavity_data_bytes.decode('utf-8'))
# HTML文件
if stp_file.html_file:
html_bytes = await storage_manager.download_file(
'html_files',
stp_file.html_file.object_key
)
result['html_content'] = html_bytes.decode('utf-8')
except Exception as e:
logger.error(f"从MinIO获取数据失败: {e}")
# 3. 从PostgreSQL获取特征和建议
features = await session.execute(
select(FeatureDetection).where(FeatureDetection.stp_file_id == stp_file_id)
)
result['features'] = [
{
'feature_type': f.feature_type,
'confidence': f.confidence,
'location': f.location,
'dimensions': f.dimensions,
'parameters': f.parameters
}
for f in features.scalars().all()
]
recommendations = await session.execute(
select(DesignRecommendation).where(DesignRecommendation.stp_file_id == stp_file_id)
)
result['recommendations'] = [
{
'rec_type': r.rec_type,
'priority': r.priority,
'description': r.description,
'reason': r.reason,
'parameters': r.parameters
}
for r in recommendations.scalars().all()
]
return result
async def delete_stp_file_cascade(self, session: AsyncSession,
stp_file_id: int):
"""级联删除STP文件及其所有关联数据"""
stp_file = await session.get(STPFile, stp_file_id)
if not stp_file:
raise ValueError(f"STP文件不存在: {stp_file_id}")
# 1. 删除MinIO中的文件
try:
if stp_file.object_key:
await storage_manager.delete_file('stp_files', stp_file.object_key)
except Exception as e:
logger.error(f"删除MinIO文件失败: {e}")
try:
if stp_file.geometry_data:
await storage_manager.delete_file('geometry_data', stp_file.geometry_data.object_key)
except Exception as e:
logger.error(f"删除几何数据失败: {e}")
try:
if stp_file.mold_cavity_data:
await storage_manager.delete_file('mold_cavities', stp_file.mold_cavity_data.detailed_object_key)
except Exception as e:
logger.error(f"删除型腔数据失败: {e}")
try:
if stp_file.html_file:
await storage_manager.delete_file('html_files', stp_file.html_file.object_key)
except Exception as e:
logger.error(f"删除HTML文件失败: {e}")
# 2. 级联删除PostgreSQL记录(通过外键自动处理)
await session.delete(stp_file)
await session.commit()
logger.info(f"STP文件及其关联数据已删除: {stp_file_id}")
# 全局存储集成服务实例
storage_integration = StorageIntegrationService()
@@ -459,7 +459,9 @@ class StorageIntegrationService:
storage_bucket=upload_result['bucket'],
filename=filename,
file_path=file_path, # 保留本地路径
html_content=html_content, # 保留内容以兼容
# 停止双写完整 HTML 进 PG:读取路径走 RustFS(html_json.content),
# PG 仅存对象键与文件名,避免大文本撑爆表
html_content=None,
visualization_type=visualization_type
)
@@ -738,20 +740,24 @@ class StorageIntegrationService:
result = await session.execute(query)
latest_files = result.unique().scalars().all()
# 一次性聚合每个文件名的上传次数(替代逐文件 count 的 N+1 查询)
count_subquery = (
select(STPFile.original_filename, func.count().label("upload_count"))
.group_by(STPFile.original_filename)
)
if user_id:
count_subquery = count_subquery.where(STPFile.user_id == user_id)
count_result = await session.execute(count_subquery)
upload_counts = {
row.original_filename: row.upload_count for row in count_result
}
# 获取每个文件名的上传次数
file_groups = []
for f in latest_files:
count_query = select(func.count()).where(
STPFile.original_filename == f.original_filename
)
if user_id:
count_query = count_query.where(STPFile.user_id == user_id)
count_result = await session.execute(count_query)
upload_count = count_result.scalar()
task_id = f.processing_tasks[0].task_id if f.processing_tasks else None
upload_count = upload_counts.get(f.original_filename, 1)
file_groups.append({
'filename': f.original_filename,
-296
View File
@@ -1,296 +0,0 @@
# services/storage_service.py
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, update
from datetime import datetime
import hashlib
import json
from pathlib import Path
from typing import Optional, Dict, Any
from shared.models.database import STPFile, GeometryData, HTMLFile, ProcessingTask
from shared.utils.logger import get_logger
from shared.models.database import MoldCavityData
logger = get_logger(__name__)
class StorageService:
"""数据存储服务"""
def __init__(self, db_session: AsyncSession):
self.db_session = db_session
async def save_stp_file(
self,
filename: str,
file_path: str,
file_size: int,
file_content: Optional[bytes] = None
) -> STPFile:
"""保存STP文件信息到数据库"""
try:
# 计算文件哈希
file_hash = self._calculate_file_hash(file_path, file_content)
# 检查是否已存在相同文件
existing_file = await self.db_session.execute(
select(STPFile).where(STPFile.file_hash == file_hash)
)
existing_file = existing_file.scalar_one_or_none()
if existing_file:
logger.info(f"文件已存在,跳过保存: {filename}")
return existing_file
# 创建新的STP文件记录
stp_file = STPFile(
filename=filename,
original_filename=filename,
file_path=file_path,
file_size=file_size,
file_hash=file_hash,
file_content=file_content,
upload_time=datetime.now(),
status="pending",
# 必填字段提供默认值
object_key=f"stp_files/{file_hash}",
storage_bucket="default",
object_url=None
)
self.db_session.add(stp_file)
await self.db_session.commit()
await self.db_session.refresh(stp_file)
logger.info(f"STP文件保存成功: {filename} (ID: {stp_file.id})")
return stp_file
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存STP文件失败: {e}")
raise
async def save_geometry_data(
self,
stp_file_id: int,
geometry_json: Dict[str, Any],
analysis_method: str
) -> GeometryData:
"""保存几何数据JSON到数据库"""
try:
# 提取关键几何属性用于快速查询
volume = geometry_json.get("volume")
surface_area = geometry_json.get("surface_area")
bounding_box = geometry_json.get("bounding_box", {})
geometry_data = GeometryData(
stp_file_id=stp_file_id,
analysis_method=analysis_method,
volume=volume,
surface_area=surface_area,
bounding_box_min=bounding_box.get("min"),
bounding_box_max=bounding_box.get("max"),
created_time=datetime.now(),
# 必填字段提供默认值
object_key=f"geometry_data/{stp_file_id}",
storage_bucket="default",
object_url=None
)
self.db_session.add(geometry_data)
await self.db_session.commit()
await self.db_session.refresh(geometry_data)
logger.info(f"几何数据保存成功: STP文件ID {stp_file_id}")
return geometry_data
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存几何数据失败: {e}")
raise
async def save_html_file(
self,
stp_file_id: int,
filename: str,
file_path: str,
html_content: Optional[str] = None,
visualization_type: str = "3d_viewer"
) -> HTMLFile:
"""保存HTML文件信息到数据库"""
try:
html_file = HTMLFile(
stp_file_id=stp_file_id,
filename=filename,
file_path=file_path,
html_content=html_content,
visualization_type=visualization_type,
has_interactive_elements=True,
generated_time=datetime.now(),
# 必填字段提供默认值
object_key=f"html_files/{stp_file_id}",
storage_bucket="default",
object_url=None
)
self.db_session.add(html_file)
await self.db_session.commit()
await self.db_session.refresh(html_file)
logger.info(f"HTML文件保存成功: {filename} (STP文件ID: {stp_file_id})")
return html_file
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存HTML文件失败: {e}")
raise
async def create_processing_task(
self,
task_id: str,
stp_file_id: int,
task_type: str = "stp_parsing"
) -> ProcessingTask:
"""创建处理任务记录"""
try:
task = ProcessingTask(
task_id=task_id,
stp_file_id=stp_file_id,
task_type=task_type,
status="pending",
started_time=datetime.now()
)
self.db_session.add(task)
await self.db_session.commit()
await self.db_session.refresh(task)
logger.info(f"处理任务创建成功: {task_id}")
return task
except Exception as e:
await self.db_session.rollback()
logger.error(f"创建处理任务失败: {e}")
raise
async def update_task_status(
self,
task_id: str,
status: str,
progress: Optional[int] = None,
current_step: Optional[str] = None,
error_message: Optional[str] = None
):
"""更新任务状态"""
try:
update_data = {
"status": status,
"completed_time": datetime.now() if status in ["completed", "failed"] else None,
"error_message": error_message
}
if progress is not None:
update_data["progress"] = progress
if current_step is not None:
update_data["current_step"] = current_step
await self.db_session.execute(
update(ProcessingTask)
.where(ProcessingTask.task_id == task_id)
.values(**update_data)
)
await self.db_session.commit()
logger.info(f"任务状态更新: {task_id} -> {status}")
except Exception as e:
await self.db_session.rollback()
logger.error(f"更新任务状态失败: {e}")
raise
async def update_stp_file_status(self, stp_file_id: int, status: str):
"""更新STP文件状态"""
try:
await self.db_session.execute(
update(STPFile)
.where(STPFile.id == stp_file_id)
.values(
status=status,
processed_time=datetime.now() if status in ["completed", "failed"] else None
)
)
await self.db_session.commit()
logger.info(f"STP文件状态更新: ID {stp_file_id} -> {status}")
except Exception as e:
await self.db_session.rollback()
logger.error(f"更新STP文件状态失败: {e}")
raise
async def get_stp_file_by_id(self, stp_file_id: int) -> Optional[STPFile]:
"""根据ID获取STP文件"""
try:
result = await self.db_session.execute(
select(STPFile).where(STPFile.id == stp_file_id)
)
return result.scalar_one_or_none()
except Exception as e:
logger.error(f"获取STP文件失败: {e}")
return None
async def get_geometry_data_by_stp_file_id(self, stp_file_id: int) -> Optional[GeometryData]:
"""根据STP文件ID获取几何数据"""
try:
result = await self.db_session.execute(
select(GeometryData).where(GeometryData.stp_file_id == stp_file_id)
)
return result.scalar_one_or_none()
except Exception as e:
logger.error(f"获取几何数据失败: {e}")
return None
def _calculate_file_hash(self, file_path: str, file_content: Optional[bytes] = None) -> str:
"""计算文件哈希值"""
sha256_hash = hashlib.sha256()
if file_content:
sha256_hash.update(file_content)
else:
# 从文件路径读取内容计算哈希
with open(file_path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
sha256_hash.update(chunk)
return sha256_hash.hexdigest()
async def save_mold_cavity_data(
self,
stp_file_id: int,
cavity_json: Dict[str, Any],
key_info: Dict[str, Any]
) -> MoldCavityData:
"""保存模具型腔数据"""
try:
mold_data = MoldCavityData(
stp_file_id=stp_file_id,
cavity_key_info=key_info,
shrinkage_rate=cavity_json["metadata"]["shrinkage_rate"],
draft_angle=cavity_json["metadata"]["draft_angle"],
generated_time=datetime.now(),
# 必填字段提供默认值
detailed_object_key=f"mold_cavity/{stp_file_id}",
storage_bucket="default"
)
self.db_session.add(mold_data)
await self.db_session.commit()
await self.db_session.refresh(mold_data)
logger.info(f"模具型腔数据保存成功: STP文件ID {stp_file_id}")
return mold_data
except Exception as e:
await self.db_session.rollback()
logger.error(f"保存模具型腔数据失败: {e}")
raise
@@ -0,0 +1,51 @@
# services/task_dispatcher.py
"""后台处理任务分派器 - 统一 upload/batch 路由的 Celery/asyncio 分派逻辑。
修复两个问题:
1. fire-and-forget:asyncio.create_task 返回值未持有引用,任务可能被 GC 中途回收,
异常也无从浮现(python 官方文档明确警告的模式);
2. 复制粘贴:upload_router 与 batch_router 各自维护一份相同的分派代码,易漂移。
"""
import asyncio
from shared.utils.logger import get_logger
logger = get_logger(__name__)
try:
from celery_tasks import process_stp_task
_use_celery = True
except ImportError:
process_stp_task = None
_use_celery = False
# 持有后台任务强引用,防止被 GC 回收;完成后自动移出
_background_tasks: set = set()
# API 进程内并发处理上限(celery 路径由 worker 并发数控制,不走这里)。
# asyncio.Semaphore 自 3.10 起惰性绑定事件循环,模块级创建安全;
# 本模块仅在 API 进程(单一事件循环)导入使用。
_dispatch_semaphore = asyncio.Semaphore(2)
async def _run_with_limit(task_id: str, file_path: str, stp_file_id: int, process_params: dict):
async with _dispatch_semaphore:
from moldinsight.services.processing_service import processing_service
await processing_service.process_file_with_storage(
task_id, file_path, stp_file_id, process_params
)
def dispatch_processing(task_id: str, file_path: str, stp_file_id: int, process_params: dict):
"""调度 STP 处理任务:优先 Celery(进程隔离),否则 API 进程内 asyncio 后台执行。"""
if _use_celery:
process_stp_task.delay(task_id, file_path, stp_file_id, process_params)
logger.info(f"[DISPATCH] Celery 任务已调度: task_id={task_id}")
return
task = asyncio.create_task(
_run_with_limit(task_id, file_path, stp_file_id, process_params)
)
_background_tasks.add(task)
task.add_done_callback(_background_tasks.discard)
logger.info(f"[DISPATCH] 进程内后台处理: task_id={task_id} (celery 未安装)")
+54 -13
View File
@@ -1,7 +1,9 @@
# services/task_query_service.py
"""任务状态查询服务 — 从 task_router.py 中的持久化任务组装逻辑抽取"""
"""任务状态查询服务 - 从 task_router.py 中的持久化任务组装逻辑抽取"""
from typing import Optional, Dict, Any, List
import time
from collections import OrderedDict
from typing import Optional, Dict, Any, List, Tuple
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -16,12 +18,47 @@ logger = get_logger(__name__)
class TaskQueryService:
"""任务状态查询与视图组装"""
"""任务状态查询与视图组装
完成态任务的 PG+RustFS 组装成本高(全量下载 geometry/型腔/网格/HTML JSON),
而状态轮询高频触发。对 completed/failed 视图加进程内 LRU+TTL 缓存:
- processing 视图不缓存(数据持续变化,且通常由 Redis 直接提供);
- completed/failed 视图不可变(仅 parameters 会被 export/cam 端点更新,
更新方负责调用 invalidate_task_view 显式失效)。
"""
_VIEW_CACHE_TTL_SECONDS = 60.0
_VIEW_CACHE_MAX_ENTRIES = 16 # 视图为 MB 级 dict,上限控制内存占用
_view_cache: "OrderedDict[str, Tuple[float, Dict[str, Any]]]" = OrderedDict()
@classmethod
def _cache_get(cls, task_id: str) -> Optional[Dict[str, Any]]:
entry = cls._view_cache.get(task_id)
if entry is None:
return None
cached_at, view = entry
if time.monotonic() - cached_at > cls._VIEW_CACHE_TTL_SECONDS:
cls._view_cache.pop(task_id, None)
return None
cls._view_cache.move_to_end(task_id)
return view
@classmethod
def _cache_set(cls, task_id: str, view: Dict[str, Any]):
cls._view_cache[task_id] = (time.monotonic(), view)
cls._view_cache.move_to_end(task_id)
while len(cls._view_cache) > cls._VIEW_CACHE_MAX_ENTRIES:
cls._view_cache.popitem(last=False)
@classmethod
def invalidate_task_view(cls, task_id: str):
"""任务 parameters 被更新后调用(export-mold / cam 等),使缓存视图失效。"""
cls._view_cache.pop(task_id, None)
@staticmethod
async def get_task_view(db_session: AsyncSession, task_id: str) -> Optional[Dict[str, Any]]:
"""
获取任务视图 — 优先返回 Redis 缓存,否则从 PostgreSQL + RustFS 组装
获取任务视图 - 优先返回 Redis 缓存,否则从 PostgreSQL + RustFS 组装
Returns:
任务视图字典,如果任务不存在返回 None
@@ -34,7 +71,13 @@ class TaskQueryService:
logger.info(f"返回缓存任务状态:{task_id} - {status}")
return task
# 2. 持久化任务(已完成/失败,或服务重启后的任务)
# 2. 完成态视图缓存(命中则免 RustFS 全量下载)
cached = TaskQueryService._cache_get(task_id)
if cached is not None:
logger.info(f"返回缓存任务视图: {task_id}")
return cached
# 3. 持久化任务(已完成/失败,或服务重启后的任务)
storage_service = StorageIntegrationService()
# 查询任务和文件元数据(预加载 html_file 关联)
@@ -70,15 +113,9 @@ class TaskQueryService:
mesh_summary = await TaskQueryService._get_mesh_summary(db_session, stp_file.id)
# 构造 html_file 路径(与即时分析的 /html/xxx.html 格式保持一致)
# joinedload 已预加载 stp_file.html_file,无需重复单独查询
html_file_url = None
html_file_record = None
try:
html_file_record = await db_session.execute(
select(HTMLFile).where(HTMLFile.stp_file_id == stp_file.id)
)
html_file_record = html_file_record.scalar_one_or_none()
except Exception:
pass
html_file_record = stp_file.html_file
if html_file_record and html_file_record.filename:
html_file_url = f"/html/{html_file_record.filename}"
if cavity_view.get("html_file"):
@@ -133,6 +170,10 @@ class TaskQueryService:
"error": processing_task.error_message or stp_file.error_message or None,
}
# 仅缓存不可变的终态视图(processing 视图持续变化不缓存)
if processing_task.status in ("completed", "failed"):
TaskQueryService._cache_set(task_id, task_view)
logger.info(f"返回持久化任务状态: {task_id} - {processing_task.status}")
return task_view