fix(deploy): celery worker --pool=solo——prefork 守护进程禁止 spawn OCC 子进程
Celery prefork 的 worker 子进程是 daemonic,multiprocessing 硬性禁止 守护进程再生子进程:OccProcessPool(spawn 拉起 OCC 常驻工作进程)在 celery worker 内首次 run 即报 "daemonic processes are not allowed to have children",解析任务全部 failed。该池在 FastAPI 主进程内无此限制 ——celery 路径的分析从未真正工作过。 改用 --pool=solo:任务在 celery 主进程(非守护)内执行,允许 spawn; 天然单任务串行,与 processing_service / redis_task_manager / db_manager 等模块级单例的 asyncio.Lock(不可跨循环并发)匹配。吞吐扩展走横向 加容器副本。CELERY_CONCURRENCY / MAX_TASKS_PER_CHILD 随之移除。 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -83,12 +83,17 @@ services:
|
||||
dockerfile: deploy/Dockerfile.moldinsight
|
||||
image: gemold-backend:latest
|
||||
container_name: gemold_celery
|
||||
command: ["sh", "-c", "cd /app/src && exec celery -A celery_app worker --concurrency=${CELERY_CONCURRENCY:-2} --max-tasks-per-child=${CELERY_MAX_TASKS_PER_CHILD:-50} --loglevel=info"]
|
||||
# --pool=solo:任务在 celery 主进程内执行。prefork 的 worker 子进程是
|
||||
# daemonic,multiprocessing 禁止守护进程再生子进程,OCC 常驻进程池
|
||||
# (OccProcessPool spawn 拉起)在 prefork 下必报
|
||||
# "daemonic processes are not allowed to have children"(2026-09-26)。
|
||||
# solo 天然单任务串行,与模块级单例(asyncio.Lock 不可跨循环并发)匹配;
|
||||
# 需要吞吐时横向加容器副本,而不是调并发。
|
||||
command: ["sh", "-c", "cd /app/src && exec celery -A celery_app worker --pool=solo --loglevel=info"]
|
||||
environment:
|
||||
<<: *base_env
|
||||
# Celery/OCC 吞吐调优(OCC_THROUGHPUT 方案 A)
|
||||
CELERY_CONCURRENCY: ${CELERY_CONCURRENCY:-2}
|
||||
CELERY_MAX_TASKS_PER_CHILD: ${CELERY_MAX_TASKS_PER_CHILD:-50}
|
||||
# 旧 CELERY_CONCURRENCY / CELERY_MAX_TASKS_PER_CHILD 已随 --pool=solo 移除:
|
||||
# solo 单进程串行,二者不适用(.env 里残留定义会被 compose 静默忽略)
|
||||
volumes:
|
||||
- uploads_data:/app/uploads
|
||||
# 注:本文件无 backend service,celery 必须等 moldinsight API 就绪后再启动
|
||||
|
||||
+10
-4
@@ -109,12 +109,18 @@ services:
|
||||
# cd /app/src 是必须的:celery_app / celery_tasks 是 src/ 的子模块,
|
||||
# 而其 import 用的是裸模块名(celery_app.py 内 include=["celery_tasks"]),
|
||||
# 所以 worker 必须从 src/ 目录启动;不依赖 celery 版本是否支持 --workdir
|
||||
command: ["sh", "-c", "cd /app/src && exec celery -A celery_app worker --concurrency=${CELERY_CONCURRENCY:-2} --max-tasks-per-child=${CELERY_MAX_TASKS_PER_CHILD:-50} --loglevel=info"]
|
||||
# --pool=solo:任务在 celery 主进程内执行。prefork 的 worker 子进程是
|
||||
# daemonic,multiprocessing 禁止守护进程再生子进程,OCC 常驻进程池
|
||||
# (OccProcessPool spawn 拉起)在 prefork 下必报
|
||||
# "daemonic processes are not allowed to have children"(2026-09-26)。
|
||||
# solo 天然单任务串行,与 processing_service / redis_task_manager /
|
||||
# db_manager 等模块级单例(其 asyncio.Lock 不可跨循环并发)也匹配;
|
||||
# 需要吞吐时横向加容器副本,而不是调并发。
|
||||
command: ["sh", "-c", "cd /app/src && exec celery -A celery_app worker --pool=solo --loglevel=info"]
|
||||
environment:
|
||||
<<: *base_env
|
||||
# Celery/OCC 吞吐调优(OCC_THROUGHPUT 方案 A)
|
||||
CELERY_CONCURRENCY: ${CELERY_CONCURRENCY:-2}
|
||||
CELERY_MAX_TASKS_PER_CHILD: ${CELERY_MAX_TASKS_PER_CHILD:-50}
|
||||
# 旧 CELERY_CONCURRENCY / CELERY_MAX_TASKS_PER_CHILD 已随 --pool=solo 移除:
|
||||
# solo 单进程串行,二者不适用(.env 里残留定义会被 compose 静默忽略)
|
||||
# uploads_data 共享卷(D6 过渡兜底):RustFS 异常时 worker 回退本地路径下载。
|
||||
# D11 后 worker 不再写 HTML 产物(直传 RustFS 报告键),无需 html_data 卷
|
||||
volumes:
|
||||
|
||||
Reference in New Issue
Block a user