feat(job-registry): remplace /api/admin/internal/trigger-*

remplace /api/admin/internal/trigger-* to /api/admin/jobs/{...}/trigger
remove OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS
This commit is contained in:
Edouard Vanbelle
2026-07-27 23:40:40 +02:00
parent dfedde54a4
commit f66f7fa31f
24 changed files with 155 additions and 464 deletions
+5 -5
View File
@@ -30,7 +30,7 @@
# same `v_dest_drive_id` variable, so (a) passing implies
# file rows used the same value and (b) cross-checks it.
#
# Sweep convergence: `/api/admin/internal/trigger-sweep` is the
# Sweep convergence: `/api/admin/jobs/storage_reconcile/trigger` is the
# deterministic synchronisation point — without it the
# fire-and-forget delta hook may not yet have updated the cached
# `used_bytes` when we read it.
@@ -136,7 +136,7 @@ file_id: jsonpath "$.id"
# numbers, the late hook adds its delta on top, and used_bytes ends
# up high by exactly one file's size. Symptom: expected 32, got 64.
# Real fix is await'ing the hook inline server-side.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -171,7 +171,7 @@ HTTP 200
[Captures]
shared_file_id: jsonpath "$.successful[0].id"
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -259,7 +259,7 @@ HTTP 201
# the file's size into the cached counter. 200 ms is well above
# the tokio task latency on any reasonable box; the deterministic
# fix would be intra-transaction hooks, deferred until D7.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -348,7 +348,7 @@ jsonpath "$.name" == "dc-subtree-inner"
# the Step 6 file copy (32) = 96. Anything other than (96, 96)
# would mean the file INSERT in copy_folder_tree used the wrong
# drive_id.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
+5 -5
View File
@@ -17,7 +17,7 @@
# a file inside and watching the destination drive's
# `used_bytes` jump by the descendant's size (not 0).
#
# Sweep convergence: `/api/admin/internal/trigger-sweep` is the
# Sweep convergence: `/api/admin/jobs/storage_reconcile/trigger` is the
# deterministic synchronisation point — it recomputes every
# drive's cached `used_bytes` from `SUM(file.size) WHERE
# drive_id = d.id`. If the file/folder move didn't update
@@ -125,7 +125,7 @@ file_id: jsonpath "$.id"
# Baseline used_bytes after the upload settles. Trigger-sweep is
# the deterministic sync point — but only after the spawn'd hook
# has had a chance to land (bug_trigger_sweep_vs_spawn_hook_race.md).
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -161,7 +161,7 @@ Content-Type: application/json
HTTP 200
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -245,7 +245,7 @@ nested_file_id: jsonpath "$.id"
# size. Symptom: expected 64, got 96 (one extra hook landed late).
# Real fix is await'ing the hook inline server-side; until then this
# delay deflakes the test.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -278,7 +278,7 @@ HTTP 200
# shared: nested hello-copy.txt now charged here (32)
# Anything other than (32, 32) means the descendant file's
# drive_id wasn't cascaded by the trigger.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
+13 -15
View File
@@ -120,7 +120,7 @@ small_file_id: jsonpath "$.id"
# Ed's 2026-07-17 design call: the sweep is the escape hatch
# for tests / operators that need immediate cache freshness;
# per-write invalidation would nuke the cache on every upload.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -157,7 +157,7 @@ HTTP 201
# `used_bytes` climbs to 64 (32 + 32). Same trigger-sweep pattern
# as the first assertion — the delta is fire-and-forget and the
# listing cache lags until the sweep invalidates it.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -194,7 +194,7 @@ HTTP 507
# consumed by the intervening GET which re-populated the cache
# with the pre-refused-write value. Sweep + re-check for
# determinism.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
@@ -238,7 +238,7 @@ HTTP 201
# Unlimited drive's `used_bytes` climbs to the file's exact size
# (5 MiB = 5_242_880 bytes). Trigger-sweep pattern (see above).
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -266,9 +266,7 @@ jsonpath "$[?(@.id=='{{unlimited_drive_id}}')].used_bytes" == 5242880
# tight drive; the 5 MiB is in the unlimited one).
# b) Permanently delete via empty-trash.
# c) Trigger the reconciliation sweep on demand —
# `/api/admin/internal/trigger-sweep` is gated by
# `OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true`
# (set in `tests/common/server.env`).
# `POST /api/admin/jobs/storage_reconcile/trigger`.
# d) `GET /api/drives` now shows the corrected counter.
# ─────────────────────────────────────────────────────────────
DELETE {{base_url}}/api/files/{{small_file_id}}
@@ -286,7 +284,7 @@ HTTP 200
# Sweep is fire-and-forget on a ticker (default 600 s). Run it now
# so the assertion below is deterministic instead of polling.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
@@ -312,7 +310,7 @@ jsonpath "$[?(@.id=='{{unlimited_drive_id}}')].used_bytes" == 5242880
# the route in production configs; here we just confirm a
# non-admin caller is refused even when the feature is on.
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{owner_token}}
HTTP 403
@@ -322,14 +320,14 @@ HTTP 403
# summary shape. Hard count is non-deterministic (depends on the
# grace window vs the test's elapsed time), so we only check the
# response shape.
POST {{base_url}}/api/admin/internal/trigger-gc
POST {{base_url}}/api/admin/jobs/dedup_gc/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
[Asserts]
jsonpath "$.ok" == true
jsonpath "$.blobs_deleted" exists
jsonpath "$.bytes_freed" exists
jsonpath "$.outcome.count" exists
jsonpath "$.outcome.extra.bytes_reclaimed" exists
# ─────────────────────────────────────────────────────────────
@@ -417,7 +415,7 @@ HTTP 200
# operations above never wrote anything. Trigger-sweep so the
# check reads live SQL (see the class doc on the earlier
# sweep + GET pair for the design rationale).
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
@@ -572,7 +570,7 @@ HTTP 201
soft_shrink_file_id: jsonpath "$.id"
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -656,7 +654,7 @@ Authorization: Bearer {{owner_token}}
HTTP 200
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
+16 -16
View File
@@ -12,12 +12,10 @@
# assertions here catch it before the daemon runs against real
# data.
#
# Uses the `POST /api/admin/internal/trigger-grant-cleanup`
# admin endpoint (gated by
# `OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true`, on for the
# api-test suite). `?force=true` collapses the grace window to
# zero for the call so we can plant a past-dated grant and
# immediately observe it purged, without waiting 15+ days.
# Uses the `POST /api/admin/jobs/grant_cleanup/trigger` admin
# endpoint (production surface, always on). `?force=true` collapses
# the grace window to zero for the call so we can plant a past-dated
# grant and immediately observe it purged, without waiting 15+ days.
# =============================================================
@@ -150,16 +148,17 @@ permanent_grant_id: jsonpath "$.grants[0].id"
# Expect `grants_deleted >= 1` (the past-dated row),
# `grace_days == 0`, `forced == true`.
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/admin/internal/trigger-grant-cleanup?force=true
POST {{base_url}}/api/admin/jobs/grant_cleanup/trigger?force=true
Authorization: Bearer {{alice_token}}
HTTP 200
[Asserts]
jsonpath "$.ok" == true
jsonpath "$.forced" == true
jsonpath "$.grace_days" == 0
jsonpath "$.outcome.outcome" == "ok"
jsonpath "$.outcome.extra.forced" == true
jsonpath "$.outcome.extra.grace_days" == 0
# At least the expired-fixture row we just planted.
jsonpath "$.grants_deleted" >= 1
jsonpath "$.outcome.count" >= 1
# ─────────────────────────────────────────────────────────────
@@ -194,28 +193,29 @@ jsonpath "$[?(@.id=='{{permanent_grant_id}}')].role" == "viewer"
# false in SQL, so it's already correct; but a
# mistyped predicate could regress).
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/admin/internal/trigger-grant-cleanup?force=true
POST {{base_url}}/api/admin/jobs/grant_cleanup/trigger?force=true
Authorization: Bearer {{alice_token}}
HTTP 200
[Asserts]
jsonpath "$.grants_deleted" == 0
jsonpath "$.outcome.count" == 0
# ─────────────────────────────────────────────────────────────
# Step 9 — Unforced trigger. Grace = configured value (15).
# No new expired grants planted, so purge is a no-op.
# ─────────────────────────────────────────────────────────────
POST {{base_url}}/api/admin/internal/trigger-grant-cleanup
POST {{base_url}}/api/admin/jobs/grant_cleanup/trigger
Authorization: Bearer {{alice_token}}
HTTP 200
[Asserts]
jsonpath "$.ok" == true
jsonpath "$.forced" == false
jsonpath "$.outcome.outcome" == "ok"
jsonpath "$.outcome.extra.forced" == false
# Response echoes the configured grace (15 days by default).
jsonpath "$.grace_days" == 15
jsonpath "$.grants_deleted" == 0
jsonpath "$.outcome.extra.grace_days" == 15
jsonpath "$.outcome.count" == 0
# Permanent grant still there after the unforced call.
+18 -23
View File
@@ -265,34 +265,29 @@ log "API confirms trash is empty."
# its `clear_trash_in` path, but that GC honours the 1-hour orphan-grace
# window — a blob orphaned seconds ago survives the inline sweep. The
# regular periodic sweep would catch it eventually, but tests need the
# disk state to be quiescent NOW. The two admin-internal triggers below
# (gated by `OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true`, set in
# tests/common/server.env) make this deterministic:
# disk state to be quiescent NOW. The two JobRegistry admin triggers
# below (production surface, always on) make this deterministic:
#
# 1. trigger-sweep — reconciles users.storage_used_bytes and
# drives.used_bytes from SUM(size) — keeps the
# cached counters honest for any quota
# assertions that follow.
# 2. trigger-gc?force=true — same `garbage_collect()` as the inline
# call, but `force=true` bypasses the orphan
# grace so freshly-orphaned blobs ARE reaped.
# Safe here because the test has no concurrent
# uploaders to race the row-delete → unlink
# window the grace normally protects.
#
# Without `force=true`, the test would have to wait an hour for the
# probe blob's `orphaned_at` timestamp to age past the grace window —
# why this script was disabled until the admin-internal triggers
# landed (commit `74b33744`).
# 1. storage_reconcile — reconciles users.storage_used_bytes and
# drives.used_bytes from SUM(size) — keeps
# the cached counters honest for any quota
# assertions that follow.
# 2. dedup_gc?force=true — same `garbage_collect()` as the inline
# call, but `force=true` bypasses the
# orphan grace so freshly-orphaned blobs
# ARE reaped. Safe here because the test
# has no concurrent uploaders to race the
# row-delete → unlink window the grace
# normally protects.
curl -sf -X POST -H "$AUTH" "$base_url/api/admin/internal/trigger-sweep" >/dev/null \
|| fail "trigger-sweep failed (is OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true?)"
curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/storage_reconcile/trigger" >/dev/null \
|| fail "storage_reconcile trigger failed"
log "Reconciliation sweep triggered."
GC_RESULT=$(curl -sf -X POST -H "$AUTH" "$base_url/api/admin/internal/trigger-gc?force=true")
GC_RESULT=$(curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/dedup_gc/trigger?force=true")
[[ -z "$GC_RESULT" ]] && fail "trigger-gc returned an empty body"
GC_BLOBS=$(echo "$GC_RESULT" | jq -r '.blobs_deleted')
GC_BYTES=$(echo "$GC_RESULT" | jq -r '.bytes_freed')
GC_BLOBS=$(echo "$GC_RESULT" | jq -r '.outcome.count')
GC_BYTES=$(echo "$GC_RESULT" | jq -r '.outcome.extra.bytes_reclaimed')
log "GC reaped $GC_BLOBS blob(s), $GC_BYTES byte(s) freed."
# ── 4. Disk verification ──────────────────────────────────────────────────────
+9 -11
View File
@@ -19,13 +19,11 @@
# 4. Sweep self-heals — after trashing the personal file and
# `trigger-sweep`, `/me.storage_used_bytes` returns to 0.
#
# `trigger-sweep` is the deterministic synchronisation point:
# it runs the drive-side sweep then the user-side sweep
# (`StorageUsageService::start_reconciliation_job`), so both
# cached counters are authoritative ground-truth by the time
# the assertion fires. Gated by
# `OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true`
# (set in `tests/common/server.env`).
# `POST /api/admin/jobs/storage_reconcile/trigger` is the
# deterministic synchronisation point: it runs the drive-side sweep
# then the user-side sweep (both under the periodic scheduler), so
# both cached counters are authoritative ground-truth by the time
# the assertion fires. Production surface — always on, no gate.
#
# Self-contained: provisions `ue_owner` so it can run alongside
# the rest of the suite.
@@ -141,7 +139,7 @@ HTTP 201
# acts as the synchronisation point for the user-envelope
# assertion below — the sweep is the authoritative
# ground-truth for both drive- and user-side counters.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
[Options]
delay: 200ms
@@ -161,7 +159,7 @@ jsonpath "$[?(@.id=='{{shared_drive_id}}')].used_bytes" == 32
# If the delta path incorrectly fired the user counter, the sweep
# would still correct it back to 0 (the new SQL excludes shared
# drives) — this also validates the sweep formula.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
@@ -209,7 +207,7 @@ jsonpath "$.storage_used_bytes" == 32
# Confirm the sweep agrees with the delta — both code paths must
# give the same number.
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
@@ -239,7 +237,7 @@ Authorization: Bearer {{owner_token}}
HTTP 200
POST {{base_url}}/api/admin/internal/trigger-sweep
POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger
Authorization: Bearer {{admin_token}}
HTTP 200
@@ -28,12 +28,6 @@ OXICLOUD_OIDC_ENABLED=false
OXICLOUD_NEXTCLOUD_ENABLED=true
# Test-only sweep triggers (`/api/admin/internal/trigger-sweep`,
# `/api/admin/internal/trigger-gc`). Off by default in production;
# the Hurl suite needs them to assert post-delete quota convergence
# without waiting out the 600 s reconciliation tick.
OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true
RUST_LOG="warn,audit=info,sqlx::migrate=info"
#RUST_LOG="warn,audit=info,oxicloud::quota=debug"
#RUST_LOG=debug
@@ -79,7 +73,6 @@ OXICLOUD_MAGIC_LINK_SEND_PER_IP_PER_HOUR=50
# permits IP spoofing for tests
OXICLOUD_TRUST_PROXY_CIDR=0.0.0.0/0
OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true
# /webdav/ will points directly to list of drives
OXICLOUD_WEBDAV_DRIVE_LISTING_PREFIX=""
-1
View File
@@ -31,7 +31,6 @@ OXICLOUD_ENABLE_MUSIC=true
OXICLOUD_EXPOSE_SYSTEM_USERS=true
OXICLOUD_WOPI_ENABLED=false
OXICLOUD_NEXTCLOUD_ENABLED=true
OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true
RUST_LOG="warn,audit=info,oxicloud::infrastructure::services::oidc_service=info,oxicloud::application::services::auth_application_service=info"
-1
View File
@@ -30,7 +30,6 @@ OXICLOUD_ENABLE_MUSIC=true
OXICLOUD_EXPOSE_SYSTEM_USERS=true
OXICLOUD_WOPI_ENABLED=false
OXICLOUD_NEXTCLOUD_ENABLED=true
OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true
RUST_LOG="warn,audit=info,oxicloud::infrastructure::services::oidc_service=info,oxicloud::application::services::auth_application_service=info"
-7
View File
@@ -31,12 +31,6 @@ OXICLOUD_OIDC_ENABLED=false
OXICLOUD_NEXTCLOUD_ENABLED=true
# Test-only sweep triggers (`/api/admin/internal/trigger-sweep`,
# `/api/admin/internal/trigger-gc`). Off by default in production;
# the Hurl suite needs them to assert post-delete quota convergence
# without waiting out the 600 s reconciliation tick.
OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true
RUST_LOG="warn,audit=info,sqlx::migrate=info"
#RUST_LOG="warn,audit=info,oxicloud::quota=debug"
#RUST_LOG=debug
@@ -107,4 +101,3 @@ OXICLOUD_MAGIC_LINK_SEND_PER_IP_PER_HOUR=50
# permits IP spoofing for tests
OXICLOUD_TRUST_PROXY_CIDR=0.0.0.0/0
OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true