diff --git a/tests/api/backend_migration_blackhole.hurl b/tests/api/backend_migration_blackhole.hurl new file mode 100644 index 00000000..44337b57 --- /dev/null +++ b/tests/api/backend_migration_blackhole.hurl @@ -0,0 +1,165 @@ +# ============================================================= +# OxiCloud – backend_migration against an endpoint that never answers +# +# The regression test asked for by +# `docs/plan/jobs-handling-recoverable-error.md` §Testing: assert that +# an unreachable backend lands the run in **Paused**, that +# `error_message` names the cause, and that it gets there in **bounded +# time rather than hanging**. +# +# ## The failure this pins +# +# A backend that *fails* was always handled — classified, retried, +# paused. A backend that never *answers* was handled by nothing. Pull a +# network on an established connection and there is no RST and no ICMP; +# the socket blocks until the OS abandons retransmission, on the order +# of fifteen minutes. Throughout that window the job is neither running +# nor failed: no error, so no retry, no log line, no pause. It looks +# exactly like a slow migration. +# +# Worse, before the classification fixes the `blob_exists` source probe +# reported such a failure as PERMANENT, which took the +# record-a-finding-and-continue branch — the cursor advanced past the +# blob, and with `failed` still 0 the run could reach `finish_completed` +# and flip the pointer to a target missing everything the outage +# covered. A migration reporting success having silently dropped +# whatever was unreachable at the time. +# +# ## Why `s3_blackhole` and not `s3_stub` +# +# `s3_stub` points at `127.0.0.1:9999`, where nothing listens, so the +# connection is REFUSED instantly. That path was never broken. A test +# built on it would pass with no timeout configured anywhere and pin +# nothing. +# +# `s3_blackhole` points at `192.0.2.1` — TEST-NET-1 (RFC 5737), +# reserved for documentation and guaranteed unrouted. A SYN goes +# unanswered, which is the hang. See `tests/common/server.env`. +# +# ## Why this is safe inside the shared suite +# +# The migration fails at `target.initialize()`, which runs BEFORE +# `migration_readonly` is engaged (`backend_migration_service.rs`, the +# comment on the target-init pause). So this file cannot leave the +# server read-only for whatever runs after it — the reason this shape +# was chosen over a mid-copy failure, which would hold the freeze. +# +# The run is cancelled at the end regardless, so the DB is left with no +# non-terminal `backend_migration` row. +# +# Prerequisites: setup.hurl must have run (admin user exists). +# ============================================================= + + +# ───────────────────────────────────────────────────────────── +# Step 1 — Log in as admin. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/auth/login +Content-Type: application/json +{ + "username": "{{username}}", + "password": "{{password}}" +} + +HTTP 200 +[Captures] +admin_token: jsonpath "$.access_token" + + +# ───────────────────────────────────────────────────────────── +# Step 2 — Trigger the migration at the black hole. +# +# The trigger is synchronous, so the response IS the outcome. +# +# `outcome.outcome == "ok"` is deliberate and not a contradiction: a +# retryable pause is carried as `Ok` because the handler did its job and +# stopped cleanly at a checkpoint. `extra.paused` / `extra.retryable` +# are what distinguish it, which is exactly why the scheduler log line +# had to stop projecting a paused run as a clean one. +# +# The `duration` assert is the heart of this file. Everything else here +# would also pass against the old hanging behaviour — given fifteen +# minutes. This is the only assertion that fails if the bound is ever +# removed, so treat it as load-bearing rather than a performance nicety. +# +# 120s: comfortably above the observed ~31s (the SDK's own attempts +# stacked on the 10s connect timeout) and far below the ~15min the +# unbounded socket would take. Deliberately loose — a slow CI runner +# must not make this flaky, and the failure it guards against is three +# orders of magnitude away, not adjacent. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/jobs/backend_migration/trigger?storage=s3_blackhole +Authorization: Bearer {{admin_token}} + +HTTP 200 +[Captures] +blackhole_run_id: jsonpath "$.outcome.extra.run_id" +[Asserts] +duration < 120000 +jsonpath "$.ok" == true +jsonpath "$.outcome.outcome" == "ok" +jsonpath "$.outcome.extra.paused" == true +jsonpath "$.outcome.extra.retryable" == true +jsonpath "$.outcome.extra.run_id" exists +# The reason must name what went wrong, not merely that something did. +# An operator reading only this string has to be able to tell an +# unreachable backend from a wrong bucket — the first is worth waiting +# out, the second never resolves on its own. +jsonpath "$.outcome.extra.reason" contains "Transient Backend" +jsonpath "$.outcome.extra.reason" contains "target backend init" + + +# ───────────────────────────────────────────────────────────── +# Step 3 — The run row must agree with the outcome. +# +# `Paused`, not `Failed`: the distinction is the whole plan. Failed is +# terminal and needs a human to decide what happened; Paused resumes +# and finishes the migration once the backend returns. +# +# `completed_at` must be absent — the run is not over. A paused row +# carrying a completion timestamp would make every "how long did this +# take" query lie, and would read as finished in the admin panel. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/admin/jobs/backend_migration/runs/{{blackhole_run_id}} +Authorization: Bearer {{admin_token}} + +HTTP 200 +[Asserts] +jsonpath "$.status" == "Paused" +jsonpath "$.error_message" exists +jsonpath "$.error_message" contains "Transient Backend" +jsonpath "$.completed_at" not exists + + +# ───────────────────────────────────────────────────────────── +# Step 4 — Teardown: cancel the paused run. +# +# Mandatory, not tidiness. `open_or_start` picks up the latest +# non-terminal row for a job name, so a `Paused` row left behind would +# be RESUMED by the next `backend_migration` trigger in the suite — +# silently retargeting that run at the black hole and failing a test +# that has nothing to do with this file. Hurl files share one database. +# +# Cancel is also the path that releases `migration_readonly` for a +# paused row (nothing to release here — this run never engaged it — +# but the call is idempotent). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/jobs/backend_migration/cancel +Authorization: Bearer {{admin_token}} + +HTTP 200 +[Asserts] +jsonpath "$.cancelled" == true +jsonpath "$.run_id" == "{{blackhole_run_id}}" + + +# ───────────────────────────────────────────────────────────── +# Step 5 — Confirm the row is terminal, so the next trigger in the +# suite starts fresh instead of resuming ours. +# ───────────────────────────────────────────────────────────── +GET {{base_url}}/api/admin/jobs/backend_migration/runs/{{blackhole_run_id}} +Authorization: Bearer {{admin_token}} + +HTTP 200 +[Asserts] +jsonpath "$.status" == "Cancelled" diff --git a/tests/api/run.sh b/tests/api/run.sh index 10643e8d..7aa66375 100755 --- a/tests/api/run.sh +++ b/tests/api/run.sh @@ -227,6 +227,13 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test "$API_DIR/nfc_normalization.hurl" \ "$API_DIR/wopi_authz.hurl" \ "$API_DIR/wopi_shared_drive.hurl" \ + `# Second-to-last. The slowest file in the suite BY DESIGN: it waits` \ + `# out an unreachable endpoint (~31s) to prove the wait is bounded, so` \ + `# that cost belongs at the end rather than in the middle. It leaves no` \ + `# read-only freeze behind — the migration fails at target init, before` \ + `# the gate is engaged — and cancels its own run, so the shared DB is` \ + `# clean for whatever follows.` \ + "$API_DIR/backend_migration_blackhole.hurl" \ `# LAST, deliberately — and kept last even though it no longer cuts` \ `# the storage pointer over. It is the only scenario that depends on a` \ `# second service (Azurite on 10000), so if that container is missing` \ diff --git a/tests/common/server.env b/tests/common/server.env index a661242d..77c6760d 100644 --- a/tests/common/server.env +++ b/tests/common/server.env @@ -42,7 +42,7 @@ OXICLOUD_NEXTCLOUD_ENABLED=true # `s3_stub` is declared but never activated — it lets storage_config.hurl # assert the entries table has more than one row without needing a # real S3 backend. -OXICLOUD_STORAGE_ENTRIES=local_main,s3_stub,azurite +OXICLOUD_STORAGE_ENTRIES=local_main,s3_stub,azurite,s3_blackhole OXICLOUD_STORAGE_local_main_BACKEND=local OXICLOUD_STORAGE_s3_stub_BACKEND=s3 OXICLOUD_STORAGE_s3_stub_S3_BUCKET=oxicloud-test-stub @@ -51,6 +51,36 @@ OXICLOUD_STORAGE_s3_stub_S3_ENDPOINT_URL=http://127.0.0.1:9999 OXICLOUD_STORAGE_s3_stub_S3_ACCESS_KEY=stub OXICLOUD_STORAGE_s3_stub_S3_SECRET_KEY=stub +# `s3_blackhole` — an endpoint that never answers, as opposed to +# `s3_stub` above which refuses instantly. +# +# The distinction is the entire point. `127.0.0.1:9999` has nothing +# listening, so a connection is REFUSED: the kernel returns ECONNREFUSED +# immediately and the SDK reports an error straight away. That path was +# always handled. A test built on it would pass even with no timeout +# configured anywhere. +# +# `192.0.2.1` is TEST-NET-1 (RFC 5737), reserved for documentation and +# guaranteed not to be routed. A SYN to it goes unanswered — no RST, no +# ICMP — which is the failure that used to hang: with no bound, a socket +# read blocks until the OS abandons retransmission, on the order of +# fifteen minutes, during which the job is neither running nor failed. +# +# Declared but NEVER activated, like the other two. `backend_migration +# ?storage=s3_blackhole` reaches it explicitly. +# +# If a CI network answers 192.0.2.1 with ICMP unreachable, the failure +# degrades to the refused shape and `backend_migration_blackhole.hurl` +# still passes — both classify transient and both pause. It would simply +# stop pinning the timeout specifically. The assert on elapsed time in +# that file is what would notice. +OXICLOUD_STORAGE_s3_blackhole_BACKEND=s3 +OXICLOUD_STORAGE_s3_blackhole_S3_BUCKET=oxicloud-blackhole +OXICLOUD_STORAGE_s3_blackhole_S3_REGION=us-east-1 +OXICLOUD_STORAGE_s3_blackhole_S3_ENDPOINT_URL=http://192.0.2.1:9999 +OXICLOUD_STORAGE_s3_blackhole_S3_ACCESS_KEY=blackhole +OXICLOUD_STORAGE_s3_blackhole_S3_SECRET_KEY=blackhole + # `azurite` — a REAL, reachable Azure backend, unlike `s3_stub` above. # It points at the Azurite emulator started by spawn-db.sh, which speaks # the actual Blob REST API, so this is the only way to exercise the Azure